diff --git a/.gitattributes b/.gitattributes index e049502a8747cf91fc77ce26f22ecf4f6924a342..261276a3d29876775a9f15ab8131ba38ba28ff0d 100644 --- a/.gitattributes +++ b/.gitattributes @@ -41,3 +41,7 @@ f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_2 f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/wandb/wandb/run-20261011_021843-lzgz3u05/run-lzgz3u05.wandb filter=lfs diff=lfs merge=lfs -text f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/wandb/wandb/run-20261010_231200-gdb9n0eb/run-gdb9n0eb.wandb filter=lfs diff=lfs merge=lfs -text f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/wandb/wandb/run-20261011_160551-acr8mx9p/run-acr8mx9p.wandb filter=lfs diff=lfs merge=lfs -text +f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints/iter_0002384/__18_0.distcp filter=lfs diff=lfs merge=lfs -text +f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints/iter_0002384/__29_1.distcp filter=lfs diff=lfs merge=lfs -text +f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints/iter_0002384/__21_0.distcp filter=lfs diff=lfs merge=lfs -text +f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints/iter_0002384/__49_0.distcp filter=lfs diff=lfs merge=lfs -text diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/hf_export_logs/convert_iter1920.log b/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/hf_export_logs/convert_iter1920.log new file mode 100644 index 0000000000000000000000000000000000000000..7d6a27c49cfba148d373ce4f51df00128034f88a --- /dev/null +++ b/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/hf_export_logs/convert_iter1920.log @@ -0,0 +1,2665 @@ +++ realpath /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/scripts/mcore_gdn_moe/../../../.. ++ MEGATRON_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher ++ CONVERTOR_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor ++ REPAIR_SCRIPT=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/utils/repair_gdn_checkpoint.py ++ RENDER_CONFIG_SCRIPT=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/utils/render_gdn_hf_config.py ++ export PYTHON_ENV_ROOT=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud ++ PYTHON_ENV_ROOT=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud ++ PYTHON_BIN=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python ++ '[' '!' -x /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python ']' ++ '[' '!' -x /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python ']' ++ '[' '!' -x /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python ']' ++ '[' '!' -x /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python ']' ++ export PYTHON_BIN ++ /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python -c 'import torch' ++ export PATH=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin:/root/.kimi-code/bin:/root/.local/bin:/mnt/hdd/minyingqian/mihomo/bin:/root/.grok/bin:/usr/local/bin:/root/miniconda3/bin:/root/.local/bin:/usr/local/bin:/root/miniconda3/bin:/root/miniconda3/bin:/mnt/ssd/liuzian/pandoc/current/bin:/mnt/ssd/cache_tmp/.local/share/../bin:/mnt/ssd/yulan/bin:/mnt/ssd/yulan/bin:/root/.local/bin:/usr/local/bin:/root/miniconda3/bin:/root/.local/bin:/usr/local/bin:/root/miniconda3/bin:/mnt/ssd/cache_tmp/.local/bin:/root/.local/bin:/root/.local/bin:/mnt/hdd/minyingqian/mihomo/bin:/root/.grok/bin:/usr/local/bin:/root/miniconda3/bin:/root/.local/bin:/usr/local/bin:/root/miniconda3/bin:/root/miniconda3/bin:/mnt/ssd/liuzian/pandoc/current/bin:/mnt/ssd/cache_tmp/.local/bin:/mnt/ssd/yulan/bin:/mnt/ssd/yulan/bin:/root/.local/bin:/root/miniconda3/bin:/root/miniconda3/condabin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/mnt/ssd/yulan/cache/installs/nsight-compute/2026.2:/mnt/ssd/yulan/cache/installs/nsight-systems/2026.3.1/bin:/mnt/ssd/yulan/cache/installs/nsight-compute/2026.2:/mnt/ssd/yulan/cache/installs/nsight-systems/2026.3.1/bin ++ PATH=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin:/root/.kimi-code/bin:/root/.local/bin:/mnt/hdd/minyingqian/mihomo/bin:/root/.grok/bin:/usr/local/bin:/root/miniconda3/bin:/root/.local/bin:/usr/local/bin:/root/miniconda3/bin:/root/miniconda3/bin:/mnt/ssd/liuzian/pandoc/current/bin:/mnt/ssd/cache_tmp/.local/share/../bin:/mnt/ssd/yulan/bin:/mnt/ssd/yulan/bin:/root/.local/bin:/usr/local/bin:/root/miniconda3/bin:/root/.local/bin:/usr/local/bin:/root/miniconda3/bin:/mnt/ssd/cache_tmp/.local/bin:/root/.local/bin:/root/.local/bin:/mnt/hdd/minyingqian/mihomo/bin:/root/.grok/bin:/usr/local/bin:/root/miniconda3/bin:/root/.local/bin:/usr/local/bin:/root/miniconda3/bin:/root/miniconda3/bin:/mnt/ssd/liuzian/pandoc/current/bin:/mnt/ssd/cache_tmp/.local/bin:/mnt/ssd/yulan/bin:/mnt/ssd/yulan/bin:/root/.local/bin:/root/miniconda3/bin:/root/miniconda3/condabin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/mnt/ssd/yulan/cache/installs/nsight-compute/2026.2:/mnt/ssd/yulan/cache/installs/nsight-systems/2026.3.1/bin:/mnt/ssd/yulan/cache/installs/nsight-compute/2026.2:/mnt/ssd/yulan/cache/installs/nsight-systems/2026.3.1/bin ++ export VIRTUAL_ENV=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud ++ VIRTUAL_ENV=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud ++ echo ✅MEGATRON_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher +✅MEGATRON_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher ++ echo ✅PYTHON_BIN=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python +✅PYTHON_BIN=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python ++ export CUDA_LAUNCH_BLOCKING=1 ++ CUDA_LAUNCH_BLOCKING=1 ++ export CUDA_VISIBLE_DEVICES=0 ++ CUDA_VISIBLE_DEVICES=0 ++ export KUBERNETES_CONTAINER_RESOURCE_GPU=1 ++ KUBERNETES_CONTAINER_RESOURCE_GPU=1 ++ export TP=1 ++ TP=1 ++ export PP=1 ++ PP=1 ++ export ETP=1 ++ ETP=1 ++ export EP=1 ++ EP=1 ++ export 'MODEL_PARALLEL_ARGS=--tensor-model-parallel-size 1 --pipeline-model-parallel-size 1 --expert-tensor-parallel-size 1 --expert-model-parallel-size 1' ++ MODEL_PARALLEL_ARGS='--tensor-model-parallel-size 1 --pipeline-model-parallel-size 1 --expert-tensor-parallel-size 1 --expert-model-parallel-size 1' ++ WORKSPACE_TMP_DIR=/tmp/mcore2hf_ws_1920 ++ mkdir -p /tmp/mcore2hf_ws_1920 ++ MEGATRON_CHECKPOINT_PATH=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/load_view1920 ++ echo MEGATRON_CHECKPOINT_PATH=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/load_view1920 +MEGATRON_CHECKPOINT_PATH=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/load_view1920 ++ MANUAL_ITER= ++ CONVERT_ALL=false ++ FORCE_RECONVERT=false ++ DROP_MTP=false ++ POSITIONAL_ARGS=() ++ [[ 3 -gt 0 ]] ++ case $1 in ++ POSITIONAL_ARGS+=("$1") ++ shift ++ [[ 2 -gt 0 ]] ++ case $1 in ++ '[' -z 1920 ']' ++ MANUAL_ITER=1920 ++ shift 2 ++ [[ 0 -gt 0 ]] ++ set -- /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/load_view1920 ++ '[' -e /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/load_view1920 ']' +++ readlink -f /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/load_view1920 ++ MEGATRON_CHECKPOINT_PATH=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/load_view1920 ++ echo ✅MEGATRON_CHECKPOINT_PATH=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/load_view1920 +✅MEGATRON_CHECKPOINT_PATH=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/load_view1920 +++ basename /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/load_view1920 ++ CHECKPOINT_BASENAME=load_view1920 ++ [[ load_view1920 =~ ^iter_([0-9]+)$ ]] ++ '[' false = true ']' ++ HF_CONFIG_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/hf-configs/YuLan-Mini-for-gdn-convert ++ echo HF_CONFIG_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/hf-configs/YuLan-Mini-for-gdn-convert +HF_CONFIG_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/hf-configs/YuLan-Mini-for-gdn-convert ++ '[' '!' -f /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/hf-configs/YuLan-Mini-for-gdn-convert/config.template.json ']' +++ mktemp -d /tmp/mcore2hf_ws_1920/mcore2hf-config.XXXXXX ++ REAL_CONFIG_PATH=/tmp/mcore2hf_ws_1920/mcore2hf-config.pNLl8c ++ LOAD_VIEW_DIR= ++ trap cleanup_real_config_path EXIT ++ export PYTHONPATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher: ++ PYTHONPATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher: ++ export PYTHONPATH=/tmp/mcore2hf_ws_1920/mcore2hf-config.pNLl8c:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher: ++ PYTHONPATH=/tmp/mcore2hf_ws_1920/mcore2hf-config.pNLl8c:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher: +++ /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/utils/parse-linear-vars-from-mcore-config.py /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/load_view1920 +W1005 16:20:29.395000 317722 torch/utils/cpp_extension.py:2425] TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +W1005 16:20:29.395000 317722 torch/utils/cpp_extension.py:2425] If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'] to specific architectures. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. ++ eval 'export LINEAR_NUM_KEY_HEADS=8 +export LINEAR_NUM_VALUE_HEADS=32 +export LINEAR_ATTN_FREQ=[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1] +export FFN_HIDDEN_SIZE=4800 +export NUM_LAYERS=56 +export HIDDEN_SIZE=2048 +export NUM_ATTN_HEADS=32 +export NUM_QUERY_GROUPS=8 +export MAX_POSITION_EMBEDDINGS=131072 +export KV_CHANNELS=64 +export LINEAR_CONV_KERNEL_DIM=4 +export LINEAR_KEY_HEAD_DIM=64 +export LINEAR_VALUE_HEAD_DIM=64 +export MOE_FFN_HIDDEN_SIZE=512 +export NUM_EXPERTS=192 +export MOE_ROUTER_TOPK=4 +export MOE_ROUTER_SCORE_FUNCTION=softmax +export MOE_ROUTER_ENABLE_EXPERT_BIAS=false +export MOE_LAYER_FREQ=[0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1] +export MLP_ONLY_LAYERS=[0,1] +export MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512 +export ROPE_THETA=10000.0 +export POSITION_EMBEDDING_TYPE=none +export ATTENTION_OUTPUT_GATE=true +export MTP_NUM_LAYERS=0 +export MOE_ROUTER_SQRT_GATE=true +export GATED_NORM=true +export GATED_NORM_RANK=16 +export GATED_NORM_GATE_SCALE=1.0 +export GATED_NORM_UP_BIAS=true +export FINAL_GATED_NORM=true +export ACTIVATION_FUNC_CLAMP_VALUE=7.0 +export ACTIVATION_FUNC_CLAMP_MODE=soft +export FINAL_LOGITS_CLAMP_VALUE=30.0 +export MOE_ROUTER_DTYPE=fp32 +export TORCH_DTYPE=bfloat16 +export VOCAB_SIZE=99072' +++ export LINEAR_NUM_KEY_HEADS=8 +++ LINEAR_NUM_KEY_HEADS=8 +++ export LINEAR_NUM_VALUE_HEADS=32 +++ LINEAR_NUM_VALUE_HEADS=32 +++ export 'LINEAR_ATTN_FREQ=[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1]' +++ LINEAR_ATTN_FREQ='[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1]' +++ export FFN_HIDDEN_SIZE=4800 +++ FFN_HIDDEN_SIZE=4800 +++ export NUM_LAYERS=56 +++ NUM_LAYERS=56 +++ export HIDDEN_SIZE=2048 +++ HIDDEN_SIZE=2048 +++ export NUM_ATTN_HEADS=32 +++ NUM_ATTN_HEADS=32 +++ export NUM_QUERY_GROUPS=8 +++ NUM_QUERY_GROUPS=8 +++ export MAX_POSITION_EMBEDDINGS=131072 +++ MAX_POSITION_EMBEDDINGS=131072 +++ export KV_CHANNELS=64 +++ KV_CHANNELS=64 +++ export LINEAR_CONV_KERNEL_DIM=4 +++ LINEAR_CONV_KERNEL_DIM=4 +++ export LINEAR_KEY_HEAD_DIM=64 +++ LINEAR_KEY_HEAD_DIM=64 +++ export LINEAR_VALUE_HEAD_DIM=64 +++ LINEAR_VALUE_HEAD_DIM=64 +++ export MOE_FFN_HIDDEN_SIZE=512 +++ MOE_FFN_HIDDEN_SIZE=512 +++ export NUM_EXPERTS=192 +++ NUM_EXPERTS=192 +++ export MOE_ROUTER_TOPK=4 +++ MOE_ROUTER_TOPK=4 +++ export MOE_ROUTER_SCORE_FUNCTION=softmax +++ MOE_ROUTER_SCORE_FUNCTION=softmax +++ export MOE_ROUTER_ENABLE_EXPERT_BIAS=false +++ MOE_ROUTER_ENABLE_EXPERT_BIAS=false +++ export 'MOE_LAYER_FREQ=[0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]' +++ MOE_LAYER_FREQ='[0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]' +++ export 'MLP_ONLY_LAYERS=[0,1]' +++ MLP_ONLY_LAYERS='[0,1]' +++ export MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512 +++ MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512 +++ export ROPE_THETA=10000.0 +++ ROPE_THETA=10000.0 +++ export POSITION_EMBEDDING_TYPE=none +++ POSITION_EMBEDDING_TYPE=none +++ export ATTENTION_OUTPUT_GATE=true +++ ATTENTION_OUTPUT_GATE=true +++ export MTP_NUM_LAYERS=0 +++ MTP_NUM_LAYERS=0 +++ export MOE_ROUTER_SQRT_GATE=true +++ MOE_ROUTER_SQRT_GATE=true +++ export GATED_NORM=true +++ GATED_NORM=true +++ export GATED_NORM_RANK=16 +++ GATED_NORM_RANK=16 +++ export GATED_NORM_GATE_SCALE=1.0 +++ GATED_NORM_GATE_SCALE=1.0 +++ export GATED_NORM_UP_BIAS=true +++ GATED_NORM_UP_BIAS=true +++ export FINAL_GATED_NORM=true +++ FINAL_GATED_NORM=true +++ export ACTIVATION_FUNC_CLAMP_VALUE=7.0 +++ ACTIVATION_FUNC_CLAMP_VALUE=7.0 +++ export ACTIVATION_FUNC_CLAMP_MODE=soft +++ ACTIVATION_FUNC_CLAMP_MODE=soft +++ export FINAL_LOGITS_CLAMP_VALUE=30.0 +++ FINAL_LOGITS_CLAMP_VALUE=30.0 +++ export MOE_ROUTER_DTYPE=fp32 +++ MOE_ROUTER_DTYPE=fp32 +++ export TORCH_DTYPE=bfloat16 +++ TORCH_DTYPE=bfloat16 +++ export VOCAB_SIZE=99072 +++ VOCAB_SIZE=99072 ++ echo LINEAR_NUM_KEY_HEADS=8 +LINEAR_NUM_KEY_HEADS=8 ++ echo LINEAR_NUM_VALUE_HEADS=32 +LINEAR_NUM_VALUE_HEADS=32 ++ echo 'LINEAR_ATTN_FREQ=[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1]' +LINEAR_ATTN_FREQ=[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1] ++ echo 'NUM_LAYERS=56 HIDDEN_SIZE=2048' +NUM_LAYERS=56 HIDDEN_SIZE=2048 ++ echo 'NUM_ATTN_HEADS=32 NUM_QUERY_GROUPS=8' +NUM_ATTN_HEADS=32 NUM_QUERY_GROUPS=8 ++ echo 'MAX_POSITION_EMBEDDINGS=131072 KV_CHANNELS=64' +MAX_POSITION_EMBEDDINGS=131072 KV_CHANNELS=64 ++ echo 'FFN_HIDDEN_SIZE=4800 MOE_FFN_HIDDEN_SIZE=512' +FFN_HIDDEN_SIZE=4800 MOE_FFN_HIDDEN_SIZE=512 ++ echo 'NUM_EXPERTS=192 MOE_ROUTER_TOPK=4' +NUM_EXPERTS=192 MOE_ROUTER_TOPK=4 ++ echo MOE_ROUTER_SCORE_FUNCTION=softmax +MOE_ROUTER_SCORE_FUNCTION=softmax ++ echo 'MLP_ONLY_LAYERS=[0,1]' +MLP_ONLY_LAYERS=[0,1] ++ echo 'MOE_LAYER_FREQ=[0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]' +MOE_LAYER_FREQ=[0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1] ++ echo MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512 +MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512 ++ echo 'LINEAR_CONV_KERNEL_DIM=4 LINEAR_KEY_HEAD_DIM=64 LINEAR_VALUE_HEAD_DIM=64' +LINEAR_CONV_KERNEL_DIM=4 LINEAR_KEY_HEAD_DIM=64 LINEAR_VALUE_HEAD_DIM=64 ++ echo ATTENTION_OUTPUT_GATE=true +ATTENTION_OUTPUT_GATE=true ++ echo ROPE_THETA=10000.0 +ROPE_THETA=10000.0 ++ echo POSITION_EMBEDDING_TYPE=none +POSITION_EMBEDDING_TYPE=none ++ echo MOE_ROUTER_SQRT_GATE=true +MOE_ROUTER_SQRT_GATE=true +++ /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/utils/parse-mom-vars-from-mcore-config.py /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/load_view1920 +W1005 16:20:44.157000 318176 torch/utils/cpp_extension.py:2425] TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +W1005 16:20:44.157000 318176 torch/utils/cpp_extension.py:2425] If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'] to specific architectures. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. ++ eval 'export MOM_ENABLED=false +export MOM_DO_MASK_SCALE=true +export MOM_TARGET_FFN_HIDDEN_SIZE=512' +++ export MOM_ENABLED=false +++ MOM_ENABLED=false +++ export MOM_DO_MASK_SCALE=true +++ MOM_DO_MASK_SCALE=true +++ export MOM_TARGET_FFN_HIDDEN_SIZE=512 +++ MOM_TARGET_FFN_HIDDEN_SIZE=512 ++ echo 'MOM_ENABLED=false MOM_DO_MASK_SCALE=true MOM_TARGET_FFN_HIDDEN_SIZE=512' +MOM_ENABLED=false MOM_DO_MASK_SCALE=true MOM_TARGET_FFN_HIDDEN_SIZE=512 ++ '[' false = true ']' ++ '[' 192 -gt 0 ']' ++ '[' 512 -le 0 ']' ++ '[' 10000.0 = 0.0 ']' +++ /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python - ++ ROPE_THETA=10000 ++ '[' -z 131072 ']' ++ '[' 131072 = 0 ']' ++ FFN_TOKEN_SHIFT= ++ FFN_INTERMEDIATE_TOKEN_SHIFT= ++ ATTN_TOKEN_SHIFT= ++ ATTN_Q_TOKEN_SHIFT= ++ ATTN_K_TOKEN_SHIFT= ++ ATTN_V_TOKEN_SHIFT= ++ TOKEN_SHIFT_CONV_SIZE=4 ++ TOKEN_SHIFT_CONV_INIT=default ++ export FFN_TOKEN_SHIFT FFN_INTERMEDIATE_TOKEN_SHIFT ATTN_TOKEN_SHIFT ++ export ATTN_Q_TOKEN_SHIFT ATTN_K_TOKEN_SHIFT ATTN_V_TOKEN_SHIFT ++ export TOKEN_SHIFT_CONV_SIZE TOKEN_SHIFT_CONV_INIT ++ export ROPE_THETA MAX_POSITION_EMBEDDINGS POSITION_EMBEDDING_TYPE ATTENTION_OUTPUT_GATE MOE_ROUTER_SQRT_GATE ++ ATTENTION_OUTPUT_GATE=true ++ /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/utils/render_gdn_hf_config.py --checkpoint-path /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/load_view1920 --template-path /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/hf-configs/YuLan-Mini-for-gdn-convert --output-dir /tmp/mcore2hf_ws_1920/mcore2hf-config.pNLl8c +W1005 16:20:56.928000 318423 torch/utils/cpp_extension.py:2425] TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +W1005 16:20:56.928000 318423 torch/utils/cpp_extension.py:2425] If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'] to specific architectures. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. ++ echo '✅config.json: /tmp/mcore2hf_ws_1920/mcore2hf-config.pNLl8c/config.json' +✅config.json: /tmp/mcore2hf_ws_1920/mcore2hf-config.pNLl8c/config.json ++ /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python - /tmp/mcore2hf_ws_1920/mcore2hf-config.pNLl8c/config.json +{ + "hidden_size": 2048, + "intermediate_size": 4800, + "num_hidden_layers": 56, + "num_attention_heads": 32, + "num_key_value_heads": 8, + "head_dim": 64, + "max_position_embeddings": 131072, + "rope_theta": 10000, + "attn_position_embedding_type": "nope", + "rnn_position_embedding_type": "nope", + "linear_num_key_heads": 8, + "linear_num_value_heads": 32, + "linear_key_head_dim": 64, + "linear_value_head_dim": 64, + "num_experts": 192, + "num_experts_per_tok": 4, + "attn_output_gate": true +} ++ sleep 5 ++ MODEL_SIZE=2.6b ++ LOAD_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/load_view1920 ++ MG2HF=true ++ USE_CUDA=false ++ PR=bf16 ++ HF_DIR=/tmp/mcore2hf_ws_1920/mcore2hf-config.pNLl8c ++ export DROP_MTP ++ PREPARED_LOAD_DIR= ++ prepare_load_dir_for_iter /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/load_view1920 1920 ++ local SOURCE_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/load_view1920 ++ local ITER_NUM=1920 ++ '[' -z 1920 ']' +++ printf %07d 1920 ++ local ITER_BASENAME=iter_0001920 ++ local SOURCE_ITER_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/load_view1920/iter_0001920 ++ '[' '!' -d /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/load_view1920/iter_0001920 ']' ++ local LATEST_VALUE= ++ '[' -f /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/load_view1920/latest_checkpointed_iteration.txt ']' +++ cat /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/load_view1920/latest_checkpointed_iteration.txt ++ LATEST_VALUE=1920 ++ '[' 1920 = 1920 ']' ++ PREPARED_LOAD_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/load_view1920 ++ return 0 ++ LOAD_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/load_view1920 ++ echo LOAD_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/load_view1920 +LOAD_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/load_view1920 ++ '[' false = true ']' ++ '[' false = true ']' ++ '[' '!' -f /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/load_view1920/latest_checkpointed_iteration.txt ']' +++ cat /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/load_view1920/latest_checkpointed_iteration.txt ++ LATEST_CHECKPOINT_ITERATION=1920 ++ echo -e '\033[32mLatest checkpoint iteration: 1920\033[0m\n' +Latest checkpoint iteration: 1920 + ++ I=1920 ++ [[ 1920 =~ ^[0-9]+$ ]] +++ printf %07d 1920 ++ ITER_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/load_view1920/iter_0001920 ++ '[' '!' -f /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/load_view1920/iter_0001920/common.pt ']' ++ SAVE_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/load_view1920/iter_1920-hf ++ '[' -n /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a ']' ++ SAVE_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/iter_1920-hf ++ '[' -n '' ']' ++ mkdir -p /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/iter_1920-hf ++ bash ./scripts/mcore_gdn_moe/convert.sh 2.6b /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/load_view1920 /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/iter_1920-hf true false bf16 /tmp/mcore2hf_ws_1920/mcore2hf-config.pNLl8c 1920 ++ tee /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/iter_1920-hf/hf2mcore.log +✅Enable attention output gate +\[\e[33mWarning adding --moe-shared-expert-gate for YuLan model. If you are not YuLan, please check this flag.\e[0m\] +torchrun --nproc_per_node 1 --nnodes 1 --node_rank 0 --master_addr localhost --master_port 50461 /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/impl/convert.py --tokenizer-type HuggingFaceTokenizer --tokenizer-model /tmp/mcore2hf_ws_1920/mcore2hf-config.pNLl8c --hf-dir /tmp/mcore2hf_ws_1920/mcore2hf-config.pNLl8c --mcore2hf --bf16 --normalization RMSNorm --swiglu --disable-bias-linear --seq-length 1 --max-position-embeddings 131072 --attention-backend auto --position-embedding-type none --kv-channels 64 --group-query-attention --add-qkv-bias --num-layers 56 --hidden-size 2048 --ffn-hidden-size 4800 --num-attention-heads 32 --rotary-base 10000 --rotary-percent 1.00 --num-query-groups 8 --normalization RMSNorm --norm-epsilon 1e-6 --untie-embeddings-and-output-weights --attention-output-gate --linear-attention-type gated_delta_net --linear-attention-freq [1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1] --linear-conv-kernel-dim 4 --linear-key-head-dim 64 --linear-value-head-dim 64 --linear-num-key-heads 8 --linear-num-value-heads 32 --num-experts 192 --moe-router-score-function softmax --moe-token-dispatcher-type alltoall --moe-router-topk 4 --moe-ffn-hidden-size 512 --moe-layer-freq [0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1] --moe-router-dtype fp32 --moe-shared-expert-intermediate-size 512 --moe-shared-expert-gate --moe-router-score-function softmax --moe-router-sqrt-gate --micro-batch-size 1 --global-batch-size 1024 --train-iters 500000 --weight-decay 0.1 --adam-beta1 0.9 --adam-beta2 0.95 --init-method-std 0.006 --clip-grad 1.0 --lr 2.0e-5 --lr-decay-style cosine --min-lr 6.0e-6 --lr-warmup-fraction .001 --lr-decay-iters 430000 --no-gradient-accumulation-fusion --bf16 --tensor-model-parallel-size 1 --pipeline-model-parallel-size 1 --expert-tensor-parallel-size 1 --expert-model-parallel-size 1 --log-interval 100 --save-interval 10000 --eval-interval 1000 --eval-iters 10 --model-type GPT --load-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/load_view1920 --save-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/iter_1920-hf --dist-ckpt-optim-fully-reshardable --skip-train --use-cpu-initialization --no-load-optim --no-load-rng --logging-level 1 --attention-backend auto --debug --max-shard-size 20GB --synchronizer mcore_gdn_moe --pretrain-script mcore_gdn_moe.model_provider --ckpt-step 1920 --gdn-split-in-proj --gated-norm --gated-norm-lora-rank 16 --distributed-backend gloo +W1005 16:21:16.620000 318917 torch/utils/cpp_extension.py:2425] TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +W1005 16:21:16.620000 318917 torch/utils/cpp_extension.py:2425] If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'] to specific architectures. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +Warning: Pai-Megatron-Patch arguments not available, some arguments may not be recognized +using world size: 1, data-parallel size: 1, context-parallel size: 1, hierarchical context-parallel sizes: None, tensor-model-parallel size: 1, pipeline-model-parallel size: 1 +Number of virtual stages per pipeline stage: None +accumulate and all-reduce gradients in fp32 for bfloat16 data type. +using torch.bfloat16 for parameters ... +------------------------ arguments ------------------------ + account_for_embedding_in_pipeline_split ......... False + account_for_loss_in_pipeline_split .............. False + accumulate_allreduce_grads_in_fp32 .............. True + activation_func_clamp_mode ...................... hard + activation_func_clamp_value ..................... None + adam_beta1 ...................................... 0.9 + adam_beta2 ...................................... 0.95 + adam_bias_correction ............................ True + adam_eps ........................................ 1e-08 + adamh_eps ....................................... 1e-08 + adamh_radius .................................... None + adamh_scope ..................................... lm_head + adamw_lr_mup_scaler ............................. False + add_bias_linear ................................. False + add_position_embedding .......................... True + add_qkv_bias .................................... True + adlr_autoresume ................................. False + adlr_autoresume_interval ........................ 1000 + align_grad_reduce ............................... True + align_param_gather .............................. False + allow_ambiguous_pad_tokens ...................... False + app_tag_run_name ................................ None + app_tag_run_version ............................. 0.0.0 + apply_layernorm_1p .............................. False + apply_query_key_layer_scaling ................... False + apply_residual_connection_post_layernorm ........ False + apply_rope_fusion ............................... False + async_save ...................................... None + async_tensor_model_parallel_allreduce ........... True + attention_backend ............................... AttnBackend.auto + attention_dropout ............................... 0.1 + attention_output_gate ........................... True + attention_output_gate_upcycling ................. False + attention_softmax_in_fp32 ....................... False + attn_k_token_shift .............................. None + attn_output_gate ................................ None + attn_output_gate_rand_init ...................... False + attn_q_token_shift .............................. None + attn_token_shift ................................ None + attn_v_token_shift .............................. None + auto_detect_ckpt_format ......................... False + auto_generate_cu_seqlens ........................ False + auto_model ...................................... AutoModelForCausalLM + barrier_with_L1_time ............................ True + benchmark_eval .................................. False + benchmark_global_batch .......................... None + benchmark_interval .............................. None + benchmark_micro_batch ........................... None + benchmark_sequence_length ....................... None + benchmark_tasks ................................. None + bert_binary_head ................................ True + bert_embedder_type .............................. megatron + bert_load ....................................... None + bf16 ............................................ True + bias_dropout_fusion ............................. True + bias_gelu_fusion ................................ False + bias_swiglu_fusion .............................. True + biencoder_projection_dim ........................ 0 + biencoder_shared_query_context_model ............ False + block_data_path ................................. None + cache_mla_latents ............................... False + calc_ft_timeouts ................................ False + calculate_per_token_loss ........................ False + check_for_large_grads ........................... False + check_for_nan_in_loss_and_grad .................. True + check_for_spiky_loss ............................ False + check_weight_hash_across_dp_replicas_interval ... None + ckpt_assume_constant_structure .................. False + ckpt_convert_format ............................. None + ckpt_convert_save ............................... None + ckpt_convert_update_legacy_dist_opt_format ...... False + ckpt_format ..................................... torch_dist + ckpt_fully_parallel_load ........................ False + ckpt_fully_parallel_save ........................ True + ckpt_fully_parallel_save_deprecated ............. False + ckpt_step ....................................... 1920 + classes_fraction ................................ 1.0 + clip_grad ....................................... 1.0 + clone_scatter_output_in_embedding ............... True + config_logger_dir ............................... + consumed_train_samples .......................... 0 + consumed_valid_samples .......................... 0 + context_parallel_size ........................... 1 + continual_pretrain .............................. False + continual_pretrain_preserve_opt_param_scheduler . False + corrected_weight_decay .......................... False + cp_comm_type .................................... ['p2p'] + create_attention_mask_in_dataloader ............. True + cross_entropy_fusion_impl ....................... native + cross_entropy_loss_fusion ....................... False + cuda_graph_impl ................................. none + cuda_graph_scope ................................ [] + cuda_graph_warmup_steps ......................... 3 + data_args_path .................................. None + data_cache_path ................................. None + data_parallel_random_init ....................... False + data_parallel_sharding_strategy ................. no_shard + data_parallel_size .............................. 1 + data_path ....................................... None + data_per_class_fraction ......................... 1.0 + data_sharding ................................... True + dataloader_defer_npy_index_mmap ................. False + dataloader_fast_cache_load ...................... False + dataloader_type ................................. single + dc_shortconv .................................... False + dc_shortconv_optimizer_state .................... conservative + dc_shortconv_upcycling .......................... False + ddp_average_in_collective ....................... False + ddp_bucket_size ................................. None + ddp_num_buckets ................................. None + ddp_pad_buckets_for_high_nccl_busbw ............. False + debug ........................................... True + decode_only_cuda_graphs ......................... False + decoder_first_pipeline_num_layers ............... None + decoder_last_pipeline_num_layers ................ None + decoder_num_layers .............................. None + decoder_seq_length .............................. None + decoupled_lr .................................... None + decoupled_min_lr ................................ None + decrease_batch_size_if_needed ................... False + defer_embedding_wgrad_compute ................... False + defer_grad_checks ............................... False + delay_wgrad_compute ............................. False + deprecated_use_mcore_models ..................... False + deterministic_mode .............................. False + dino_bottleneck_size ............................ 256 + dino_freeze_last_layer .......................... 1 + dino_head_hidden_size ........................... 2048 + dino_local_crops_number ......................... 10 + dino_local_img_size ............................. 96 + dino_norm_last_layer ............................ False + dino_teacher_temp ............................... 0.07 + dino_warmup_teacher_temp ........................ 0.04 + dino_warmup_teacher_temp_epochs ................. 30 + disable_attn_output_gate ........................ False + disable_bf16_reduced_precision_matmul ........... False + disable_chunked_prefill ......................... False + disable_explicit_attention_mask ................. False + disable_mamba_mem_eff_path ...................... False + disable_straggler_on_startup .................... False + disable_symmetric_registration .................. False + dist_ckpt_format_deprecated ..................... None + dist_ckpt_optim_fully_reshardable ............... True + dist_ckpt_save_pre_mcore_014 .................... False + dist_ckpt_strictness ............................ assume_ok_unexpected + distrib_optim_fully_reshardable_mem_efficient ... False + distribute_saved_activations .................... False + distributed_backend ............................. gloo + distributed_timeout_minutes ..................... 10 + distributed_timeout_seconds_after_init .......... None + document_packing_algorithm ...................... ogbfd + document_packing_dp_size ........................ None + drop_mtp ........................................ False + dryrun .......................................... False + dump_param_to_param_group_map ................... None + dynamic_context_parallel ........................ False + dynamic_context_parallel_sizes .................. None + dynamic_cp_candidate_sizes ...................... None + dynamic_cp_gdn_policy ........................... None + dynamic_cp_moe_policy ........................... None + dynamic_cp_on_error ............................. error + dynamic_cp_scheduler ............................ default_dynamic_cp + dynamic_cp_wave_coexecution ..................... False + dynamic_cp_workspace_max_tokens ................. None + emb_deviation_loss_coeff ........................ 0 + emb_deviation_type .............................. None + embedding_init_method_std ....................... None + embedding_path .................................. None + empty_unused_memory_level ....................... 0 + enable_cuda_graph ............................... False + enable_debug_logging ............................ False + enable_experimental ............................. False + enable_ft_package ............................... False + enable_full_sharding_in_hsdp .................... False + enable_gloo_process_groups ...................... True + enable_msc ...................................... True + enable_one_logger ............................... True + encoder_num_layers .............................. 56 + encoder_seq_length .............................. 1 + end_weight_decay ................................ 0.1 + eod_mask_loss ................................... False + error_injection_rate ............................ 0 + error_injection_type ............................ transient_error + eval_interval ................................... 1000 + eval_iters ...................................... 10 + evidence_data_path .............................. None + exit_duration_in_mins ........................... None + exit_interval ................................... None + exit_on_missing_checkpoint ...................... False + exit_signal_handler ............................. False + exp_avg_dtype ................................... torch.float32 + exp_avg_sq_dtype ................................ torch.float32 + expert_model_parallel_size ...................... 1 + expert_tensor_parallel_size ..................... 1 + export_force_local_attention .................... False + export_kd_cfg ................................... None + export_kd_teacher_ckpt_format ................... None + export_kd_teacher_load .......................... None + export_kv_cache_quant ........................... False + export_legacy_megatron .......................... False + export_model_type ............................... GPTModel + export_moe_apply_probs_on_input ................. False + export_offline_model ............................ False + export_qk_l2_norm ............................... False + export_quant_cfg ................................ None + export_real_quant_cfg ........................... None + export_te_mcore_model ........................... False + external_cuda_graph ............................. False + fast_thd_a2a .................................... False + ffn_hidden_size ................................. 4800 + ffn_intermediate_token_shift .................... None + ffn_token_shift ................................. None + final_logits_clamp_value ........................ None + fine_grained_activation_offloading .............. False + finetune ........................................ False + finetune_data_split ............................. train + finetune_hf_dataset ............................. None + first_last_layers_bf16 .......................... False + flash_decode .................................... False + fp16 ............................................ False + fp16_lm_cross_entropy ........................... False + fp32_residual_connection ........................ False + fp4 ............................................. None + fp4_param ....................................... False + fp4_recipe ...................................... nvfp4 + fp8 ............................................. None + fp8_amax_compute_algo ........................... most_recent + fp8_amax_history_len ............................ 1 + fp8_interval .................................... 1 + fp8_margin ...................................... 0 + fp8_param_gather ................................ False + fp8_recipe ...................................... delayed + fp8_wgrad ....................................... True + freeze_layernorm_weight ......................... False + freeze_non_mamba ................................ False + fsdp_double_buffer .............................. False + full_validation ................................. False + gated_norm ...................................... True + gated_norm_lora_rank ............................ 16 + gated_norm_upcycling ............................ False + gdn_alpha_weight_scale .......................... 1.0 + gdn_alpha_weight_scale_mode ..................... matrix + gdn_backend ..................................... fla + gdn_fused_split_forward ......................... False + gdn_gated_norm_backend .......................... torch + gdn_lr_mult_alpha ............................... 1.0 + gdn_lr_mult_beta ................................ 1.0 + gdn_lr_mult_k ................................... 1.0 + gdn_lr_mult_q ................................... 1.0 + gdn_lr_mult_v ................................... 1.0 + gdn_postinit_alog_shift ......................... 0.0 + gdn_postinit_alpha_scale ........................ 1.0 + gdn_postinit_key_scale .......................... 1.0 + gdn_pre_gdr_fusion .............................. False + gdn_pre_gdr_fusion_deterministic_weight_gradient False + gdn_pre_gdr_fusion_high_precision_backward ...... False + gdn_shortconv_separate_gate ..................... False + gdn_shortconv_separate_gate_upcycling ........... False + gdn_split_in_proj ............................... True + geglu ........................................... False + global_batch_size ............................... 1024 + glu_linear_offset ............................... 0.0 + grad_reduce_in_bf16 ............................. False + gradient_accumulation_fusion .................... False + gradient_reduce_div_fusion ...................... True + group_query_attention ........................... True + grpo_clamp_eps_lower ............................ 0.01 + grpo_clamp_eps_upper ............................ 0.01 + grpo_default_temperature ........................ 1.0 + grpo_default_top_p .............................. 0 + grpo_entropy_term_weight ........................ 0.0 + grpo_filter_groups_with_same_reward ............. False + grpo_group_size ................................. 2 + grpo_iterations ................................. 2 + grpo_kl_beta .................................... 0.001 + grpo_prompts_per_step ........................... 32 + head_lr_mult .................................... 1.0 + heterogeneous_layers_config_encoded_json ........ None + heterogeneous_layers_config_path ................ None + hf_dir .......................................... /tmp/mcore2hf_ws_1920/mcore2hf-config.pNLl8c + hidden_dropout .................................. 0.1 + hidden_size ..................................... 2048 + hidden_states_shrink ............................ 1.0 + hierarchical_context_parallel_sizes ............. None + high_priority_stream_groups ..................... [] + hybrid_adamh .................................... False + hybrid_attention_ratio .......................... 0.0 + hybrid_context_parallel ......................... False + hybrid_mlp_ratio ................................ 0.0 + hybrid_override_pattern ......................... None + hysteresis ...................................... 2 + ict_head_size ................................... None + ict_load ........................................ None + img_h ........................................... 224 + img_w ........................................... 224 + increase_log_level_interval ..................... 1000 + increase_log_level_iters ........................ 5 + indexer_batch_size .............................. 128 + indexer_log_interval ............................ 1000 + inference_batch_times_seqlen_threshold .......... -1 + inference_dynamic_batching ...................... False + inference_dynamic_batching_block_size ........... 256 + inference_dynamic_batching_buffer_guaranteed_fraction 0.2 + inference_dynamic_batching_buffer_overflow_factor None + inference_dynamic_batching_buffer_size_gb ....... 40.0 + inference_dynamic_batching_max_requests_override None + inference_dynamic_batching_max_tokens_override .. None + inference_dynamic_batching_num_cuda_graphs ...... 16 + inference_dynamic_batching_track_paused_request_events False + inference_dynamic_batching_unified_memory_level . 0 + inference_max_batch_size ........................ 8 + inference_max_seq_length ........................ 2560 + inference_rng_tracker ........................... False + init_method_std ................................. 0.006 + init_method_xavier_uniform ...................... False + init_model_with_meta_device ..................... False + initial_loss_scale .............................. 4294967296 + inprocess_active_world_size ..................... 1 + inprocess_barrier_timeout ....................... 120 + inprocess_completion_timeout .................... 120 + inprocess_empty_cuda_cache ...................... False + inprocess_granularity ........................... node + inprocess_hard_timeout .......................... 90 + inprocess_heartbeat_interval .................... 30 + inprocess_heartbeat_timeout ..................... 60 + inprocess_last_call_wait ........................ 1 + inprocess_max_iterations ........................ None + inprocess_monitor_process_interval .............. 1.0 + inprocess_monitor_thread_interval ............... 1.0 + inprocess_progress_watchdog_interval ............ 1.0 + inprocess_restart ............................... False + inprocess_soft_timeout .......................... 60 + inprocess_termination_grace_time ................ 1 + is_hybrid_model ................................. False + iter_per_epoch .................................. 1250 + iterations_to_skip .............................. [] + keep_fp8_transpose_cache ........................ False + kitchen_config_file ............................. None + kitchen_recipe_number ........................... None + kv_channels ..................................... 64 + kv_lora_rank .................................... 32 + langrl_env_config ............................... None + langrl_external_server .......................... False + langrl_inference_server_conversation_template ... None + langrl_inference_server_type .................... inplace_megatron + lazy_mpu_init ................................... None + legacy_tokenizer ................................ False + linear_attention_freq ........................... [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 0, 0, 1, 1, 1, 1, 1, 1] + linear_attention_type ........................... gated_delta_net + linear_conv_kernel_dim .......................... 4 + linear_cross_entropy_accumulate_grad_in_fp32 .... True + linear_cross_entropy_num_chunks ................. 8 + linear_key_head_dim ............................. 64 + linear_num_key_heads ............................ 8 + linear_num_value_heads .......................... 32 + linear_rope_mode ................................ none + linear_value_head_dim ........................... 64 + lm_head_alpha ................................... 1.0 + lm_loss_normalization ........................... legacy + load ............................................ None + load_complemental_dataset ....................... None + load_dir ........................................ /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/load_view1920 + load_main_params_from_ckpt ...................... None + local_rank ...................................... 0 + log_attn_logits ................................. False + log_energy ...................................... False + log_hidden_states ............................... [] + log_interval .................................... 100 + log_logits_z_loss ............................... False + log_loss_scale_to_tensorboard ................... True + log_memory_to_tensorboard ....................... False + log_metrics_interval ............................ None + log_moe_activations ............................. False + log_moe_params .................................. False + log_moe_router_metrics .......................... False + log_num_zeros_in_grad ........................... False + log_padding_stats ............................... False + log_params ...................................... [] + log_params_norm ................................. False + log_per_layer_grad_rms .......................... False + log_per_module_grad_rms ......................... False + log_per_module_update_rms ....................... False + log_progress .................................... False + log_spike_diagnostics ........................... False + log_straggler ................................... False + log_throughput .................................. False + log_timers_to_tensorboard ....................... False + log_update_rms .................................. False + log_validation_ppl_to_tensorboard ............... False + log_world_size_to_tensorboard ................... False + logging_level ................................... 1 + loss_scale ...................................... None + loss_scale_window ............................... 1000 + lr .............................................. 2e-05 + lr_decay_iters .................................. 430000 + lr_decay_samples ................................ None + lr_decay_style .................................. cosine + lr_warmup_fraction .............................. 0.001 + lr_warmup_init .................................. 0.0 + lr_warmup_iters ................................. 0 + lr_warmup_samples ............................... 0 + lr_wsd_decay_iters .............................. None + lr_wsd_decay_samples ............................ None + lr_wsd_decay_style .............................. exponential + main_grads_dtype ................................ torch.float32 + main_params_dtype ............................... torch.float32 + make_vocab_size_divisible_by .................... 128 + mamba_disable_cp ................................ False + mamba_expand .................................... 2 + mamba_head_dim .................................. 64 + mamba_num_groups ................................ 8 + mamba_num_heads ................................. None + mamba_state_dim ................................. 128 + manual_gc ....................................... False + manual_gc_eval .................................. True + manual_gc_interval .............................. 0 + mask_factor ..................................... 1.0 + mask_prob ....................................... 0.15 + mask_type ....................................... random + masked_softmax_fusion ........................... True + max_position_embeddings ......................... 131072 + max_seqlen_per_cp_rank .......................... None + max_shard_size .................................. 20GB + max_tokens_to_oom ............................... 12000 + mcore2hf ........................................ True + memory_snapshot_path ............................ None + merge_file ...................................... None + micro_batch_size ................................ 1 + microbatch_group_size_per_vp_stage .............. None + mid_level_dataset_surplus ....................... 0.005 + min_loss_scale .................................. 1.0 + min_lr .......................................... 6e-06 + min_offloaded_tensor_size ....................... 1048576 + mlp_chunks_for_prefill .......................... 1 + mmap_bin_files .................................. True + mock_data ....................................... False + model_type ...................................... GPT + modelopt_enabled ................................ False + moe_apply_probs_on_input ........................ False + moe_aux_loss_coeff .............................. 0.0 + moe_deepep_num_sms .............................. 20 + moe_enable_deepep ............................... False + moe_expert_capacity_factor ...................... None + moe_extended_tp ................................. False + moe_ffn_hidden_size ............................. 512 + moe_flex_dispatcher_backend ..................... deepep + moe_gaux_overlap ................................ False + moe_grouped_gemm ................................ False + moe_hybridep_num_sms ............................ 16 + moe_input_jitter_eps ............................ None + moe_layer_freq .................................. [0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] + moe_layer_recompute ............................. False + moe_monitor_layers .............................. + moe_monitor_log_interval ........................ None + moe_pad_expert_input_to_capacity ................ False + moe_pad_experts_for_cuda_graph_inference ........ False + moe_per_layer_logging ........................... False + moe_permute_backend ............................. auto + moe_permute_fusion .............................. False + moe_router_aux_loss_fusion ...................... auto + moe_router_bias_update_method ................... sign + moe_router_bias_update_rate ..................... 0.001 + moe_router_dtype ................................ fp32 + moe_router_enable_expert_bias ................... False + moe_router_force_load_balancing ................. False + moe_router_fusion ............................... False + moe_router_gemm_backend ......................... torch + moe_router_group_topk ........................... None + moe_router_load_balancing_type .................. aux_loss + moe_router_num_groups ........................... None + moe_router_padding_for_fp8 ...................... False + moe_router_padding_for_quantization ............. False + moe_router_pre_softmax .......................... False + moe_router_score_function ....................... softmax + moe_router_sqrt_gate ............................ True + moe_router_topk ................................. 4 + moe_router_topk_scaling_factor .................. None + moe_shared_expert_gate .......................... True + moe_shared_expert_intermediate_size ............. 512 + moe_shared_expert_overlap ....................... False + moe_token_dispatcher_type ....................... alltoall + moe_token_drop_policy ........................... probs + moe_triton_dispatcher_exclude_padding_tokens .... False + moe_triton_mega_activation_sms .................. 4 + moe_triton_mega_combine_sms ..................... 32 + moe_triton_mega_compute_sms ..................... 24 + moe_triton_mega_dispatch_sms .................... 48 + moe_triton_mega_num_warps ....................... 16 + moe_triton_mega_reduce_sms ...................... 32 + moe_triton_mega_tail_sms ........................ 16 + moe_triton_shared_expert_overlap ................ False + moe_triton_workspace_expand ..................... 4.0 + moe_upcycling_granularity ....................... 1 + moe_use_legacy_grouped_gemm ..................... False + moe_use_upcycling ............................... False + moe_z_loss_coeff ................................ None + mrope_section ................................... None + mscale .......................................... 1.0 + mscale_all_dim .................................. 0.0 + mtp_linear_attention_type ....................... None + mtp_loss_scaling_factor ......................... 0.1 + mtp_num_layers .................................. None + multi_latent_attention .......................... False + multiple_validation_sets ........................ False + muon_ball_momentum .............................. 0.9 + muon_ball_msign_steps ........................... 5 + muon_ball_power_iteration_steps ................. 10 + muon_ball_qkv_split_mode ........................ component + muon_ball_radius_mode ........................... spectral_mup + muon_ball_retract_alpha ......................... 0.05 + muon_ball_retract_mode .......................... hard + muon_ball_scale_mode ............................ spectral_mup + muon_ball_split_fc1 ............................. True + muon_ball_split_moe_experts ..................... True + muon_ball_split_qkv ............................. True + muon_ball_use_nesterov .......................... True + muon_extra_scale_factor ......................... 1.0 + muon_fp32_matmul_prec ........................... medium + muon_momentum ................................... 0.9 + muon_num_ns_steps ............................... 5 + muon_qkv_split_mode ............................. component + muon_scale_mode ................................. spectral_mup + muon_scale_vectorized_mode ...................... full + muon_split_fc1 .................................. True + muon_split_moe_experts .......................... True + muon_split_qkv .................................. True + muon_tp_mode .................................... blockwise + muon_use_nesterov ............................... False + muon_vectorize .................................. [] + muon_vectorize_attn_dim ......................... hidden_size + mup_lr_mult ..................................... 1.0 + nccl_all_reduce_for_prefill ..................... False + nccl_communicator_config_path ................... None + nccl_ub ......................................... False + no_load_optim ................................... True + no_load_rng ..................................... True + no_load_scheduler ............................... None + no_persist_layer_norm ........................... False + no_rope_freq .................................... None + no_save_optim ................................... None + no_save_rng ..................................... None + no_save_step_one ................................ None + no_weight_decay_cond_type ....................... None + non_persistent_ckpt_type ........................ None + non_persistent_global_ckpt_dir .................. None + non_persistent_keep_last ........................ 1 + non_persistent_local_ckpt_algo .................. fully_parallel + non_persistent_local_ckpt_dir ................... None + non_persistent_save_interval .................... None + norm_epsilon .................................... 1e-06 + normalization ................................... RMSNorm + num_attention_heads ............................. 32 + num_channels .................................... 3 + num_classes ..................................... 1000 + num_dataset_builder_threads ..................... 1 + num_distributed_optimizer_instances ............. 1 + num_experts ..................................... 192 + num_hf_saver .................................... None + num_layers ...................................... 56 + num_layers_at_end_in_bf16 ....................... 1 + num_layers_at_start_in_bf16 ..................... 1 + num_layers_per_virtual_pipeline_stage ........... None + num_query_groups ................................ 8 + num_virtual_stages_per_pipeline_rank ............ None + num_workers ..................................... 2 + object_storage_cache_path ....................... None + offload_modules ................................. [] + one_logger_async ................................ False + one_logger_project .............................. megatron-lm + one_logger_run_name ............................. None + onnx_safe ....................................... None + openai_gelu ..................................... False + optimizer ....................................... adam + optimizer_cpu_offload ........................... False + optimizer_offload_fraction ...................... 1.0 + output_bert_embeddings .......................... False + output_logits_z_loss_coeff ...................... None + overlap_cpu_optimizer_d2h_h2d ................... False + overlap_grad_reduce ............................. False + overlap_moe_expert_parallel_comm ................ False + overlap_p2p_comm ................................ False + overlap_p2p_comm_warmup_flush ................... False + overlap_param_gather ............................ False + overlap_param_gather_with_optimizer_step ........ False + override_hf_eod_token_id ........................ None + override_opt_param_scheduler .................... False + packing_tokens_per_shard ........................ 100000000000 + padded_vocab_size ............................... None + params_dtype .................................... torch.bfloat16 + patch_dim ....................................... 16 + per_split_data_args_path ........................ None + perform_initialization .......................... True + perform_rl_step ................................. False + pin_cpu_grads ................................... True + pin_cpu_params .................................. True + pipeline_model_parallel_comm_backend ............ None + pipeline_model_parallel_layout .................. None + pipeline_model_parallel_size .................... 1 + position_embedding_type ......................... none + pp_aware_finegrained_recompute .................. False + pp_aware_finegrained_recompute_global_layers .... None + pp_aware_finegrained_recompute_pp_ranks ......... None + prefetch_factor ................................. None + pretrain_script ................................. mcore_gdn_moe.model_provider + pretrained_checkpoint ........................... None + previous_stage_seq_length ....................... None + profile ......................................... False + profile_microbatch_count ........................ 2 + profile_microbatch_dir .......................... None + profile_microbatch_iteration .................... None + profile_microbatch_start ........................ 0 + profile_ranks ................................... [0] + profile_step_end ................................ None + profile_step_start .............................. None + q_lora_rank ..................................... None + qk_head_dim ..................................... 128 + qk_l2_norm ...................................... False + qk_layernorm .................................... False + qk_pos_emb_head_dim ............................. 64 + query_in_block_prob ............................. 0.1 + quick_geglu ..................................... False + rampup_batch_size ............................... None + rank ............................................ 0 + recompute_granularity ........................... None + recompute_method ................................ None + recompute_modules ............................... None + recompute_num_layers ............................ None + record_memory_history ........................... False + relative_attention_max_distance ................. 128 + relative_attention_num_buckets .................. 32 + reparam_checkpoint .............................. None + reparam_fallback_value .......................... None + reparam_keys .................................... None + replication ..................................... False + replication_factor .............................. 2 + replication_jump ................................ None + rerun_mode ...................................... validate_results + reset_attention_mask ............................ False + reset_iteration_one_to_zero ..................... False + reset_position_ids .............................. False + result_rejected_tracker_filename ................ None + retriever_report_topk_accuracies ................ [] + retriever_score_scaling ......................... False + retriever_seq_length ............................ 256 + retro_add_retriever ............................. False + retro_attention_gate ............................ 1 + retro_cyclic_train_iters ........................ None + retro_encoder_attention_dropout ................. 0.1 + retro_encoder_hidden_dropout .................... 0.1 + retro_encoder_layers ............................ 2 + retro_num_neighbors ............................. 2 + retro_num_retrieved_chunks ...................... 2 + retro_project_dir ............................... None + retro_verify_neighbor_count ..................... True + reuse_grad_buf_for_mxfp8_param_ag ............... False + reuse_training_communicators .................... False + rl_calculate_intra_group_similarity ............. False + rl_importance_sampling_truncation_coef .......... None + rl_inference_logprobs_is_correction ............. False + rl_offload_kv_cache_during_training ............. False + rl_offload_optimizer_during_inference ........... False + rl_partial_rollouts ............................. False + rl_prompts_per_eval ............................. 32 + rl_remove_kv_cache_during_training .............. False + rl_reset_cuda_graphs ............................ False + rl_sequence_packing_algo ........................ fifo + rl_sequence_packing_bin_size .................... 8192 + rl_use_sequence_packing ......................... False + rope_scaling_factor ............................. 8.0 + rope_type ....................................... None + rotary_base ..................................... 10000 + rotary_interleaved .............................. False + rotary_percent .................................. 1.0 + rotary_scaling_factor ........................... 1.0 + rotary_seq_len_interpolation_factor ............. None + run_workload_inspector_server ................... False + runtime_audit_dir ............................... None + sample_rate ..................................... 1.0 + save ............................................ None + save_after_load ................................. False + save_dir ........................................ /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/iter_1920-hf + save_interval ................................... 10000 + save_retain_interval ............................ None + scale_emb ....................................... 1.0 + scatter_gather_tensors_in_pipeline .............. True + seed ............................................ 1234 + seq_length ...................................... 1 + sequence_packing_scheduler ...................... default_dynamic_cp + sequence_parallel ............................... False + sft ............................................. False + sft_tokenizer_prompt_format ..................... nemotron-h-aligned + sgd_momentum .................................... 0.9 + sharp_enabled_group ............................. None + short_seq_prob .................................. 0.1 + skip_train ...................................... True + skipped_train_samples ........................... 0 + softmax_type .................................... vanilla + spec ............................................ None + spectra_clip .................................... 1.0 + spectra_grad_clip ............................... None + spectra_hyperball ............................... False + spectra_ns_steps ................................ 10 + spectra_scope ................................... matrices + spectral_ball_momentum .......................... 0.9 + spectral_ball_msign_steps ....................... 8 + spectral_ball_power_iteration_steps ............. 20 + spectral_ball_qkv_split_mode .................... component + spectral_ball_radius_mode ....................... spectral_mup + spectral_ball_retract_alpha ..................... 0.05 + spectral_ball_retract_mode ...................... hard + spectral_ball_scale_mode ........................ spectral_mup + spectral_ball_solver ............................ bisection + spectral_ball_solver_max_iterations ............. 20 + spectral_ball_solver_tolerance_f ................ 1e-08 + spectral_ball_split_fc1 ......................... True + spectral_ball_split_moe_experts ................. True + spectral_ball_split_qkv ......................... True + spectral_ball_use_nesterov ...................... True + spectral_mup_init ............................... False + spike_diagnostics_dir ........................... None + spike_diagnostics_end ........................... 21 + spike_diagnostics_start ......................... 2 + split ........................................... None + split_expert_init ............................... True + split_fc1_init .................................. True + split_qkv_init .................................. True + split_qkv_init_mode ............................. group + sqreglu ......................................... False + squared_relu .................................... False + start_samples ................................... None + start_weight_decay .............................. 0.1 + straggler_ctrlr_port ............................ 65535 + straggler_minmax_count .......................... 1 + strict_fsdp_dtensor_load ........................ True + suggested_communication_unit_size ............... None + swanlab_exp_name ................................ + swanlab_project ................................. + swanlab_save_dir ................................ + swanlab_workspace ............................... + swiglu .......................................... True + swin_backbone_type .............................. tiny + symmetric_ar_type ............................... None + synchronizer .................................... mcore_gdn_moe + target_ckpt_format .............................. torch_dist + te_rng_tracker .................................. False + teacher_model_config ............................ None + tensor_model_parallel_size ...................... 1 + tensorboard_dir ................................. None + tensorboard_log_interval ........................ 1 + tensorboard_queue_size .......................... 1000 + test_data_path .................................. None + test_mode ....................................... False + tiktoken_num_special_tokens ..................... 1000 + tiktoken_pattern ................................ None + tiktoken_special_tokens ......................... None + timing_log_level ................................ 0 + timing_log_option ............................... minmax + titles_data_path ................................ None + token_shift_conv_init ........................... default + token_shift_conv_size ........................... 4 + tokenizer_metadata .............................. None + tokenizer_model ................................. /tmp/mcore2hf_ws_1920/mcore2hf-config.pNLl8c + tokenizer_type .................................. HuggingFaceTokenizer + torch_fsdp2_reshard_after_forward ............... True + tp_comm_bootstrap_backend ....................... nccl + tp_comm_bulk_dgrad .............................. True + tp_comm_bulk_wgrad .............................. True + tp_comm_overlap ................................. False + tp_comm_overlap_ag .............................. True + tp_comm_overlap_cfg ............................. None + tp_comm_overlap_rs .............................. True + tp_comm_overlap_rs_dgrad ........................ False + tp_comm_split_ag ................................ True + tp_comm_split_rs ................................ True + train_data_path ................................. None + train_iters ..................................... 500000 + train_samples ................................... None + train_sync_interval ............................. None + train_until_stopped ............................. False + transformer_impl ................................ transformer_engine + transformer_pipeline_model_parallel_size ........ 1 + trust_remote_code ............................... False + untie_embeddings_and_output_weights ............. True + use_checkpoint_args ............................. False + use_checkpoint_opt_param_scheduler .............. False + use_cpu_initialization .......................... True + use_dist_ckpt ................................... True + use_dist_ckpt_deprecated ........................ False + use_distributed_optimizer ....................... False + use_flash_attn .................................. False + use_fused_weighted_squared_relu ................. False + use_gpu ......................................... False + use_legacy_models ............................... False + use_megatron_fsdp ............................... False + use_mp_args_from_checkpoint_args ................ False + use_one_sent_docs ............................... False + use_persistent_ckpt_worker ...................... False + use_precision_aware_optimizer ................... False + use_pytorch_profiler ............................ False + use_ring_exchange_p2p ........................... False + use_rope_scaling ................................ False + use_rotary_position_embeddings .................. False + use_sharp ....................................... False + use_te_activation_func .......................... False + use_tokenizer_model_from_checkpoint_args ........ True + use_torch_fsdp2 ................................. False + use_torch_optimizer_for_cpu_offload ............. False + use_tp_pp_dp_mapping ............................ False + v_head_dim ...................................... 128 + valid_data_path ................................. None + variable_seq_lengths ............................ False + virtual_pipeline_model_parallel_size ............ None + vision_backbone_type ............................ vit + vision_pretraining .............................. False + vision_pretraining_type ......................... classify + vocab_extra_ids ................................. 0 + vocab_file ...................................... None + vocab_size ...................................... None + wandb_entity .................................... + wandb_exp_name .................................. + wandb_project ................................... + wandb_save_dir .................................. + weight_decay .................................... 0.1 + weight_decay_incr_style ......................... constant + wgrad_deferral_limit ............................ 0 + window_attn_skip_freq ........................... None + window_size ..................................... None + word_embedding_dropout_prob ..................... 0.0 + world_size ...................................... 1 + yaml_cfg ........................................ None +-------------------- end of arguments --------------------- +INFO:megatron.core.num_microbatches_calculator:setting number of microbatches to constant 1024 +> building HuggingFaceTokenizer tokenizer ... + > padded vocab (size: 99000) with 72 dummy tokens (new size: 99072) +WARNING: one_logger package is required to enable e2e metrics tracking. please go to https://confluence.nvidia.com/display/MLWFO/Package+Repositories for details to install it +INFO:megatron.training.initialize:Setting logging level to 1 +WARNING:megatron.core.rerun_state_machine:RerunStateMachine initialized in mode validate_results +> initializing torch distributed ... +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +> initialized tensor model parallel with size 1 +> initialized pipeline model parallel with size 1 +> setting random seeds to 1234 ... +> compiling dataset index builder ... +make: Entering directory '/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/datasets' +make: Nothing to be done for 'default'. +make: Leaving directory '/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/datasets' +>>> done with dataset index builder. Compilation time: 0.129 seconds +WARNING: constraints for invoking optimized fused softmax kernel are not met. We default back to unfused kernel invocations. +> compiling and loading fused kernels ... +>>> done with compiling and loading fused kernels. Compilation time: 0.003 seconds +WORLD_SIZE: 1, RANK: 0, LOCAL_RANK: 0 +[convert] creating synchronizer: mode=mcore2hf +building GPT model ... +[GPT_LAYER_SPEC] log_hidden_states=[] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=[] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=[] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=[] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/transformers/modeling_rope_utils.py:1036: FutureWarning: `rope_config_validation` is deprecated and has been removed. Its functionality has been moved to RotaryEmbeddingConfigMixin.validate_rope method. PreTrainedConfig inherits this class, so please call self.validate_rope() instead. Also, make sure to use the new rope_parameters syntax. You can call self.standardize_rope_params() in the meantime. + warnings.warn( +[transformers] `layer_type_validation` is deprecated and will be removed in v5.20. Use `PreTrainedConfig.validate_layer_type` instead +[transformers] `layer_type_validation` is deprecated and will be removed in v5.20. Use `PreTrainedConfig.validate_layer_type` instead +[transformers] `torch_dtype` is deprecated! Use `dtype` instead! +[ERROR] `cache_position` is part of Qwen3NextModel.forward's signature, but not documented. Make sure to add it to the docstring of the function in /mnt/ssd/cache_tmp/huggingface/modules/transformers_modules/mcore2hf_hyphen_config_dot_pNLl8c/10c6759df797195a/modeling_qwen3_next.py. +[ERROR] `cache_position` is part of Qwen3NextForCausalLM.forward's signature, but not documented. Make sure to add it to the docstring of the function in /mnt/ssd/cache_tmp/huggingface/modules/transformers_modules/mcore2hf_hyphen_config_dot_pNLl8c/10c6759df797195a/modeling_qwen3_next.py. +[transformers] `torch_dtype` is deprecated! Use `dtype` instead! +INFO:transformers_modules.mcore2hf_hyphen_config_dot_pNLl8c.10c6759df797195a.modeling_qwen3_next:[Qwen3Next custom] attn_position_embedding_type=nope, rnn_position_embedding_type=nope, attn_logits_scaling=None +Qwen3NextForCausalLM( + (model): Qwen3NextModel( + (embed_tokens): Embedding(99072, 2048, padding_idx=102) + (layers): ModuleList( + (0-1): 2 x Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=4800, bias=False) + (up_proj): Linear(in_features=2048, out_features=4800, bias=False) + (down_proj): Linear(in_features=4800, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (2-11): 10 x Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (12): Qwen3NextDecoderLayer( + (self_attn): Qwen3NextAttention( + (q_proj): Linear(in_features=2048, out_features=4096, bias=True) + (k_proj): Linear(in_features=2048, out_features=512, bias=True) + (v_proj): Linear(in_features=2048, out_features=512, bias=True) + (o_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (13-19): 7 x Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (20-21): 2 x Qwen3NextDecoderLayer( + (self_attn): Qwen3NextAttention( + (q_proj): Linear(in_features=2048, out_features=4096, bias=True) + (k_proj): Linear(in_features=2048, out_features=512, bias=True) + (v_proj): Linear(in_features=2048, out_features=512, bias=True) + (o_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (22): Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (23): Qwen3NextDecoderLayer( + (self_attn): Qwen3NextAttention( + (q_proj): Linear(in_features=2048, out_features=4096, bias=True) + (k_proj): Linear(in_features=2048, out_features=512, bias=True) + (v_proj): Linear(in_features=2048, out_features=512, bias=True) + (o_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (24-45): 22 x Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (46): Qwen3NextDecoderLayer( + (self_attn): Qwen3NextAttention( + (q_proj): Linear(in_features=2048, out_features=4096, bias=True) + (k_proj): Linear(in_features=2048, out_features=512, bias=True) + (v_proj): Linear(in_features=2048, out_features=512, bias=True) + (o_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (47): Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (48-49): 2 x Qwen3NextDecoderLayer( + (self_attn): Qwen3NextAttention( + (q_proj): Linear(in_features=2048, out_features=4096, bias=True) + (k_proj): Linear(in_features=2048, out_features=512, bias=True) + (v_proj): Linear(in_features=2048, out_features=512, bias=True) + (o_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (50-55): 6 x Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + ) + (norm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (rotary_emb): Qwen3NextRotaryEmbedding() + ) + (lm_head): Linear(in_features=2048, out_features=99072, bias=False) +) +GPTModel( + (embedding): LanguageModelEmbedding( + (word_embeddings): VocabParallelEmbedding() + (embedding_dropout): Dropout(p=0.1, inplace=False) + ) + (decoder): TransformerBlock( + (layers): ModuleList( + (0-1): 2 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=9600, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=4800, out_features=2048, bias=False, TP=1) + ) + ) + (2-11): 10 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (12): TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): SelfAttention( + (core_attention): TEDotProductAttention( + (flash_attention): FlashAttention() + (fused_attention): FusedAttention() + (unfused_attention): UnfusedDotProductAttention( + (scale_mask_softmax): FusedScaleMaskSoftmax() + (attention_dropout): Dropout(p=0.1, inplace=False) + ) + ) + (linear_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (linear_qkv): TEColumnParallelLinear(in_features=2048, out_features=5120, bias=True, TP=1) + (q_layernorm): IdentityOp() + (k_layernorm): IdentityOp() + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (13-19): 7 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (20-21): 2 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): SelfAttention( + (core_attention): TEDotProductAttention( + (flash_attention): FlashAttention() + (fused_attention): FusedAttention() + (unfused_attention): UnfusedDotProductAttention( + (scale_mask_softmax): FusedScaleMaskSoftmax() + (attention_dropout): Dropout(p=0.1, inplace=False) + ) + ) + (linear_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (linear_qkv): TEColumnParallelLinear(in_features=2048, out_features=5120, bias=True, TP=1) + (q_layernorm): IdentityOp() + (k_layernorm): IdentityOp() + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (22): TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (23): TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): SelfAttention( + (core_attention): TEDotProductAttention( + (flash_attention): FlashAttention() + (fused_attention): FusedAttention() + (unfused_attention): UnfusedDotProductAttention( + (scale_mask_softmax): FusedScaleMaskSoftmax() + (attention_dropout): Dropout(p=0.1, inplace=False) + ) + ) + (linear_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (linear_qkv): TEColumnParallelLinear(in_features=2048, out_features=5120, bias=True, TP=1) + (q_layernorm): IdentityOp() + (k_layernorm): IdentityOp() + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (24-45): 22 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (46): TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): SelfAttention( + (core_attention): TEDotProductAttention( + (flash_attention): FlashAttention() + (fused_attention): FusedAttention() + (unfused_attention): UnfusedDotProductAttention( + (scale_mask_softmax): FusedScaleMaskSoftmax() + (attention_dropout): Dropout(p=0.1, inplace=False) + ) + ) + (linear_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (linear_qkv): TEColumnParallelLinear(in_features=2048, out_features=5120, bias=True, TP=1) + (q_layernorm): IdentityOp() + (k_layernorm): IdentityOp() + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (47): TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (48-49): 2 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): SelfAttention( + (core_attention): TEDotProductAttention( + (flash_attention): FlashAttention() + (fused_attention): FusedAttention() + (unfused_attention): UnfusedDotProductAttention( + (scale_mask_softmax): FusedScaleMaskSoftmax() + (attention_dropout): Dropout(p=0.1, inplace=False) + ) + ) + (linear_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (linear_qkv): TEColumnParallelLinear(in_features=2048, out_features=5120, bias=True, TP=1) + (q_layernorm): IdentityOp() + (k_layernorm): IdentityOp() + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (50-55): 6 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + ) + (final_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (output_layer): ColumnParallelLinear(in_features=2048, out_features=99072, bias=False, TP=1) +) +DEBUG:multistorageclient.config:Searching MSC config file paths: ['/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/msc/msc_config.yaml', '/mnt/ssd/cache_tmp/.config/msc/config.yaml', '/mnt/ssd/cache_tmp/.config/msc/config.json', '/root/.msc_config.yaml', '/root/.msc_config.json', '/root/.config/msc/config.yaml', '/root/.config/msc/config.json', '/etc/xdg/msc/config.yaml', '/etc/xdg/msc/config.json', '/etc/msc_config.yaml', '/etc/msc_config.json'] +DEBUG:multistorageclient.config:Using MSC config file: /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/msc/msc_config.yaml +DEBUG:multistorageclient.config:Searching MSC config file paths: ['/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/msc/msc_config.yaml', '/mnt/ssd/cache_tmp/.config/msc/config.yaml', '/mnt/ssd/cache_tmp/.config/msc/config.json', '/root/.msc_config.yaml', '/root/.msc_config.json', '/root/.config/msc/config.yaml', '/root/.config/msc/config.json', '/etc/xdg/msc/config.yaml', '/etc/xdg/msc/config.json', '/etc/msc_config.yaml', '/etc/msc_config.json'] +DEBUG:multistorageclient.config:Using MSC config file: /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/msc/msc_config.yaml +DEBUG:multistorageclient.client.single:No metadata provider configured, auto-commit will not be enabled +DEBUG:multistorageclient.client.client:StorageClient '__filesystem__' using SingleStorageClient +sharded_state_dict metadata loaded from the checkpoint: {'singleton_local_shards': True, 'distrib_optim_sharding_type': 'dp_reshardable', 'chained_optim_avoid_prefix': True} +Job sharding has changed: Rerun state will be ignored + loading distributed checkpoint from /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/load_view1920 at iteration 1920 +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/dist_checkpointing/strategies/torch.py:956: FutureWarning: `load_state_dict` is deprecated and will be removed in future versions. Please use `load` instead. + checkpoint.load_state_dict( + checkpoint version 3.0 + successfully loaded checkpoint from /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/load_view1920 [ t 1/1, p 1/1 ] at iteration 1920 +[convert] synchronizer created in 386.30s +[convert] starting sync_params +INFO:root:Converting layer 0 is_gdn=True is_not_moe=True +INFO:root:Converting layer 1 is_gdn=True is_not_moe=True +INFO:root:Converting layer 2 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.07666015625 mean=-0.0004863739013671875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.1162109375 mean=0.0020599365234375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 3 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.07763671875 mean=-0.0001773834228515625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.123046875 mean=1.055002212524414e-05 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 4 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.08447265625 mean=-0.0002689361572265625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.09130859375 mean=0.00127410888671875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 5 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.087890625 mean=-0.00022602081298828125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.09326171875 mean=0.0020751953125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 6 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.095703125 mean=-0.000213623046875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.08642578125 mean=-0.001373291015625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 7 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.091796875 mean=-4.6253204345703125e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.07177734375 mean=-0.000335693359375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 8 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.09375 mean=-6.580352783203125e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.07177734375 mean=-1.823902130126953e-05 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 9 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.08837890625 mean=2.586841583251953e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0654296875 mean=0.000278472900390625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 10 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.08447265625 mean=5.1975250244140625e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0634765625 mean=-0.00115966796875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 11 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.08203125 mean=7.62939453125e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.07421875 mean=-0.00018596649169921875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 12 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 12: args.attention_output_gate=True +WARNING:root:Layer 12: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.076171875 mean=0.00023555755615234375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.061767578125 mean=-0.000659942626953125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 13 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.07177734375 mean=0.00019073486328125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.050048828125 mean=-0.001220703125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 14 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.076171875 mean=0.0001926422119140625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.05419921875 mean=-0.00116729736328125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 15 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.07373046875 mean=0.000263214111328125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.05810546875 mean=-0.001129150390625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 16 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.072265625 mean=0.000339508056640625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.057861328125 mean=-0.00115203857421875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 17 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0693359375 mean=0.0002765655517578125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0556640625 mean=-0.0017852783203125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 18 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0654296875 mean=0.0002727508544921875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.05908203125 mean=-0.002777099609375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 19 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.068359375 mean=0.0006103515625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.057861328125 mean=-0.00193023681640625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 20 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 20: args.attention_output_gate=True +WARNING:root:Layer 20: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.06689453125 mean=0.000392913818359375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.059326171875 mean=-0.0018157958984375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 21 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 21: args.attention_output_gate=True +WARNING:root:Layer 21: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.059814453125 mean=0.000446319580078125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.05078125 mean=-0.0025787353515625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 22 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.05517578125 mean=0.0003223419189453125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.052734375 mean=-0.00323486328125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 23 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 23: args.attention_output_gate=True +WARNING:root:Layer 23: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.053466796875 mean=0.00048065185546875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.045166015625 mean=-0.0029296875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 24 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.052490234375 mean=0.0003871917724609375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.04296875 mean=-0.0031280517578125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 25 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.049560546875 mean=0.00031280517578125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.041015625 mean=-0.0021820068359375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 26 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.04931640625 mean=0.0002593994140625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.03857421875 mean=-0.0024566650390625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 27 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.051513671875 mean=0.0004119873046875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0361328125 mean=-0.0024871826171875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 28 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0478515625 mean=0.000339508056640625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.031982421875 mean=-0.001251220703125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 29 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.04833984375 mean=0.0004673004150390625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.028076171875 mean=-0.00121307373046875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 30 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.05126953125 mean=0.00072479248046875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0299072265625 mean=-0.0012359619140625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 31 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.048828125 mean=0.00046539306640625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.02978515625 mean=-0.00096893310546875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 32 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.046630859375 mean=0.00010585784912109375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0281982421875 mean=-0.000331878662109375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 33 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.050048828125 mean=7.724761962890625e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0302734375 mean=-0.0003204345703125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 34 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.05224609375 mean=0.0001468658447265625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0299072265625 mean=-0.00013256072998046875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 35 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.057373046875 mean=0.0001087188720703125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.03466796875 mean=0.000125885009765625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 36 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0595703125 mean=7.05718994140625e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.044677734375 mean=0.000736236572265625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 37 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0595703125 mean=0.0001220703125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.04345703125 mean=0.001129150390625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 38 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.05810546875 mean=0.0001277923583984375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.03857421875 mean=0.00142669677734375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 39 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.056884765625 mean=9.5367431640625e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0380859375 mean=0.0010986328125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 40 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.05322265625 mean=0.00014019012451171875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.028076171875 mean=-2.7060508728027344e-05 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 41 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0498046875 mean=0.0001392364501953125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0263671875 mean=-9.107589721679688e-05 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 42 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.04443359375 mean=0.000347137451171875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0233154296875 mean=-0.000743865966796875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 43 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.038330078125 mean=0.000179290771484375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0223388671875 mean=-0.0003376007080078125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 44 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.034912109375 mean=0.0003108978271484375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0233154296875 mean=-0.0004730224609375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 45 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.035400390625 mean=0.00018978118896484375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.026123046875 mean=-0.000934600830078125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 46 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 46: args.attention_output_gate=True +WARNING:root:Layer 46: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0361328125 mean=0.000347137451171875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.022705078125 mean=-0.000762939453125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 47 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.036865234375 mean=0.00019168853759765625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.027099609375 mean=-0.0009002685546875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 48 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 48: args.attention_output_gate=True +WARNING:root:Layer 48: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.034423828125 mean=-7.724761962890625e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0220947265625 mean=-0.000530242919921875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 49 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 49: args.attention_output_gate=True +WARNING:root:Layer 49: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.03466796875 mean=7.009506225585938e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.026611328125 mean=-0.00130462646484375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 50 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.036865234375 mean=-1.7523765563964844e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0267333984375 mean=-0.00091552734375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 51 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.03662109375 mean=7.772445678710938e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0264892578125 mean=-0.00128936767578125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 52 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.041259765625 mean=0.00023746490478515625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.03369140625 mean=-0.00160980224609375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 53 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0380859375 mean=6.103515625e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.051513671875 mean=-0.00238037109375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 54 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.038818359375 mean=6.723403930664062e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.042724609375 mean=-0.0024261474609375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 55 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.04248046875 mean=-0.00016021728515625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.045166015625 mean=0.0026702880859375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +[convert] finished sync_params in 0.93s +[convert] starting check_and_save -> /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/iter_1920-hf +DEBUG:root:[RANK 0] 0 send op & 0 recv op. +INFO:root:[Iters 0 RANK 0] starts synchronizing parameters with other ranks... +INFO:root:[Iters 0 RANK 0] finishes synchronizing +[Iters 0 RANK 0] model-00001-of-00004.safetensors is saved. +DEBUG:root:[Iters 0 RANK 0] joined +DEBUG:root:[RANK 0] 0 send op & 0 recv op. +INFO:root:[Iters 1 RANK 0] starts synchronizing parameters with other ranks... +INFO:root:[Iters 1 RANK 0] finishes synchronizing +[Iters 1 RANK 0] model-00002-of-00004.safetensors is saved. +DEBUG:root:[Iters 1 RANK 0] joined +DEBUG:root:[RANK 0] 0 send op & 0 recv op. +INFO:root:[Iters 2 RANK 0] starts synchronizing parameters with other ranks... +INFO:root:[Iters 2 RANK 0] finishes synchronizing +[Iters 2 RANK 0] model-00003-of-00004.safetensors is saved. +DEBUG:root:[Iters 2 RANK 0] joined +DEBUG:root:[RANK 0] 0 send op & 0 recv op. +INFO:root:[Iters 3 RANK 0] starts synchronizing parameters with other ranks... +INFO:root:[Iters 3 RANK 0] finishes synchronizing +[Iters 3 RANK 0] model-00004-of-00004.safetensors is saved. +DEBUG:root:[Iters 3 RANK 0] joined +[convert] finished check_and_save in 77.77s +Conversion finished in 466.5846004486084 seconds. ++ echo '✅Conversion completed: /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/iter_1920-hf' +✅Conversion completed: /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/iter_1920-hf ++ cleanup_real_config_path ++ rm -rf /tmp/mcore2hf_ws_1920/mcore2hf-config.pNLl8c ++ '[' -n '' ']' diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/hf_export_logs/convert_iter2840.log b/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/hf_export_logs/convert_iter2840.log new file mode 100644 index 0000000000000000000000000000000000000000..63e8391440113fbc05edf2c4dc8a47ef79dabee7 --- /dev/null +++ b/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/hf_export_logs/convert_iter2840.log @@ -0,0 +1,2665 @@ +++ realpath /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/scripts/mcore_gdn_moe/../../../.. ++ MEGATRON_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher ++ CONVERTOR_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor ++ REPAIR_SCRIPT=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/utils/repair_gdn_checkpoint.py ++ RENDER_CONFIG_SCRIPT=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/utils/render_gdn_hf_config.py ++ export PYTHON_ENV_ROOT=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud ++ PYTHON_ENV_ROOT=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud ++ PYTHON_BIN=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python ++ '[' '!' -x /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python ']' ++ '[' '!' -x /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python ']' ++ '[' '!' -x /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python ']' ++ '[' '!' -x /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python ']' ++ export PYTHON_BIN ++ /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python -c 'import torch' ++ export PATH=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin:/root/.kimi-code/bin:/root/.local/bin:/mnt/hdd/minyingqian/mihomo/bin:/root/.grok/bin:/usr/local/bin:/root/miniconda3/bin:/root/.local/bin:/usr/local/bin:/root/miniconda3/bin:/root/miniconda3/bin:/mnt/ssd/liuzian/pandoc/current/bin:/mnt/ssd/cache_tmp/.local/share/../bin:/mnt/ssd/yulan/bin:/mnt/ssd/yulan/bin:/root/.local/bin:/usr/local/bin:/root/miniconda3/bin:/root/.local/bin:/usr/local/bin:/root/miniconda3/bin:/mnt/ssd/cache_tmp/.local/bin:/root/.local/bin:/root/.local/bin:/mnt/hdd/minyingqian/mihomo/bin:/root/.grok/bin:/usr/local/bin:/root/miniconda3/bin:/root/.local/bin:/usr/local/bin:/root/miniconda3/bin:/root/miniconda3/bin:/mnt/ssd/liuzian/pandoc/current/bin:/mnt/ssd/cache_tmp/.local/bin:/mnt/ssd/yulan/bin:/mnt/ssd/yulan/bin:/root/.local/bin:/root/miniconda3/bin:/root/miniconda3/condabin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/mnt/ssd/yulan/cache/installs/nsight-compute/2026.2:/mnt/ssd/yulan/cache/installs/nsight-systems/2026.3.1/bin:/mnt/ssd/yulan/cache/installs/nsight-compute/2026.2:/mnt/ssd/yulan/cache/installs/nsight-systems/2026.3.1/bin ++ PATH=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin:/root/.kimi-code/bin:/root/.local/bin:/mnt/hdd/minyingqian/mihomo/bin:/root/.grok/bin:/usr/local/bin:/root/miniconda3/bin:/root/.local/bin:/usr/local/bin:/root/miniconda3/bin:/root/miniconda3/bin:/mnt/ssd/liuzian/pandoc/current/bin:/mnt/ssd/cache_tmp/.local/share/../bin:/mnt/ssd/yulan/bin:/mnt/ssd/yulan/bin:/root/.local/bin:/usr/local/bin:/root/miniconda3/bin:/root/.local/bin:/usr/local/bin:/root/miniconda3/bin:/mnt/ssd/cache_tmp/.local/bin:/root/.local/bin:/root/.local/bin:/mnt/hdd/minyingqian/mihomo/bin:/root/.grok/bin:/usr/local/bin:/root/miniconda3/bin:/root/.local/bin:/usr/local/bin:/root/miniconda3/bin:/root/miniconda3/bin:/mnt/ssd/liuzian/pandoc/current/bin:/mnt/ssd/cache_tmp/.local/bin:/mnt/ssd/yulan/bin:/mnt/ssd/yulan/bin:/root/.local/bin:/root/miniconda3/bin:/root/miniconda3/condabin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/mnt/ssd/yulan/cache/installs/nsight-compute/2026.2:/mnt/ssd/yulan/cache/installs/nsight-systems/2026.3.1/bin:/mnt/ssd/yulan/cache/installs/nsight-compute/2026.2:/mnt/ssd/yulan/cache/installs/nsight-systems/2026.3.1/bin ++ export VIRTUAL_ENV=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud ++ VIRTUAL_ENV=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud ++ echo ✅MEGATRON_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher +✅MEGATRON_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher ++ echo ✅PYTHON_BIN=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python +✅PYTHON_BIN=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python ++ export CUDA_LAUNCH_BLOCKING=1 ++ CUDA_LAUNCH_BLOCKING=1 ++ export CUDA_VISIBLE_DEVICES=0 ++ CUDA_VISIBLE_DEVICES=0 ++ export KUBERNETES_CONTAINER_RESOURCE_GPU=1 ++ KUBERNETES_CONTAINER_RESOURCE_GPU=1 ++ export TP=1 ++ TP=1 ++ export PP=1 ++ PP=1 ++ export ETP=1 ++ ETP=1 ++ export EP=1 ++ EP=1 ++ export 'MODEL_PARALLEL_ARGS=--tensor-model-parallel-size 1 --pipeline-model-parallel-size 1 --expert-tensor-parallel-size 1 --expert-model-parallel-size 1' ++ MODEL_PARALLEL_ARGS='--tensor-model-parallel-size 1 --pipeline-model-parallel-size 1 --expert-tensor-parallel-size 1 --expert-model-parallel-size 1' ++ WORKSPACE_TMP_DIR=/tmp/mcore2hf_ws_2840 ++ mkdir -p /tmp/mcore2hf_ws_2840 ++ MEGATRON_CHECKPOINT_PATH=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/load_view2840 ++ echo MEGATRON_CHECKPOINT_PATH=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/load_view2840 +MEGATRON_CHECKPOINT_PATH=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/load_view2840 ++ MANUAL_ITER= ++ CONVERT_ALL=false ++ FORCE_RECONVERT=false ++ DROP_MTP=false ++ POSITIONAL_ARGS=() ++ [[ 3 -gt 0 ]] ++ case $1 in ++ POSITIONAL_ARGS+=("$1") ++ shift ++ [[ 2 -gt 0 ]] ++ case $1 in ++ '[' -z 2840 ']' ++ MANUAL_ITER=2840 ++ shift 2 ++ [[ 0 -gt 0 ]] ++ set -- /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/load_view2840 ++ '[' -e /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/load_view2840 ']' +++ readlink -f /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/load_view2840 ++ MEGATRON_CHECKPOINT_PATH=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/load_view2840 ++ echo ✅MEGATRON_CHECKPOINT_PATH=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/load_view2840 +✅MEGATRON_CHECKPOINT_PATH=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/load_view2840 +++ basename /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/load_view2840 ++ CHECKPOINT_BASENAME=load_view2840 ++ [[ load_view2840 =~ ^iter_([0-9]+)$ ]] ++ '[' false = true ']' ++ HF_CONFIG_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/hf-configs/YuLan-Mini-for-gdn-convert ++ echo HF_CONFIG_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/hf-configs/YuLan-Mini-for-gdn-convert +HF_CONFIG_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/hf-configs/YuLan-Mini-for-gdn-convert ++ '[' '!' -f /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/hf-configs/YuLan-Mini-for-gdn-convert/config.template.json ']' +++ mktemp -d /tmp/mcore2hf_ws_2840/mcore2hf-config.XXXXXX ++ REAL_CONFIG_PATH=/tmp/mcore2hf_ws_2840/mcore2hf-config.jBseUr ++ LOAD_VIEW_DIR= ++ trap cleanup_real_config_path EXIT ++ export PYTHONPATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher: ++ PYTHONPATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher: ++ export PYTHONPATH=/tmp/mcore2hf_ws_2840/mcore2hf-config.jBseUr:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher: ++ PYTHONPATH=/tmp/mcore2hf_ws_2840/mcore2hf-config.jBseUr:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher: +++ /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/utils/parse-linear-vars-from-mcore-config.py /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/load_view2840 +W1006 15:19:07.467000 744823 torch/utils/cpp_extension.py:2425] TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +W1006 15:19:07.467000 744823 torch/utils/cpp_extension.py:2425] If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'] to specific architectures. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. ++ eval 'export LINEAR_NUM_KEY_HEADS=8 +export LINEAR_NUM_VALUE_HEADS=32 +export LINEAR_ATTN_FREQ=[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1] +export FFN_HIDDEN_SIZE=4800 +export NUM_LAYERS=56 +export HIDDEN_SIZE=2048 +export NUM_ATTN_HEADS=32 +export NUM_QUERY_GROUPS=8 +export MAX_POSITION_EMBEDDINGS=131072 +export KV_CHANNELS=64 +export LINEAR_CONV_KERNEL_DIM=4 +export LINEAR_KEY_HEAD_DIM=64 +export LINEAR_VALUE_HEAD_DIM=64 +export MOE_FFN_HIDDEN_SIZE=512 +export NUM_EXPERTS=192 +export MOE_ROUTER_TOPK=4 +export MOE_ROUTER_SCORE_FUNCTION=softmax +export MOE_ROUTER_ENABLE_EXPERT_BIAS=false +export MOE_LAYER_FREQ=[0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1] +export MLP_ONLY_LAYERS=[0,1] +export MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512 +export ROPE_THETA=10000.0 +export POSITION_EMBEDDING_TYPE=none +export ATTENTION_OUTPUT_GATE=true +export MTP_NUM_LAYERS=0 +export MOE_ROUTER_SQRT_GATE=true +export GATED_NORM=true +export GATED_NORM_RANK=16 +export GATED_NORM_GATE_SCALE=1.0 +export GATED_NORM_UP_BIAS=true +export FINAL_GATED_NORM=true +export ACTIVATION_FUNC_CLAMP_VALUE=7.0 +export ACTIVATION_FUNC_CLAMP_MODE=soft +export FINAL_LOGITS_CLAMP_VALUE=30.0 +export MOE_ROUTER_DTYPE=fp32 +export TORCH_DTYPE=bfloat16 +export VOCAB_SIZE=99072' +++ export LINEAR_NUM_KEY_HEADS=8 +++ LINEAR_NUM_KEY_HEADS=8 +++ export LINEAR_NUM_VALUE_HEADS=32 +++ LINEAR_NUM_VALUE_HEADS=32 +++ export 'LINEAR_ATTN_FREQ=[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1]' +++ LINEAR_ATTN_FREQ='[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1]' +++ export FFN_HIDDEN_SIZE=4800 +++ FFN_HIDDEN_SIZE=4800 +++ export NUM_LAYERS=56 +++ NUM_LAYERS=56 +++ export HIDDEN_SIZE=2048 +++ HIDDEN_SIZE=2048 +++ export NUM_ATTN_HEADS=32 +++ NUM_ATTN_HEADS=32 +++ export NUM_QUERY_GROUPS=8 +++ NUM_QUERY_GROUPS=8 +++ export MAX_POSITION_EMBEDDINGS=131072 +++ MAX_POSITION_EMBEDDINGS=131072 +++ export KV_CHANNELS=64 +++ KV_CHANNELS=64 +++ export LINEAR_CONV_KERNEL_DIM=4 +++ LINEAR_CONV_KERNEL_DIM=4 +++ export LINEAR_KEY_HEAD_DIM=64 +++ LINEAR_KEY_HEAD_DIM=64 +++ export LINEAR_VALUE_HEAD_DIM=64 +++ LINEAR_VALUE_HEAD_DIM=64 +++ export MOE_FFN_HIDDEN_SIZE=512 +++ MOE_FFN_HIDDEN_SIZE=512 +++ export NUM_EXPERTS=192 +++ NUM_EXPERTS=192 +++ export MOE_ROUTER_TOPK=4 +++ MOE_ROUTER_TOPK=4 +++ export MOE_ROUTER_SCORE_FUNCTION=softmax +++ MOE_ROUTER_SCORE_FUNCTION=softmax +++ export MOE_ROUTER_ENABLE_EXPERT_BIAS=false +++ MOE_ROUTER_ENABLE_EXPERT_BIAS=false +++ export 'MOE_LAYER_FREQ=[0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]' +++ MOE_LAYER_FREQ='[0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]' +++ export 'MLP_ONLY_LAYERS=[0,1]' +++ MLP_ONLY_LAYERS='[0,1]' +++ export MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512 +++ MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512 +++ export ROPE_THETA=10000.0 +++ ROPE_THETA=10000.0 +++ export POSITION_EMBEDDING_TYPE=none +++ POSITION_EMBEDDING_TYPE=none +++ export ATTENTION_OUTPUT_GATE=true +++ ATTENTION_OUTPUT_GATE=true +++ export MTP_NUM_LAYERS=0 +++ MTP_NUM_LAYERS=0 +++ export MOE_ROUTER_SQRT_GATE=true +++ MOE_ROUTER_SQRT_GATE=true +++ export GATED_NORM=true +++ GATED_NORM=true +++ export GATED_NORM_RANK=16 +++ GATED_NORM_RANK=16 +++ export GATED_NORM_GATE_SCALE=1.0 +++ GATED_NORM_GATE_SCALE=1.0 +++ export GATED_NORM_UP_BIAS=true +++ GATED_NORM_UP_BIAS=true +++ export FINAL_GATED_NORM=true +++ FINAL_GATED_NORM=true +++ export ACTIVATION_FUNC_CLAMP_VALUE=7.0 +++ ACTIVATION_FUNC_CLAMP_VALUE=7.0 +++ export ACTIVATION_FUNC_CLAMP_MODE=soft +++ ACTIVATION_FUNC_CLAMP_MODE=soft +++ export FINAL_LOGITS_CLAMP_VALUE=30.0 +++ FINAL_LOGITS_CLAMP_VALUE=30.0 +++ export MOE_ROUTER_DTYPE=fp32 +++ MOE_ROUTER_DTYPE=fp32 +++ export TORCH_DTYPE=bfloat16 +++ TORCH_DTYPE=bfloat16 +++ export VOCAB_SIZE=99072 +++ VOCAB_SIZE=99072 ++ echo LINEAR_NUM_KEY_HEADS=8 +LINEAR_NUM_KEY_HEADS=8 ++ echo LINEAR_NUM_VALUE_HEADS=32 +LINEAR_NUM_VALUE_HEADS=32 ++ echo 'LINEAR_ATTN_FREQ=[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1]' +LINEAR_ATTN_FREQ=[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1] ++ echo 'NUM_LAYERS=56 HIDDEN_SIZE=2048' +NUM_LAYERS=56 HIDDEN_SIZE=2048 ++ echo 'NUM_ATTN_HEADS=32 NUM_QUERY_GROUPS=8' +NUM_ATTN_HEADS=32 NUM_QUERY_GROUPS=8 ++ echo 'MAX_POSITION_EMBEDDINGS=131072 KV_CHANNELS=64' +MAX_POSITION_EMBEDDINGS=131072 KV_CHANNELS=64 ++ echo 'FFN_HIDDEN_SIZE=4800 MOE_FFN_HIDDEN_SIZE=512' +FFN_HIDDEN_SIZE=4800 MOE_FFN_HIDDEN_SIZE=512 ++ echo 'NUM_EXPERTS=192 MOE_ROUTER_TOPK=4' +NUM_EXPERTS=192 MOE_ROUTER_TOPK=4 ++ echo MOE_ROUTER_SCORE_FUNCTION=softmax +MOE_ROUTER_SCORE_FUNCTION=softmax ++ echo 'MLP_ONLY_LAYERS=[0,1]' +MLP_ONLY_LAYERS=[0,1] ++ echo 'MOE_LAYER_FREQ=[0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]' +MOE_LAYER_FREQ=[0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1] ++ echo MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512 +MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512 ++ echo 'LINEAR_CONV_KERNEL_DIM=4 LINEAR_KEY_HEAD_DIM=64 LINEAR_VALUE_HEAD_DIM=64' +LINEAR_CONV_KERNEL_DIM=4 LINEAR_KEY_HEAD_DIM=64 LINEAR_VALUE_HEAD_DIM=64 ++ echo ATTENTION_OUTPUT_GATE=true +ATTENTION_OUTPUT_GATE=true ++ echo ROPE_THETA=10000.0 +ROPE_THETA=10000.0 ++ echo POSITION_EMBEDDING_TYPE=none +POSITION_EMBEDDING_TYPE=none ++ echo MOE_ROUTER_SQRT_GATE=true +MOE_ROUTER_SQRT_GATE=true +++ /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/utils/parse-mom-vars-from-mcore-config.py /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/load_view2840 +W1006 15:19:22.549000 745279 torch/utils/cpp_extension.py:2425] TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +W1006 15:19:22.549000 745279 torch/utils/cpp_extension.py:2425] If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'] to specific architectures. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. ++ eval 'export MOM_ENABLED=false +export MOM_DO_MASK_SCALE=true +export MOM_TARGET_FFN_HIDDEN_SIZE=512' +++ export MOM_ENABLED=false +++ MOM_ENABLED=false +++ export MOM_DO_MASK_SCALE=true +++ MOM_DO_MASK_SCALE=true +++ export MOM_TARGET_FFN_HIDDEN_SIZE=512 +++ MOM_TARGET_FFN_HIDDEN_SIZE=512 ++ echo 'MOM_ENABLED=false MOM_DO_MASK_SCALE=true MOM_TARGET_FFN_HIDDEN_SIZE=512' +MOM_ENABLED=false MOM_DO_MASK_SCALE=true MOM_TARGET_FFN_HIDDEN_SIZE=512 ++ '[' false = true ']' ++ '[' 192 -gt 0 ']' ++ '[' 512 -le 0 ']' ++ '[' 10000.0 = 0.0 ']' +++ /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python - ++ ROPE_THETA=10000 ++ '[' -z 131072 ']' ++ '[' 131072 = 0 ']' ++ FFN_TOKEN_SHIFT= ++ FFN_INTERMEDIATE_TOKEN_SHIFT= ++ ATTN_TOKEN_SHIFT= ++ ATTN_Q_TOKEN_SHIFT= ++ ATTN_K_TOKEN_SHIFT= ++ ATTN_V_TOKEN_SHIFT= ++ TOKEN_SHIFT_CONV_SIZE=4 ++ TOKEN_SHIFT_CONV_INIT=default ++ export FFN_TOKEN_SHIFT FFN_INTERMEDIATE_TOKEN_SHIFT ATTN_TOKEN_SHIFT ++ export ATTN_Q_TOKEN_SHIFT ATTN_K_TOKEN_SHIFT ATTN_V_TOKEN_SHIFT ++ export TOKEN_SHIFT_CONV_SIZE TOKEN_SHIFT_CONV_INIT ++ export ROPE_THETA MAX_POSITION_EMBEDDINGS POSITION_EMBEDDING_TYPE ATTENTION_OUTPUT_GATE MOE_ROUTER_SQRT_GATE ++ ATTENTION_OUTPUT_GATE=true ++ /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/utils/render_gdn_hf_config.py --checkpoint-path /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/load_view2840 --template-path /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/hf-configs/YuLan-Mini-for-gdn-convert --output-dir /tmp/mcore2hf_ws_2840/mcore2hf-config.jBseUr +W1006 15:19:35.665000 745604 torch/utils/cpp_extension.py:2425] TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +W1006 15:19:35.665000 745604 torch/utils/cpp_extension.py:2425] If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'] to specific architectures. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. ++ echo '✅config.json: /tmp/mcore2hf_ws_2840/mcore2hf-config.jBseUr/config.json' +✅config.json: /tmp/mcore2hf_ws_2840/mcore2hf-config.jBseUr/config.json ++ /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python - /tmp/mcore2hf_ws_2840/mcore2hf-config.jBseUr/config.json +{ + "hidden_size": 2048, + "intermediate_size": 4800, + "num_hidden_layers": 56, + "num_attention_heads": 32, + "num_key_value_heads": 8, + "head_dim": 64, + "max_position_embeddings": 131072, + "rope_theta": 10000, + "attn_position_embedding_type": "nope", + "rnn_position_embedding_type": "nope", + "linear_num_key_heads": 8, + "linear_num_value_heads": 32, + "linear_key_head_dim": 64, + "linear_value_head_dim": 64, + "num_experts": 192, + "num_experts_per_tok": 4, + "attn_output_gate": true +} ++ sleep 5 ++ MODEL_SIZE=2.6b ++ LOAD_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/load_view2840 ++ MG2HF=true ++ USE_CUDA=false ++ PR=bf16 ++ HF_DIR=/tmp/mcore2hf_ws_2840/mcore2hf-config.jBseUr ++ export DROP_MTP ++ PREPARED_LOAD_DIR= ++ prepare_load_dir_for_iter /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/load_view2840 2840 ++ local SOURCE_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/load_view2840 ++ local ITER_NUM=2840 ++ '[' -z 2840 ']' +++ printf %07d 2840 ++ local ITER_BASENAME=iter_0002840 ++ local SOURCE_ITER_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/load_view2840/iter_0002840 ++ '[' '!' -d /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/load_view2840/iter_0002840 ']' ++ local LATEST_VALUE= ++ '[' -f /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/load_view2840/latest_checkpointed_iteration.txt ']' +++ cat /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/load_view2840/latest_checkpointed_iteration.txt ++ LATEST_VALUE=2840 ++ '[' 2840 = 2840 ']' ++ PREPARED_LOAD_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/load_view2840 ++ return 0 ++ LOAD_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/load_view2840 ++ echo LOAD_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/load_view2840 +LOAD_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/load_view2840 ++ '[' false = true ']' ++ '[' false = true ']' ++ '[' '!' -f /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/load_view2840/latest_checkpointed_iteration.txt ']' +++ cat /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/load_view2840/latest_checkpointed_iteration.txt ++ LATEST_CHECKPOINT_ITERATION=2840 ++ echo -e '\033[32mLatest checkpoint iteration: 2840\033[0m\n' +Latest checkpoint iteration: 2840 + ++ I=2840 ++ [[ 2840 =~ ^[0-9]+$ ]] +++ printf %07d 2840 ++ ITER_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/load_view2840/iter_0002840 ++ '[' '!' -f /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/load_view2840/iter_0002840/common.pt ']' ++ SAVE_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/load_view2840/iter_2840-hf ++ '[' -n /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a ']' ++ SAVE_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/iter_2840-hf ++ '[' -n '' ']' ++ mkdir -p /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/iter_2840-hf ++ bash ./scripts/mcore_gdn_moe/convert.sh 2.6b /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/load_view2840 /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/iter_2840-hf true false bf16 /tmp/mcore2hf_ws_2840/mcore2hf-config.jBseUr 2840 ++ tee /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/iter_2840-hf/hf2mcore.log +✅Enable attention output gate +\[\e[33mWarning adding --moe-shared-expert-gate for YuLan model. If you are not YuLan, please check this flag.\e[0m\] +torchrun --nproc_per_node 1 --nnodes 1 --node_rank 0 --master_addr localhost --master_port 41143 /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/impl/convert.py --tokenizer-type HuggingFaceTokenizer --tokenizer-model /tmp/mcore2hf_ws_2840/mcore2hf-config.jBseUr --hf-dir /tmp/mcore2hf_ws_2840/mcore2hf-config.jBseUr --mcore2hf --bf16 --normalization RMSNorm --swiglu --disable-bias-linear --seq-length 1 --max-position-embeddings 131072 --attention-backend auto --position-embedding-type none --kv-channels 64 --group-query-attention --add-qkv-bias --num-layers 56 --hidden-size 2048 --ffn-hidden-size 4800 --num-attention-heads 32 --rotary-base 10000 --rotary-percent 1.00 --num-query-groups 8 --normalization RMSNorm --norm-epsilon 1e-6 --untie-embeddings-and-output-weights --attention-output-gate --linear-attention-type gated_delta_net --linear-attention-freq [1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1] --linear-conv-kernel-dim 4 --linear-key-head-dim 64 --linear-value-head-dim 64 --linear-num-key-heads 8 --linear-num-value-heads 32 --num-experts 192 --moe-router-score-function softmax --moe-token-dispatcher-type alltoall --moe-router-topk 4 --moe-ffn-hidden-size 512 --moe-layer-freq [0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1] --moe-router-dtype fp32 --moe-shared-expert-intermediate-size 512 --moe-shared-expert-gate --moe-router-score-function softmax --moe-router-sqrt-gate --micro-batch-size 1 --global-batch-size 1024 --train-iters 500000 --weight-decay 0.1 --adam-beta1 0.9 --adam-beta2 0.95 --init-method-std 0.006 --clip-grad 1.0 --lr 2.0e-5 --lr-decay-style cosine --min-lr 6.0e-6 --lr-warmup-fraction .001 --lr-decay-iters 430000 --no-gradient-accumulation-fusion --bf16 --tensor-model-parallel-size 1 --pipeline-model-parallel-size 1 --expert-tensor-parallel-size 1 --expert-model-parallel-size 1 --log-interval 100 --save-interval 10000 --eval-interval 1000 --eval-iters 10 --model-type GPT --load-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/load_view2840 --save-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/iter_2840-hf --dist-ckpt-optim-fully-reshardable --skip-train --use-cpu-initialization --no-load-optim --no-load-rng --logging-level 1 --attention-backend auto --debug --max-shard-size 20GB --synchronizer mcore_gdn_moe --pretrain-script mcore_gdn_moe.model_provider --ckpt-step 2840 --gdn-split-in-proj --gated-norm --gated-norm-lora-rank 16 --distributed-backend gloo +W1006 15:19:56.232000 746011 torch/utils/cpp_extension.py:2425] TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +W1006 15:19:56.232000 746011 torch/utils/cpp_extension.py:2425] If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'] to specific architectures. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +Warning: Pai-Megatron-Patch arguments not available, some arguments may not be recognized +using world size: 1, data-parallel size: 1, context-parallel size: 1, hierarchical context-parallel sizes: None, tensor-model-parallel size: 1, pipeline-model-parallel size: 1 +Number of virtual stages per pipeline stage: None +accumulate and all-reduce gradients in fp32 for bfloat16 data type. +using torch.bfloat16 for parameters ... +------------------------ arguments ------------------------ + account_for_embedding_in_pipeline_split ......... False + account_for_loss_in_pipeline_split .............. False + accumulate_allreduce_grads_in_fp32 .............. True + activation_func_clamp_mode ...................... hard + activation_func_clamp_value ..................... None + adam_beta1 ...................................... 0.9 + adam_beta2 ...................................... 0.95 + adam_bias_correction ............................ True + adam_eps ........................................ 1e-08 + adamh_eps ....................................... 1e-08 + adamh_radius .................................... None + adamh_scope ..................................... lm_head + adamw_lr_mup_scaler ............................. False + add_bias_linear ................................. False + add_position_embedding .......................... True + add_qkv_bias .................................... True + adlr_autoresume ................................. False + adlr_autoresume_interval ........................ 1000 + align_grad_reduce ............................... True + align_param_gather .............................. False + allow_ambiguous_pad_tokens ...................... False + app_tag_run_name ................................ None + app_tag_run_version ............................. 0.0.0 + apply_layernorm_1p .............................. False + apply_query_key_layer_scaling ................... False + apply_residual_connection_post_layernorm ........ False + apply_rope_fusion ............................... False + async_save ...................................... None + async_tensor_model_parallel_allreduce ........... True + attention_backend ............................... AttnBackend.auto + attention_dropout ............................... 0.1 + attention_output_gate ........................... True + attention_output_gate_upcycling ................. False + attention_softmax_in_fp32 ....................... False + attn_k_token_shift .............................. None + attn_output_gate ................................ None + attn_output_gate_rand_init ...................... False + attn_q_token_shift .............................. None + attn_token_shift ................................ None + attn_v_token_shift .............................. None + auto_detect_ckpt_format ......................... False + auto_generate_cu_seqlens ........................ False + auto_model ...................................... AutoModelForCausalLM + barrier_with_L1_time ............................ True + benchmark_eval .................................. False + benchmark_global_batch .......................... None + benchmark_interval .............................. None + benchmark_micro_batch ........................... None + benchmark_sequence_length ....................... None + benchmark_tasks ................................. None + bert_binary_head ................................ True + bert_embedder_type .............................. megatron + bert_load ....................................... None + bf16 ............................................ True + bias_dropout_fusion ............................. True + bias_gelu_fusion ................................ False + bias_swiglu_fusion .............................. True + biencoder_projection_dim ........................ 0 + biencoder_shared_query_context_model ............ False + block_data_path ................................. None + cache_mla_latents ............................... False + calc_ft_timeouts ................................ False + calculate_per_token_loss ........................ False + check_for_large_grads ........................... False + check_for_nan_in_loss_and_grad .................. True + check_for_spiky_loss ............................ False + check_weight_hash_across_dp_replicas_interval ... None + ckpt_assume_constant_structure .................. False + ckpt_convert_format ............................. None + ckpt_convert_save ............................... None + ckpt_convert_update_legacy_dist_opt_format ...... False + ckpt_format ..................................... torch_dist + ckpt_fully_parallel_load ........................ False + ckpt_fully_parallel_save ........................ True + ckpt_fully_parallel_save_deprecated ............. False + ckpt_step ....................................... 2840 + classes_fraction ................................ 1.0 + clip_grad ....................................... 1.0 + clone_scatter_output_in_embedding ............... True + config_logger_dir ............................... + consumed_train_samples .......................... 0 + consumed_valid_samples .......................... 0 + context_parallel_size ........................... 1 + continual_pretrain .............................. False + continual_pretrain_preserve_opt_param_scheduler . False + corrected_weight_decay .......................... False + cp_comm_type .................................... ['p2p'] + create_attention_mask_in_dataloader ............. True + cross_entropy_fusion_impl ....................... native + cross_entropy_loss_fusion ....................... False + cuda_graph_impl ................................. none + cuda_graph_scope ................................ [] + cuda_graph_warmup_steps ......................... 3 + data_args_path .................................. None + data_cache_path ................................. None + data_parallel_random_init ....................... False + data_parallel_sharding_strategy ................. no_shard + data_parallel_size .............................. 1 + data_path ....................................... None + data_per_class_fraction ......................... 1.0 + data_sharding ................................... True + dataloader_defer_npy_index_mmap ................. False + dataloader_fast_cache_load ...................... False + dataloader_type ................................. single + dc_shortconv .................................... False + dc_shortconv_optimizer_state .................... conservative + dc_shortconv_upcycling .......................... False + ddp_average_in_collective ....................... False + ddp_bucket_size ................................. None + ddp_num_buckets ................................. None + ddp_pad_buckets_for_high_nccl_busbw ............. False + debug ........................................... True + decode_only_cuda_graphs ......................... False + decoder_first_pipeline_num_layers ............... None + decoder_last_pipeline_num_layers ................ None + decoder_num_layers .............................. None + decoder_seq_length .............................. None + decoupled_lr .................................... None + decoupled_min_lr ................................ None + decrease_batch_size_if_needed ................... False + defer_embedding_wgrad_compute ................... False + defer_grad_checks ............................... False + delay_wgrad_compute ............................. False + deprecated_use_mcore_models ..................... False + deterministic_mode .............................. False + dino_bottleneck_size ............................ 256 + dino_freeze_last_layer .......................... 1 + dino_head_hidden_size ........................... 2048 + dino_local_crops_number ......................... 10 + dino_local_img_size ............................. 96 + dino_norm_last_layer ............................ False + dino_teacher_temp ............................... 0.07 + dino_warmup_teacher_temp ........................ 0.04 + dino_warmup_teacher_temp_epochs ................. 30 + disable_attn_output_gate ........................ False + disable_bf16_reduced_precision_matmul ........... False + disable_chunked_prefill ......................... False + disable_explicit_attention_mask ................. False + disable_mamba_mem_eff_path ...................... False + disable_straggler_on_startup .................... False + disable_symmetric_registration .................. False + dist_ckpt_format_deprecated ..................... None + dist_ckpt_optim_fully_reshardable ............... True + dist_ckpt_save_pre_mcore_014 .................... False + dist_ckpt_strictness ............................ assume_ok_unexpected + distrib_optim_fully_reshardable_mem_efficient ... False + distribute_saved_activations .................... False + distributed_backend ............................. gloo + distributed_timeout_minutes ..................... 10 + distributed_timeout_seconds_after_init .......... None + document_packing_algorithm ...................... ogbfd + document_packing_dp_size ........................ None + drop_mtp ........................................ False + dryrun .......................................... False + dump_param_to_param_group_map ................... None + dynamic_context_parallel ........................ False + dynamic_context_parallel_sizes .................. None + dynamic_cp_candidate_sizes ...................... None + dynamic_cp_gdn_policy ........................... None + dynamic_cp_moe_policy ........................... None + dynamic_cp_on_error ............................. error + dynamic_cp_scheduler ............................ default_dynamic_cp + dynamic_cp_wave_coexecution ..................... False + dynamic_cp_workspace_max_tokens ................. None + emb_deviation_loss_coeff ........................ 0 + emb_deviation_type .............................. None + embedding_init_method_std ....................... None + embedding_path .................................. None + empty_unused_memory_level ....................... 0 + enable_cuda_graph ............................... False + enable_debug_logging ............................ False + enable_experimental ............................. False + enable_ft_package ............................... False + enable_full_sharding_in_hsdp .................... False + enable_gloo_process_groups ...................... True + enable_msc ...................................... True + enable_one_logger ............................... True + encoder_num_layers .............................. 56 + encoder_seq_length .............................. 1 + end_weight_decay ................................ 0.1 + eod_mask_loss ................................... False + error_injection_rate ............................ 0 + error_injection_type ............................ transient_error + eval_interval ................................... 1000 + eval_iters ...................................... 10 + evidence_data_path .............................. None + exit_duration_in_mins ........................... None + exit_interval ................................... None + exit_on_missing_checkpoint ...................... False + exit_signal_handler ............................. False + exp_avg_dtype ................................... torch.float32 + exp_avg_sq_dtype ................................ torch.float32 + expert_model_parallel_size ...................... 1 + expert_tensor_parallel_size ..................... 1 + export_force_local_attention .................... False + export_kd_cfg ................................... None + export_kd_teacher_ckpt_format ................... None + export_kd_teacher_load .......................... None + export_kv_cache_quant ........................... False + export_legacy_megatron .......................... False + export_model_type ............................... GPTModel + export_moe_apply_probs_on_input ................. False + export_offline_model ............................ False + export_qk_l2_norm ............................... False + export_quant_cfg ................................ None + export_real_quant_cfg ........................... None + export_te_mcore_model ........................... False + external_cuda_graph ............................. False + fast_thd_a2a .................................... False + ffn_hidden_size ................................. 4800 + ffn_intermediate_token_shift .................... None + ffn_token_shift ................................. None + final_logits_clamp_value ........................ None + fine_grained_activation_offloading .............. False + finetune ........................................ False + finetune_data_split ............................. train + finetune_hf_dataset ............................. None + first_last_layers_bf16 .......................... False + flash_decode .................................... False + fp16 ............................................ False + fp16_lm_cross_entropy ........................... False + fp32_residual_connection ........................ False + fp4 ............................................. None + fp4_param ....................................... False + fp4_recipe ...................................... nvfp4 + fp8 ............................................. None + fp8_amax_compute_algo ........................... most_recent + fp8_amax_history_len ............................ 1 + fp8_interval .................................... 1 + fp8_margin ...................................... 0 + fp8_param_gather ................................ False + fp8_recipe ...................................... delayed + fp8_wgrad ....................................... True + freeze_layernorm_weight ......................... False + freeze_non_mamba ................................ False + fsdp_double_buffer .............................. False + full_validation ................................. False + gated_norm ...................................... True + gated_norm_lora_rank ............................ 16 + gated_norm_upcycling ............................ False + gdn_alpha_weight_scale .......................... 1.0 + gdn_alpha_weight_scale_mode ..................... matrix + gdn_backend ..................................... fla + gdn_fused_split_forward ......................... False + gdn_gated_norm_backend .......................... torch + gdn_lr_mult_alpha ............................... 1.0 + gdn_lr_mult_beta ................................ 1.0 + gdn_lr_mult_k ................................... 1.0 + gdn_lr_mult_q ................................... 1.0 + gdn_lr_mult_v ................................... 1.0 + gdn_postinit_alog_shift ......................... 0.0 + gdn_postinit_alpha_scale ........................ 1.0 + gdn_postinit_key_scale .......................... 1.0 + gdn_pre_gdr_fusion .............................. False + gdn_pre_gdr_fusion_deterministic_weight_gradient False + gdn_pre_gdr_fusion_high_precision_backward ...... False + gdn_shortconv_separate_gate ..................... False + gdn_shortconv_separate_gate_upcycling ........... False + gdn_split_in_proj ............................... True + geglu ........................................... False + global_batch_size ............................... 1024 + glu_linear_offset ............................... 0.0 + grad_reduce_in_bf16 ............................. False + gradient_accumulation_fusion .................... False + gradient_reduce_div_fusion ...................... True + group_query_attention ........................... True + grpo_clamp_eps_lower ............................ 0.01 + grpo_clamp_eps_upper ............................ 0.01 + grpo_default_temperature ........................ 1.0 + grpo_default_top_p .............................. 0 + grpo_entropy_term_weight ........................ 0.0 + grpo_filter_groups_with_same_reward ............. False + grpo_group_size ................................. 2 + grpo_iterations ................................. 2 + grpo_kl_beta .................................... 0.001 + grpo_prompts_per_step ........................... 32 + head_lr_mult .................................... 1.0 + heterogeneous_layers_config_encoded_json ........ None + heterogeneous_layers_config_path ................ None + hf_dir .......................................... /tmp/mcore2hf_ws_2840/mcore2hf-config.jBseUr + hidden_dropout .................................. 0.1 + hidden_size ..................................... 2048 + hidden_states_shrink ............................ 1.0 + hierarchical_context_parallel_sizes ............. None + high_priority_stream_groups ..................... [] + hybrid_adamh .................................... False + hybrid_attention_ratio .......................... 0.0 + hybrid_context_parallel ......................... False + hybrid_mlp_ratio ................................ 0.0 + hybrid_override_pattern ......................... None + hysteresis ...................................... 2 + ict_head_size ................................... None + ict_load ........................................ None + img_h ........................................... 224 + img_w ........................................... 224 + increase_log_level_interval ..................... 1000 + increase_log_level_iters ........................ 5 + indexer_batch_size .............................. 128 + indexer_log_interval ............................ 1000 + inference_batch_times_seqlen_threshold .......... -1 + inference_dynamic_batching ...................... False + inference_dynamic_batching_block_size ........... 256 + inference_dynamic_batching_buffer_guaranteed_fraction 0.2 + inference_dynamic_batching_buffer_overflow_factor None + inference_dynamic_batching_buffer_size_gb ....... 40.0 + inference_dynamic_batching_max_requests_override None + inference_dynamic_batching_max_tokens_override .. None + inference_dynamic_batching_num_cuda_graphs ...... 16 + inference_dynamic_batching_track_paused_request_events False + inference_dynamic_batching_unified_memory_level . 0 + inference_max_batch_size ........................ 8 + inference_max_seq_length ........................ 2560 + inference_rng_tracker ........................... False + init_method_std ................................. 0.006 + init_method_xavier_uniform ...................... False + init_model_with_meta_device ..................... False + initial_loss_scale .............................. 4294967296 + inprocess_active_world_size ..................... 1 + inprocess_barrier_timeout ....................... 120 + inprocess_completion_timeout .................... 120 + inprocess_empty_cuda_cache ...................... False + inprocess_granularity ........................... node + inprocess_hard_timeout .......................... 90 + inprocess_heartbeat_interval .................... 30 + inprocess_heartbeat_timeout ..................... 60 + inprocess_last_call_wait ........................ 1 + inprocess_max_iterations ........................ None + inprocess_monitor_process_interval .............. 1.0 + inprocess_monitor_thread_interval ............... 1.0 + inprocess_progress_watchdog_interval ............ 1.0 + inprocess_restart ............................... False + inprocess_soft_timeout .......................... 60 + inprocess_termination_grace_time ................ 1 + is_hybrid_model ................................. False + iter_per_epoch .................................. 1250 + iterations_to_skip .............................. [] + keep_fp8_transpose_cache ........................ False + kitchen_config_file ............................. None + kitchen_recipe_number ........................... None + kv_channels ..................................... 64 + kv_lora_rank .................................... 32 + langrl_env_config ............................... None + langrl_external_server .......................... False + langrl_inference_server_conversation_template ... None + langrl_inference_server_type .................... inplace_megatron + lazy_mpu_init ................................... None + legacy_tokenizer ................................ False + linear_attention_freq ........................... [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 0, 0, 1, 1, 1, 1, 1, 1] + linear_attention_type ........................... gated_delta_net + linear_conv_kernel_dim .......................... 4 + linear_cross_entropy_accumulate_grad_in_fp32 .... True + linear_cross_entropy_num_chunks ................. 8 + linear_key_head_dim ............................. 64 + linear_num_key_heads ............................ 8 + linear_num_value_heads .......................... 32 + linear_rope_mode ................................ none + linear_value_head_dim ........................... 64 + lm_head_alpha ................................... 1.0 + lm_loss_normalization ........................... legacy + load ............................................ None + load_complemental_dataset ....................... None + load_dir ........................................ /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/load_view2840 + load_main_params_from_ckpt ...................... None + local_rank ...................................... 0 + log_attn_logits ................................. False + log_energy ...................................... False + log_hidden_states ............................... [] + log_interval .................................... 100 + log_logits_z_loss ............................... False + log_loss_scale_to_tensorboard ................... True + log_memory_to_tensorboard ....................... False + log_metrics_interval ............................ None + log_moe_activations ............................. False + log_moe_params .................................. False + log_moe_router_metrics .......................... False + log_num_zeros_in_grad ........................... False + log_padding_stats ............................... False + log_params ...................................... [] + log_params_norm ................................. False + log_per_layer_grad_rms .......................... False + log_per_module_grad_rms ......................... False + log_per_module_update_rms ....................... False + log_progress .................................... False + log_spike_diagnostics ........................... False + log_straggler ................................... False + log_throughput .................................. False + log_timers_to_tensorboard ....................... False + log_update_rms .................................. False + log_validation_ppl_to_tensorboard ............... False + log_world_size_to_tensorboard ................... False + logging_level ................................... 1 + loss_scale ...................................... None + loss_scale_window ............................... 1000 + lr .............................................. 2e-05 + lr_decay_iters .................................. 430000 + lr_decay_samples ................................ None + lr_decay_style .................................. cosine + lr_warmup_fraction .............................. 0.001 + lr_warmup_init .................................. 0.0 + lr_warmup_iters ................................. 0 + lr_warmup_samples ............................... 0 + lr_wsd_decay_iters .............................. None + lr_wsd_decay_samples ............................ None + lr_wsd_decay_style .............................. exponential + main_grads_dtype ................................ torch.float32 + main_params_dtype ............................... torch.float32 + make_vocab_size_divisible_by .................... 128 + mamba_disable_cp ................................ False + mamba_expand .................................... 2 + mamba_head_dim .................................. 64 + mamba_num_groups ................................ 8 + mamba_num_heads ................................. None + mamba_state_dim ................................. 128 + manual_gc ....................................... False + manual_gc_eval .................................. True + manual_gc_interval .............................. 0 + mask_factor ..................................... 1.0 + mask_prob ....................................... 0.15 + mask_type ....................................... random + masked_softmax_fusion ........................... True + max_position_embeddings ......................... 131072 + max_seqlen_per_cp_rank .......................... None + max_shard_size .................................. 20GB + max_tokens_to_oom ............................... 12000 + mcore2hf ........................................ True + memory_snapshot_path ............................ None + merge_file ...................................... None + micro_batch_size ................................ 1 + microbatch_group_size_per_vp_stage .............. None + mid_level_dataset_surplus ....................... 0.005 + min_loss_scale .................................. 1.0 + min_lr .......................................... 6e-06 + min_offloaded_tensor_size ....................... 1048576 + mlp_chunks_for_prefill .......................... 1 + mmap_bin_files .................................. True + mock_data ....................................... False + model_type ...................................... GPT + modelopt_enabled ................................ False + moe_apply_probs_on_input ........................ False + moe_aux_loss_coeff .............................. 0.0 + moe_deepep_num_sms .............................. 20 + moe_enable_deepep ............................... False + moe_expert_capacity_factor ...................... None + moe_extended_tp ................................. False + moe_ffn_hidden_size ............................. 512 + moe_flex_dispatcher_backend ..................... deepep + moe_gaux_overlap ................................ False + moe_grouped_gemm ................................ False + moe_hybridep_num_sms ............................ 16 + moe_input_jitter_eps ............................ None + moe_layer_freq .................................. [0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] + moe_layer_recompute ............................. False + moe_monitor_layers .............................. + moe_monitor_log_interval ........................ None + moe_pad_expert_input_to_capacity ................ False + moe_pad_experts_for_cuda_graph_inference ........ False + moe_per_layer_logging ........................... False + moe_permute_backend ............................. auto + moe_permute_fusion .............................. False + moe_router_aux_loss_fusion ...................... auto + moe_router_bias_update_method ................... sign + moe_router_bias_update_rate ..................... 0.001 + moe_router_dtype ................................ fp32 + moe_router_enable_expert_bias ................... False + moe_router_force_load_balancing ................. False + moe_router_fusion ............................... False + moe_router_gemm_backend ......................... torch + moe_router_group_topk ........................... None + moe_router_load_balancing_type .................. aux_loss + moe_router_num_groups ........................... None + moe_router_padding_for_fp8 ...................... False + moe_router_padding_for_quantization ............. False + moe_router_pre_softmax .......................... False + moe_router_score_function ....................... softmax + moe_router_sqrt_gate ............................ True + moe_router_topk ................................. 4 + moe_router_topk_scaling_factor .................. None + moe_shared_expert_gate .......................... True + moe_shared_expert_intermediate_size ............. 512 + moe_shared_expert_overlap ....................... False + moe_token_dispatcher_type ....................... alltoall + moe_token_drop_policy ........................... probs + moe_triton_dispatcher_exclude_padding_tokens .... False + moe_triton_mega_activation_sms .................. 4 + moe_triton_mega_combine_sms ..................... 32 + moe_triton_mega_compute_sms ..................... 24 + moe_triton_mega_dispatch_sms .................... 48 + moe_triton_mega_num_warps ....................... 16 + moe_triton_mega_reduce_sms ...................... 32 + moe_triton_mega_tail_sms ........................ 16 + moe_triton_shared_expert_overlap ................ False + moe_triton_workspace_expand ..................... 4.0 + moe_upcycling_granularity ....................... 1 + moe_use_legacy_grouped_gemm ..................... False + moe_use_upcycling ............................... False + moe_z_loss_coeff ................................ None + mrope_section ................................... None + mscale .......................................... 1.0 + mscale_all_dim .................................. 0.0 + mtp_linear_attention_type ....................... None + mtp_loss_scaling_factor ......................... 0.1 + mtp_num_layers .................................. None + multi_latent_attention .......................... False + multiple_validation_sets ........................ False + muon_ball_momentum .............................. 0.9 + muon_ball_msign_steps ........................... 5 + muon_ball_power_iteration_steps ................. 10 + muon_ball_qkv_split_mode ........................ component + muon_ball_radius_mode ........................... spectral_mup + muon_ball_retract_alpha ......................... 0.05 + muon_ball_retract_mode .......................... hard + muon_ball_scale_mode ............................ spectral_mup + muon_ball_split_fc1 ............................. True + muon_ball_split_moe_experts ..................... True + muon_ball_split_qkv ............................. True + muon_ball_use_nesterov .......................... True + muon_extra_scale_factor ......................... 1.0 + muon_fp32_matmul_prec ........................... medium + muon_momentum ................................... 0.9 + muon_num_ns_steps ............................... 5 + muon_qkv_split_mode ............................. component + muon_scale_mode ................................. spectral_mup + muon_scale_vectorized_mode ...................... full + muon_split_fc1 .................................. True + muon_split_moe_experts .......................... True + muon_split_qkv .................................. True + muon_tp_mode .................................... blockwise + muon_use_nesterov ............................... False + muon_vectorize .................................. [] + muon_vectorize_attn_dim ......................... hidden_size + mup_lr_mult ..................................... 1.0 + nccl_all_reduce_for_prefill ..................... False + nccl_communicator_config_path ................... None + nccl_ub ......................................... False + no_load_optim ................................... True + no_load_rng ..................................... True + no_load_scheduler ............................... None + no_persist_layer_norm ........................... False + no_rope_freq .................................... None + no_save_optim ................................... None + no_save_rng ..................................... None + no_save_step_one ................................ None + no_weight_decay_cond_type ....................... None + non_persistent_ckpt_type ........................ None + non_persistent_global_ckpt_dir .................. None + non_persistent_keep_last ........................ 1 + non_persistent_local_ckpt_algo .................. fully_parallel + non_persistent_local_ckpt_dir ................... None + non_persistent_save_interval .................... None + norm_epsilon .................................... 1e-06 + normalization ................................... RMSNorm + num_attention_heads ............................. 32 + num_channels .................................... 3 + num_classes ..................................... 1000 + num_dataset_builder_threads ..................... 1 + num_distributed_optimizer_instances ............. 1 + num_experts ..................................... 192 + num_hf_saver .................................... None + num_layers ...................................... 56 + num_layers_at_end_in_bf16 ....................... 1 + num_layers_at_start_in_bf16 ..................... 1 + num_layers_per_virtual_pipeline_stage ........... None + num_query_groups ................................ 8 + num_virtual_stages_per_pipeline_rank ............ None + num_workers ..................................... 2 + object_storage_cache_path ....................... None + offload_modules ................................. [] + one_logger_async ................................ False + one_logger_project .............................. megatron-lm + one_logger_run_name ............................. None + onnx_safe ....................................... None + openai_gelu ..................................... False + optimizer ....................................... adam + optimizer_cpu_offload ........................... False + optimizer_offload_fraction ...................... 1.0 + output_bert_embeddings .......................... False + output_logits_z_loss_coeff ...................... None + overlap_cpu_optimizer_d2h_h2d ................... False + overlap_grad_reduce ............................. False + overlap_moe_expert_parallel_comm ................ False + overlap_p2p_comm ................................ False + overlap_p2p_comm_warmup_flush ................... False + overlap_param_gather ............................ False + overlap_param_gather_with_optimizer_step ........ False + override_hf_eod_token_id ........................ None + override_opt_param_scheduler .................... False + packing_tokens_per_shard ........................ 100000000000 + padded_vocab_size ............................... None + params_dtype .................................... torch.bfloat16 + patch_dim ....................................... 16 + per_split_data_args_path ........................ None + perform_initialization .......................... True + perform_rl_step ................................. False + pin_cpu_grads ................................... True + pin_cpu_params .................................. True + pipeline_model_parallel_comm_backend ............ None + pipeline_model_parallel_layout .................. None + pipeline_model_parallel_size .................... 1 + position_embedding_type ......................... none + pp_aware_finegrained_recompute .................. False + pp_aware_finegrained_recompute_global_layers .... None + pp_aware_finegrained_recompute_pp_ranks ......... None + prefetch_factor ................................. None + pretrain_script ................................. mcore_gdn_moe.model_provider + pretrained_checkpoint ........................... None + previous_stage_seq_length ....................... None + profile ......................................... False + profile_microbatch_count ........................ 2 + profile_microbatch_dir .......................... None + profile_microbatch_iteration .................... None + profile_microbatch_start ........................ 0 + profile_ranks ................................... [0] + profile_step_end ................................ None + profile_step_start .............................. None + q_lora_rank ..................................... None + qk_head_dim ..................................... 128 + qk_l2_norm ...................................... False + qk_layernorm .................................... False + qk_pos_emb_head_dim ............................. 64 + query_in_block_prob ............................. 0.1 + quick_geglu ..................................... False + rampup_batch_size ............................... None + rank ............................................ 0 + recompute_granularity ........................... None + recompute_method ................................ None + recompute_modules ............................... None + recompute_num_layers ............................ None + record_memory_history ........................... False + relative_attention_max_distance ................. 128 + relative_attention_num_buckets .................. 32 + reparam_checkpoint .............................. None + reparam_fallback_value .......................... None + reparam_keys .................................... None + replication ..................................... False + replication_factor .............................. 2 + replication_jump ................................ None + rerun_mode ...................................... validate_results + reset_attention_mask ............................ False + reset_iteration_one_to_zero ..................... False + reset_position_ids .............................. False + result_rejected_tracker_filename ................ None + retriever_report_topk_accuracies ................ [] + retriever_score_scaling ......................... False + retriever_seq_length ............................ 256 + retro_add_retriever ............................. False + retro_attention_gate ............................ 1 + retro_cyclic_train_iters ........................ None + retro_encoder_attention_dropout ................. 0.1 + retro_encoder_hidden_dropout .................... 0.1 + retro_encoder_layers ............................ 2 + retro_num_neighbors ............................. 2 + retro_num_retrieved_chunks ...................... 2 + retro_project_dir ............................... None + retro_verify_neighbor_count ..................... True + reuse_grad_buf_for_mxfp8_param_ag ............... False + reuse_training_communicators .................... False + rl_calculate_intra_group_similarity ............. False + rl_importance_sampling_truncation_coef .......... None + rl_inference_logprobs_is_correction ............. False + rl_offload_kv_cache_during_training ............. False + rl_offload_optimizer_during_inference ........... False + rl_partial_rollouts ............................. False + rl_prompts_per_eval ............................. 32 + rl_remove_kv_cache_during_training .............. False + rl_reset_cuda_graphs ............................ False + rl_sequence_packing_algo ........................ fifo + rl_sequence_packing_bin_size .................... 8192 + rl_use_sequence_packing ......................... False + rope_scaling_factor ............................. 8.0 + rope_type ....................................... None + rotary_base ..................................... 10000 + rotary_interleaved .............................. False + rotary_percent .................................. 1.0 + rotary_scaling_factor ........................... 1.0 + rotary_seq_len_interpolation_factor ............. None + run_workload_inspector_server ................... False + runtime_audit_dir ............................... None + sample_rate ..................................... 1.0 + save ............................................ None + save_after_load ................................. False + save_dir ........................................ /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/iter_2840-hf + save_interval ................................... 10000 + save_retain_interval ............................ None + scale_emb ....................................... 1.0 + scatter_gather_tensors_in_pipeline .............. True + seed ............................................ 1234 + seq_length ...................................... 1 + sequence_packing_scheduler ...................... default_dynamic_cp + sequence_parallel ............................... False + sft ............................................. False + sft_tokenizer_prompt_format ..................... nemotron-h-aligned + sgd_momentum .................................... 0.9 + sharp_enabled_group ............................. None + short_seq_prob .................................. 0.1 + skip_train ...................................... True + skipped_train_samples ........................... 0 + softmax_type .................................... vanilla + spec ............................................ None + spectra_clip .................................... 1.0 + spectra_grad_clip ............................... None + spectra_hyperball ............................... False + spectra_ns_steps ................................ 10 + spectra_scope ................................... matrices + spectral_ball_momentum .......................... 0.9 + spectral_ball_msign_steps ....................... 8 + spectral_ball_power_iteration_steps ............. 20 + spectral_ball_qkv_split_mode .................... component + spectral_ball_radius_mode ....................... spectral_mup + spectral_ball_retract_alpha ..................... 0.05 + spectral_ball_retract_mode ...................... hard + spectral_ball_scale_mode ........................ spectral_mup + spectral_ball_solver ............................ bisection + spectral_ball_solver_max_iterations ............. 20 + spectral_ball_solver_tolerance_f ................ 1e-08 + spectral_ball_split_fc1 ......................... True + spectral_ball_split_moe_experts ................. True + spectral_ball_split_qkv ......................... True + spectral_ball_use_nesterov ...................... True + spectral_mup_init ............................... False + spike_diagnostics_dir ........................... None + spike_diagnostics_end ........................... 21 + spike_diagnostics_start ......................... 2 + split ........................................... None + split_expert_init ............................... True + split_fc1_init .................................. True + split_qkv_init .................................. True + split_qkv_init_mode ............................. group + sqreglu ......................................... False + squared_relu .................................... False + start_samples ................................... None + start_weight_decay .............................. 0.1 + straggler_ctrlr_port ............................ 65535 + straggler_minmax_count .......................... 1 + strict_fsdp_dtensor_load ........................ True + suggested_communication_unit_size ............... None + swanlab_exp_name ................................ + swanlab_project ................................. + swanlab_save_dir ................................ + swanlab_workspace ............................... + swiglu .......................................... True + swin_backbone_type .............................. tiny + symmetric_ar_type ............................... None + synchronizer .................................... mcore_gdn_moe + target_ckpt_format .............................. torch_dist + te_rng_tracker .................................. False + teacher_model_config ............................ None + tensor_model_parallel_size ...................... 1 + tensorboard_dir ................................. None + tensorboard_log_interval ........................ 1 + tensorboard_queue_size .......................... 1000 + test_data_path .................................. None + test_mode ....................................... False + tiktoken_num_special_tokens ..................... 1000 + tiktoken_pattern ................................ None + tiktoken_special_tokens ......................... None + timing_log_level ................................ 0 + timing_log_option ............................... minmax + titles_data_path ................................ None + token_shift_conv_init ........................... default + token_shift_conv_size ........................... 4 + tokenizer_metadata .............................. None + tokenizer_model ................................. /tmp/mcore2hf_ws_2840/mcore2hf-config.jBseUr + tokenizer_type .................................. HuggingFaceTokenizer + torch_fsdp2_reshard_after_forward ............... True + tp_comm_bootstrap_backend ....................... nccl + tp_comm_bulk_dgrad .............................. True + tp_comm_bulk_wgrad .............................. True + tp_comm_overlap ................................. False + tp_comm_overlap_ag .............................. True + tp_comm_overlap_cfg ............................. None + tp_comm_overlap_rs .............................. True + tp_comm_overlap_rs_dgrad ........................ False + tp_comm_split_ag ................................ True + tp_comm_split_rs ................................ True + train_data_path ................................. None + train_iters ..................................... 500000 + train_samples ................................... None + train_sync_interval ............................. None + train_until_stopped ............................. False + transformer_impl ................................ transformer_engine + transformer_pipeline_model_parallel_size ........ 1 + trust_remote_code ............................... False + untie_embeddings_and_output_weights ............. True + use_checkpoint_args ............................. False + use_checkpoint_opt_param_scheduler .............. False + use_cpu_initialization .......................... True + use_dist_ckpt ................................... True + use_dist_ckpt_deprecated ........................ False + use_distributed_optimizer ....................... False + use_flash_attn .................................. False + use_fused_weighted_squared_relu ................. False + use_gpu ......................................... False + use_legacy_models ............................... False + use_megatron_fsdp ............................... False + use_mp_args_from_checkpoint_args ................ False + use_one_sent_docs ............................... False + use_persistent_ckpt_worker ...................... False + use_precision_aware_optimizer ................... False + use_pytorch_profiler ............................ False + use_ring_exchange_p2p ........................... False + use_rope_scaling ................................ False + use_rotary_position_embeddings .................. False + use_sharp ....................................... False + use_te_activation_func .......................... False + use_tokenizer_model_from_checkpoint_args ........ True + use_torch_fsdp2 ................................. False + use_torch_optimizer_for_cpu_offload ............. False + use_tp_pp_dp_mapping ............................ False + v_head_dim ...................................... 128 + valid_data_path ................................. None + variable_seq_lengths ............................ False + virtual_pipeline_model_parallel_size ............ None + vision_backbone_type ............................ vit + vision_pretraining .............................. False + vision_pretraining_type ......................... classify + vocab_extra_ids ................................. 0 + vocab_file ...................................... None + vocab_size ...................................... None + wandb_entity .................................... + wandb_exp_name .................................. + wandb_project ................................... + wandb_save_dir .................................. + weight_decay .................................... 0.1 + weight_decay_incr_style ......................... constant + wgrad_deferral_limit ............................ 0 + window_attn_skip_freq ........................... None + window_size ..................................... None + word_embedding_dropout_prob ..................... 0.0 + world_size ...................................... 1 + yaml_cfg ........................................ None +-------------------- end of arguments --------------------- +INFO:megatron.core.num_microbatches_calculator:setting number of microbatches to constant 1024 +> building HuggingFaceTokenizer tokenizer ... + > padded vocab (size: 99000) with 72 dummy tokens (new size: 99072) +WARNING: one_logger package is required to enable e2e metrics tracking. please go to https://confluence.nvidia.com/display/MLWFO/Package+Repositories for details to install it +INFO:megatron.training.initialize:Setting logging level to 1 +WARNING:megatron.core.rerun_state_machine:RerunStateMachine initialized in mode validate_results +> initializing torch distributed ... +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +> initialized tensor model parallel with size 1 +> initialized pipeline model parallel with size 1 +> setting random seeds to 1234 ... +> compiling dataset index builder ... +make: Entering directory '/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/datasets' +make: Nothing to be done for 'default'. +make: Leaving directory '/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/datasets' +>>> done with dataset index builder. Compilation time: 0.130 seconds +WARNING: constraints for invoking optimized fused softmax kernel are not met. We default back to unfused kernel invocations. +> compiling and loading fused kernels ... +>>> done with compiling and loading fused kernels. Compilation time: 0.003 seconds +WORLD_SIZE: 1, RANK: 0, LOCAL_RANK: 0 +[convert] creating synchronizer: mode=mcore2hf +building GPT model ... +[GPT_LAYER_SPEC] log_hidden_states=[] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=[] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=[] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=[] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/transformers/modeling_rope_utils.py:1036: FutureWarning: `rope_config_validation` is deprecated and has been removed. Its functionality has been moved to RotaryEmbeddingConfigMixin.validate_rope method. PreTrainedConfig inherits this class, so please call self.validate_rope() instead. Also, make sure to use the new rope_parameters syntax. You can call self.standardize_rope_params() in the meantime. + warnings.warn( +[transformers] `layer_type_validation` is deprecated and will be removed in v5.20. Use `PreTrainedConfig.validate_layer_type` instead +[transformers] `layer_type_validation` is deprecated and will be removed in v5.20. Use `PreTrainedConfig.validate_layer_type` instead +[transformers] `torch_dtype` is deprecated! Use `dtype` instead! +[ERROR] `cache_position` is part of Qwen3NextModel.forward's signature, but not documented. Make sure to add it to the docstring of the function in /mnt/ssd/cache_tmp/huggingface/modules/transformers_modules/mcore2hf_hyphen_config_dot_jBseUr/10c6759df797195a/modeling_qwen3_next.py. +[ERROR] `cache_position` is part of Qwen3NextForCausalLM.forward's signature, but not documented. Make sure to add it to the docstring of the function in /mnt/ssd/cache_tmp/huggingface/modules/transformers_modules/mcore2hf_hyphen_config_dot_jBseUr/10c6759df797195a/modeling_qwen3_next.py. +[transformers] `torch_dtype` is deprecated! Use `dtype` instead! +INFO:transformers_modules.mcore2hf_hyphen_config_dot_jBseUr.10c6759df797195a.modeling_qwen3_next:[Qwen3Next custom] attn_position_embedding_type=nope, rnn_position_embedding_type=nope, attn_logits_scaling=None +Qwen3NextForCausalLM( + (model): Qwen3NextModel( + (embed_tokens): Embedding(99072, 2048, padding_idx=102) + (layers): ModuleList( + (0-1): 2 x Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=4800, bias=False) + (up_proj): Linear(in_features=2048, out_features=4800, bias=False) + (down_proj): Linear(in_features=4800, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (2-11): 10 x Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (12): Qwen3NextDecoderLayer( + (self_attn): Qwen3NextAttention( + (q_proj): Linear(in_features=2048, out_features=4096, bias=True) + (k_proj): Linear(in_features=2048, out_features=512, bias=True) + (v_proj): Linear(in_features=2048, out_features=512, bias=True) + (o_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (13-19): 7 x Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (20-21): 2 x Qwen3NextDecoderLayer( + (self_attn): Qwen3NextAttention( + (q_proj): Linear(in_features=2048, out_features=4096, bias=True) + (k_proj): Linear(in_features=2048, out_features=512, bias=True) + (v_proj): Linear(in_features=2048, out_features=512, bias=True) + (o_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (22): Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (23): Qwen3NextDecoderLayer( + (self_attn): Qwen3NextAttention( + (q_proj): Linear(in_features=2048, out_features=4096, bias=True) + (k_proj): Linear(in_features=2048, out_features=512, bias=True) + (v_proj): Linear(in_features=2048, out_features=512, bias=True) + (o_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (24-45): 22 x Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (46): Qwen3NextDecoderLayer( + (self_attn): Qwen3NextAttention( + (q_proj): Linear(in_features=2048, out_features=4096, bias=True) + (k_proj): Linear(in_features=2048, out_features=512, bias=True) + (v_proj): Linear(in_features=2048, out_features=512, bias=True) + (o_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (47): Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (48-49): 2 x Qwen3NextDecoderLayer( + (self_attn): Qwen3NextAttention( + (q_proj): Linear(in_features=2048, out_features=4096, bias=True) + (k_proj): Linear(in_features=2048, out_features=512, bias=True) + (v_proj): Linear(in_features=2048, out_features=512, bias=True) + (o_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (50-55): 6 x Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + ) + (norm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (rotary_emb): Qwen3NextRotaryEmbedding() + ) + (lm_head): Linear(in_features=2048, out_features=99072, bias=False) +) +GPTModel( + (embedding): LanguageModelEmbedding( + (word_embeddings): VocabParallelEmbedding() + (embedding_dropout): Dropout(p=0.1, inplace=False) + ) + (decoder): TransformerBlock( + (layers): ModuleList( + (0-1): 2 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=9600, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=4800, out_features=2048, bias=False, TP=1) + ) + ) + (2-11): 10 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (12): TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): SelfAttention( + (core_attention): TEDotProductAttention( + (flash_attention): FlashAttention() + (fused_attention): FusedAttention() + (unfused_attention): UnfusedDotProductAttention( + (scale_mask_softmax): FusedScaleMaskSoftmax() + (attention_dropout): Dropout(p=0.1, inplace=False) + ) + ) + (linear_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (linear_qkv): TEColumnParallelLinear(in_features=2048, out_features=5120, bias=True, TP=1) + (q_layernorm): IdentityOp() + (k_layernorm): IdentityOp() + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (13-19): 7 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (20-21): 2 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): SelfAttention( + (core_attention): TEDotProductAttention( + (flash_attention): FlashAttention() + (fused_attention): FusedAttention() + (unfused_attention): UnfusedDotProductAttention( + (scale_mask_softmax): FusedScaleMaskSoftmax() + (attention_dropout): Dropout(p=0.1, inplace=False) + ) + ) + (linear_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (linear_qkv): TEColumnParallelLinear(in_features=2048, out_features=5120, bias=True, TP=1) + (q_layernorm): IdentityOp() + (k_layernorm): IdentityOp() + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (22): TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (23): TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): SelfAttention( + (core_attention): TEDotProductAttention( + (flash_attention): FlashAttention() + (fused_attention): FusedAttention() + (unfused_attention): UnfusedDotProductAttention( + (scale_mask_softmax): FusedScaleMaskSoftmax() + (attention_dropout): Dropout(p=0.1, inplace=False) + ) + ) + (linear_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (linear_qkv): TEColumnParallelLinear(in_features=2048, out_features=5120, bias=True, TP=1) + (q_layernorm): IdentityOp() + (k_layernorm): IdentityOp() + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (24-45): 22 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (46): TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): SelfAttention( + (core_attention): TEDotProductAttention( + (flash_attention): FlashAttention() + (fused_attention): FusedAttention() + (unfused_attention): UnfusedDotProductAttention( + (scale_mask_softmax): FusedScaleMaskSoftmax() + (attention_dropout): Dropout(p=0.1, inplace=False) + ) + ) + (linear_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (linear_qkv): TEColumnParallelLinear(in_features=2048, out_features=5120, bias=True, TP=1) + (q_layernorm): IdentityOp() + (k_layernorm): IdentityOp() + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (47): TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (48-49): 2 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): SelfAttention( + (core_attention): TEDotProductAttention( + (flash_attention): FlashAttention() + (fused_attention): FusedAttention() + (unfused_attention): UnfusedDotProductAttention( + (scale_mask_softmax): FusedScaleMaskSoftmax() + (attention_dropout): Dropout(p=0.1, inplace=False) + ) + ) + (linear_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (linear_qkv): TEColumnParallelLinear(in_features=2048, out_features=5120, bias=True, TP=1) + (q_layernorm): IdentityOp() + (k_layernorm): IdentityOp() + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (50-55): 6 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + ) + (final_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (output_layer): ColumnParallelLinear(in_features=2048, out_features=99072, bias=False, TP=1) +) +DEBUG:multistorageclient.config:Searching MSC config file paths: ['/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/msc/msc_config.yaml', '/mnt/ssd/cache_tmp/.config/msc/config.yaml', '/mnt/ssd/cache_tmp/.config/msc/config.json', '/root/.msc_config.yaml', '/root/.msc_config.json', '/root/.config/msc/config.yaml', '/root/.config/msc/config.json', '/etc/xdg/msc/config.yaml', '/etc/xdg/msc/config.json', '/etc/msc_config.yaml', '/etc/msc_config.json'] +DEBUG:multistorageclient.config:Using MSC config file: /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/msc/msc_config.yaml +DEBUG:multistorageclient.config:Searching MSC config file paths: ['/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/msc/msc_config.yaml', '/mnt/ssd/cache_tmp/.config/msc/config.yaml', '/mnt/ssd/cache_tmp/.config/msc/config.json', '/root/.msc_config.yaml', '/root/.msc_config.json', '/root/.config/msc/config.yaml', '/root/.config/msc/config.json', '/etc/xdg/msc/config.yaml', '/etc/xdg/msc/config.json', '/etc/msc_config.yaml', '/etc/msc_config.json'] +DEBUG:multistorageclient.config:Using MSC config file: /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/msc/msc_config.yaml +DEBUG:multistorageclient.client.single:No metadata provider configured, auto-commit will not be enabled +DEBUG:multistorageclient.client.client:StorageClient '__filesystem__' using SingleStorageClient +sharded_state_dict metadata loaded from the checkpoint: {'singleton_local_shards': True, 'distrib_optim_sharding_type': 'dp_reshardable', 'chained_optim_avoid_prefix': True} +Job sharding has changed: Rerun state will be ignored + loading distributed checkpoint from /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/load_view2840 at iteration 2840 +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/dist_checkpointing/strategies/torch.py:956: FutureWarning: `load_state_dict` is deprecated and will be removed in future versions. Please use `load` instead. + checkpoint.load_state_dict( + checkpoint version 3.0 + successfully loaded checkpoint from /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/load_view2840 [ t 1/1, p 1/1 ] at iteration 2840 +[convert] synchronizer created in 377.71s +[convert] starting sync_params +INFO:root:Converting layer 0 is_gdn=True is_not_moe=True +INFO:root:Converting layer 1 is_gdn=True is_not_moe=True +INFO:root:Converting layer 2 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.07666015625 mean=-0.00048828125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.11474609375 mean=0.001983642578125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 3 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.078125 mean=-0.000179290771484375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.12109375 mean=1.3172626495361328e-05 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 4 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0849609375 mean=-0.00026702880859375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.08935546875 mean=0.00135040283203125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 5 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.08837890625 mean=-0.0002307891845703125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0908203125 mean=0.0023651123046875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 6 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.09619140625 mean=-0.00021266937255859375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.083984375 mean=-0.00128936767578125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 7 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.091796875 mean=-4.458427429199219e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0703125 mean=-0.0002613067626953125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 8 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.09423828125 mean=-6.0558319091796875e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.06982421875 mean=0.00011682510375976562 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 9 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.08837890625 mean=2.9087066650390625e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.06396484375 mean=0.00015735626220703125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 10 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0849609375 mean=4.315376281738281e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.06103515625 mean=-0.000675201416015625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 11 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.08251953125 mean=7.343292236328125e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.07275390625 mean=-4.947185516357422e-06 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 12 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 12: args.attention_output_gate=True +WARNING:root:Layer 12: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.07666015625 mean=0.00023555755615234375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0595703125 mean=-0.000732421875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 13 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.072265625 mean=0.00018978118896484375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.048583984375 mean=-0.001251220703125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 14 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.076171875 mean=0.000186920166015625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.05224609375 mean=-0.0011138916015625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 15 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.07373046875 mean=0.0002593994140625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.056396484375 mean=-0.00128936767578125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 16 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.07275390625 mean=0.0003337860107421875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0556640625 mean=-0.001220703125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 17 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.06982421875 mean=0.000270843505859375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0537109375 mean=-0.0018157958984375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 18 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.06591796875 mean=0.0002593994140625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.056396484375 mean=-0.0026397705078125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 19 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.068359375 mean=0.000598907470703125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.056396484375 mean=-0.00180816650390625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 20 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 20: args.attention_output_gate=True +WARNING:root:Layer 20: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0673828125 mean=0.000385284423828125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.056884765625 mean=-0.0016326904296875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 21 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 21: args.attention_output_gate=True +WARNING:root:Layer 21: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.060302734375 mean=0.000431060791015625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.04833984375 mean=-0.002410888671875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 22 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.055419921875 mean=0.0003223419189453125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.051025390625 mean=-0.0034332275390625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 23 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 23: args.attention_output_gate=True +WARNING:root:Layer 23: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0537109375 mean=0.0004673004150390625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.043701171875 mean=-0.0026397705078125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 24 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.052490234375 mean=0.000392913818359375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.041015625 mean=-0.00299072265625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 25 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0498046875 mean=0.000339508056640625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.03955078125 mean=-0.0021209716796875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 26 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.049560546875 mean=0.0002651214599609375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.036865234375 mean=-0.00225830078125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 27 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0517578125 mean=0.000415802001953125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.03466796875 mean=-0.0021209716796875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 28 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.048095703125 mean=0.000354766845703125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0308837890625 mean=-0.00107574462890625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 29 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.048583984375 mean=0.000484466552734375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.027099609375 mean=-0.0012359619140625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 30 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.051513671875 mean=0.000743865966796875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0286865234375 mean=-0.00106048583984375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 31 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.049072265625 mean=0.000484466552734375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.02880859375 mean=-0.000843048095703125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 32 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.046875 mean=0.00012111663818359375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0272216796875 mean=-0.0005035400390625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 33 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.05029296875 mean=8.869171142578125e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0296630859375 mean=-0.000213623046875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 34 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.052490234375 mean=0.00015735626220703125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0291748046875 mean=-1.9788742065429688e-05 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 35 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0576171875 mean=0.0001068115234375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0341796875 mean=2.1219253540039062e-05 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 36 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.059814453125 mean=6.389617919921875e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.044189453125 mean=0.00066375732421875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 37 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.059814453125 mean=0.00011777877807617188 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.04345703125 mean=0.00112152099609375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 38 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.058349609375 mean=0.0001220703125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.038330078125 mean=0.00133514404296875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 39 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.056884765625 mean=9.202957153320312e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.03759765625 mean=0.0009918212890625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 40 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.053466796875 mean=0.0001354217529296875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0274658203125 mean=0.00023365020751953125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 41 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.050048828125 mean=0.0001392364501953125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0257568359375 mean=0.00022983551025390625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 42 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.04443359375 mean=0.0003528594970703125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0225830078125 mean=-0.000514984130859375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 43 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.03857421875 mean=0.0001926422119140625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0220947265625 mean=-0.0003643035888671875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 44 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.03515625 mean=0.0003337860107421875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.022705078125 mean=-0.000293731689453125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 45 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.03564453125 mean=0.00017261505126953125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.025390625 mean=-0.00090789794921875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 46 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 46: args.attention_output_gate=True +WARNING:root:Layer 46: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.036376953125 mean=0.0003681182861328125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0223388671875 mean=-0.000736236572265625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 47 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.037109375 mean=0.00019359588623046875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0263671875 mean=-0.000896453857421875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 48 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 48: args.attention_output_gate=True +WARNING:root:Layer 48: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.034423828125 mean=-6.723403930664062e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0213623046875 mean=-0.0004425048828125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 49 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 49: args.attention_output_gate=True +WARNING:root:Layer 49: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.03466796875 mean=5.435943603515625e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0255126953125 mean=-0.0010986328125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 50 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.03662109375 mean=-2.574920654296875e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.025634765625 mean=-0.000728607177734375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 51 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.037109375 mean=6.151199340820312e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0257568359375 mean=-0.00121307373046875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 52 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.041015625 mean=0.000244140625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.03271484375 mean=-0.00148773193359375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 53 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.038330078125 mean=5.9604644775390625e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.05126953125 mean=-0.00262451171875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 54 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0390625 mean=5.507469177246094e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0419921875 mean=-0.002288818359375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 55 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.042724609375 mean=-0.00015354156494140625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.044677734375 mean=0.0026397705078125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +[convert] finished sync_params in 0.93s +[convert] starting check_and_save -> /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/iter_2840-hf +DEBUG:root:[RANK 0] 0 send op & 0 recv op. +INFO:root:[Iters 0 RANK 0] starts synchronizing parameters with other ranks... +INFO:root:[Iters 0 RANK 0] finishes synchronizing +[Iters 0 RANK 0] model-00001-of-00004.safetensors is saved. +DEBUG:root:[Iters 0 RANK 0] joined +DEBUG:root:[RANK 0] 0 send op & 0 recv op. +INFO:root:[Iters 1 RANK 0] starts synchronizing parameters with other ranks... +INFO:root:[Iters 1 RANK 0] finishes synchronizing +[Iters 1 RANK 0] model-00002-of-00004.safetensors is saved. +DEBUG:root:[Iters 1 RANK 0] joined +DEBUG:root:[RANK 0] 0 send op & 0 recv op. +INFO:root:[Iters 2 RANK 0] starts synchronizing parameters with other ranks... +INFO:root:[Iters 2 RANK 0] finishes synchronizing +[Iters 2 RANK 0] model-00003-of-00004.safetensors is saved. +DEBUG:root:[Iters 2 RANK 0] joined +DEBUG:root:[RANK 0] 0 send op & 0 recv op. +INFO:root:[Iters 3 RANK 0] starts synchronizing parameters with other ranks... +INFO:root:[Iters 3 RANK 0] finishes synchronizing +[Iters 3 RANK 0] model-00004-of-00004.safetensors is saved. +DEBUG:root:[Iters 3 RANK 0] joined +[convert] finished check_and_save in 74.84s +Conversion finished in 454.8206191062927 seconds. ++ echo '✅Conversion completed: /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/iter_2840-hf' +✅Conversion completed: /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/iter_2840-hf ++ cleanup_real_config_path ++ rm -rf /tmp/mcore2hf_ws_2840/mcore2hf-config.jBseUr ++ '[' -n '' ']' diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/hf_export_logs/convert_iter3840.log b/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/hf_export_logs/convert_iter3840.log new file mode 100644 index 0000000000000000000000000000000000000000..99813786eedc2a606c8c7c6779d39cef22a250a6 --- /dev/null +++ b/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/hf_export_logs/convert_iter3840.log @@ -0,0 +1,2665 @@ +++ realpath /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/scripts/mcore_gdn_moe/../../../.. ++ MEGATRON_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher ++ CONVERTOR_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor ++ REPAIR_SCRIPT=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/utils/repair_gdn_checkpoint.py ++ RENDER_CONFIG_SCRIPT=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/utils/render_gdn_hf_config.py ++ export PYTHON_ENV_ROOT=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud ++ PYTHON_ENV_ROOT=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud ++ PYTHON_BIN=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python ++ '[' '!' -x /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python ']' ++ '[' '!' -x /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python ']' ++ '[' '!' -x /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python ']' ++ '[' '!' -x /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python ']' ++ export PYTHON_BIN ++ /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python -c 'import torch' ++ export PATH=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin:/root/.kimi-code/bin:/root/.local/bin:/mnt/hdd/minyingqian/mihomo/bin:/root/.grok/bin:/usr/local/bin:/root/miniconda3/bin:/root/.local/bin:/usr/local/bin:/root/miniconda3/bin:/root/miniconda3/bin:/mnt/ssd/liuzian/pandoc/current/bin:/mnt/ssd/cache_tmp/.local/share/../bin:/mnt/ssd/yulan/bin:/mnt/ssd/yulan/bin:/root/.local/bin:/usr/local/bin:/root/miniconda3/bin:/root/.local/bin:/usr/local/bin:/root/miniconda3/bin:/mnt/ssd/cache_tmp/.local/bin:/root/.local/bin:/root/.local/bin:/mnt/hdd/minyingqian/mihomo/bin:/root/.grok/bin:/usr/local/bin:/root/miniconda3/bin:/root/.local/bin:/usr/local/bin:/root/miniconda3/bin:/root/miniconda3/bin:/mnt/ssd/liuzian/pandoc/current/bin:/mnt/ssd/cache_tmp/.local/bin:/mnt/ssd/yulan/bin:/mnt/ssd/yulan/bin:/root/.local/bin:/root/miniconda3/bin:/root/miniconda3/condabin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/mnt/ssd/yulan/cache/installs/nsight-compute/2026.2:/mnt/ssd/yulan/cache/installs/nsight-systems/2026.3.1/bin:/mnt/ssd/yulan/cache/installs/nsight-compute/2026.2:/mnt/ssd/yulan/cache/installs/nsight-systems/2026.3.1/bin ++ PATH=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin:/root/.kimi-code/bin:/root/.local/bin:/mnt/hdd/minyingqian/mihomo/bin:/root/.grok/bin:/usr/local/bin:/root/miniconda3/bin:/root/.local/bin:/usr/local/bin:/root/miniconda3/bin:/root/miniconda3/bin:/mnt/ssd/liuzian/pandoc/current/bin:/mnt/ssd/cache_tmp/.local/share/../bin:/mnt/ssd/yulan/bin:/mnt/ssd/yulan/bin:/root/.local/bin:/usr/local/bin:/root/miniconda3/bin:/root/.local/bin:/usr/local/bin:/root/miniconda3/bin:/mnt/ssd/cache_tmp/.local/bin:/root/.local/bin:/root/.local/bin:/mnt/hdd/minyingqian/mihomo/bin:/root/.grok/bin:/usr/local/bin:/root/miniconda3/bin:/root/.local/bin:/usr/local/bin:/root/miniconda3/bin:/root/miniconda3/bin:/mnt/ssd/liuzian/pandoc/current/bin:/mnt/ssd/cache_tmp/.local/bin:/mnt/ssd/yulan/bin:/mnt/ssd/yulan/bin:/root/.local/bin:/root/miniconda3/bin:/root/miniconda3/condabin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/mnt/ssd/yulan/cache/installs/nsight-compute/2026.2:/mnt/ssd/yulan/cache/installs/nsight-systems/2026.3.1/bin:/mnt/ssd/yulan/cache/installs/nsight-compute/2026.2:/mnt/ssd/yulan/cache/installs/nsight-systems/2026.3.1/bin ++ export VIRTUAL_ENV=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud ++ VIRTUAL_ENV=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud ++ echo ✅MEGATRON_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher +✅MEGATRON_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher ++ echo ✅PYTHON_BIN=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python +✅PYTHON_BIN=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python ++ export CUDA_LAUNCH_BLOCKING=1 ++ CUDA_LAUNCH_BLOCKING=1 ++ export CUDA_VISIBLE_DEVICES=0 ++ CUDA_VISIBLE_DEVICES=0 ++ export KUBERNETES_CONTAINER_RESOURCE_GPU=1 ++ KUBERNETES_CONTAINER_RESOURCE_GPU=1 ++ export TP=1 ++ TP=1 ++ export PP=1 ++ PP=1 ++ export ETP=1 ++ ETP=1 ++ export EP=1 ++ EP=1 ++ export 'MODEL_PARALLEL_ARGS=--tensor-model-parallel-size 1 --pipeline-model-parallel-size 1 --expert-tensor-parallel-size 1 --expert-model-parallel-size 1' ++ MODEL_PARALLEL_ARGS='--tensor-model-parallel-size 1 --pipeline-model-parallel-size 1 --expert-tensor-parallel-size 1 --expert-model-parallel-size 1' ++ WORKSPACE_TMP_DIR=/tmp/mcore2hf_ws_3840 ++ mkdir -p /tmp/mcore2hf_ws_3840 ++ MEGATRON_CHECKPOINT_PATH=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/load_view3840 ++ echo MEGATRON_CHECKPOINT_PATH=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/load_view3840 +MEGATRON_CHECKPOINT_PATH=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/load_view3840 ++ MANUAL_ITER= ++ CONVERT_ALL=false ++ FORCE_RECONVERT=false ++ DROP_MTP=false ++ POSITIONAL_ARGS=() ++ [[ 3 -gt 0 ]] ++ case $1 in ++ POSITIONAL_ARGS+=("$1") ++ shift ++ [[ 2 -gt 0 ]] ++ case $1 in ++ '[' -z 3840 ']' ++ MANUAL_ITER=3840 ++ shift 2 ++ [[ 0 -gt 0 ]] ++ set -- /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/load_view3840 ++ '[' -e /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/load_view3840 ']' +++ readlink -f /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/load_view3840 ++ MEGATRON_CHECKPOINT_PATH=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/load_view3840 ++ echo ✅MEGATRON_CHECKPOINT_PATH=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/load_view3840 +✅MEGATRON_CHECKPOINT_PATH=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/load_view3840 +++ basename /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/load_view3840 ++ CHECKPOINT_BASENAME=load_view3840 ++ [[ load_view3840 =~ ^iter_([0-9]+)$ ]] ++ '[' false = true ']' ++ HF_CONFIG_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/hf-configs/YuLan-Mini-for-gdn-convert ++ echo HF_CONFIG_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/hf-configs/YuLan-Mini-for-gdn-convert +HF_CONFIG_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/hf-configs/YuLan-Mini-for-gdn-convert ++ '[' '!' -f /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/hf-configs/YuLan-Mini-for-gdn-convert/config.template.json ']' +++ mktemp -d /tmp/mcore2hf_ws_3840/mcore2hf-config.XXXXXX ++ REAL_CONFIG_PATH=/tmp/mcore2hf_ws_3840/mcore2hf-config.XdGUQ3 ++ LOAD_VIEW_DIR= ++ trap cleanup_real_config_path EXIT ++ export PYTHONPATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher: ++ PYTHONPATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher: ++ export PYTHONPATH=/tmp/mcore2hf_ws_3840/mcore2hf-config.XdGUQ3:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher: ++ PYTHONPATH=/tmp/mcore2hf_ws_3840/mcore2hf-config.XdGUQ3:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher: +++ /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/utils/parse-linear-vars-from-mcore-config.py /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/load_view3840 +W1007 16:41:01.985000 306937 torch/utils/cpp_extension.py:2425] TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +W1007 16:41:01.985000 306937 torch/utils/cpp_extension.py:2425] If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'] to specific architectures. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. ++ eval 'export LINEAR_NUM_KEY_HEADS=8 +export LINEAR_NUM_VALUE_HEADS=32 +export LINEAR_ATTN_FREQ=[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1] +export FFN_HIDDEN_SIZE=4800 +export NUM_LAYERS=56 +export HIDDEN_SIZE=2048 +export NUM_ATTN_HEADS=32 +export NUM_QUERY_GROUPS=8 +export MAX_POSITION_EMBEDDINGS=131072 +export KV_CHANNELS=64 +export LINEAR_CONV_KERNEL_DIM=4 +export LINEAR_KEY_HEAD_DIM=64 +export LINEAR_VALUE_HEAD_DIM=64 +export MOE_FFN_HIDDEN_SIZE=512 +export NUM_EXPERTS=192 +export MOE_ROUTER_TOPK=4 +export MOE_ROUTER_SCORE_FUNCTION=softmax +export MOE_ROUTER_ENABLE_EXPERT_BIAS=false +export MOE_LAYER_FREQ=[0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1] +export MLP_ONLY_LAYERS=[0,1] +export MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512 +export ROPE_THETA=10000.0 +export POSITION_EMBEDDING_TYPE=none +export ATTENTION_OUTPUT_GATE=true +export MTP_NUM_LAYERS=0 +export MOE_ROUTER_SQRT_GATE=true +export GATED_NORM=true +export GATED_NORM_RANK=16 +export GATED_NORM_GATE_SCALE=1.0 +export GATED_NORM_UP_BIAS=true +export FINAL_GATED_NORM=true +export ACTIVATION_FUNC_CLAMP_VALUE=7.0 +export ACTIVATION_FUNC_CLAMP_MODE=soft +export FINAL_LOGITS_CLAMP_VALUE=30.0 +export MOE_ROUTER_DTYPE=fp32 +export TORCH_DTYPE=bfloat16 +export VOCAB_SIZE=99072' +++ export LINEAR_NUM_KEY_HEADS=8 +++ LINEAR_NUM_KEY_HEADS=8 +++ export LINEAR_NUM_VALUE_HEADS=32 +++ LINEAR_NUM_VALUE_HEADS=32 +++ export 'LINEAR_ATTN_FREQ=[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1]' +++ LINEAR_ATTN_FREQ='[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1]' +++ export FFN_HIDDEN_SIZE=4800 +++ FFN_HIDDEN_SIZE=4800 +++ export NUM_LAYERS=56 +++ NUM_LAYERS=56 +++ export HIDDEN_SIZE=2048 +++ HIDDEN_SIZE=2048 +++ export NUM_ATTN_HEADS=32 +++ NUM_ATTN_HEADS=32 +++ export NUM_QUERY_GROUPS=8 +++ NUM_QUERY_GROUPS=8 +++ export MAX_POSITION_EMBEDDINGS=131072 +++ MAX_POSITION_EMBEDDINGS=131072 +++ export KV_CHANNELS=64 +++ KV_CHANNELS=64 +++ export LINEAR_CONV_KERNEL_DIM=4 +++ LINEAR_CONV_KERNEL_DIM=4 +++ export LINEAR_KEY_HEAD_DIM=64 +++ LINEAR_KEY_HEAD_DIM=64 +++ export LINEAR_VALUE_HEAD_DIM=64 +++ LINEAR_VALUE_HEAD_DIM=64 +++ export MOE_FFN_HIDDEN_SIZE=512 +++ MOE_FFN_HIDDEN_SIZE=512 +++ export NUM_EXPERTS=192 +++ NUM_EXPERTS=192 +++ export MOE_ROUTER_TOPK=4 +++ MOE_ROUTER_TOPK=4 +++ export MOE_ROUTER_SCORE_FUNCTION=softmax +++ MOE_ROUTER_SCORE_FUNCTION=softmax +++ export MOE_ROUTER_ENABLE_EXPERT_BIAS=false +++ MOE_ROUTER_ENABLE_EXPERT_BIAS=false +++ export 'MOE_LAYER_FREQ=[0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]' +++ MOE_LAYER_FREQ='[0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]' +++ export 'MLP_ONLY_LAYERS=[0,1]' +++ MLP_ONLY_LAYERS='[0,1]' +++ export MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512 +++ MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512 +++ export ROPE_THETA=10000.0 +++ ROPE_THETA=10000.0 +++ export POSITION_EMBEDDING_TYPE=none +++ POSITION_EMBEDDING_TYPE=none +++ export ATTENTION_OUTPUT_GATE=true +++ ATTENTION_OUTPUT_GATE=true +++ export MTP_NUM_LAYERS=0 +++ MTP_NUM_LAYERS=0 +++ export MOE_ROUTER_SQRT_GATE=true +++ MOE_ROUTER_SQRT_GATE=true +++ export GATED_NORM=true +++ GATED_NORM=true +++ export GATED_NORM_RANK=16 +++ GATED_NORM_RANK=16 +++ export GATED_NORM_GATE_SCALE=1.0 +++ GATED_NORM_GATE_SCALE=1.0 +++ export GATED_NORM_UP_BIAS=true +++ GATED_NORM_UP_BIAS=true +++ export FINAL_GATED_NORM=true +++ FINAL_GATED_NORM=true +++ export ACTIVATION_FUNC_CLAMP_VALUE=7.0 +++ ACTIVATION_FUNC_CLAMP_VALUE=7.0 +++ export ACTIVATION_FUNC_CLAMP_MODE=soft +++ ACTIVATION_FUNC_CLAMP_MODE=soft +++ export FINAL_LOGITS_CLAMP_VALUE=30.0 +++ FINAL_LOGITS_CLAMP_VALUE=30.0 +++ export MOE_ROUTER_DTYPE=fp32 +++ MOE_ROUTER_DTYPE=fp32 +++ export TORCH_DTYPE=bfloat16 +++ TORCH_DTYPE=bfloat16 +++ export VOCAB_SIZE=99072 +++ VOCAB_SIZE=99072 ++ echo LINEAR_NUM_KEY_HEADS=8 +LINEAR_NUM_KEY_HEADS=8 ++ echo LINEAR_NUM_VALUE_HEADS=32 +LINEAR_NUM_VALUE_HEADS=32 ++ echo 'LINEAR_ATTN_FREQ=[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1]' +LINEAR_ATTN_FREQ=[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1] ++ echo 'NUM_LAYERS=56 HIDDEN_SIZE=2048' +NUM_LAYERS=56 HIDDEN_SIZE=2048 ++ echo 'NUM_ATTN_HEADS=32 NUM_QUERY_GROUPS=8' +NUM_ATTN_HEADS=32 NUM_QUERY_GROUPS=8 ++ echo 'MAX_POSITION_EMBEDDINGS=131072 KV_CHANNELS=64' +MAX_POSITION_EMBEDDINGS=131072 KV_CHANNELS=64 ++ echo 'FFN_HIDDEN_SIZE=4800 MOE_FFN_HIDDEN_SIZE=512' +FFN_HIDDEN_SIZE=4800 MOE_FFN_HIDDEN_SIZE=512 ++ echo 'NUM_EXPERTS=192 MOE_ROUTER_TOPK=4' +NUM_EXPERTS=192 MOE_ROUTER_TOPK=4 ++ echo MOE_ROUTER_SCORE_FUNCTION=softmax +MOE_ROUTER_SCORE_FUNCTION=softmax ++ echo 'MLP_ONLY_LAYERS=[0,1]' +MLP_ONLY_LAYERS=[0,1] ++ echo 'MOE_LAYER_FREQ=[0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]' +MOE_LAYER_FREQ=[0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1] ++ echo MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512 +MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512 ++ echo 'LINEAR_CONV_KERNEL_DIM=4 LINEAR_KEY_HEAD_DIM=64 LINEAR_VALUE_HEAD_DIM=64' +LINEAR_CONV_KERNEL_DIM=4 LINEAR_KEY_HEAD_DIM=64 LINEAR_VALUE_HEAD_DIM=64 ++ echo ATTENTION_OUTPUT_GATE=true +ATTENTION_OUTPUT_GATE=true ++ echo ROPE_THETA=10000.0 +ROPE_THETA=10000.0 ++ echo POSITION_EMBEDDING_TYPE=none +POSITION_EMBEDDING_TYPE=none ++ echo MOE_ROUTER_SQRT_GATE=true +MOE_ROUTER_SQRT_GATE=true +++ /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/utils/parse-mom-vars-from-mcore-config.py /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/load_view3840 +W1007 16:41:16.866000 307418 torch/utils/cpp_extension.py:2425] TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +W1007 16:41:16.866000 307418 torch/utils/cpp_extension.py:2425] If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'] to specific architectures. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. ++ eval 'export MOM_ENABLED=false +export MOM_DO_MASK_SCALE=true +export MOM_TARGET_FFN_HIDDEN_SIZE=512' +++ export MOM_ENABLED=false +++ MOM_ENABLED=false +++ export MOM_DO_MASK_SCALE=true +++ MOM_DO_MASK_SCALE=true +++ export MOM_TARGET_FFN_HIDDEN_SIZE=512 +++ MOM_TARGET_FFN_HIDDEN_SIZE=512 ++ echo 'MOM_ENABLED=false MOM_DO_MASK_SCALE=true MOM_TARGET_FFN_HIDDEN_SIZE=512' +MOM_ENABLED=false MOM_DO_MASK_SCALE=true MOM_TARGET_FFN_HIDDEN_SIZE=512 ++ '[' false = true ']' ++ '[' 192 -gt 0 ']' ++ '[' 512 -le 0 ']' ++ '[' 10000.0 = 0.0 ']' +++ /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python - ++ ROPE_THETA=10000 ++ '[' -z 131072 ']' ++ '[' 131072 = 0 ']' ++ FFN_TOKEN_SHIFT= ++ FFN_INTERMEDIATE_TOKEN_SHIFT= ++ ATTN_TOKEN_SHIFT= ++ ATTN_Q_TOKEN_SHIFT= ++ ATTN_K_TOKEN_SHIFT= ++ ATTN_V_TOKEN_SHIFT= ++ TOKEN_SHIFT_CONV_SIZE=4 ++ TOKEN_SHIFT_CONV_INIT=default ++ export FFN_TOKEN_SHIFT FFN_INTERMEDIATE_TOKEN_SHIFT ATTN_TOKEN_SHIFT ++ export ATTN_Q_TOKEN_SHIFT ATTN_K_TOKEN_SHIFT ATTN_V_TOKEN_SHIFT ++ export TOKEN_SHIFT_CONV_SIZE TOKEN_SHIFT_CONV_INIT ++ export ROPE_THETA MAX_POSITION_EMBEDDINGS POSITION_EMBEDDING_TYPE ATTENTION_OUTPUT_GATE MOE_ROUTER_SQRT_GATE ++ ATTENTION_OUTPUT_GATE=true ++ /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/utils/render_gdn_hf_config.py --checkpoint-path /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/load_view3840 --template-path /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/hf-configs/YuLan-Mini-for-gdn-convert --output-dir /tmp/mcore2hf_ws_3840/mcore2hf-config.XdGUQ3 +W1007 16:41:30.067000 307625 torch/utils/cpp_extension.py:2425] TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +W1007 16:41:30.067000 307625 torch/utils/cpp_extension.py:2425] If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'] to specific architectures. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. ++ echo '✅config.json: /tmp/mcore2hf_ws_3840/mcore2hf-config.XdGUQ3/config.json' +✅config.json: /tmp/mcore2hf_ws_3840/mcore2hf-config.XdGUQ3/config.json ++ /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python - /tmp/mcore2hf_ws_3840/mcore2hf-config.XdGUQ3/config.json +{ + "hidden_size": 2048, + "intermediate_size": 4800, + "num_hidden_layers": 56, + "num_attention_heads": 32, + "num_key_value_heads": 8, + "head_dim": 64, + "max_position_embeddings": 131072, + "rope_theta": 10000, + "attn_position_embedding_type": "nope", + "rnn_position_embedding_type": "nope", + "linear_num_key_heads": 8, + "linear_num_value_heads": 32, + "linear_key_head_dim": 64, + "linear_value_head_dim": 64, + "num_experts": 192, + "num_experts_per_tok": 4, + "attn_output_gate": true +} ++ sleep 5 ++ MODEL_SIZE=2.6b ++ LOAD_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/load_view3840 ++ MG2HF=true ++ USE_CUDA=false ++ PR=bf16 ++ HF_DIR=/tmp/mcore2hf_ws_3840/mcore2hf-config.XdGUQ3 ++ export DROP_MTP ++ PREPARED_LOAD_DIR= ++ prepare_load_dir_for_iter /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/load_view3840 3840 ++ local SOURCE_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/load_view3840 ++ local ITER_NUM=3840 ++ '[' -z 3840 ']' +++ printf %07d 3840 ++ local ITER_BASENAME=iter_0003840 ++ local SOURCE_ITER_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/load_view3840/iter_0003840 ++ '[' '!' -d /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/load_view3840/iter_0003840 ']' ++ local LATEST_VALUE= ++ '[' -f /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/load_view3840/latest_checkpointed_iteration.txt ']' +++ cat /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/load_view3840/latest_checkpointed_iteration.txt ++ LATEST_VALUE=3840 ++ '[' 3840 = 3840 ']' ++ PREPARED_LOAD_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/load_view3840 ++ return 0 ++ LOAD_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/load_view3840 ++ echo LOAD_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/load_view3840 +LOAD_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/load_view3840 ++ '[' false = true ']' ++ '[' false = true ']' ++ '[' '!' -f /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/load_view3840/latest_checkpointed_iteration.txt ']' +++ cat /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/load_view3840/latest_checkpointed_iteration.txt ++ LATEST_CHECKPOINT_ITERATION=3840 ++ echo -e '\033[32mLatest checkpoint iteration: 3840\033[0m\n' +Latest checkpoint iteration: 3840 + ++ I=3840 ++ [[ 3840 =~ ^[0-9]+$ ]] +++ printf %07d 3840 ++ ITER_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/load_view3840/iter_0003840 ++ '[' '!' -f /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/load_view3840/iter_0003840/common.pt ']' ++ SAVE_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/load_view3840/iter_3840-hf ++ '[' -n /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a ']' ++ SAVE_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/iter_3840-hf ++ '[' -n '' ']' ++ mkdir -p /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/iter_3840-hf ++ bash ./scripts/mcore_gdn_moe/convert.sh 2.6b /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/load_view3840 /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/iter_3840-hf true false bf16 /tmp/mcore2hf_ws_3840/mcore2hf-config.XdGUQ3 3840 ++ tee /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/iter_3840-hf/hf2mcore.log +✅Enable attention output gate +\[\e[33mWarning adding --moe-shared-expert-gate for YuLan model. If you are not YuLan, please check this flag.\e[0m\] +torchrun --nproc_per_node 1 --nnodes 1 --node_rank 0 --master_addr localhost --master_port 47505 /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/impl/convert.py --tokenizer-type HuggingFaceTokenizer --tokenizer-model /tmp/mcore2hf_ws_3840/mcore2hf-config.XdGUQ3 --hf-dir /tmp/mcore2hf_ws_3840/mcore2hf-config.XdGUQ3 --mcore2hf --bf16 --normalization RMSNorm --swiglu --disable-bias-linear --seq-length 1 --max-position-embeddings 131072 --attention-backend auto --position-embedding-type none --kv-channels 64 --group-query-attention --add-qkv-bias --num-layers 56 --hidden-size 2048 --ffn-hidden-size 4800 --num-attention-heads 32 --rotary-base 10000 --rotary-percent 1.00 --num-query-groups 8 --normalization RMSNorm --norm-epsilon 1e-6 --untie-embeddings-and-output-weights --attention-output-gate --linear-attention-type gated_delta_net --linear-attention-freq [1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1] --linear-conv-kernel-dim 4 --linear-key-head-dim 64 --linear-value-head-dim 64 --linear-num-key-heads 8 --linear-num-value-heads 32 --num-experts 192 --moe-router-score-function softmax --moe-token-dispatcher-type alltoall --moe-router-topk 4 --moe-ffn-hidden-size 512 --moe-layer-freq [0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1] --moe-router-dtype fp32 --moe-shared-expert-intermediate-size 512 --moe-shared-expert-gate --moe-router-score-function softmax --moe-router-sqrt-gate --micro-batch-size 1 --global-batch-size 1024 --train-iters 500000 --weight-decay 0.1 --adam-beta1 0.9 --adam-beta2 0.95 --init-method-std 0.006 --clip-grad 1.0 --lr 2.0e-5 --lr-decay-style cosine --min-lr 6.0e-6 --lr-warmup-fraction .001 --lr-decay-iters 430000 --no-gradient-accumulation-fusion --bf16 --tensor-model-parallel-size 1 --pipeline-model-parallel-size 1 --expert-tensor-parallel-size 1 --expert-model-parallel-size 1 --log-interval 100 --save-interval 10000 --eval-interval 1000 --eval-iters 10 --model-type GPT --load-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/load_view3840 --save-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/iter_3840-hf --dist-ckpt-optim-fully-reshardable --skip-train --use-cpu-initialization --no-load-optim --no-load-rng --logging-level 1 --attention-backend auto --debug --max-shard-size 20GB --synchronizer mcore_gdn_moe --pretrain-script mcore_gdn_moe.model_provider --ckpt-step 3840 --gdn-split-in-proj --gated-norm --gated-norm-lora-rank 16 --distributed-backend gloo +W1007 16:41:50.578000 308152 torch/utils/cpp_extension.py:2425] TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +W1007 16:41:50.578000 308152 torch/utils/cpp_extension.py:2425] If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'] to specific architectures. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +Warning: Pai-Megatron-Patch arguments not available, some arguments may not be recognized +using world size: 1, data-parallel size: 1, context-parallel size: 1, hierarchical context-parallel sizes: None, tensor-model-parallel size: 1, pipeline-model-parallel size: 1 +Number of virtual stages per pipeline stage: None +accumulate and all-reduce gradients in fp32 for bfloat16 data type. +using torch.bfloat16 for parameters ... +------------------------ arguments ------------------------ + account_for_embedding_in_pipeline_split ......... False + account_for_loss_in_pipeline_split .............. False + accumulate_allreduce_grads_in_fp32 .............. True + activation_func_clamp_mode ...................... hard + activation_func_clamp_value ..................... None + adam_beta1 ...................................... 0.9 + adam_beta2 ...................................... 0.95 + adam_bias_correction ............................ True + adam_eps ........................................ 1e-08 + adamh_eps ....................................... 1e-08 + adamh_radius .................................... None + adamh_scope ..................................... lm_head + adamw_lr_mup_scaler ............................. False + add_bias_linear ................................. False + add_position_embedding .......................... True + add_qkv_bias .................................... True + adlr_autoresume ................................. False + adlr_autoresume_interval ........................ 1000 + align_grad_reduce ............................... True + align_param_gather .............................. False + allow_ambiguous_pad_tokens ...................... False + app_tag_run_name ................................ None + app_tag_run_version ............................. 0.0.0 + apply_layernorm_1p .............................. False + apply_query_key_layer_scaling ................... False + apply_residual_connection_post_layernorm ........ False + apply_rope_fusion ............................... False + async_save ...................................... None + async_tensor_model_parallel_allreduce ........... True + attention_backend ............................... AttnBackend.auto + attention_dropout ............................... 0.1 + attention_output_gate ........................... True + attention_output_gate_upcycling ................. False + attention_softmax_in_fp32 ....................... False + attn_k_token_shift .............................. None + attn_output_gate ................................ None + attn_output_gate_rand_init ...................... False + attn_q_token_shift .............................. None + attn_token_shift ................................ None + attn_v_token_shift .............................. None + auto_detect_ckpt_format ......................... False + auto_generate_cu_seqlens ........................ False + auto_model ...................................... AutoModelForCausalLM + barrier_with_L1_time ............................ True + benchmark_eval .................................. False + benchmark_global_batch .......................... None + benchmark_interval .............................. None + benchmark_micro_batch ........................... None + benchmark_sequence_length ....................... None + benchmark_tasks ................................. None + bert_binary_head ................................ True + bert_embedder_type .............................. megatron + bert_load ....................................... None + bf16 ............................................ True + bias_dropout_fusion ............................. True + bias_gelu_fusion ................................ False + bias_swiglu_fusion .............................. True + biencoder_projection_dim ........................ 0 + biencoder_shared_query_context_model ............ False + block_data_path ................................. None + cache_mla_latents ............................... False + calc_ft_timeouts ................................ False + calculate_per_token_loss ........................ False + check_for_large_grads ........................... False + check_for_nan_in_loss_and_grad .................. True + check_for_spiky_loss ............................ False + check_weight_hash_across_dp_replicas_interval ... None + ckpt_assume_constant_structure .................. False + ckpt_convert_format ............................. None + ckpt_convert_save ............................... None + ckpt_convert_update_legacy_dist_opt_format ...... False + ckpt_format ..................................... torch_dist + ckpt_fully_parallel_load ........................ False + ckpt_fully_parallel_save ........................ True + ckpt_fully_parallel_save_deprecated ............. False + ckpt_step ....................................... 3840 + classes_fraction ................................ 1.0 + clip_grad ....................................... 1.0 + clone_scatter_output_in_embedding ............... True + config_logger_dir ............................... + consumed_train_samples .......................... 0 + consumed_valid_samples .......................... 0 + context_parallel_size ........................... 1 + continual_pretrain .............................. False + continual_pretrain_preserve_opt_param_scheduler . False + corrected_weight_decay .......................... False + cp_comm_type .................................... ['p2p'] + create_attention_mask_in_dataloader ............. True + cross_entropy_fusion_impl ....................... native + cross_entropy_loss_fusion ....................... False + cuda_graph_impl ................................. none + cuda_graph_scope ................................ [] + cuda_graph_warmup_steps ......................... 3 + data_args_path .................................. None + data_cache_path ................................. None + data_parallel_random_init ....................... False + data_parallel_sharding_strategy ................. no_shard + data_parallel_size .............................. 1 + data_path ....................................... None + data_per_class_fraction ......................... 1.0 + data_sharding ................................... True + dataloader_defer_npy_index_mmap ................. False + dataloader_fast_cache_load ...................... False + dataloader_type ................................. single + dc_shortconv .................................... False + dc_shortconv_optimizer_state .................... conservative + dc_shortconv_upcycling .......................... False + ddp_average_in_collective ....................... False + ddp_bucket_size ................................. None + ddp_num_buckets ................................. None + ddp_pad_buckets_for_high_nccl_busbw ............. False + debug ........................................... True + decode_only_cuda_graphs ......................... False + decoder_first_pipeline_num_layers ............... None + decoder_last_pipeline_num_layers ................ None + decoder_num_layers .............................. None + decoder_seq_length .............................. None + decoupled_lr .................................... None + decoupled_min_lr ................................ None + decrease_batch_size_if_needed ................... False + defer_embedding_wgrad_compute ................... False + defer_grad_checks ............................... False + delay_wgrad_compute ............................. False + deprecated_use_mcore_models ..................... False + deterministic_mode .............................. False + dino_bottleneck_size ............................ 256 + dino_freeze_last_layer .......................... 1 + dino_head_hidden_size ........................... 2048 + dino_local_crops_number ......................... 10 + dino_local_img_size ............................. 96 + dino_norm_last_layer ............................ False + dino_teacher_temp ............................... 0.07 + dino_warmup_teacher_temp ........................ 0.04 + dino_warmup_teacher_temp_epochs ................. 30 + disable_attn_output_gate ........................ False + disable_bf16_reduced_precision_matmul ........... False + disable_chunked_prefill ......................... False + disable_explicit_attention_mask ................. False + disable_mamba_mem_eff_path ...................... False + disable_straggler_on_startup .................... False + disable_symmetric_registration .................. False + dist_ckpt_format_deprecated ..................... None + dist_ckpt_optim_fully_reshardable ............... True + dist_ckpt_save_pre_mcore_014 .................... False + dist_ckpt_strictness ............................ assume_ok_unexpected + distrib_optim_fully_reshardable_mem_efficient ... False + distribute_saved_activations .................... False + distributed_backend ............................. gloo + distributed_timeout_minutes ..................... 10 + distributed_timeout_seconds_after_init .......... None + document_packing_algorithm ...................... ogbfd + document_packing_dp_size ........................ None + drop_mtp ........................................ False + dryrun .......................................... False + dump_param_to_param_group_map ................... None + dynamic_context_parallel ........................ False + dynamic_context_parallel_sizes .................. None + dynamic_cp_candidate_sizes ...................... None + dynamic_cp_gdn_policy ........................... None + dynamic_cp_moe_policy ........................... None + dynamic_cp_on_error ............................. error + dynamic_cp_scheduler ............................ default_dynamic_cp + dynamic_cp_wave_coexecution ..................... False + dynamic_cp_workspace_max_tokens ................. None + emb_deviation_loss_coeff ........................ 0 + emb_deviation_type .............................. None + embedding_init_method_std ....................... None + embedding_path .................................. None + empty_unused_memory_level ....................... 0 + enable_cuda_graph ............................... False + enable_debug_logging ............................ False + enable_experimental ............................. False + enable_ft_package ............................... False + enable_full_sharding_in_hsdp .................... False + enable_gloo_process_groups ...................... True + enable_msc ...................................... True + enable_one_logger ............................... True + encoder_num_layers .............................. 56 + encoder_seq_length .............................. 1 + end_weight_decay ................................ 0.1 + eod_mask_loss ................................... False + error_injection_rate ............................ 0 + error_injection_type ............................ transient_error + eval_interval ................................... 1000 + eval_iters ...................................... 10 + evidence_data_path .............................. None + exit_duration_in_mins ........................... None + exit_interval ................................... None + exit_on_missing_checkpoint ...................... False + exit_signal_handler ............................. False + exp_avg_dtype ................................... torch.float32 + exp_avg_sq_dtype ................................ torch.float32 + expert_model_parallel_size ...................... 1 + expert_tensor_parallel_size ..................... 1 + export_force_local_attention .................... False + export_kd_cfg ................................... None + export_kd_teacher_ckpt_format ................... None + export_kd_teacher_load .......................... None + export_kv_cache_quant ........................... False + export_legacy_megatron .......................... False + export_model_type ............................... GPTModel + export_moe_apply_probs_on_input ................. False + export_offline_model ............................ False + export_qk_l2_norm ............................... False + export_quant_cfg ................................ None + export_real_quant_cfg ........................... None + export_te_mcore_model ........................... False + external_cuda_graph ............................. False + fast_thd_a2a .................................... False + ffn_hidden_size ................................. 4800 + ffn_intermediate_token_shift .................... None + ffn_token_shift ................................. None + final_logits_clamp_value ........................ None + fine_grained_activation_offloading .............. False + finetune ........................................ False + finetune_data_split ............................. train + finetune_hf_dataset ............................. None + first_last_layers_bf16 .......................... False + flash_decode .................................... False + fp16 ............................................ False + fp16_lm_cross_entropy ........................... False + fp32_residual_connection ........................ False + fp4 ............................................. None + fp4_param ....................................... False + fp4_recipe ...................................... nvfp4 + fp8 ............................................. None + fp8_amax_compute_algo ........................... most_recent + fp8_amax_history_len ............................ 1 + fp8_interval .................................... 1 + fp8_margin ...................................... 0 + fp8_param_gather ................................ False + fp8_recipe ...................................... delayed + fp8_wgrad ....................................... True + freeze_layernorm_weight ......................... False + freeze_non_mamba ................................ False + fsdp_double_buffer .............................. False + full_validation ................................. False + gated_norm ...................................... True + gated_norm_lora_rank ............................ 16 + gated_norm_upcycling ............................ False + gdn_alpha_weight_scale .......................... 1.0 + gdn_alpha_weight_scale_mode ..................... matrix + gdn_backend ..................................... fla + gdn_fused_split_forward ......................... False + gdn_gated_norm_backend .......................... torch + gdn_lr_mult_alpha ............................... 1.0 + gdn_lr_mult_beta ................................ 1.0 + gdn_lr_mult_k ................................... 1.0 + gdn_lr_mult_q ................................... 1.0 + gdn_lr_mult_v ................................... 1.0 + gdn_postinit_alog_shift ......................... 0.0 + gdn_postinit_alpha_scale ........................ 1.0 + gdn_postinit_key_scale .......................... 1.0 + gdn_pre_gdr_fusion .............................. False + gdn_pre_gdr_fusion_deterministic_weight_gradient False + gdn_pre_gdr_fusion_high_precision_backward ...... False + gdn_shortconv_separate_gate ..................... False + gdn_shortconv_separate_gate_upcycling ........... False + gdn_split_in_proj ............................... True + geglu ........................................... False + global_batch_size ............................... 1024 + glu_linear_offset ............................... 0.0 + grad_reduce_in_bf16 ............................. False + gradient_accumulation_fusion .................... False + gradient_reduce_div_fusion ...................... True + group_query_attention ........................... True + grpo_clamp_eps_lower ............................ 0.01 + grpo_clamp_eps_upper ............................ 0.01 + grpo_default_temperature ........................ 1.0 + grpo_default_top_p .............................. 0 + grpo_entropy_term_weight ........................ 0.0 + grpo_filter_groups_with_same_reward ............. False + grpo_group_size ................................. 2 + grpo_iterations ................................. 2 + grpo_kl_beta .................................... 0.001 + grpo_prompts_per_step ........................... 32 + head_lr_mult .................................... 1.0 + heterogeneous_layers_config_encoded_json ........ None + heterogeneous_layers_config_path ................ None + hf_dir .......................................... /tmp/mcore2hf_ws_3840/mcore2hf-config.XdGUQ3 + hidden_dropout .................................. 0.1 + hidden_size ..................................... 2048 + hidden_states_shrink ............................ 1.0 + hierarchical_context_parallel_sizes ............. None + high_priority_stream_groups ..................... [] + hybrid_adamh .................................... False + hybrid_attention_ratio .......................... 0.0 + hybrid_context_parallel ......................... False + hybrid_mlp_ratio ................................ 0.0 + hybrid_override_pattern ......................... None + hysteresis ...................................... 2 + ict_head_size ................................... None + ict_load ........................................ None + img_h ........................................... 224 + img_w ........................................... 224 + increase_log_level_interval ..................... 1000 + increase_log_level_iters ........................ 5 + indexer_batch_size .............................. 128 + indexer_log_interval ............................ 1000 + inference_batch_times_seqlen_threshold .......... -1 + inference_dynamic_batching ...................... False + inference_dynamic_batching_block_size ........... 256 + inference_dynamic_batching_buffer_guaranteed_fraction 0.2 + inference_dynamic_batching_buffer_overflow_factor None + inference_dynamic_batching_buffer_size_gb ....... 40.0 + inference_dynamic_batching_max_requests_override None + inference_dynamic_batching_max_tokens_override .. None + inference_dynamic_batching_num_cuda_graphs ...... 16 + inference_dynamic_batching_track_paused_request_events False + inference_dynamic_batching_unified_memory_level . 0 + inference_max_batch_size ........................ 8 + inference_max_seq_length ........................ 2560 + inference_rng_tracker ........................... False + init_method_std ................................. 0.006 + init_method_xavier_uniform ...................... False + init_model_with_meta_device ..................... False + initial_loss_scale .............................. 4294967296 + inprocess_active_world_size ..................... 1 + inprocess_barrier_timeout ....................... 120 + inprocess_completion_timeout .................... 120 + inprocess_empty_cuda_cache ...................... False + inprocess_granularity ........................... node + inprocess_hard_timeout .......................... 90 + inprocess_heartbeat_interval .................... 30 + inprocess_heartbeat_timeout ..................... 60 + inprocess_last_call_wait ........................ 1 + inprocess_max_iterations ........................ None + inprocess_monitor_process_interval .............. 1.0 + inprocess_monitor_thread_interval ............... 1.0 + inprocess_progress_watchdog_interval ............ 1.0 + inprocess_restart ............................... False + inprocess_soft_timeout .......................... 60 + inprocess_termination_grace_time ................ 1 + is_hybrid_model ................................. False + iter_per_epoch .................................. 1250 + iterations_to_skip .............................. [] + keep_fp8_transpose_cache ........................ False + kitchen_config_file ............................. None + kitchen_recipe_number ........................... None + kv_channels ..................................... 64 + kv_lora_rank .................................... 32 + langrl_env_config ............................... None + langrl_external_server .......................... False + langrl_inference_server_conversation_template ... None + langrl_inference_server_type .................... inplace_megatron + lazy_mpu_init ................................... None + legacy_tokenizer ................................ False + linear_attention_freq ........................... [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 0, 0, 1, 1, 1, 1, 1, 1] + linear_attention_type ........................... gated_delta_net + linear_conv_kernel_dim .......................... 4 + linear_cross_entropy_accumulate_grad_in_fp32 .... True + linear_cross_entropy_num_chunks ................. 8 + linear_key_head_dim ............................. 64 + linear_num_key_heads ............................ 8 + linear_num_value_heads .......................... 32 + linear_rope_mode ................................ none + linear_value_head_dim ........................... 64 + lm_head_alpha ................................... 1.0 + lm_loss_normalization ........................... legacy + load ............................................ None + load_complemental_dataset ....................... None + load_dir ........................................ /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/load_view3840 + load_main_params_from_ckpt ...................... None + local_rank ...................................... 0 + log_attn_logits ................................. False + log_energy ...................................... False + log_hidden_states ............................... [] + log_interval .................................... 100 + log_logits_z_loss ............................... False + log_loss_scale_to_tensorboard ................... True + log_memory_to_tensorboard ....................... False + log_metrics_interval ............................ None + log_moe_activations ............................. False + log_moe_params .................................. False + log_moe_router_metrics .......................... False + log_num_zeros_in_grad ........................... False + log_padding_stats ............................... False + log_params ...................................... [] + log_params_norm ................................. False + log_per_layer_grad_rms .......................... False + log_per_module_grad_rms ......................... False + log_per_module_update_rms ....................... False + log_progress .................................... False + log_spike_diagnostics ........................... False + log_straggler ................................... False + log_throughput .................................. False + log_timers_to_tensorboard ....................... False + log_update_rms .................................. False + log_validation_ppl_to_tensorboard ............... False + log_world_size_to_tensorboard ................... False + logging_level ................................... 1 + loss_scale ...................................... None + loss_scale_window ............................... 1000 + lr .............................................. 2e-05 + lr_decay_iters .................................. 430000 + lr_decay_samples ................................ None + lr_decay_style .................................. cosine + lr_warmup_fraction .............................. 0.001 + lr_warmup_init .................................. 0.0 + lr_warmup_iters ................................. 0 + lr_warmup_samples ............................... 0 + lr_wsd_decay_iters .............................. None + lr_wsd_decay_samples ............................ None + lr_wsd_decay_style .............................. exponential + main_grads_dtype ................................ torch.float32 + main_params_dtype ............................... torch.float32 + make_vocab_size_divisible_by .................... 128 + mamba_disable_cp ................................ False + mamba_expand .................................... 2 + mamba_head_dim .................................. 64 + mamba_num_groups ................................ 8 + mamba_num_heads ................................. None + mamba_state_dim ................................. 128 + manual_gc ....................................... False + manual_gc_eval .................................. True + manual_gc_interval .............................. 0 + mask_factor ..................................... 1.0 + mask_prob ....................................... 0.15 + mask_type ....................................... random + masked_softmax_fusion ........................... True + max_position_embeddings ......................... 131072 + max_seqlen_per_cp_rank .......................... None + max_shard_size .................................. 20GB + max_tokens_to_oom ............................... 12000 + mcore2hf ........................................ True + memory_snapshot_path ............................ None + merge_file ...................................... None + micro_batch_size ................................ 1 + microbatch_group_size_per_vp_stage .............. None + mid_level_dataset_surplus ....................... 0.005 + min_loss_scale .................................. 1.0 + min_lr .......................................... 6e-06 + min_offloaded_tensor_size ....................... 1048576 + mlp_chunks_for_prefill .......................... 1 + mmap_bin_files .................................. True + mock_data ....................................... False + model_type ...................................... GPT + modelopt_enabled ................................ False + moe_apply_probs_on_input ........................ False + moe_aux_loss_coeff .............................. 0.0 + moe_deepep_num_sms .............................. 20 + moe_enable_deepep ............................... False + moe_expert_capacity_factor ...................... None + moe_extended_tp ................................. False + moe_ffn_hidden_size ............................. 512 + moe_flex_dispatcher_backend ..................... deepep + moe_gaux_overlap ................................ False + moe_grouped_gemm ................................ False + moe_hybridep_num_sms ............................ 16 + moe_input_jitter_eps ............................ None + moe_layer_freq .................................. [0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] + moe_layer_recompute ............................. False + moe_monitor_layers .............................. + moe_monitor_log_interval ........................ None + moe_pad_expert_input_to_capacity ................ False + moe_pad_experts_for_cuda_graph_inference ........ False + moe_per_layer_logging ........................... False + moe_permute_backend ............................. auto + moe_permute_fusion .............................. False + moe_router_aux_loss_fusion ...................... auto + moe_router_bias_update_method ................... sign + moe_router_bias_update_rate ..................... 0.001 + moe_router_dtype ................................ fp32 + moe_router_enable_expert_bias ................... False + moe_router_force_load_balancing ................. False + moe_router_fusion ............................... False + moe_router_gemm_backend ......................... torch + moe_router_group_topk ........................... None + moe_router_load_balancing_type .................. aux_loss + moe_router_num_groups ........................... None + moe_router_padding_for_fp8 ...................... False + moe_router_padding_for_quantization ............. False + moe_router_pre_softmax .......................... False + moe_router_score_function ....................... softmax + moe_router_sqrt_gate ............................ True + moe_router_topk ................................. 4 + moe_router_topk_scaling_factor .................. None + moe_shared_expert_gate .......................... True + moe_shared_expert_intermediate_size ............. 512 + moe_shared_expert_overlap ....................... False + moe_token_dispatcher_type ....................... alltoall + moe_token_drop_policy ........................... probs + moe_triton_dispatcher_exclude_padding_tokens .... False + moe_triton_mega_activation_sms .................. 4 + moe_triton_mega_combine_sms ..................... 32 + moe_triton_mega_compute_sms ..................... 24 + moe_triton_mega_dispatch_sms .................... 48 + moe_triton_mega_num_warps ....................... 16 + moe_triton_mega_reduce_sms ...................... 32 + moe_triton_mega_tail_sms ........................ 16 + moe_triton_shared_expert_overlap ................ False + moe_triton_workspace_expand ..................... 4.0 + moe_upcycling_granularity ....................... 1 + moe_use_legacy_grouped_gemm ..................... False + moe_use_upcycling ............................... False + moe_z_loss_coeff ................................ None + mrope_section ................................... None + mscale .......................................... 1.0 + mscale_all_dim .................................. 0.0 + mtp_linear_attention_type ....................... None + mtp_loss_scaling_factor ......................... 0.1 + mtp_num_layers .................................. None + multi_latent_attention .......................... False + multiple_validation_sets ........................ False + muon_ball_momentum .............................. 0.9 + muon_ball_msign_steps ........................... 5 + muon_ball_power_iteration_steps ................. 10 + muon_ball_qkv_split_mode ........................ component + muon_ball_radius_mode ........................... spectral_mup + muon_ball_retract_alpha ......................... 0.05 + muon_ball_retract_mode .......................... hard + muon_ball_scale_mode ............................ spectral_mup + muon_ball_split_fc1 ............................. True + muon_ball_split_moe_experts ..................... True + muon_ball_split_qkv ............................. True + muon_ball_use_nesterov .......................... True + muon_extra_scale_factor ......................... 1.0 + muon_fp32_matmul_prec ........................... medium + muon_momentum ................................... 0.9 + muon_num_ns_steps ............................... 5 + muon_qkv_split_mode ............................. component + muon_scale_mode ................................. spectral_mup + muon_scale_vectorized_mode ...................... full + muon_split_fc1 .................................. True + muon_split_moe_experts .......................... True + muon_split_qkv .................................. True + muon_tp_mode .................................... blockwise + muon_use_nesterov ............................... False + muon_vectorize .................................. [] + muon_vectorize_attn_dim ......................... hidden_size + mup_lr_mult ..................................... 1.0 + nccl_all_reduce_for_prefill ..................... False + nccl_communicator_config_path ................... None + nccl_ub ......................................... False + no_load_optim ................................... True + no_load_rng ..................................... True + no_load_scheduler ............................... None + no_persist_layer_norm ........................... False + no_rope_freq .................................... None + no_save_optim ................................... None + no_save_rng ..................................... None + no_save_step_one ................................ None + no_weight_decay_cond_type ....................... None + non_persistent_ckpt_type ........................ None + non_persistent_global_ckpt_dir .................. None + non_persistent_keep_last ........................ 1 + non_persistent_local_ckpt_algo .................. fully_parallel + non_persistent_local_ckpt_dir ................... None + non_persistent_save_interval .................... None + norm_epsilon .................................... 1e-06 + normalization ................................... RMSNorm + num_attention_heads ............................. 32 + num_channels .................................... 3 + num_classes ..................................... 1000 + num_dataset_builder_threads ..................... 1 + num_distributed_optimizer_instances ............. 1 + num_experts ..................................... 192 + num_hf_saver .................................... None + num_layers ...................................... 56 + num_layers_at_end_in_bf16 ....................... 1 + num_layers_at_start_in_bf16 ..................... 1 + num_layers_per_virtual_pipeline_stage ........... None + num_query_groups ................................ 8 + num_virtual_stages_per_pipeline_rank ............ None + num_workers ..................................... 2 + object_storage_cache_path ....................... None + offload_modules ................................. [] + one_logger_async ................................ False + one_logger_project .............................. megatron-lm + one_logger_run_name ............................. None + onnx_safe ....................................... None + openai_gelu ..................................... False + optimizer ....................................... adam + optimizer_cpu_offload ........................... False + optimizer_offload_fraction ...................... 1.0 + output_bert_embeddings .......................... False + output_logits_z_loss_coeff ...................... None + overlap_cpu_optimizer_d2h_h2d ................... False + overlap_grad_reduce ............................. False + overlap_moe_expert_parallel_comm ................ False + overlap_p2p_comm ................................ False + overlap_p2p_comm_warmup_flush ................... False + overlap_param_gather ............................ False + overlap_param_gather_with_optimizer_step ........ False + override_hf_eod_token_id ........................ None + override_opt_param_scheduler .................... False + packing_tokens_per_shard ........................ 100000000000 + padded_vocab_size ............................... None + params_dtype .................................... torch.bfloat16 + patch_dim ....................................... 16 + per_split_data_args_path ........................ None + perform_initialization .......................... True + perform_rl_step ................................. False + pin_cpu_grads ................................... True + pin_cpu_params .................................. True + pipeline_model_parallel_comm_backend ............ None + pipeline_model_parallel_layout .................. None + pipeline_model_parallel_size .................... 1 + position_embedding_type ......................... none + pp_aware_finegrained_recompute .................. False + pp_aware_finegrained_recompute_global_layers .... None + pp_aware_finegrained_recompute_pp_ranks ......... None + prefetch_factor ................................. None + pretrain_script ................................. mcore_gdn_moe.model_provider + pretrained_checkpoint ........................... None + previous_stage_seq_length ....................... None + profile ......................................... False + profile_microbatch_count ........................ 2 + profile_microbatch_dir .......................... None + profile_microbatch_iteration .................... None + profile_microbatch_start ........................ 0 + profile_ranks ................................... [0] + profile_step_end ................................ None + profile_step_start .............................. None + q_lora_rank ..................................... None + qk_head_dim ..................................... 128 + qk_l2_norm ...................................... False + qk_layernorm .................................... False + qk_pos_emb_head_dim ............................. 64 + query_in_block_prob ............................. 0.1 + quick_geglu ..................................... False + rampup_batch_size ............................... None + rank ............................................ 0 + recompute_granularity ........................... None + recompute_method ................................ None + recompute_modules ............................... None + recompute_num_layers ............................ None + record_memory_history ........................... False + relative_attention_max_distance ................. 128 + relative_attention_num_buckets .................. 32 + reparam_checkpoint .............................. None + reparam_fallback_value .......................... None + reparam_keys .................................... None + replication ..................................... False + replication_factor .............................. 2 + replication_jump ................................ None + rerun_mode ...................................... validate_results + reset_attention_mask ............................ False + reset_iteration_one_to_zero ..................... False + reset_position_ids .............................. False + result_rejected_tracker_filename ................ None + retriever_report_topk_accuracies ................ [] + retriever_score_scaling ......................... False + retriever_seq_length ............................ 256 + retro_add_retriever ............................. False + retro_attention_gate ............................ 1 + retro_cyclic_train_iters ........................ None + retro_encoder_attention_dropout ................. 0.1 + retro_encoder_hidden_dropout .................... 0.1 + retro_encoder_layers ............................ 2 + retro_num_neighbors ............................. 2 + retro_num_retrieved_chunks ...................... 2 + retro_project_dir ............................... None + retro_verify_neighbor_count ..................... True + reuse_grad_buf_for_mxfp8_param_ag ............... False + reuse_training_communicators .................... False + rl_calculate_intra_group_similarity ............. False + rl_importance_sampling_truncation_coef .......... None + rl_inference_logprobs_is_correction ............. False + rl_offload_kv_cache_during_training ............. False + rl_offload_optimizer_during_inference ........... False + rl_partial_rollouts ............................. False + rl_prompts_per_eval ............................. 32 + rl_remove_kv_cache_during_training .............. False + rl_reset_cuda_graphs ............................ False + rl_sequence_packing_algo ........................ fifo + rl_sequence_packing_bin_size .................... 8192 + rl_use_sequence_packing ......................... False + rope_scaling_factor ............................. 8.0 + rope_type ....................................... None + rotary_base ..................................... 10000 + rotary_interleaved .............................. False + rotary_percent .................................. 1.0 + rotary_scaling_factor ........................... 1.0 + rotary_seq_len_interpolation_factor ............. None + run_workload_inspector_server ................... False + runtime_audit_dir ............................... None + sample_rate ..................................... 1.0 + save ............................................ None + save_after_load ................................. False + save_dir ........................................ /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/iter_3840-hf + save_interval ................................... 10000 + save_retain_interval ............................ None + scale_emb ....................................... 1.0 + scatter_gather_tensors_in_pipeline .............. True + seed ............................................ 1234 + seq_length ...................................... 1 + sequence_packing_scheduler ...................... default_dynamic_cp + sequence_parallel ............................... False + sft ............................................. False + sft_tokenizer_prompt_format ..................... nemotron-h-aligned + sgd_momentum .................................... 0.9 + sharp_enabled_group ............................. None + short_seq_prob .................................. 0.1 + skip_train ...................................... True + skipped_train_samples ........................... 0 + softmax_type .................................... vanilla + spec ............................................ None + spectra_clip .................................... 1.0 + spectra_grad_clip ............................... None + spectra_hyperball ............................... False + spectra_ns_steps ................................ 10 + spectra_scope ................................... matrices + spectral_ball_momentum .......................... 0.9 + spectral_ball_msign_steps ....................... 8 + spectral_ball_power_iteration_steps ............. 20 + spectral_ball_qkv_split_mode .................... component + spectral_ball_radius_mode ....................... spectral_mup + spectral_ball_retract_alpha ..................... 0.05 + spectral_ball_retract_mode ...................... hard + spectral_ball_scale_mode ........................ spectral_mup + spectral_ball_solver ............................ bisection + spectral_ball_solver_max_iterations ............. 20 + spectral_ball_solver_tolerance_f ................ 1e-08 + spectral_ball_split_fc1 ......................... True + spectral_ball_split_moe_experts ................. True + spectral_ball_split_qkv ......................... True + spectral_ball_use_nesterov ...................... True + spectral_mup_init ............................... False + spike_diagnostics_dir ........................... None + spike_diagnostics_end ........................... 21 + spike_diagnostics_start ......................... 2 + split ........................................... None + split_expert_init ............................... True + split_fc1_init .................................. True + split_qkv_init .................................. True + split_qkv_init_mode ............................. group + sqreglu ......................................... False + squared_relu .................................... False + start_samples ................................... None + start_weight_decay .............................. 0.1 + straggler_ctrlr_port ............................ 65535 + straggler_minmax_count .......................... 1 + strict_fsdp_dtensor_load ........................ True + suggested_communication_unit_size ............... None + swanlab_exp_name ................................ + swanlab_project ................................. + swanlab_save_dir ................................ + swanlab_workspace ............................... + swiglu .......................................... True + swin_backbone_type .............................. tiny + symmetric_ar_type ............................... None + synchronizer .................................... mcore_gdn_moe + target_ckpt_format .............................. torch_dist + te_rng_tracker .................................. False + teacher_model_config ............................ None + tensor_model_parallel_size ...................... 1 + tensorboard_dir ................................. None + tensorboard_log_interval ........................ 1 + tensorboard_queue_size .......................... 1000 + test_data_path .................................. None + test_mode ....................................... False + tiktoken_num_special_tokens ..................... 1000 + tiktoken_pattern ................................ None + tiktoken_special_tokens ......................... None + timing_log_level ................................ 0 + timing_log_option ............................... minmax + titles_data_path ................................ None + token_shift_conv_init ........................... default + token_shift_conv_size ........................... 4 + tokenizer_metadata .............................. None + tokenizer_model ................................. /tmp/mcore2hf_ws_3840/mcore2hf-config.XdGUQ3 + tokenizer_type .................................. HuggingFaceTokenizer + torch_fsdp2_reshard_after_forward ............... True + tp_comm_bootstrap_backend ....................... nccl + tp_comm_bulk_dgrad .............................. True + tp_comm_bulk_wgrad .............................. True + tp_comm_overlap ................................. False + tp_comm_overlap_ag .............................. True + tp_comm_overlap_cfg ............................. None + tp_comm_overlap_rs .............................. True + tp_comm_overlap_rs_dgrad ........................ False + tp_comm_split_ag ................................ True + tp_comm_split_rs ................................ True + train_data_path ................................. None + train_iters ..................................... 500000 + train_samples ................................... None + train_sync_interval ............................. None + train_until_stopped ............................. False + transformer_impl ................................ transformer_engine + transformer_pipeline_model_parallel_size ........ 1 + trust_remote_code ............................... False + untie_embeddings_and_output_weights ............. True + use_checkpoint_args ............................. False + use_checkpoint_opt_param_scheduler .............. False + use_cpu_initialization .......................... True + use_dist_ckpt ................................... True + use_dist_ckpt_deprecated ........................ False + use_distributed_optimizer ....................... False + use_flash_attn .................................. False + use_fused_weighted_squared_relu ................. False + use_gpu ......................................... False + use_legacy_models ............................... False + use_megatron_fsdp ............................... False + use_mp_args_from_checkpoint_args ................ False + use_one_sent_docs ............................... False + use_persistent_ckpt_worker ...................... False + use_precision_aware_optimizer ................... False + use_pytorch_profiler ............................ False + use_ring_exchange_p2p ........................... False + use_rope_scaling ................................ False + use_rotary_position_embeddings .................. False + use_sharp ....................................... False + use_te_activation_func .......................... False + use_tokenizer_model_from_checkpoint_args ........ True + use_torch_fsdp2 ................................. False + use_torch_optimizer_for_cpu_offload ............. False + use_tp_pp_dp_mapping ............................ False + v_head_dim ...................................... 128 + valid_data_path ................................. None + variable_seq_lengths ............................ False + virtual_pipeline_model_parallel_size ............ None + vision_backbone_type ............................ vit + vision_pretraining .............................. False + vision_pretraining_type ......................... classify + vocab_extra_ids ................................. 0 + vocab_file ...................................... None + vocab_size ...................................... None + wandb_entity .................................... + wandb_exp_name .................................. + wandb_project ................................... + wandb_save_dir .................................. + weight_decay .................................... 0.1 + weight_decay_incr_style ......................... constant + wgrad_deferral_limit ............................ 0 + window_attn_skip_freq ........................... None + window_size ..................................... None + word_embedding_dropout_prob ..................... 0.0 + world_size ...................................... 1 + yaml_cfg ........................................ None +-------------------- end of arguments --------------------- +INFO:megatron.core.num_microbatches_calculator:setting number of microbatches to constant 1024 +> building HuggingFaceTokenizer tokenizer ... + > padded vocab (size: 99000) with 72 dummy tokens (new size: 99072) +WARNING: one_logger package is required to enable e2e metrics tracking. please go to https://confluence.nvidia.com/display/MLWFO/Package+Repositories for details to install it +INFO:megatron.training.initialize:Setting logging level to 1 +WARNING:megatron.core.rerun_state_machine:RerunStateMachine initialized in mode validate_results +> initializing torch distributed ... +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +> initialized tensor model parallel with size 1 +> initialized pipeline model parallel with size 1 +> setting random seeds to 1234 ... +> compiling dataset index builder ... +make: Entering directory '/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/datasets' +make: Nothing to be done for 'default'. +make: Leaving directory '/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/datasets' +>>> done with dataset index builder. Compilation time: 0.134 seconds +WARNING: constraints for invoking optimized fused softmax kernel are not met. We default back to unfused kernel invocations. +> compiling and loading fused kernels ... +>>> done with compiling and loading fused kernels. Compilation time: 0.003 seconds +WORLD_SIZE: 1, RANK: 0, LOCAL_RANK: 0 +[convert] creating synchronizer: mode=mcore2hf +building GPT model ... +[GPT_LAYER_SPEC] log_hidden_states=[] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=[] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=[] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=[] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/transformers/modeling_rope_utils.py:1036: FutureWarning: `rope_config_validation` is deprecated and has been removed. Its functionality has been moved to RotaryEmbeddingConfigMixin.validate_rope method. PreTrainedConfig inherits this class, so please call self.validate_rope() instead. Also, make sure to use the new rope_parameters syntax. You can call self.standardize_rope_params() in the meantime. + warnings.warn( +[transformers] `layer_type_validation` is deprecated and will be removed in v5.20. Use `PreTrainedConfig.validate_layer_type` instead +[transformers] `layer_type_validation` is deprecated and will be removed in v5.20. Use `PreTrainedConfig.validate_layer_type` instead +[transformers] `torch_dtype` is deprecated! Use `dtype` instead! +[ERROR] `cache_position` is part of Qwen3NextModel.forward's signature, but not documented. Make sure to add it to the docstring of the function in /mnt/ssd/cache_tmp/huggingface/modules/transformers_modules/mcore2hf_hyphen_config_dot_XdGUQ3/10c6759df797195a/modeling_qwen3_next.py. +[ERROR] `cache_position` is part of Qwen3NextForCausalLM.forward's signature, but not documented. Make sure to add it to the docstring of the function in /mnt/ssd/cache_tmp/huggingface/modules/transformers_modules/mcore2hf_hyphen_config_dot_XdGUQ3/10c6759df797195a/modeling_qwen3_next.py. +[transformers] `torch_dtype` is deprecated! Use `dtype` instead! +INFO:transformers_modules.mcore2hf_hyphen_config_dot_XdGUQ3.10c6759df797195a.modeling_qwen3_next:[Qwen3Next custom] attn_position_embedding_type=nope, rnn_position_embedding_type=nope, attn_logits_scaling=None +Qwen3NextForCausalLM( + (model): Qwen3NextModel( + (embed_tokens): Embedding(99072, 2048, padding_idx=102) + (layers): ModuleList( + (0-1): 2 x Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=4800, bias=False) + (up_proj): Linear(in_features=2048, out_features=4800, bias=False) + (down_proj): Linear(in_features=4800, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (2-11): 10 x Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (12): Qwen3NextDecoderLayer( + (self_attn): Qwen3NextAttention( + (q_proj): Linear(in_features=2048, out_features=4096, bias=True) + (k_proj): Linear(in_features=2048, out_features=512, bias=True) + (v_proj): Linear(in_features=2048, out_features=512, bias=True) + (o_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (13-19): 7 x Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (20-21): 2 x Qwen3NextDecoderLayer( + (self_attn): Qwen3NextAttention( + (q_proj): Linear(in_features=2048, out_features=4096, bias=True) + (k_proj): Linear(in_features=2048, out_features=512, bias=True) + (v_proj): Linear(in_features=2048, out_features=512, bias=True) + (o_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (22): Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (23): Qwen3NextDecoderLayer( + (self_attn): Qwen3NextAttention( + (q_proj): Linear(in_features=2048, out_features=4096, bias=True) + (k_proj): Linear(in_features=2048, out_features=512, bias=True) + (v_proj): Linear(in_features=2048, out_features=512, bias=True) + (o_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (24-45): 22 x Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (46): Qwen3NextDecoderLayer( + (self_attn): Qwen3NextAttention( + (q_proj): Linear(in_features=2048, out_features=4096, bias=True) + (k_proj): Linear(in_features=2048, out_features=512, bias=True) + (v_proj): Linear(in_features=2048, out_features=512, bias=True) + (o_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (47): Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (48-49): 2 x Qwen3NextDecoderLayer( + (self_attn): Qwen3NextAttention( + (q_proj): Linear(in_features=2048, out_features=4096, bias=True) + (k_proj): Linear(in_features=2048, out_features=512, bias=True) + (v_proj): Linear(in_features=2048, out_features=512, bias=True) + (o_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (50-55): 6 x Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + ) + (norm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (rotary_emb): Qwen3NextRotaryEmbedding() + ) + (lm_head): Linear(in_features=2048, out_features=99072, bias=False) +) +GPTModel( + (embedding): LanguageModelEmbedding( + (word_embeddings): VocabParallelEmbedding() + (embedding_dropout): Dropout(p=0.1, inplace=False) + ) + (decoder): TransformerBlock( + (layers): ModuleList( + (0-1): 2 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=9600, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=4800, out_features=2048, bias=False, TP=1) + ) + ) + (2-11): 10 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (12): TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): SelfAttention( + (core_attention): TEDotProductAttention( + (flash_attention): FlashAttention() + (fused_attention): FusedAttention() + (unfused_attention): UnfusedDotProductAttention( + (scale_mask_softmax): FusedScaleMaskSoftmax() + (attention_dropout): Dropout(p=0.1, inplace=False) + ) + ) + (linear_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (linear_qkv): TEColumnParallelLinear(in_features=2048, out_features=5120, bias=True, TP=1) + (q_layernorm): IdentityOp() + (k_layernorm): IdentityOp() + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (13-19): 7 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (20-21): 2 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): SelfAttention( + (core_attention): TEDotProductAttention( + (flash_attention): FlashAttention() + (fused_attention): FusedAttention() + (unfused_attention): UnfusedDotProductAttention( + (scale_mask_softmax): FusedScaleMaskSoftmax() + (attention_dropout): Dropout(p=0.1, inplace=False) + ) + ) + (linear_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (linear_qkv): TEColumnParallelLinear(in_features=2048, out_features=5120, bias=True, TP=1) + (q_layernorm): IdentityOp() + (k_layernorm): IdentityOp() + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (22): TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (23): TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): SelfAttention( + (core_attention): TEDotProductAttention( + (flash_attention): FlashAttention() + (fused_attention): FusedAttention() + (unfused_attention): UnfusedDotProductAttention( + (scale_mask_softmax): FusedScaleMaskSoftmax() + (attention_dropout): Dropout(p=0.1, inplace=False) + ) + ) + (linear_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (linear_qkv): TEColumnParallelLinear(in_features=2048, out_features=5120, bias=True, TP=1) + (q_layernorm): IdentityOp() + (k_layernorm): IdentityOp() + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (24-45): 22 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (46): TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): SelfAttention( + (core_attention): TEDotProductAttention( + (flash_attention): FlashAttention() + (fused_attention): FusedAttention() + (unfused_attention): UnfusedDotProductAttention( + (scale_mask_softmax): FusedScaleMaskSoftmax() + (attention_dropout): Dropout(p=0.1, inplace=False) + ) + ) + (linear_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (linear_qkv): TEColumnParallelLinear(in_features=2048, out_features=5120, bias=True, TP=1) + (q_layernorm): IdentityOp() + (k_layernorm): IdentityOp() + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (47): TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (48-49): 2 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): SelfAttention( + (core_attention): TEDotProductAttention( + (flash_attention): FlashAttention() + (fused_attention): FusedAttention() + (unfused_attention): UnfusedDotProductAttention( + (scale_mask_softmax): FusedScaleMaskSoftmax() + (attention_dropout): Dropout(p=0.1, inplace=False) + ) + ) + (linear_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (linear_qkv): TEColumnParallelLinear(in_features=2048, out_features=5120, bias=True, TP=1) + (q_layernorm): IdentityOp() + (k_layernorm): IdentityOp() + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (50-55): 6 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + ) + (final_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (output_layer): ColumnParallelLinear(in_features=2048, out_features=99072, bias=False, TP=1) +) +DEBUG:multistorageclient.config:Searching MSC config file paths: ['/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/msc/msc_config.yaml', '/mnt/ssd/cache_tmp/.config/msc/config.yaml', '/mnt/ssd/cache_tmp/.config/msc/config.json', '/root/.msc_config.yaml', '/root/.msc_config.json', '/root/.config/msc/config.yaml', '/root/.config/msc/config.json', '/etc/xdg/msc/config.yaml', '/etc/xdg/msc/config.json', '/etc/msc_config.yaml', '/etc/msc_config.json'] +DEBUG:multistorageclient.config:Using MSC config file: /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/msc/msc_config.yaml +DEBUG:multistorageclient.config:Searching MSC config file paths: ['/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/msc/msc_config.yaml', '/mnt/ssd/cache_tmp/.config/msc/config.yaml', '/mnt/ssd/cache_tmp/.config/msc/config.json', '/root/.msc_config.yaml', '/root/.msc_config.json', '/root/.config/msc/config.yaml', '/root/.config/msc/config.json', '/etc/xdg/msc/config.yaml', '/etc/xdg/msc/config.json', '/etc/msc_config.yaml', '/etc/msc_config.json'] +DEBUG:multistorageclient.config:Using MSC config file: /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/msc/msc_config.yaml +DEBUG:multistorageclient.client.single:No metadata provider configured, auto-commit will not be enabled +DEBUG:multistorageclient.client.client:StorageClient '__filesystem__' using SingleStorageClient +sharded_state_dict metadata loaded from the checkpoint: {'singleton_local_shards': True, 'distrib_optim_sharding_type': 'dp_reshardable', 'chained_optim_avoid_prefix': True} +Job sharding has changed: Rerun state will be ignored + loading distributed checkpoint from /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/load_view3840 at iteration 3840 +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/dist_checkpointing/strategies/torch.py:956: FutureWarning: `load_state_dict` is deprecated and will be removed in future versions. Please use `load` instead. + checkpoint.load_state_dict( + checkpoint version 3.0 + successfully loaded checkpoint from /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/load_view3840 [ t 1/1, p 1/1 ] at iteration 3840 +[convert] synchronizer created in 377.01s +[convert] starting sync_params +INFO:root:Converting layer 0 is_gdn=True is_not_moe=True +INFO:root:Converting layer 1 is_gdn=True is_not_moe=True +INFO:root:Converting layer 2 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0771484375 mean=-0.000499725341796875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.1123046875 mean=0.0018768310546875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 3 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.07861328125 mean=-0.00017833709716796875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.1181640625 mean=0.000213623046875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 4 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.08544921875 mean=-0.00028228759765625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.08740234375 mean=0.0009918212890625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 5 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0888671875 mean=-0.000244140625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.087890625 mean=0.0019989013671875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 6 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0966796875 mean=-0.0002193450927734375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.08154296875 mean=-0.00118255615234375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 7 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.09228515625 mean=-5.0067901611328125e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.068359375 mean=-0.00038909912109375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 8 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0947265625 mean=-4.982948303222656e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.068359375 mean=-0.0001964569091796875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 9 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0888671875 mean=2.2172927856445312e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.061767578125 mean=-3.790855407714844e-05 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 10 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.08544921875 mean=2.944469451904297e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.058837890625 mean=-0.000904083251953125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 11 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0830078125 mean=6.961822509765625e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.07080078125 mean=-0.0003223419189453125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 12 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 12: args.attention_output_gate=True +WARNING:root:Layer 12: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.07666015625 mean=0.0002307891845703125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.057861328125 mean=-0.000728607177734375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 13 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.07275390625 mean=0.0001773834228515625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.046875 mean=-0.00147247314453125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 14 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.07666015625 mean=0.00017642974853515625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.05029296875 mean=-0.000873565673828125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 15 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.07421875 mean=0.0002593994140625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.05419921875 mean=-0.00121307373046875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 16 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.07275390625 mean=0.00030517578125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.053466796875 mean=-0.000942230224609375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 17 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.06982421875 mean=0.000270843505859375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.05126953125 mean=-0.001983642578125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 18 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.06640625 mean=0.000263214111328125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0537109375 mean=-0.0025177001953125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 19 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.06884765625 mean=0.00060272216796875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.054443359375 mean=-0.00189971923828125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 20 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 20: args.attention_output_gate=True +WARNING:root:Layer 20: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.06787109375 mean=0.000385284423828125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.05419921875 mean=-0.00133514404296875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 21 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 21: args.attention_output_gate=True +WARNING:root:Layer 21: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.060546875 mean=0.000431060791015625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.04638671875 mean=-0.0023040771484375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 22 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.055908203125 mean=0.0003376007080078125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.049072265625 mean=-0.0031585693359375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 23 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 23: args.attention_output_gate=True +WARNING:root:Layer 23: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.053955078125 mean=0.0004749298095703125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.04150390625 mean=-0.002532958984375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 24 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.052978515625 mean=0.000408172607421875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0390625 mean=-0.00262451171875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 25 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.05029296875 mean=0.0003452301025390625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.03759765625 mean=-0.00186920166015625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 26 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0498046875 mean=0.000286102294921875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.034912109375 mean=-0.00213623046875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 27 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.052001953125 mean=0.0004177093505859375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.032958984375 mean=-0.0021209716796875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 28 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.04833984375 mean=0.0003509521484375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0296630859375 mean=-0.0010986328125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 29 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.049072265625 mean=0.000492095947265625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0260009765625 mean=-0.0011749267578125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 30 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0517578125 mean=0.000762939453125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.02734375 mean=-0.0011444091796875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 31 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.04931640625 mean=0.00049591064453125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.02783203125 mean=-0.000713348388671875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 32 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.047119140625 mean=0.00011587142944335938 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0264892578125 mean=-0.000194549560546875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 33 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.050537109375 mean=8.296966552734375e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0289306640625 mean=-9.822845458984375e-05 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 34 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.052734375 mean=0.000156402587890625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0284423828125 mean=-3.647804260253906e-05 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 35 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.05810546875 mean=9.870529174804688e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.033447265625 mean=0.0004329681396484375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 36 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.06005859375 mean=6.628036499023438e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0439453125 mean=0.00084686279296875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 37 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.06005859375 mean=0.00012111663818359375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.043212890625 mean=0.001251220703125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 38 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.05859375 mean=0.0001239776611328125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.037841796875 mean=0.001373291015625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 39 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.05712890625 mean=9.059906005859375e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.037353515625 mean=0.001190185546875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 40 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0537109375 mean=0.00013828277587890625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.027099609375 mean=0.000347137451171875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 41 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.05029296875 mean=0.000141143798828125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0252685546875 mean=0.00012683868408203125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 42 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.044921875 mean=0.000347137451171875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0218505859375 mean=-0.000682830810546875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 43 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.038818359375 mean=0.0001926422119140625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.021484375 mean=-0.00028228759765625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 44 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.03564453125 mean=0.0003509521484375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.02197265625 mean=-0.0004405975341796875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 45 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0361328125 mean=0.0001583099365234375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.024658203125 mean=-0.000873565673828125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 46 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 46: args.attention_output_gate=True +WARNING:root:Layer 46: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.03662109375 mean=0.000392913818359375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.021728515625 mean=-0.00064849853515625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 47 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.037353515625 mean=0.00021457672119140625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0255126953125 mean=-0.0008544921875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 48 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 48: args.attention_output_gate=True +WARNING:root:Layer 48: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.034912109375 mean=-7.772445678710938e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0206298828125 mean=-0.00054168701171875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 49 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 49: args.attention_output_gate=True +WARNING:root:Layer 49: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.034912109375 mean=2.2411346435546875e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0242919921875 mean=-0.001190185546875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 50 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.03662109375 mean=-4.8160552978515625e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.024658203125 mean=-0.000888824462890625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 51 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.037353515625 mean=4.6253204345703125e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0250244140625 mean=-0.00115203857421875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 52 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.04052734375 mean=0.00023365020751953125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.031494140625 mean=-0.0013885498046875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 53 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.03857421875 mean=6.914138793945312e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.051025390625 mean=-0.0023193359375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 54 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.039306640625 mean=4.553794860839844e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.041015625 mean=-0.00213623046875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 55 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.043212890625 mean=-0.00013828277587890625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.044189453125 mean=0.0025482177734375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +[convert] finished sync_params in 0.94s +[convert] starting check_and_save -> /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/iter_3840-hf +DEBUG:root:[RANK 0] 0 send op & 0 recv op. +INFO:root:[Iters 0 RANK 0] starts synchronizing parameters with other ranks... +INFO:root:[Iters 0 RANK 0] finishes synchronizing +[Iters 0 RANK 0] model-00001-of-00004.safetensors is saved. +DEBUG:root:[Iters 0 RANK 0] joined +DEBUG:root:[RANK 0] 0 send op & 0 recv op. +INFO:root:[Iters 1 RANK 0] starts synchronizing parameters with other ranks... +INFO:root:[Iters 1 RANK 0] finishes synchronizing +[Iters 1 RANK 0] model-00002-of-00004.safetensors is saved. +DEBUG:root:[Iters 1 RANK 0] joined +DEBUG:root:[RANK 0] 0 send op & 0 recv op. +INFO:root:[Iters 2 RANK 0] starts synchronizing parameters with other ranks... +INFO:root:[Iters 2 RANK 0] finishes synchronizing +[Iters 2 RANK 0] model-00003-of-00004.safetensors is saved. +DEBUG:root:[Iters 2 RANK 0] joined +DEBUG:root:[RANK 0] 0 send op & 0 recv op. +INFO:root:[Iters 3 RANK 0] starts synchronizing parameters with other ranks... +INFO:root:[Iters 3 RANK 0] finishes synchronizing +[Iters 3 RANK 0] model-00004-of-00004.safetensors is saved. +DEBUG:root:[Iters 3 RANK 0] joined +[convert] finished check_and_save in 80.50s +Conversion finished in 459.85870456695557 seconds. ++ echo '✅Conversion completed: /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/iter_3840-hf' +✅Conversion completed: /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/iter_3840-hf ++ cleanup_real_config_path ++ rm -rf /tmp/mcore2hf_ws_3840/mcore2hf-config.XdGUQ3 ++ '[' -n '' ']' diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/hf_export_logs/convert_iter4760.log b/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/hf_export_logs/convert_iter4760.log new file mode 100644 index 0000000000000000000000000000000000000000..aaf54d3b603f0443e8503bfc6ffd0202cdb5738a --- /dev/null +++ b/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/hf_export_logs/convert_iter4760.log @@ -0,0 +1,2665 @@ +++ realpath /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/scripts/mcore_gdn_moe/../../../.. ++ MEGATRON_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher ++ CONVERTOR_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor ++ REPAIR_SCRIPT=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/utils/repair_gdn_checkpoint.py ++ RENDER_CONFIG_SCRIPT=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/utils/render_gdn_hf_config.py ++ export PYTHON_ENV_ROOT=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud ++ PYTHON_ENV_ROOT=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud ++ PYTHON_BIN=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python ++ '[' '!' -x /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python ']' ++ '[' '!' -x /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python ']' ++ '[' '!' -x /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python ']' ++ '[' '!' -x /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python ']' ++ export PYTHON_BIN ++ /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python -c 'import torch' ++ export PATH=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin:/root/.kimi-code/bin:/root/.local/bin:/mnt/hdd/minyingqian/mihomo/bin:/root/.grok/bin:/usr/local/bin:/root/miniconda3/bin:/root/.local/bin:/usr/local/bin:/root/miniconda3/bin:/root/miniconda3/bin:/mnt/ssd/liuzian/pandoc/current/bin:/mnt/ssd/cache_tmp/.local/share/../bin:/mnt/ssd/yulan/bin:/mnt/ssd/yulan/bin:/root/.local/bin:/usr/local/bin:/root/miniconda3/bin:/root/.local/bin:/usr/local/bin:/root/miniconda3/bin:/mnt/ssd/cache_tmp/.local/bin:/root/.local/bin:/root/.local/bin:/mnt/hdd/minyingqian/mihomo/bin:/root/.grok/bin:/usr/local/bin:/root/miniconda3/bin:/root/.local/bin:/usr/local/bin:/root/miniconda3/bin:/root/miniconda3/bin:/mnt/ssd/liuzian/pandoc/current/bin:/mnt/ssd/cache_tmp/.local/bin:/mnt/ssd/yulan/bin:/mnt/ssd/yulan/bin:/root/.local/bin:/root/miniconda3/bin:/root/miniconda3/condabin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/mnt/ssd/yulan/cache/installs/nsight-compute/2026.2:/mnt/ssd/yulan/cache/installs/nsight-systems/2026.3.1/bin:/mnt/ssd/yulan/cache/installs/nsight-compute/2026.2:/mnt/ssd/yulan/cache/installs/nsight-systems/2026.3.1/bin ++ PATH=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin:/root/.kimi-code/bin:/root/.local/bin:/mnt/hdd/minyingqian/mihomo/bin:/root/.grok/bin:/usr/local/bin:/root/miniconda3/bin:/root/.local/bin:/usr/local/bin:/root/miniconda3/bin:/root/miniconda3/bin:/mnt/ssd/liuzian/pandoc/current/bin:/mnt/ssd/cache_tmp/.local/share/../bin:/mnt/ssd/yulan/bin:/mnt/ssd/yulan/bin:/root/.local/bin:/usr/local/bin:/root/miniconda3/bin:/root/.local/bin:/usr/local/bin:/root/miniconda3/bin:/mnt/ssd/cache_tmp/.local/bin:/root/.local/bin:/root/.local/bin:/mnt/hdd/minyingqian/mihomo/bin:/root/.grok/bin:/usr/local/bin:/root/miniconda3/bin:/root/.local/bin:/usr/local/bin:/root/miniconda3/bin:/root/miniconda3/bin:/mnt/ssd/liuzian/pandoc/current/bin:/mnt/ssd/cache_tmp/.local/bin:/mnt/ssd/yulan/bin:/mnt/ssd/yulan/bin:/root/.local/bin:/root/miniconda3/bin:/root/miniconda3/condabin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/mnt/ssd/yulan/cache/installs/nsight-compute/2026.2:/mnt/ssd/yulan/cache/installs/nsight-systems/2026.3.1/bin:/mnt/ssd/yulan/cache/installs/nsight-compute/2026.2:/mnt/ssd/yulan/cache/installs/nsight-systems/2026.3.1/bin ++ export VIRTUAL_ENV=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud ++ VIRTUAL_ENV=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud ++ echo ✅MEGATRON_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher +✅MEGATRON_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher ++ echo ✅PYTHON_BIN=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python +✅PYTHON_BIN=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python ++ export CUDA_LAUNCH_BLOCKING=1 ++ CUDA_LAUNCH_BLOCKING=1 ++ export CUDA_VISIBLE_DEVICES=0 ++ CUDA_VISIBLE_DEVICES=0 ++ export KUBERNETES_CONTAINER_RESOURCE_GPU=1 ++ KUBERNETES_CONTAINER_RESOURCE_GPU=1 ++ export TP=1 ++ TP=1 ++ export PP=1 ++ PP=1 ++ export ETP=1 ++ ETP=1 ++ export EP=1 ++ EP=1 ++ export 'MODEL_PARALLEL_ARGS=--tensor-model-parallel-size 1 --pipeline-model-parallel-size 1 --expert-tensor-parallel-size 1 --expert-model-parallel-size 1' ++ MODEL_PARALLEL_ARGS='--tensor-model-parallel-size 1 --pipeline-model-parallel-size 1 --expert-tensor-parallel-size 1 --expert-model-parallel-size 1' ++ WORKSPACE_TMP_DIR=/tmp/mcore2hf_ws_4760 ++ mkdir -p /tmp/mcore2hf_ws_4760 ++ MEGATRON_CHECKPOINT_PATH=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/load_view4760 ++ echo MEGATRON_CHECKPOINT_PATH=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/load_view4760 +MEGATRON_CHECKPOINT_PATH=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/load_view4760 ++ MANUAL_ITER= ++ CONVERT_ALL=false ++ FORCE_RECONVERT=false ++ DROP_MTP=false ++ POSITIONAL_ARGS=() ++ [[ 3 -gt 0 ]] ++ case $1 in ++ POSITIONAL_ARGS+=("$1") ++ shift ++ [[ 2 -gt 0 ]] ++ case $1 in ++ '[' -z 4760 ']' ++ MANUAL_ITER=4760 ++ shift 2 ++ [[ 0 -gt 0 ]] ++ set -- /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/load_view4760 ++ '[' -e /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/load_view4760 ']' +++ readlink -f /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/load_view4760 ++ MEGATRON_CHECKPOINT_PATH=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/load_view4760 ++ echo ✅MEGATRON_CHECKPOINT_PATH=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/load_view4760 +✅MEGATRON_CHECKPOINT_PATH=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/load_view4760 +++ basename /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/load_view4760 ++ CHECKPOINT_BASENAME=load_view4760 ++ [[ load_view4760 =~ ^iter_([0-9]+)$ ]] ++ '[' false = true ']' ++ HF_CONFIG_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/hf-configs/YuLan-Mini-for-gdn-convert ++ echo HF_CONFIG_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/hf-configs/YuLan-Mini-for-gdn-convert +HF_CONFIG_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/hf-configs/YuLan-Mini-for-gdn-convert ++ '[' '!' -f /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/hf-configs/YuLan-Mini-for-gdn-convert/config.template.json ']' +++ mktemp -d /tmp/mcore2hf_ws_4760/mcore2hf-config.XXXXXX ++ REAL_CONFIG_PATH=/tmp/mcore2hf_ws_4760/mcore2hf-config.9zTuxl ++ LOAD_VIEW_DIR= ++ trap cleanup_real_config_path EXIT ++ export PYTHONPATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher: ++ PYTHONPATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher: ++ export PYTHONPATH=/tmp/mcore2hf_ws_4760/mcore2hf-config.9zTuxl:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher: ++ PYTHONPATH=/tmp/mcore2hf_ws_4760/mcore2hf-config.9zTuxl:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher: +++ /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/utils/parse-linear-vars-from-mcore-config.py /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/load_view4760 +W1008 16:17:41.637000 950108 torch/utils/cpp_extension.py:2425] TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +W1008 16:17:41.637000 950108 torch/utils/cpp_extension.py:2425] If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'] to specific architectures. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. ++ eval 'export LINEAR_NUM_KEY_HEADS=8 +export LINEAR_NUM_VALUE_HEADS=32 +export LINEAR_ATTN_FREQ=[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1] +export FFN_HIDDEN_SIZE=4800 +export NUM_LAYERS=56 +export HIDDEN_SIZE=2048 +export NUM_ATTN_HEADS=32 +export NUM_QUERY_GROUPS=8 +export MAX_POSITION_EMBEDDINGS=131072 +export KV_CHANNELS=64 +export LINEAR_CONV_KERNEL_DIM=4 +export LINEAR_KEY_HEAD_DIM=64 +export LINEAR_VALUE_HEAD_DIM=64 +export MOE_FFN_HIDDEN_SIZE=512 +export NUM_EXPERTS=192 +export MOE_ROUTER_TOPK=4 +export MOE_ROUTER_SCORE_FUNCTION=softmax +export MOE_ROUTER_ENABLE_EXPERT_BIAS=false +export MOE_LAYER_FREQ=[0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1] +export MLP_ONLY_LAYERS=[0,1] +export MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512 +export ROPE_THETA=10000.0 +export POSITION_EMBEDDING_TYPE=none +export ATTENTION_OUTPUT_GATE=true +export MTP_NUM_LAYERS=0 +export MOE_ROUTER_SQRT_GATE=true +export GATED_NORM=true +export GATED_NORM_RANK=16 +export GATED_NORM_GATE_SCALE=1.0 +export GATED_NORM_UP_BIAS=true +export FINAL_GATED_NORM=true +export ACTIVATION_FUNC_CLAMP_VALUE=7.0 +export ACTIVATION_FUNC_CLAMP_MODE=soft +export FINAL_LOGITS_CLAMP_VALUE=30.0 +export MOE_ROUTER_DTYPE=fp32 +export TORCH_DTYPE=bfloat16 +export VOCAB_SIZE=99072' +++ export LINEAR_NUM_KEY_HEADS=8 +++ LINEAR_NUM_KEY_HEADS=8 +++ export LINEAR_NUM_VALUE_HEADS=32 +++ LINEAR_NUM_VALUE_HEADS=32 +++ export 'LINEAR_ATTN_FREQ=[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1]' +++ LINEAR_ATTN_FREQ='[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1]' +++ export FFN_HIDDEN_SIZE=4800 +++ FFN_HIDDEN_SIZE=4800 +++ export NUM_LAYERS=56 +++ NUM_LAYERS=56 +++ export HIDDEN_SIZE=2048 +++ HIDDEN_SIZE=2048 +++ export NUM_ATTN_HEADS=32 +++ NUM_ATTN_HEADS=32 +++ export NUM_QUERY_GROUPS=8 +++ NUM_QUERY_GROUPS=8 +++ export MAX_POSITION_EMBEDDINGS=131072 +++ MAX_POSITION_EMBEDDINGS=131072 +++ export KV_CHANNELS=64 +++ KV_CHANNELS=64 +++ export LINEAR_CONV_KERNEL_DIM=4 +++ LINEAR_CONV_KERNEL_DIM=4 +++ export LINEAR_KEY_HEAD_DIM=64 +++ LINEAR_KEY_HEAD_DIM=64 +++ export LINEAR_VALUE_HEAD_DIM=64 +++ LINEAR_VALUE_HEAD_DIM=64 +++ export MOE_FFN_HIDDEN_SIZE=512 +++ MOE_FFN_HIDDEN_SIZE=512 +++ export NUM_EXPERTS=192 +++ NUM_EXPERTS=192 +++ export MOE_ROUTER_TOPK=4 +++ MOE_ROUTER_TOPK=4 +++ export MOE_ROUTER_SCORE_FUNCTION=softmax +++ MOE_ROUTER_SCORE_FUNCTION=softmax +++ export MOE_ROUTER_ENABLE_EXPERT_BIAS=false +++ MOE_ROUTER_ENABLE_EXPERT_BIAS=false +++ export 'MOE_LAYER_FREQ=[0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]' +++ MOE_LAYER_FREQ='[0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]' +++ export 'MLP_ONLY_LAYERS=[0,1]' +++ MLP_ONLY_LAYERS='[0,1]' +++ export MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512 +++ MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512 +++ export ROPE_THETA=10000.0 +++ ROPE_THETA=10000.0 +++ export POSITION_EMBEDDING_TYPE=none +++ POSITION_EMBEDDING_TYPE=none +++ export ATTENTION_OUTPUT_GATE=true +++ ATTENTION_OUTPUT_GATE=true +++ export MTP_NUM_LAYERS=0 +++ MTP_NUM_LAYERS=0 +++ export MOE_ROUTER_SQRT_GATE=true +++ MOE_ROUTER_SQRT_GATE=true +++ export GATED_NORM=true +++ GATED_NORM=true +++ export GATED_NORM_RANK=16 +++ GATED_NORM_RANK=16 +++ export GATED_NORM_GATE_SCALE=1.0 +++ GATED_NORM_GATE_SCALE=1.0 +++ export GATED_NORM_UP_BIAS=true +++ GATED_NORM_UP_BIAS=true +++ export FINAL_GATED_NORM=true +++ FINAL_GATED_NORM=true +++ export ACTIVATION_FUNC_CLAMP_VALUE=7.0 +++ ACTIVATION_FUNC_CLAMP_VALUE=7.0 +++ export ACTIVATION_FUNC_CLAMP_MODE=soft +++ ACTIVATION_FUNC_CLAMP_MODE=soft +++ export FINAL_LOGITS_CLAMP_VALUE=30.0 +++ FINAL_LOGITS_CLAMP_VALUE=30.0 +++ export MOE_ROUTER_DTYPE=fp32 +++ MOE_ROUTER_DTYPE=fp32 +++ export TORCH_DTYPE=bfloat16 +++ TORCH_DTYPE=bfloat16 +++ export VOCAB_SIZE=99072 +++ VOCAB_SIZE=99072 ++ echo LINEAR_NUM_KEY_HEADS=8 +LINEAR_NUM_KEY_HEADS=8 ++ echo LINEAR_NUM_VALUE_HEADS=32 +LINEAR_NUM_VALUE_HEADS=32 ++ echo 'LINEAR_ATTN_FREQ=[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1]' +LINEAR_ATTN_FREQ=[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1] ++ echo 'NUM_LAYERS=56 HIDDEN_SIZE=2048' +NUM_LAYERS=56 HIDDEN_SIZE=2048 ++ echo 'NUM_ATTN_HEADS=32 NUM_QUERY_GROUPS=8' +NUM_ATTN_HEADS=32 NUM_QUERY_GROUPS=8 ++ echo 'MAX_POSITION_EMBEDDINGS=131072 KV_CHANNELS=64' +MAX_POSITION_EMBEDDINGS=131072 KV_CHANNELS=64 ++ echo 'FFN_HIDDEN_SIZE=4800 MOE_FFN_HIDDEN_SIZE=512' +FFN_HIDDEN_SIZE=4800 MOE_FFN_HIDDEN_SIZE=512 ++ echo 'NUM_EXPERTS=192 MOE_ROUTER_TOPK=4' +NUM_EXPERTS=192 MOE_ROUTER_TOPK=4 ++ echo MOE_ROUTER_SCORE_FUNCTION=softmax +MOE_ROUTER_SCORE_FUNCTION=softmax ++ echo 'MLP_ONLY_LAYERS=[0,1]' +MLP_ONLY_LAYERS=[0,1] ++ echo 'MOE_LAYER_FREQ=[0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]' +MOE_LAYER_FREQ=[0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1] ++ echo MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512 +MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512 ++ echo 'LINEAR_CONV_KERNEL_DIM=4 LINEAR_KEY_HEAD_DIM=64 LINEAR_VALUE_HEAD_DIM=64' +LINEAR_CONV_KERNEL_DIM=4 LINEAR_KEY_HEAD_DIM=64 LINEAR_VALUE_HEAD_DIM=64 ++ echo ATTENTION_OUTPUT_GATE=true +ATTENTION_OUTPUT_GATE=true ++ echo ROPE_THETA=10000.0 +ROPE_THETA=10000.0 ++ echo POSITION_EMBEDDING_TYPE=none +POSITION_EMBEDDING_TYPE=none ++ echo MOE_ROUTER_SQRT_GATE=true +MOE_ROUTER_SQRT_GATE=true +++ /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/utils/parse-mom-vars-from-mcore-config.py /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/load_view4760 +W1008 16:17:56.629000 951436 torch/utils/cpp_extension.py:2425] TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +W1008 16:17:56.629000 951436 torch/utils/cpp_extension.py:2425] If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'] to specific architectures. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. ++ eval 'export MOM_ENABLED=false +export MOM_DO_MASK_SCALE=true +export MOM_TARGET_FFN_HIDDEN_SIZE=512' +++ export MOM_ENABLED=false +++ MOM_ENABLED=false +++ export MOM_DO_MASK_SCALE=true +++ MOM_DO_MASK_SCALE=true +++ export MOM_TARGET_FFN_HIDDEN_SIZE=512 +++ MOM_TARGET_FFN_HIDDEN_SIZE=512 ++ echo 'MOM_ENABLED=false MOM_DO_MASK_SCALE=true MOM_TARGET_FFN_HIDDEN_SIZE=512' +MOM_ENABLED=false MOM_DO_MASK_SCALE=true MOM_TARGET_FFN_HIDDEN_SIZE=512 ++ '[' false = true ']' ++ '[' 192 -gt 0 ']' ++ '[' 512 -le 0 ']' ++ '[' 10000.0 = 0.0 ']' +++ /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python - ++ ROPE_THETA=10000 ++ '[' -z 131072 ']' ++ '[' 131072 = 0 ']' ++ FFN_TOKEN_SHIFT= ++ FFN_INTERMEDIATE_TOKEN_SHIFT= ++ ATTN_TOKEN_SHIFT= ++ ATTN_Q_TOKEN_SHIFT= ++ ATTN_K_TOKEN_SHIFT= ++ ATTN_V_TOKEN_SHIFT= ++ TOKEN_SHIFT_CONV_SIZE=4 ++ TOKEN_SHIFT_CONV_INIT=default ++ export FFN_TOKEN_SHIFT FFN_INTERMEDIATE_TOKEN_SHIFT ATTN_TOKEN_SHIFT ++ export ATTN_Q_TOKEN_SHIFT ATTN_K_TOKEN_SHIFT ATTN_V_TOKEN_SHIFT ++ export TOKEN_SHIFT_CONV_SIZE TOKEN_SHIFT_CONV_INIT ++ export ROPE_THETA MAX_POSITION_EMBEDDINGS POSITION_EMBEDDING_TYPE ATTENTION_OUTPUT_GATE MOE_ROUTER_SQRT_GATE ++ ATTENTION_OUTPUT_GATE=true ++ /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/utils/render_gdn_hf_config.py --checkpoint-path /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/load_view4760 --template-path /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/hf-configs/YuLan-Mini-for-gdn-convert --output-dir /tmp/mcore2hf_ws_4760/mcore2hf-config.9zTuxl +W1008 16:18:09.880000 952025 torch/utils/cpp_extension.py:2425] TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +W1008 16:18:09.880000 952025 torch/utils/cpp_extension.py:2425] If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'] to specific architectures. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. ++ echo '✅config.json: /tmp/mcore2hf_ws_4760/mcore2hf-config.9zTuxl/config.json' +✅config.json: /tmp/mcore2hf_ws_4760/mcore2hf-config.9zTuxl/config.json ++ /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python - /tmp/mcore2hf_ws_4760/mcore2hf-config.9zTuxl/config.json +{ + "hidden_size": 2048, + "intermediate_size": 4800, + "num_hidden_layers": 56, + "num_attention_heads": 32, + "num_key_value_heads": 8, + "head_dim": 64, + "max_position_embeddings": 131072, + "rope_theta": 10000, + "attn_position_embedding_type": "nope", + "rnn_position_embedding_type": "nope", + "linear_num_key_heads": 8, + "linear_num_value_heads": 32, + "linear_key_head_dim": 64, + "linear_value_head_dim": 64, + "num_experts": 192, + "num_experts_per_tok": 4, + "attn_output_gate": true +} ++ sleep 5 ++ MODEL_SIZE=2.6b ++ LOAD_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/load_view4760 ++ MG2HF=true ++ USE_CUDA=false ++ PR=bf16 ++ HF_DIR=/tmp/mcore2hf_ws_4760/mcore2hf-config.9zTuxl ++ export DROP_MTP ++ PREPARED_LOAD_DIR= ++ prepare_load_dir_for_iter /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/load_view4760 4760 ++ local SOURCE_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/load_view4760 ++ local ITER_NUM=4760 ++ '[' -z 4760 ']' +++ printf %07d 4760 ++ local ITER_BASENAME=iter_0004760 ++ local SOURCE_ITER_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/load_view4760/iter_0004760 ++ '[' '!' -d /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/load_view4760/iter_0004760 ']' ++ local LATEST_VALUE= ++ '[' -f /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/load_view4760/latest_checkpointed_iteration.txt ']' +++ cat /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/load_view4760/latest_checkpointed_iteration.txt ++ LATEST_VALUE=4760 ++ '[' 4760 = 4760 ']' ++ PREPARED_LOAD_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/load_view4760 ++ return 0 ++ LOAD_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/load_view4760 ++ echo LOAD_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/load_view4760 +LOAD_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/load_view4760 ++ '[' false = true ']' ++ '[' false = true ']' ++ '[' '!' -f /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/load_view4760/latest_checkpointed_iteration.txt ']' +++ cat /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/load_view4760/latest_checkpointed_iteration.txt ++ LATEST_CHECKPOINT_ITERATION=4760 ++ echo -e '\033[32mLatest checkpoint iteration: 4760\033[0m\n' +Latest checkpoint iteration: 4760 + ++ I=4760 ++ [[ 4760 =~ ^[0-9]+$ ]] +++ printf %07d 4760 ++ ITER_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/load_view4760/iter_0004760 ++ '[' '!' -f /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/load_view4760/iter_0004760/common.pt ']' ++ SAVE_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/load_view4760/iter_4760-hf ++ '[' -n /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a ']' ++ SAVE_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/iter_4760-hf ++ '[' -n '' ']' ++ mkdir -p /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/iter_4760-hf ++ bash ./scripts/mcore_gdn_moe/convert.sh 2.6b /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/load_view4760 /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/iter_4760-hf true false bf16 /tmp/mcore2hf_ws_4760/mcore2hf-config.9zTuxl 4760 ++ tee /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/iter_4760-hf/hf2mcore.log +✅Enable attention output gate +\[\e[33mWarning adding --moe-shared-expert-gate for YuLan model. If you are not YuLan, please check this flag.\e[0m\] +torchrun --nproc_per_node 1 --nnodes 1 --node_rank 0 --master_addr localhost --master_port 42921 /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/impl/convert.py --tokenizer-type HuggingFaceTokenizer --tokenizer-model /tmp/mcore2hf_ws_4760/mcore2hf-config.9zTuxl --hf-dir /tmp/mcore2hf_ws_4760/mcore2hf-config.9zTuxl --mcore2hf --bf16 --normalization RMSNorm --swiglu --disable-bias-linear --seq-length 1 --max-position-embeddings 131072 --attention-backend auto --position-embedding-type none --kv-channels 64 --group-query-attention --add-qkv-bias --num-layers 56 --hidden-size 2048 --ffn-hidden-size 4800 --num-attention-heads 32 --rotary-base 10000 --rotary-percent 1.00 --num-query-groups 8 --normalization RMSNorm --norm-epsilon 1e-6 --untie-embeddings-and-output-weights --attention-output-gate --linear-attention-type gated_delta_net --linear-attention-freq [1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1] --linear-conv-kernel-dim 4 --linear-key-head-dim 64 --linear-value-head-dim 64 --linear-num-key-heads 8 --linear-num-value-heads 32 --num-experts 192 --moe-router-score-function softmax --moe-token-dispatcher-type alltoall --moe-router-topk 4 --moe-ffn-hidden-size 512 --moe-layer-freq [0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1] --moe-router-dtype fp32 --moe-shared-expert-intermediate-size 512 --moe-shared-expert-gate --moe-router-score-function softmax --moe-router-sqrt-gate --micro-batch-size 1 --global-batch-size 1024 --train-iters 500000 --weight-decay 0.1 --adam-beta1 0.9 --adam-beta2 0.95 --init-method-std 0.006 --clip-grad 1.0 --lr 2.0e-5 --lr-decay-style cosine --min-lr 6.0e-6 --lr-warmup-fraction .001 --lr-decay-iters 430000 --no-gradient-accumulation-fusion --bf16 --tensor-model-parallel-size 1 --pipeline-model-parallel-size 1 --expert-tensor-parallel-size 1 --expert-model-parallel-size 1 --log-interval 100 --save-interval 10000 --eval-interval 1000 --eval-iters 10 --model-type GPT --load-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/load_view4760 --save-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/iter_4760-hf --dist-ckpt-optim-fully-reshardable --skip-train --use-cpu-initialization --no-load-optim --no-load-rng --logging-level 1 --attention-backend auto --debug --max-shard-size 20GB --synchronizer mcore_gdn_moe --pretrain-script mcore_gdn_moe.model_provider --ckpt-step 4760 --gdn-split-in-proj --gated-norm --gated-norm-lora-rank 16 --distributed-backend gloo +W1008 16:18:30.379000 952761 torch/utils/cpp_extension.py:2425] TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +W1008 16:18:30.379000 952761 torch/utils/cpp_extension.py:2425] If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'] to specific architectures. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +Warning: Pai-Megatron-Patch arguments not available, some arguments may not be recognized +using world size: 1, data-parallel size: 1, context-parallel size: 1, hierarchical context-parallel sizes: None, tensor-model-parallel size: 1, pipeline-model-parallel size: 1 +Number of virtual stages per pipeline stage: None +accumulate and all-reduce gradients in fp32 for bfloat16 data type. +using torch.bfloat16 for parameters ... +------------------------ arguments ------------------------ + account_for_embedding_in_pipeline_split ......... False + account_for_loss_in_pipeline_split .............. False + accumulate_allreduce_grads_in_fp32 .............. True + activation_func_clamp_mode ...................... hard + activation_func_clamp_value ..................... None + adam_beta1 ...................................... 0.9 + adam_beta2 ...................................... 0.95 + adam_bias_correction ............................ True + adam_eps ........................................ 1e-08 + adamh_eps ....................................... 1e-08 + adamh_radius .................................... None + adamh_scope ..................................... lm_head + adamw_lr_mup_scaler ............................. False + add_bias_linear ................................. False + add_position_embedding .......................... True + add_qkv_bias .................................... True + adlr_autoresume ................................. False + adlr_autoresume_interval ........................ 1000 + align_grad_reduce ............................... True + align_param_gather .............................. False + allow_ambiguous_pad_tokens ...................... False + app_tag_run_name ................................ None + app_tag_run_version ............................. 0.0.0 + apply_layernorm_1p .............................. False + apply_query_key_layer_scaling ................... False + apply_residual_connection_post_layernorm ........ False + apply_rope_fusion ............................... False + async_save ...................................... None + async_tensor_model_parallel_allreduce ........... True + attention_backend ............................... AttnBackend.auto + attention_dropout ............................... 0.1 + attention_output_gate ........................... True + attention_output_gate_upcycling ................. False + attention_softmax_in_fp32 ....................... False + attn_k_token_shift .............................. None + attn_output_gate ................................ None + attn_output_gate_rand_init ...................... False + attn_q_token_shift .............................. None + attn_token_shift ................................ None + attn_v_token_shift .............................. None + auto_detect_ckpt_format ......................... False + auto_generate_cu_seqlens ........................ False + auto_model ...................................... AutoModelForCausalLM + barrier_with_L1_time ............................ True + benchmark_eval .................................. False + benchmark_global_batch .......................... None + benchmark_interval .............................. None + benchmark_micro_batch ........................... None + benchmark_sequence_length ....................... None + benchmark_tasks ................................. None + bert_binary_head ................................ True + bert_embedder_type .............................. megatron + bert_load ....................................... None + bf16 ............................................ True + bias_dropout_fusion ............................. True + bias_gelu_fusion ................................ False + bias_swiglu_fusion .............................. True + biencoder_projection_dim ........................ 0 + biencoder_shared_query_context_model ............ False + block_data_path ................................. None + cache_mla_latents ............................... False + calc_ft_timeouts ................................ False + calculate_per_token_loss ........................ False + check_for_large_grads ........................... False + check_for_nan_in_loss_and_grad .................. True + check_for_spiky_loss ............................ False + check_weight_hash_across_dp_replicas_interval ... None + ckpt_assume_constant_structure .................. False + ckpt_convert_format ............................. None + ckpt_convert_save ............................... None + ckpt_convert_update_legacy_dist_opt_format ...... False + ckpt_format ..................................... torch_dist + ckpt_fully_parallel_load ........................ False + ckpt_fully_parallel_save ........................ True + ckpt_fully_parallel_save_deprecated ............. False + ckpt_step ....................................... 4760 + classes_fraction ................................ 1.0 + clip_grad ....................................... 1.0 + clone_scatter_output_in_embedding ............... True + config_logger_dir ............................... + consumed_train_samples .......................... 0 + consumed_valid_samples .......................... 0 + context_parallel_size ........................... 1 + continual_pretrain .............................. False + continual_pretrain_preserve_opt_param_scheduler . False + corrected_weight_decay .......................... False + cp_comm_type .................................... ['p2p'] + create_attention_mask_in_dataloader ............. True + cross_entropy_fusion_impl ....................... native + cross_entropy_loss_fusion ....................... False + cuda_graph_impl ................................. none + cuda_graph_scope ................................ [] + cuda_graph_warmup_steps ......................... 3 + data_args_path .................................. None + data_cache_path ................................. None + data_parallel_random_init ....................... False + data_parallel_sharding_strategy ................. no_shard + data_parallel_size .............................. 1 + data_path ....................................... None + data_per_class_fraction ......................... 1.0 + data_sharding ................................... True + dataloader_defer_npy_index_mmap ................. False + dataloader_fast_cache_load ...................... False + dataloader_type ................................. single + dc_shortconv .................................... False + dc_shortconv_optimizer_state .................... conservative + dc_shortconv_upcycling .......................... False + ddp_average_in_collective ....................... False + ddp_bucket_size ................................. None + ddp_num_buckets ................................. None + ddp_pad_buckets_for_high_nccl_busbw ............. False + debug ........................................... True + decode_only_cuda_graphs ......................... False + decoder_first_pipeline_num_layers ............... None + decoder_last_pipeline_num_layers ................ None + decoder_num_layers .............................. None + decoder_seq_length .............................. None + decoupled_lr .................................... None + decoupled_min_lr ................................ None + decrease_batch_size_if_needed ................... False + defer_embedding_wgrad_compute ................... False + defer_grad_checks ............................... False + delay_wgrad_compute ............................. False + deprecated_use_mcore_models ..................... False + deterministic_mode .............................. False + dino_bottleneck_size ............................ 256 + dino_freeze_last_layer .......................... 1 + dino_head_hidden_size ........................... 2048 + dino_local_crops_number ......................... 10 + dino_local_img_size ............................. 96 + dino_norm_last_layer ............................ False + dino_teacher_temp ............................... 0.07 + dino_warmup_teacher_temp ........................ 0.04 + dino_warmup_teacher_temp_epochs ................. 30 + disable_attn_output_gate ........................ False + disable_bf16_reduced_precision_matmul ........... False + disable_chunked_prefill ......................... False + disable_explicit_attention_mask ................. False + disable_mamba_mem_eff_path ...................... False + disable_straggler_on_startup .................... False + disable_symmetric_registration .................. False + dist_ckpt_format_deprecated ..................... None + dist_ckpt_optim_fully_reshardable ............... True + dist_ckpt_save_pre_mcore_014 .................... False + dist_ckpt_strictness ............................ assume_ok_unexpected + distrib_optim_fully_reshardable_mem_efficient ... False + distribute_saved_activations .................... False + distributed_backend ............................. gloo + distributed_timeout_minutes ..................... 10 + distributed_timeout_seconds_after_init .......... None + document_packing_algorithm ...................... ogbfd + document_packing_dp_size ........................ None + drop_mtp ........................................ False + dryrun .......................................... False + dump_param_to_param_group_map ................... None + dynamic_context_parallel ........................ False + dynamic_context_parallel_sizes .................. None + dynamic_cp_candidate_sizes ...................... None + dynamic_cp_gdn_policy ........................... None + dynamic_cp_moe_policy ........................... None + dynamic_cp_on_error ............................. error + dynamic_cp_scheduler ............................ default_dynamic_cp + dynamic_cp_wave_coexecution ..................... False + dynamic_cp_workspace_max_tokens ................. None + emb_deviation_loss_coeff ........................ 0 + emb_deviation_type .............................. None + embedding_init_method_std ....................... None + embedding_path .................................. None + empty_unused_memory_level ....................... 0 + enable_cuda_graph ............................... False + enable_debug_logging ............................ False + enable_experimental ............................. False + enable_ft_package ............................... False + enable_full_sharding_in_hsdp .................... False + enable_gloo_process_groups ...................... True + enable_msc ...................................... True + enable_one_logger ............................... True + encoder_num_layers .............................. 56 + encoder_seq_length .............................. 1 + end_weight_decay ................................ 0.1 + eod_mask_loss ................................... False + error_injection_rate ............................ 0 + error_injection_type ............................ transient_error + eval_interval ................................... 1000 + eval_iters ...................................... 10 + evidence_data_path .............................. None + exit_duration_in_mins ........................... None + exit_interval ................................... None + exit_on_missing_checkpoint ...................... False + exit_signal_handler ............................. False + exp_avg_dtype ................................... torch.float32 + exp_avg_sq_dtype ................................ torch.float32 + expert_model_parallel_size ...................... 1 + expert_tensor_parallel_size ..................... 1 + export_force_local_attention .................... False + export_kd_cfg ................................... None + export_kd_teacher_ckpt_format ................... None + export_kd_teacher_load .......................... None + export_kv_cache_quant ........................... False + export_legacy_megatron .......................... False + export_model_type ............................... GPTModel + export_moe_apply_probs_on_input ................. False + export_offline_model ............................ False + export_qk_l2_norm ............................... False + export_quant_cfg ................................ None + export_real_quant_cfg ........................... None + export_te_mcore_model ........................... False + external_cuda_graph ............................. False + fast_thd_a2a .................................... False + ffn_hidden_size ................................. 4800 + ffn_intermediate_token_shift .................... None + ffn_token_shift ................................. None + final_logits_clamp_value ........................ None + fine_grained_activation_offloading .............. False + finetune ........................................ False + finetune_data_split ............................. train + finetune_hf_dataset ............................. None + first_last_layers_bf16 .......................... False + flash_decode .................................... False + fp16 ............................................ False + fp16_lm_cross_entropy ........................... False + fp32_residual_connection ........................ False + fp4 ............................................. None + fp4_param ....................................... False + fp4_recipe ...................................... nvfp4 + fp8 ............................................. None + fp8_amax_compute_algo ........................... most_recent + fp8_amax_history_len ............................ 1 + fp8_interval .................................... 1 + fp8_margin ...................................... 0 + fp8_param_gather ................................ False + fp8_recipe ...................................... delayed + fp8_wgrad ....................................... True + freeze_layernorm_weight ......................... False + freeze_non_mamba ................................ False + fsdp_double_buffer .............................. False + full_validation ................................. False + gated_norm ...................................... True + gated_norm_lora_rank ............................ 16 + gated_norm_upcycling ............................ False + gdn_alpha_weight_scale .......................... 1.0 + gdn_alpha_weight_scale_mode ..................... matrix + gdn_backend ..................................... fla + gdn_fused_split_forward ......................... False + gdn_gated_norm_backend .......................... torch + gdn_lr_mult_alpha ............................... 1.0 + gdn_lr_mult_beta ................................ 1.0 + gdn_lr_mult_k ................................... 1.0 + gdn_lr_mult_q ................................... 1.0 + gdn_lr_mult_v ................................... 1.0 + gdn_postinit_alog_shift ......................... 0.0 + gdn_postinit_alpha_scale ........................ 1.0 + gdn_postinit_key_scale .......................... 1.0 + gdn_pre_gdr_fusion .............................. False + gdn_pre_gdr_fusion_deterministic_weight_gradient False + gdn_pre_gdr_fusion_high_precision_backward ...... False + gdn_shortconv_separate_gate ..................... False + gdn_shortconv_separate_gate_upcycling ........... False + gdn_split_in_proj ............................... True + geglu ........................................... False + global_batch_size ............................... 1024 + glu_linear_offset ............................... 0.0 + grad_reduce_in_bf16 ............................. False + gradient_accumulation_fusion .................... False + gradient_reduce_div_fusion ...................... True + group_query_attention ........................... True + grpo_clamp_eps_lower ............................ 0.01 + grpo_clamp_eps_upper ............................ 0.01 + grpo_default_temperature ........................ 1.0 + grpo_default_top_p .............................. 0 + grpo_entropy_term_weight ........................ 0.0 + grpo_filter_groups_with_same_reward ............. False + grpo_group_size ................................. 2 + grpo_iterations ................................. 2 + grpo_kl_beta .................................... 0.001 + grpo_prompts_per_step ........................... 32 + head_lr_mult .................................... 1.0 + heterogeneous_layers_config_encoded_json ........ None + heterogeneous_layers_config_path ................ None + hf_dir .......................................... /tmp/mcore2hf_ws_4760/mcore2hf-config.9zTuxl + hidden_dropout .................................. 0.1 + hidden_size ..................................... 2048 + hidden_states_shrink ............................ 1.0 + hierarchical_context_parallel_sizes ............. None + high_priority_stream_groups ..................... [] + hybrid_adamh .................................... False + hybrid_attention_ratio .......................... 0.0 + hybrid_context_parallel ......................... False + hybrid_mlp_ratio ................................ 0.0 + hybrid_override_pattern ......................... None + hysteresis ...................................... 2 + ict_head_size ................................... None + ict_load ........................................ None + img_h ........................................... 224 + img_w ........................................... 224 + increase_log_level_interval ..................... 1000 + increase_log_level_iters ........................ 5 + indexer_batch_size .............................. 128 + indexer_log_interval ............................ 1000 + inference_batch_times_seqlen_threshold .......... -1 + inference_dynamic_batching ...................... False + inference_dynamic_batching_block_size ........... 256 + inference_dynamic_batching_buffer_guaranteed_fraction 0.2 + inference_dynamic_batching_buffer_overflow_factor None + inference_dynamic_batching_buffer_size_gb ....... 40.0 + inference_dynamic_batching_max_requests_override None + inference_dynamic_batching_max_tokens_override .. None + inference_dynamic_batching_num_cuda_graphs ...... 16 + inference_dynamic_batching_track_paused_request_events False + inference_dynamic_batching_unified_memory_level . 0 + inference_max_batch_size ........................ 8 + inference_max_seq_length ........................ 2560 + inference_rng_tracker ........................... False + init_method_std ................................. 0.006 + init_method_xavier_uniform ...................... False + init_model_with_meta_device ..................... False + initial_loss_scale .............................. 4294967296 + inprocess_active_world_size ..................... 1 + inprocess_barrier_timeout ....................... 120 + inprocess_completion_timeout .................... 120 + inprocess_empty_cuda_cache ...................... False + inprocess_granularity ........................... node + inprocess_hard_timeout .......................... 90 + inprocess_heartbeat_interval .................... 30 + inprocess_heartbeat_timeout ..................... 60 + inprocess_last_call_wait ........................ 1 + inprocess_max_iterations ........................ None + inprocess_monitor_process_interval .............. 1.0 + inprocess_monitor_thread_interval ............... 1.0 + inprocess_progress_watchdog_interval ............ 1.0 + inprocess_restart ............................... False + inprocess_soft_timeout .......................... 60 + inprocess_termination_grace_time ................ 1 + is_hybrid_model ................................. False + iter_per_epoch .................................. 1250 + iterations_to_skip .............................. [] + keep_fp8_transpose_cache ........................ False + kitchen_config_file ............................. None + kitchen_recipe_number ........................... None + kv_channels ..................................... 64 + kv_lora_rank .................................... 32 + langrl_env_config ............................... None + langrl_external_server .......................... False + langrl_inference_server_conversation_template ... None + langrl_inference_server_type .................... inplace_megatron + lazy_mpu_init ................................... None + legacy_tokenizer ................................ False + linear_attention_freq ........................... [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 0, 0, 1, 1, 1, 1, 1, 1] + linear_attention_type ........................... gated_delta_net + linear_conv_kernel_dim .......................... 4 + linear_cross_entropy_accumulate_grad_in_fp32 .... True + linear_cross_entropy_num_chunks ................. 8 + linear_key_head_dim ............................. 64 + linear_num_key_heads ............................ 8 + linear_num_value_heads .......................... 32 + linear_rope_mode ................................ none + linear_value_head_dim ........................... 64 + lm_head_alpha ................................... 1.0 + lm_loss_normalization ........................... legacy + load ............................................ None + load_complemental_dataset ....................... None + load_dir ........................................ /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/load_view4760 + load_main_params_from_ckpt ...................... None + local_rank ...................................... 0 + log_attn_logits ................................. False + log_energy ...................................... False + log_hidden_states ............................... [] + log_interval .................................... 100 + log_logits_z_loss ............................... False + log_loss_scale_to_tensorboard ................... True + log_memory_to_tensorboard ....................... False + log_metrics_interval ............................ None + log_moe_activations ............................. False + log_moe_params .................................. False + log_moe_router_metrics .......................... False + log_num_zeros_in_grad ........................... False + log_padding_stats ............................... False + log_params ...................................... [] + log_params_norm ................................. False + log_per_layer_grad_rms .......................... False + log_per_module_grad_rms ......................... False + log_per_module_update_rms ....................... False + log_progress .................................... False + log_spike_diagnostics ........................... False + log_straggler ................................... False + log_throughput .................................. False + log_timers_to_tensorboard ....................... False + log_update_rms .................................. False + log_validation_ppl_to_tensorboard ............... False + log_world_size_to_tensorboard ................... False + logging_level ................................... 1 + loss_scale ...................................... None + loss_scale_window ............................... 1000 + lr .............................................. 2e-05 + lr_decay_iters .................................. 430000 + lr_decay_samples ................................ None + lr_decay_style .................................. cosine + lr_warmup_fraction .............................. 0.001 + lr_warmup_init .................................. 0.0 + lr_warmup_iters ................................. 0 + lr_warmup_samples ............................... 0 + lr_wsd_decay_iters .............................. None + lr_wsd_decay_samples ............................ None + lr_wsd_decay_style .............................. exponential + main_grads_dtype ................................ torch.float32 + main_params_dtype ............................... torch.float32 + make_vocab_size_divisible_by .................... 128 + mamba_disable_cp ................................ False + mamba_expand .................................... 2 + mamba_head_dim .................................. 64 + mamba_num_groups ................................ 8 + mamba_num_heads ................................. None + mamba_state_dim ................................. 128 + manual_gc ....................................... False + manual_gc_eval .................................. True + manual_gc_interval .............................. 0 + mask_factor ..................................... 1.0 + mask_prob ....................................... 0.15 + mask_type ....................................... random + masked_softmax_fusion ........................... True + max_position_embeddings ......................... 131072 + max_seqlen_per_cp_rank .......................... None + max_shard_size .................................. 20GB + max_tokens_to_oom ............................... 12000 + mcore2hf ........................................ True + memory_snapshot_path ............................ None + merge_file ...................................... None + micro_batch_size ................................ 1 + microbatch_group_size_per_vp_stage .............. None + mid_level_dataset_surplus ....................... 0.005 + min_loss_scale .................................. 1.0 + min_lr .......................................... 6e-06 + min_offloaded_tensor_size ....................... 1048576 + mlp_chunks_for_prefill .......................... 1 + mmap_bin_files .................................. True + mock_data ....................................... False + model_type ...................................... GPT + modelopt_enabled ................................ False + moe_apply_probs_on_input ........................ False + moe_aux_loss_coeff .............................. 0.0 + moe_deepep_num_sms .............................. 20 + moe_enable_deepep ............................... False + moe_expert_capacity_factor ...................... None + moe_extended_tp ................................. False + moe_ffn_hidden_size ............................. 512 + moe_flex_dispatcher_backend ..................... deepep + moe_gaux_overlap ................................ False + moe_grouped_gemm ................................ False + moe_hybridep_num_sms ............................ 16 + moe_input_jitter_eps ............................ None + moe_layer_freq .................................. [0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] + moe_layer_recompute ............................. False + moe_monitor_layers .............................. + moe_monitor_log_interval ........................ None + moe_pad_expert_input_to_capacity ................ False + moe_pad_experts_for_cuda_graph_inference ........ False + moe_per_layer_logging ........................... False + moe_permute_backend ............................. auto + moe_permute_fusion .............................. False + moe_router_aux_loss_fusion ...................... auto + moe_router_bias_update_method ................... sign + moe_router_bias_update_rate ..................... 0.001 + moe_router_dtype ................................ fp32 + moe_router_enable_expert_bias ................... False + moe_router_force_load_balancing ................. False + moe_router_fusion ............................... False + moe_router_gemm_backend ......................... torch + moe_router_group_topk ........................... None + moe_router_load_balancing_type .................. aux_loss + moe_router_num_groups ........................... None + moe_router_padding_for_fp8 ...................... False + moe_router_padding_for_quantization ............. False + moe_router_pre_softmax .......................... False + moe_router_score_function ....................... softmax + moe_router_sqrt_gate ............................ True + moe_router_topk ................................. 4 + moe_router_topk_scaling_factor .................. None + moe_shared_expert_gate .......................... True + moe_shared_expert_intermediate_size ............. 512 + moe_shared_expert_overlap ....................... False + moe_token_dispatcher_type ....................... alltoall + moe_token_drop_policy ........................... probs + moe_triton_dispatcher_exclude_padding_tokens .... False + moe_triton_mega_activation_sms .................. 4 + moe_triton_mega_combine_sms ..................... 32 + moe_triton_mega_compute_sms ..................... 24 + moe_triton_mega_dispatch_sms .................... 48 + moe_triton_mega_num_warps ....................... 16 + moe_triton_mega_reduce_sms ...................... 32 + moe_triton_mega_tail_sms ........................ 16 + moe_triton_shared_expert_overlap ................ False + moe_triton_workspace_expand ..................... 4.0 + moe_upcycling_granularity ....................... 1 + moe_use_legacy_grouped_gemm ..................... False + moe_use_upcycling ............................... False + moe_z_loss_coeff ................................ None + mrope_section ................................... None + mscale .......................................... 1.0 + mscale_all_dim .................................. 0.0 + mtp_linear_attention_type ....................... None + mtp_loss_scaling_factor ......................... 0.1 + mtp_num_layers .................................. None + multi_latent_attention .......................... False + multiple_validation_sets ........................ False + muon_ball_momentum .............................. 0.9 + muon_ball_msign_steps ........................... 5 + muon_ball_power_iteration_steps ................. 10 + muon_ball_qkv_split_mode ........................ component + muon_ball_radius_mode ........................... spectral_mup + muon_ball_retract_alpha ......................... 0.05 + muon_ball_retract_mode .......................... hard + muon_ball_scale_mode ............................ spectral_mup + muon_ball_split_fc1 ............................. True + muon_ball_split_moe_experts ..................... True + muon_ball_split_qkv ............................. True + muon_ball_use_nesterov .......................... True + muon_extra_scale_factor ......................... 1.0 + muon_fp32_matmul_prec ........................... medium + muon_momentum ................................... 0.9 + muon_num_ns_steps ............................... 5 + muon_qkv_split_mode ............................. component + muon_scale_mode ................................. spectral_mup + muon_scale_vectorized_mode ...................... full + muon_split_fc1 .................................. True + muon_split_moe_experts .......................... True + muon_split_qkv .................................. True + muon_tp_mode .................................... blockwise + muon_use_nesterov ............................... False + muon_vectorize .................................. [] + muon_vectorize_attn_dim ......................... hidden_size + mup_lr_mult ..................................... 1.0 + nccl_all_reduce_for_prefill ..................... False + nccl_communicator_config_path ................... None + nccl_ub ......................................... False + no_load_optim ................................... True + no_load_rng ..................................... True + no_load_scheduler ............................... None + no_persist_layer_norm ........................... False + no_rope_freq .................................... None + no_save_optim ................................... None + no_save_rng ..................................... None + no_save_step_one ................................ None + no_weight_decay_cond_type ....................... None + non_persistent_ckpt_type ........................ None + non_persistent_global_ckpt_dir .................. None + non_persistent_keep_last ........................ 1 + non_persistent_local_ckpt_algo .................. fully_parallel + non_persistent_local_ckpt_dir ................... None + non_persistent_save_interval .................... None + norm_epsilon .................................... 1e-06 + normalization ................................... RMSNorm + num_attention_heads ............................. 32 + num_channels .................................... 3 + num_classes ..................................... 1000 + num_dataset_builder_threads ..................... 1 + num_distributed_optimizer_instances ............. 1 + num_experts ..................................... 192 + num_hf_saver .................................... None + num_layers ...................................... 56 + num_layers_at_end_in_bf16 ....................... 1 + num_layers_at_start_in_bf16 ..................... 1 + num_layers_per_virtual_pipeline_stage ........... None + num_query_groups ................................ 8 + num_virtual_stages_per_pipeline_rank ............ None + num_workers ..................................... 2 + object_storage_cache_path ....................... None + offload_modules ................................. [] + one_logger_async ................................ False + one_logger_project .............................. megatron-lm + one_logger_run_name ............................. None + onnx_safe ....................................... None + openai_gelu ..................................... False + optimizer ....................................... adam + optimizer_cpu_offload ........................... False + optimizer_offload_fraction ...................... 1.0 + output_bert_embeddings .......................... False + output_logits_z_loss_coeff ...................... None + overlap_cpu_optimizer_d2h_h2d ................... False + overlap_grad_reduce ............................. False + overlap_moe_expert_parallel_comm ................ False + overlap_p2p_comm ................................ False + overlap_p2p_comm_warmup_flush ................... False + overlap_param_gather ............................ False + overlap_param_gather_with_optimizer_step ........ False + override_hf_eod_token_id ........................ None + override_opt_param_scheduler .................... False + packing_tokens_per_shard ........................ 100000000000 + padded_vocab_size ............................... None + params_dtype .................................... torch.bfloat16 + patch_dim ....................................... 16 + per_split_data_args_path ........................ None + perform_initialization .......................... True + perform_rl_step ................................. False + pin_cpu_grads ................................... True + pin_cpu_params .................................. True + pipeline_model_parallel_comm_backend ............ None + pipeline_model_parallel_layout .................. None + pipeline_model_parallel_size .................... 1 + position_embedding_type ......................... none + pp_aware_finegrained_recompute .................. False + pp_aware_finegrained_recompute_global_layers .... None + pp_aware_finegrained_recompute_pp_ranks ......... None + prefetch_factor ................................. None + pretrain_script ................................. mcore_gdn_moe.model_provider + pretrained_checkpoint ........................... None + previous_stage_seq_length ....................... None + profile ......................................... False + profile_microbatch_count ........................ 2 + profile_microbatch_dir .......................... None + profile_microbatch_iteration .................... None + profile_microbatch_start ........................ 0 + profile_ranks ................................... [0] + profile_step_end ................................ None + profile_step_start .............................. None + q_lora_rank ..................................... None + qk_head_dim ..................................... 128 + qk_l2_norm ...................................... False + qk_layernorm .................................... False + qk_pos_emb_head_dim ............................. 64 + query_in_block_prob ............................. 0.1 + quick_geglu ..................................... False + rampup_batch_size ............................... None + rank ............................................ 0 + recompute_granularity ........................... None + recompute_method ................................ None + recompute_modules ............................... None + recompute_num_layers ............................ None + record_memory_history ........................... False + relative_attention_max_distance ................. 128 + relative_attention_num_buckets .................. 32 + reparam_checkpoint .............................. None + reparam_fallback_value .......................... None + reparam_keys .................................... None + replication ..................................... False + replication_factor .............................. 2 + replication_jump ................................ None + rerun_mode ...................................... validate_results + reset_attention_mask ............................ False + reset_iteration_one_to_zero ..................... False + reset_position_ids .............................. False + result_rejected_tracker_filename ................ None + retriever_report_topk_accuracies ................ [] + retriever_score_scaling ......................... False + retriever_seq_length ............................ 256 + retro_add_retriever ............................. False + retro_attention_gate ............................ 1 + retro_cyclic_train_iters ........................ None + retro_encoder_attention_dropout ................. 0.1 + retro_encoder_hidden_dropout .................... 0.1 + retro_encoder_layers ............................ 2 + retro_num_neighbors ............................. 2 + retro_num_retrieved_chunks ...................... 2 + retro_project_dir ............................... None + retro_verify_neighbor_count ..................... True + reuse_grad_buf_for_mxfp8_param_ag ............... False + reuse_training_communicators .................... False + rl_calculate_intra_group_similarity ............. False + rl_importance_sampling_truncation_coef .......... None + rl_inference_logprobs_is_correction ............. False + rl_offload_kv_cache_during_training ............. False + rl_offload_optimizer_during_inference ........... False + rl_partial_rollouts ............................. False + rl_prompts_per_eval ............................. 32 + rl_remove_kv_cache_during_training .............. False + rl_reset_cuda_graphs ............................ False + rl_sequence_packing_algo ........................ fifo + rl_sequence_packing_bin_size .................... 8192 + rl_use_sequence_packing ......................... False + rope_scaling_factor ............................. 8.0 + rope_type ....................................... None + rotary_base ..................................... 10000 + rotary_interleaved .............................. False + rotary_percent .................................. 1.0 + rotary_scaling_factor ........................... 1.0 + rotary_seq_len_interpolation_factor ............. None + run_workload_inspector_server ................... False + runtime_audit_dir ............................... None + sample_rate ..................................... 1.0 + save ............................................ None + save_after_load ................................. False + save_dir ........................................ /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/iter_4760-hf + save_interval ................................... 10000 + save_retain_interval ............................ None + scale_emb ....................................... 1.0 + scatter_gather_tensors_in_pipeline .............. True + seed ............................................ 1234 + seq_length ...................................... 1 + sequence_packing_scheduler ...................... default_dynamic_cp + sequence_parallel ............................... False + sft ............................................. False + sft_tokenizer_prompt_format ..................... nemotron-h-aligned + sgd_momentum .................................... 0.9 + sharp_enabled_group ............................. None + short_seq_prob .................................. 0.1 + skip_train ...................................... True + skipped_train_samples ........................... 0 + softmax_type .................................... vanilla + spec ............................................ None + spectra_clip .................................... 1.0 + spectra_grad_clip ............................... None + spectra_hyperball ............................... False + spectra_ns_steps ................................ 10 + spectra_scope ................................... matrices + spectral_ball_momentum .......................... 0.9 + spectral_ball_msign_steps ....................... 8 + spectral_ball_power_iteration_steps ............. 20 + spectral_ball_qkv_split_mode .................... component + spectral_ball_radius_mode ....................... spectral_mup + spectral_ball_retract_alpha ..................... 0.05 + spectral_ball_retract_mode ...................... hard + spectral_ball_scale_mode ........................ spectral_mup + spectral_ball_solver ............................ bisection + spectral_ball_solver_max_iterations ............. 20 + spectral_ball_solver_tolerance_f ................ 1e-08 + spectral_ball_split_fc1 ......................... True + spectral_ball_split_moe_experts ................. True + spectral_ball_split_qkv ......................... True + spectral_ball_use_nesterov ...................... True + spectral_mup_init ............................... False + spike_diagnostics_dir ........................... None + spike_diagnostics_end ........................... 21 + spike_diagnostics_start ......................... 2 + split ........................................... None + split_expert_init ............................... True + split_fc1_init .................................. True + split_qkv_init .................................. True + split_qkv_init_mode ............................. group + sqreglu ......................................... False + squared_relu .................................... False + start_samples ................................... None + start_weight_decay .............................. 0.1 + straggler_ctrlr_port ............................ 65535 + straggler_minmax_count .......................... 1 + strict_fsdp_dtensor_load ........................ True + suggested_communication_unit_size ............... None + swanlab_exp_name ................................ + swanlab_project ................................. + swanlab_save_dir ................................ + swanlab_workspace ............................... + swiglu .......................................... True + swin_backbone_type .............................. tiny + symmetric_ar_type ............................... None + synchronizer .................................... mcore_gdn_moe + target_ckpt_format .............................. torch_dist + te_rng_tracker .................................. False + teacher_model_config ............................ None + tensor_model_parallel_size ...................... 1 + tensorboard_dir ................................. None + tensorboard_log_interval ........................ 1 + tensorboard_queue_size .......................... 1000 + test_data_path .................................. None + test_mode ....................................... False + tiktoken_num_special_tokens ..................... 1000 + tiktoken_pattern ................................ None + tiktoken_special_tokens ......................... None + timing_log_level ................................ 0 + timing_log_option ............................... minmax + titles_data_path ................................ None + token_shift_conv_init ........................... default + token_shift_conv_size ........................... 4 + tokenizer_metadata .............................. None + tokenizer_model ................................. /tmp/mcore2hf_ws_4760/mcore2hf-config.9zTuxl + tokenizer_type .................................. HuggingFaceTokenizer + torch_fsdp2_reshard_after_forward ............... True + tp_comm_bootstrap_backend ....................... nccl + tp_comm_bulk_dgrad .............................. True + tp_comm_bulk_wgrad .............................. True + tp_comm_overlap ................................. False + tp_comm_overlap_ag .............................. True + tp_comm_overlap_cfg ............................. None + tp_comm_overlap_rs .............................. True + tp_comm_overlap_rs_dgrad ........................ False + tp_comm_split_ag ................................ True + tp_comm_split_rs ................................ True + train_data_path ................................. None + train_iters ..................................... 500000 + train_samples ................................... None + train_sync_interval ............................. None + train_until_stopped ............................. False + transformer_impl ................................ transformer_engine + transformer_pipeline_model_parallel_size ........ 1 + trust_remote_code ............................... False + untie_embeddings_and_output_weights ............. True + use_checkpoint_args ............................. False + use_checkpoint_opt_param_scheduler .............. False + use_cpu_initialization .......................... True + use_dist_ckpt ................................... True + use_dist_ckpt_deprecated ........................ False + use_distributed_optimizer ....................... False + use_flash_attn .................................. False + use_fused_weighted_squared_relu ................. False + use_gpu ......................................... False + use_legacy_models ............................... False + use_megatron_fsdp ............................... False + use_mp_args_from_checkpoint_args ................ False + use_one_sent_docs ............................... False + use_persistent_ckpt_worker ...................... False + use_precision_aware_optimizer ................... False + use_pytorch_profiler ............................ False + use_ring_exchange_p2p ........................... False + use_rope_scaling ................................ False + use_rotary_position_embeddings .................. False + use_sharp ....................................... False + use_te_activation_func .......................... False + use_tokenizer_model_from_checkpoint_args ........ True + use_torch_fsdp2 ................................. False + use_torch_optimizer_for_cpu_offload ............. False + use_tp_pp_dp_mapping ............................ False + v_head_dim ...................................... 128 + valid_data_path ................................. None + variable_seq_lengths ............................ False + virtual_pipeline_model_parallel_size ............ None + vision_backbone_type ............................ vit + vision_pretraining .............................. False + vision_pretraining_type ......................... classify + vocab_extra_ids ................................. 0 + vocab_file ...................................... None + vocab_size ...................................... None + wandb_entity .................................... + wandb_exp_name .................................. + wandb_project ................................... + wandb_save_dir .................................. + weight_decay .................................... 0.1 + weight_decay_incr_style ......................... constant + wgrad_deferral_limit ............................ 0 + window_attn_skip_freq ........................... None + window_size ..................................... None + word_embedding_dropout_prob ..................... 0.0 + world_size ...................................... 1 + yaml_cfg ........................................ None +-------------------- end of arguments --------------------- +INFO:megatron.core.num_microbatches_calculator:setting number of microbatches to constant 1024 +> building HuggingFaceTokenizer tokenizer ... + > padded vocab (size: 99000) with 72 dummy tokens (new size: 99072) +WARNING: one_logger package is required to enable e2e metrics tracking. please go to https://confluence.nvidia.com/display/MLWFO/Package+Repositories for details to install it +INFO:megatron.training.initialize:Setting logging level to 1 +WARNING:megatron.core.rerun_state_machine:RerunStateMachine initialized in mode validate_results +> initializing torch distributed ... +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +> initialized tensor model parallel with size 1 +> initialized pipeline model parallel with size 1 +> setting random seeds to 1234 ... +> compiling dataset index builder ... +make: Entering directory '/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/datasets' +make: Nothing to be done for 'default'. +make: Leaving directory '/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/datasets' +>>> done with dataset index builder. Compilation time: 0.133 seconds +WARNING: constraints for invoking optimized fused softmax kernel are not met. We default back to unfused kernel invocations. +> compiling and loading fused kernels ... +>>> done with compiling and loading fused kernels. Compilation time: 0.003 seconds +WORLD_SIZE: 1, RANK: 0, LOCAL_RANK: 0 +[convert] creating synchronizer: mode=mcore2hf +building GPT model ... +[GPT_LAYER_SPEC] log_hidden_states=[] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=[] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=[] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=[] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/transformers/modeling_rope_utils.py:1036: FutureWarning: `rope_config_validation` is deprecated and has been removed. Its functionality has been moved to RotaryEmbeddingConfigMixin.validate_rope method. PreTrainedConfig inherits this class, so please call self.validate_rope() instead. Also, make sure to use the new rope_parameters syntax. You can call self.standardize_rope_params() in the meantime. + warnings.warn( +[transformers] `layer_type_validation` is deprecated and will be removed in v5.20. Use `PreTrainedConfig.validate_layer_type` instead +[transformers] `layer_type_validation` is deprecated and will be removed in v5.20. Use `PreTrainedConfig.validate_layer_type` instead +[transformers] `torch_dtype` is deprecated! Use `dtype` instead! +[ERROR] `cache_position` is part of Qwen3NextModel.forward's signature, but not documented. Make sure to add it to the docstring of the function in /mnt/ssd/cache_tmp/huggingface/modules/transformers_modules/mcore2hf_hyphen_config_dot_9zTuxl/10c6759df797195a/modeling_qwen3_next.py. +[ERROR] `cache_position` is part of Qwen3NextForCausalLM.forward's signature, but not documented. Make sure to add it to the docstring of the function in /mnt/ssd/cache_tmp/huggingface/modules/transformers_modules/mcore2hf_hyphen_config_dot_9zTuxl/10c6759df797195a/modeling_qwen3_next.py. +[transformers] `torch_dtype` is deprecated! Use `dtype` instead! +INFO:transformers_modules.mcore2hf_hyphen_config_dot_9zTuxl.10c6759df797195a.modeling_qwen3_next:[Qwen3Next custom] attn_position_embedding_type=nope, rnn_position_embedding_type=nope, attn_logits_scaling=None +Qwen3NextForCausalLM( + (model): Qwen3NextModel( + (embed_tokens): Embedding(99072, 2048, padding_idx=102) + (layers): ModuleList( + (0-1): 2 x Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=4800, bias=False) + (up_proj): Linear(in_features=2048, out_features=4800, bias=False) + (down_proj): Linear(in_features=4800, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (2-11): 10 x Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (12): Qwen3NextDecoderLayer( + (self_attn): Qwen3NextAttention( + (q_proj): Linear(in_features=2048, out_features=4096, bias=True) + (k_proj): Linear(in_features=2048, out_features=512, bias=True) + (v_proj): Linear(in_features=2048, out_features=512, bias=True) + (o_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (13-19): 7 x Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (20-21): 2 x Qwen3NextDecoderLayer( + (self_attn): Qwen3NextAttention( + (q_proj): Linear(in_features=2048, out_features=4096, bias=True) + (k_proj): Linear(in_features=2048, out_features=512, bias=True) + (v_proj): Linear(in_features=2048, out_features=512, bias=True) + (o_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (22): Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (23): Qwen3NextDecoderLayer( + (self_attn): Qwen3NextAttention( + (q_proj): Linear(in_features=2048, out_features=4096, bias=True) + (k_proj): Linear(in_features=2048, out_features=512, bias=True) + (v_proj): Linear(in_features=2048, out_features=512, bias=True) + (o_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (24-45): 22 x Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (46): Qwen3NextDecoderLayer( + (self_attn): Qwen3NextAttention( + (q_proj): Linear(in_features=2048, out_features=4096, bias=True) + (k_proj): Linear(in_features=2048, out_features=512, bias=True) + (v_proj): Linear(in_features=2048, out_features=512, bias=True) + (o_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (47): Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (48-49): 2 x Qwen3NextDecoderLayer( + (self_attn): Qwen3NextAttention( + (q_proj): Linear(in_features=2048, out_features=4096, bias=True) + (k_proj): Linear(in_features=2048, out_features=512, bias=True) + (v_proj): Linear(in_features=2048, out_features=512, bias=True) + (o_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (50-55): 6 x Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + ) + (norm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (rotary_emb): Qwen3NextRotaryEmbedding() + ) + (lm_head): Linear(in_features=2048, out_features=99072, bias=False) +) +GPTModel( + (embedding): LanguageModelEmbedding( + (word_embeddings): VocabParallelEmbedding() + (embedding_dropout): Dropout(p=0.1, inplace=False) + ) + (decoder): TransformerBlock( + (layers): ModuleList( + (0-1): 2 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=9600, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=4800, out_features=2048, bias=False, TP=1) + ) + ) + (2-11): 10 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (12): TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): SelfAttention( + (core_attention): TEDotProductAttention( + (flash_attention): FlashAttention() + (fused_attention): FusedAttention() + (unfused_attention): UnfusedDotProductAttention( + (scale_mask_softmax): FusedScaleMaskSoftmax() + (attention_dropout): Dropout(p=0.1, inplace=False) + ) + ) + (linear_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (linear_qkv): TEColumnParallelLinear(in_features=2048, out_features=5120, bias=True, TP=1) + (q_layernorm): IdentityOp() + (k_layernorm): IdentityOp() + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (13-19): 7 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (20-21): 2 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): SelfAttention( + (core_attention): TEDotProductAttention( + (flash_attention): FlashAttention() + (fused_attention): FusedAttention() + (unfused_attention): UnfusedDotProductAttention( + (scale_mask_softmax): FusedScaleMaskSoftmax() + (attention_dropout): Dropout(p=0.1, inplace=False) + ) + ) + (linear_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (linear_qkv): TEColumnParallelLinear(in_features=2048, out_features=5120, bias=True, TP=1) + (q_layernorm): IdentityOp() + (k_layernorm): IdentityOp() + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (22): TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (23): TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): SelfAttention( + (core_attention): TEDotProductAttention( + (flash_attention): FlashAttention() + (fused_attention): FusedAttention() + (unfused_attention): UnfusedDotProductAttention( + (scale_mask_softmax): FusedScaleMaskSoftmax() + (attention_dropout): Dropout(p=0.1, inplace=False) + ) + ) + (linear_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (linear_qkv): TEColumnParallelLinear(in_features=2048, out_features=5120, bias=True, TP=1) + (q_layernorm): IdentityOp() + (k_layernorm): IdentityOp() + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (24-45): 22 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (46): TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): SelfAttention( + (core_attention): TEDotProductAttention( + (flash_attention): FlashAttention() + (fused_attention): FusedAttention() + (unfused_attention): UnfusedDotProductAttention( + (scale_mask_softmax): FusedScaleMaskSoftmax() + (attention_dropout): Dropout(p=0.1, inplace=False) + ) + ) + (linear_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (linear_qkv): TEColumnParallelLinear(in_features=2048, out_features=5120, bias=True, TP=1) + (q_layernorm): IdentityOp() + (k_layernorm): IdentityOp() + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (47): TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (48-49): 2 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): SelfAttention( + (core_attention): TEDotProductAttention( + (flash_attention): FlashAttention() + (fused_attention): FusedAttention() + (unfused_attention): UnfusedDotProductAttention( + (scale_mask_softmax): FusedScaleMaskSoftmax() + (attention_dropout): Dropout(p=0.1, inplace=False) + ) + ) + (linear_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (linear_qkv): TEColumnParallelLinear(in_features=2048, out_features=5120, bias=True, TP=1) + (q_layernorm): IdentityOp() + (k_layernorm): IdentityOp() + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (50-55): 6 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + ) + (final_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (output_layer): ColumnParallelLinear(in_features=2048, out_features=99072, bias=False, TP=1) +) +DEBUG:multistorageclient.config:Searching MSC config file paths: ['/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/msc/msc_config.yaml', '/mnt/ssd/cache_tmp/.config/msc/config.yaml', '/mnt/ssd/cache_tmp/.config/msc/config.json', '/root/.msc_config.yaml', '/root/.msc_config.json', '/root/.config/msc/config.yaml', '/root/.config/msc/config.json', '/etc/xdg/msc/config.yaml', '/etc/xdg/msc/config.json', '/etc/msc_config.yaml', '/etc/msc_config.json'] +DEBUG:multistorageclient.config:Using MSC config file: /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/msc/msc_config.yaml +DEBUG:multistorageclient.config:Searching MSC config file paths: ['/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/msc/msc_config.yaml', '/mnt/ssd/cache_tmp/.config/msc/config.yaml', '/mnt/ssd/cache_tmp/.config/msc/config.json', '/root/.msc_config.yaml', '/root/.msc_config.json', '/root/.config/msc/config.yaml', '/root/.config/msc/config.json', '/etc/xdg/msc/config.yaml', '/etc/xdg/msc/config.json', '/etc/msc_config.yaml', '/etc/msc_config.json'] +DEBUG:multistorageclient.config:Using MSC config file: /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/msc/msc_config.yaml +DEBUG:multistorageclient.client.single:No metadata provider configured, auto-commit will not be enabled +DEBUG:multistorageclient.client.client:StorageClient '__filesystem__' using SingleStorageClient +sharded_state_dict metadata loaded from the checkpoint: {'singleton_local_shards': True, 'distrib_optim_sharding_type': 'dp_reshardable', 'chained_optim_avoid_prefix': True} +Job sharding has changed: Rerun state will be ignored + loading distributed checkpoint from /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/load_view4760 at iteration 4760 +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/dist_checkpointing/strategies/torch.py:956: FutureWarning: `load_state_dict` is deprecated and will be removed in future versions. Please use `load` instead. + checkpoint.load_state_dict( + checkpoint version 3.0 + successfully loaded checkpoint from /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/load_view4760 [ t 1/1, p 1/1 ] at iteration 4760 +[convert] synchronizer created in 390.87s +[convert] starting sync_params +INFO:root:Converting layer 0 is_gdn=True is_not_moe=True +INFO:root:Converting layer 1 is_gdn=True is_not_moe=True +INFO:root:Converting layer 2 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0771484375 mean=-0.00049591064453125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.1103515625 mean=0.00179290771484375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 3 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0791015625 mean=-0.00018215179443359375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.115234375 mean=-0.0001583099365234375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 4 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0859375 mean=-0.0002841949462890625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.08544921875 mean=0.0009613037109375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 5 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.08935546875 mean=-0.00023555755615234375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0849609375 mean=0.00189971923828125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 6 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.09716796875 mean=-0.0002422332763671875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0791015625 mean=-0.00109100341796875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 7 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0927734375 mean=-5.841255187988281e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.06640625 mean=-0.000354766845703125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 8 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0947265625 mean=-6.628036499023438e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.06640625 mean=0.0004138946533203125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 9 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.08935546875 mean=1.5854835510253906e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.06005859375 mean=-3.719329833984375e-05 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 10 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.08544921875 mean=2.0503997802734375e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.056884765625 mean=-0.0006866455078125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 11 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.08349609375 mean=7.104873657226562e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.06884765625 mean=-0.0003185272216796875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 12 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 12: args.attention_output_gate=True +WARNING:root:Layer 12: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0771484375 mean=0.00022220611572265625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.05615234375 mean=-0.000835418701171875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 13 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.07275390625 mean=0.0001678466796875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.045166015625 mean=-0.00118255615234375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 14 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0771484375 mean=0.0001621246337890625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.048828125 mean=-0.00107574462890625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 15 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.07470703125 mean=0.00025177001953125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.05224609375 mean=-0.00107574462890625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 16 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0732421875 mean=0.0002994537353515625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.051513671875 mean=-0.000865936279296875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 17 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0703125 mean=0.0002613067626953125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.049072265625 mean=-0.001861572265625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 18 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.06640625 mean=0.0002765655517578125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.05126953125 mean=-0.00225830078125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 19 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0693359375 mean=0.00060272216796875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.052734375 mean=-0.0016021728515625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 20 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 20: args.attention_output_gate=True +WARNING:root:Layer 20: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.06787109375 mean=0.000377655029296875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.052490234375 mean=-0.00146484375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 21 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 21: args.attention_output_gate=True +WARNING:root:Layer 21: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.06103515625 mean=0.000423431396484375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.044189453125 mean=-0.002227783203125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 22 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.05615234375 mean=0.000347137451171875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.047119140625 mean=-0.003173828125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 23 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 23: args.attention_output_gate=True +WARNING:root:Layer 23: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.054443359375 mean=0.00048065185546875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.040283203125 mean=-0.0023040771484375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 24 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.05322265625 mean=0.0004215240478515625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.03759765625 mean=-0.00262451171875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 25 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.050537109375 mean=0.00035858154296875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.035888671875 mean=-0.00180816650390625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 26 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.050048828125 mean=0.00030517578125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.033447265625 mean=-0.001953125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 27 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.052490234375 mean=0.00043487548828125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.031494140625 mean=-0.0019378662109375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 28 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.048583984375 mean=0.0003719329833984375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0283203125 mean=-0.00106048583984375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 29 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.04931640625 mean=0.000514984130859375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0250244140625 mean=-0.00096893310546875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 30 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.05224609375 mean=0.00080108642578125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0264892578125 mean=-0.00098419189453125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 31 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.049560546875 mean=0.00051116943359375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.02685546875 mean=-0.000579833984375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 32 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.04736328125 mean=0.0001125335693359375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.02587890625 mean=3.62396240234375e-05 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 33 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.05078125 mean=8.487701416015625e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0284423828125 mean=-0.0001544952392578125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 34 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.052978515625 mean=0.00016689300537109375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.02783203125 mean=-1.3887882232666016e-05 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 35 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.058349609375 mean=9.298324584960938e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.032958984375 mean=0.000244140625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 36 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.060302734375 mean=6.389617919921875e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.04345703125 mean=0.0009613037109375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 37 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.060302734375 mean=0.0001163482666015625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.042724609375 mean=0.00115203857421875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 38 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.058837890625 mean=0.00012302398681640625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.037353515625 mean=0.00136566162109375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 39 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.057373046875 mean=8.7738037109375e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.036865234375 mean=0.00115966796875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 40 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0537109375 mean=0.00014019012451171875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0263671875 mean=0.0001697540283203125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 41 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.050537109375 mean=0.00013637542724609375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0247802734375 mean=0.00015926361083984375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 42 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.045166015625 mean=0.000362396240234375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0211181640625 mean=-0.0004024505615234375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 43 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0390625 mean=0.00019741058349609375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0208740234375 mean=-0.000308990478515625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 44 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.035888671875 mean=0.0003814697265625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.021484375 mean=-0.00035858154296875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 45 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.036376953125 mean=0.00014591217041015625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.024169921875 mean=-0.000926971435546875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 46 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 46: args.attention_output_gate=True +WARNING:root:Layer 46: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.037109375 mean=0.0003871917724609375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.021240234375 mean=-0.000690460205078125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 47 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.03759765625 mean=0.0001964569091796875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0245361328125 mean=-0.00080108642578125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 48 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 48: args.attention_output_gate=True +WARNING:root:Layer 48: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.034912109375 mean=-6.723403930664062e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0201416015625 mean=-0.0004119873046875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 49 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 49: args.attention_output_gate=True +WARNING:root:Layer 49: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.034912109375 mean=-2.765655517578125e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.023193359375 mean=-0.00102996826171875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 50 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.036865234375 mean=-6.866455078125e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0238037109375 mean=-0.000835418701171875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 51 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.03759765625 mean=3.24249267578125e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.02392578125 mean=-0.00116729736328125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 52 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.040283203125 mean=0.0002269744873046875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0302734375 mean=-0.001434326171875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 53 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.038818359375 mean=5.316734313964844e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.050537109375 mean=-0.0021514892578125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 54 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.03955078125 mean=5.269050598144531e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.039794921875 mean=-0.0020751953125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 55 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.043701171875 mean=-0.00019073486328125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.043212890625 mean=0.002593994140625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +[convert] finished sync_params in 2.34s +[convert] starting check_and_save -> /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/iter_4760-hf +DEBUG:root:[RANK 0] 0 send op & 0 recv op. +INFO:root:[Iters 0 RANK 0] starts synchronizing parameters with other ranks... +INFO:root:[Iters 0 RANK 0] finishes synchronizing +[Iters 0 RANK 0] model-00001-of-00004.safetensors is saved. +DEBUG:root:[Iters 0 RANK 0] joined +DEBUG:root:[RANK 0] 0 send op & 0 recv op. +INFO:root:[Iters 1 RANK 0] starts synchronizing parameters with other ranks... +INFO:root:[Iters 1 RANK 0] finishes synchronizing +[Iters 1 RANK 0] model-00002-of-00004.safetensors is saved. +DEBUG:root:[Iters 1 RANK 0] joined +DEBUG:root:[RANK 0] 0 send op & 0 recv op. +INFO:root:[Iters 2 RANK 0] starts synchronizing parameters with other ranks... +INFO:root:[Iters 2 RANK 0] finishes synchronizing +[Iters 2 RANK 0] model-00003-of-00004.safetensors is saved. +DEBUG:root:[Iters 2 RANK 0] joined +DEBUG:root:[RANK 0] 0 send op & 0 recv op. +INFO:root:[Iters 3 RANK 0] starts synchronizing parameters with other ranks... +INFO:root:[Iters 3 RANK 0] finishes synchronizing +[Iters 3 RANK 0] model-00004-of-00004.safetensors is saved. +DEBUG:root:[Iters 3 RANK 0] joined +[convert] finished check_and_save in 107.62s +Conversion finished in 502.33410596847534 seconds. ++ echo '✅Conversion completed: /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/iter_4760-hf' +✅Conversion completed: /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/iter_4760-hf ++ cleanup_real_config_path ++ rm -rf /tmp/mcore2hf_ws_4760/mcore2hf-config.9zTuxl ++ '[' -n '' ']' diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/hf_export_logs/convert_iter6855.log b/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/hf_export_logs/convert_iter6855.log new file mode 100644 index 0000000000000000000000000000000000000000..d7db222dd87f6e402427f3a30699c20b49d8573f --- /dev/null +++ b/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/hf_export_logs/convert_iter6855.log @@ -0,0 +1,2665 @@ +++ realpath /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/scripts/mcore_gdn_moe/../../../.. ++ MEGATRON_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher ++ CONVERTOR_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor ++ REPAIR_SCRIPT=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/utils/repair_gdn_checkpoint.py ++ RENDER_CONFIG_SCRIPT=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/utils/render_gdn_hf_config.py ++ export PYTHON_ENV_ROOT=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud ++ PYTHON_ENV_ROOT=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud ++ PYTHON_BIN=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python ++ '[' '!' -x /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python ']' ++ '[' '!' -x /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python ']' ++ '[' '!' -x /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python ']' ++ '[' '!' -x /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python ']' ++ export PYTHON_BIN ++ /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python -c 'import torch' ++ export PATH=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin:/root/.kimi-code/bin:/root/.local/bin:/mnt/hdd/minyingqian/mihomo/bin:/root/.grok/bin:/usr/local/bin:/root/miniconda3/bin:/root/.local/bin:/usr/local/bin:/root/miniconda3/bin:/root/miniconda3/bin:/mnt/ssd/liuzian/pandoc/current/bin:/mnt/ssd/cache_tmp/.local/share/../bin:/mnt/ssd/yulan/bin:/mnt/ssd/yulan/bin:/root/.local/bin:/usr/local/bin:/root/miniconda3/bin:/root/.local/bin:/usr/local/bin:/root/miniconda3/bin:/mnt/ssd/cache_tmp/.local/bin:/root/.local/bin:/root/.local/bin:/mnt/hdd/minyingqian/mihomo/bin:/root/.grok/bin:/usr/local/bin:/root/miniconda3/bin:/root/.local/bin:/usr/local/bin:/root/miniconda3/bin:/root/miniconda3/bin:/mnt/ssd/liuzian/pandoc/current/bin:/mnt/ssd/cache_tmp/.local/bin:/mnt/ssd/yulan/bin:/mnt/ssd/yulan/bin:/root/.local/bin:/root/miniconda3/bin:/root/miniconda3/condabin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/mnt/ssd/yulan/cache/installs/nsight-compute/2026.2:/mnt/ssd/yulan/cache/installs/nsight-systems/2026.3.1/bin:/mnt/ssd/yulan/cache/installs/nsight-compute/2026.2:/mnt/ssd/yulan/cache/installs/nsight-systems/2026.3.1/bin ++ PATH=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin:/root/.kimi-code/bin:/root/.local/bin:/mnt/hdd/minyingqian/mihomo/bin:/root/.grok/bin:/usr/local/bin:/root/miniconda3/bin:/root/.local/bin:/usr/local/bin:/root/miniconda3/bin:/root/miniconda3/bin:/mnt/ssd/liuzian/pandoc/current/bin:/mnt/ssd/cache_tmp/.local/share/../bin:/mnt/ssd/yulan/bin:/mnt/ssd/yulan/bin:/root/.local/bin:/usr/local/bin:/root/miniconda3/bin:/root/.local/bin:/usr/local/bin:/root/miniconda3/bin:/mnt/ssd/cache_tmp/.local/bin:/root/.local/bin:/root/.local/bin:/mnt/hdd/minyingqian/mihomo/bin:/root/.grok/bin:/usr/local/bin:/root/miniconda3/bin:/root/.local/bin:/usr/local/bin:/root/miniconda3/bin:/root/miniconda3/bin:/mnt/ssd/liuzian/pandoc/current/bin:/mnt/ssd/cache_tmp/.local/bin:/mnt/ssd/yulan/bin:/mnt/ssd/yulan/bin:/root/.local/bin:/root/miniconda3/bin:/root/miniconda3/condabin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/mnt/ssd/yulan/cache/installs/nsight-compute/2026.2:/mnt/ssd/yulan/cache/installs/nsight-systems/2026.3.1/bin:/mnt/ssd/yulan/cache/installs/nsight-compute/2026.2:/mnt/ssd/yulan/cache/installs/nsight-systems/2026.3.1/bin ++ export VIRTUAL_ENV=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud ++ VIRTUAL_ENV=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud ++ echo ✅MEGATRON_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher +✅MEGATRON_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher ++ echo ✅PYTHON_BIN=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python +✅PYTHON_BIN=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python ++ export CUDA_LAUNCH_BLOCKING=1 ++ CUDA_LAUNCH_BLOCKING=1 ++ export CUDA_VISIBLE_DEVICES=0 ++ CUDA_VISIBLE_DEVICES=0 ++ export KUBERNETES_CONTAINER_RESOURCE_GPU=1 ++ KUBERNETES_CONTAINER_RESOURCE_GPU=1 ++ export TP=1 ++ TP=1 ++ export PP=1 ++ PP=1 ++ export ETP=1 ++ ETP=1 ++ export EP=1 ++ EP=1 ++ export 'MODEL_PARALLEL_ARGS=--tensor-model-parallel-size 1 --pipeline-model-parallel-size 1 --expert-tensor-parallel-size 1 --expert-model-parallel-size 1' ++ MODEL_PARALLEL_ARGS='--tensor-model-parallel-size 1 --pipeline-model-parallel-size 1 --expert-tensor-parallel-size 1 --expert-model-parallel-size 1' ++ WORKSPACE_TMP_DIR=/tmp/mcore2hf_ws_6855 ++ mkdir -p /tmp/mcore2hf_ws_6855 ++ MEGATRON_CHECKPOINT_PATH=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/load_view6855 ++ echo MEGATRON_CHECKPOINT_PATH=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/load_view6855 +MEGATRON_CHECKPOINT_PATH=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/load_view6855 ++ MANUAL_ITER= ++ CONVERT_ALL=false ++ FORCE_RECONVERT=false ++ DROP_MTP=false ++ POSITIONAL_ARGS=() ++ [[ 3 -gt 0 ]] ++ case $1 in ++ POSITIONAL_ARGS+=("$1") ++ shift ++ [[ 2 -gt 0 ]] ++ case $1 in ++ '[' -z 6855 ']' ++ MANUAL_ITER=6855 ++ shift 2 ++ [[ 0 -gt 0 ]] ++ set -- /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/load_view6855 ++ '[' -e /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/load_view6855 ']' +++ readlink -f /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/load_view6855 ++ MEGATRON_CHECKPOINT_PATH=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/load_view6855 ++ echo ✅MEGATRON_CHECKPOINT_PATH=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/load_view6855 +✅MEGATRON_CHECKPOINT_PATH=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/load_view6855 +++ basename /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/load_view6855 ++ CHECKPOINT_BASENAME=load_view6855 ++ [[ load_view6855 =~ ^iter_([0-9]+)$ ]] ++ '[' false = true ']' ++ HF_CONFIG_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/hf-configs/YuLan-Mini-for-gdn-convert ++ echo HF_CONFIG_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/hf-configs/YuLan-Mini-for-gdn-convert +HF_CONFIG_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/hf-configs/YuLan-Mini-for-gdn-convert ++ '[' '!' -f /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/hf-configs/YuLan-Mini-for-gdn-convert/config.template.json ']' +++ mktemp -d /tmp/mcore2hf_ws_6855/mcore2hf-config.XXXXXX ++ REAL_CONFIG_PATH=/tmp/mcore2hf_ws_6855/mcore2hf-config.RGQ3d0 ++ LOAD_VIEW_DIR= ++ trap cleanup_real_config_path EXIT ++ export PYTHONPATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher: ++ PYTHONPATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher: ++ export PYTHONPATH=/tmp/mcore2hf_ws_6855/mcore2hf-config.RGQ3d0:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher: ++ PYTHONPATH=/tmp/mcore2hf_ws_6855/mcore2hf-config.RGQ3d0:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher: +++ /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/utils/parse-linear-vars-from-mcore-config.py /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/load_view6855 +W1010 22:55:30.729000 623606 torch/utils/cpp_extension.py:2425] TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +W1010 22:55:30.729000 623606 torch/utils/cpp_extension.py:2425] If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'] to specific architectures. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. ++ eval 'export LINEAR_NUM_KEY_HEADS=8 +export LINEAR_NUM_VALUE_HEADS=32 +export LINEAR_ATTN_FREQ=[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1] +export FFN_HIDDEN_SIZE=4800 +export NUM_LAYERS=56 +export HIDDEN_SIZE=2048 +export NUM_ATTN_HEADS=32 +export NUM_QUERY_GROUPS=8 +export MAX_POSITION_EMBEDDINGS=131072 +export KV_CHANNELS=64 +export LINEAR_CONV_KERNEL_DIM=4 +export LINEAR_KEY_HEAD_DIM=64 +export LINEAR_VALUE_HEAD_DIM=64 +export MOE_FFN_HIDDEN_SIZE=512 +export NUM_EXPERTS=192 +export MOE_ROUTER_TOPK=4 +export MOE_ROUTER_SCORE_FUNCTION=softmax +export MOE_ROUTER_ENABLE_EXPERT_BIAS=false +export MOE_LAYER_FREQ=[0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1] +export MLP_ONLY_LAYERS=[0,1] +export MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512 +export ROPE_THETA=10000.0 +export POSITION_EMBEDDING_TYPE=none +export ATTENTION_OUTPUT_GATE=true +export MTP_NUM_LAYERS=0 +export MOE_ROUTER_SQRT_GATE=true +export GATED_NORM=true +export GATED_NORM_RANK=16 +export GATED_NORM_GATE_SCALE=1.0 +export GATED_NORM_UP_BIAS=true +export FINAL_GATED_NORM=true +export ACTIVATION_FUNC_CLAMP_VALUE=7.0 +export ACTIVATION_FUNC_CLAMP_MODE=soft +export FINAL_LOGITS_CLAMP_VALUE=30.0 +export MOE_ROUTER_DTYPE=fp32 +export TORCH_DTYPE=bfloat16 +export VOCAB_SIZE=99072' +++ export LINEAR_NUM_KEY_HEADS=8 +++ LINEAR_NUM_KEY_HEADS=8 +++ export LINEAR_NUM_VALUE_HEADS=32 +++ LINEAR_NUM_VALUE_HEADS=32 +++ export 'LINEAR_ATTN_FREQ=[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1]' +++ LINEAR_ATTN_FREQ='[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1]' +++ export FFN_HIDDEN_SIZE=4800 +++ FFN_HIDDEN_SIZE=4800 +++ export NUM_LAYERS=56 +++ NUM_LAYERS=56 +++ export HIDDEN_SIZE=2048 +++ HIDDEN_SIZE=2048 +++ export NUM_ATTN_HEADS=32 +++ NUM_ATTN_HEADS=32 +++ export NUM_QUERY_GROUPS=8 +++ NUM_QUERY_GROUPS=8 +++ export MAX_POSITION_EMBEDDINGS=131072 +++ MAX_POSITION_EMBEDDINGS=131072 +++ export KV_CHANNELS=64 +++ KV_CHANNELS=64 +++ export LINEAR_CONV_KERNEL_DIM=4 +++ LINEAR_CONV_KERNEL_DIM=4 +++ export LINEAR_KEY_HEAD_DIM=64 +++ LINEAR_KEY_HEAD_DIM=64 +++ export LINEAR_VALUE_HEAD_DIM=64 +++ LINEAR_VALUE_HEAD_DIM=64 +++ export MOE_FFN_HIDDEN_SIZE=512 +++ MOE_FFN_HIDDEN_SIZE=512 +++ export NUM_EXPERTS=192 +++ NUM_EXPERTS=192 +++ export MOE_ROUTER_TOPK=4 +++ MOE_ROUTER_TOPK=4 +++ export MOE_ROUTER_SCORE_FUNCTION=softmax +++ MOE_ROUTER_SCORE_FUNCTION=softmax +++ export MOE_ROUTER_ENABLE_EXPERT_BIAS=false +++ MOE_ROUTER_ENABLE_EXPERT_BIAS=false +++ export 'MOE_LAYER_FREQ=[0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]' +++ MOE_LAYER_FREQ='[0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]' +++ export 'MLP_ONLY_LAYERS=[0,1]' +++ MLP_ONLY_LAYERS='[0,1]' +++ export MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512 +++ MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512 +++ export ROPE_THETA=10000.0 +++ ROPE_THETA=10000.0 +++ export POSITION_EMBEDDING_TYPE=none +++ POSITION_EMBEDDING_TYPE=none +++ export ATTENTION_OUTPUT_GATE=true +++ ATTENTION_OUTPUT_GATE=true +++ export MTP_NUM_LAYERS=0 +++ MTP_NUM_LAYERS=0 +++ export MOE_ROUTER_SQRT_GATE=true +++ MOE_ROUTER_SQRT_GATE=true +++ export GATED_NORM=true +++ GATED_NORM=true +++ export GATED_NORM_RANK=16 +++ GATED_NORM_RANK=16 +++ export GATED_NORM_GATE_SCALE=1.0 +++ GATED_NORM_GATE_SCALE=1.0 +++ export GATED_NORM_UP_BIAS=true +++ GATED_NORM_UP_BIAS=true +++ export FINAL_GATED_NORM=true +++ FINAL_GATED_NORM=true +++ export ACTIVATION_FUNC_CLAMP_VALUE=7.0 +++ ACTIVATION_FUNC_CLAMP_VALUE=7.0 +++ export ACTIVATION_FUNC_CLAMP_MODE=soft +++ ACTIVATION_FUNC_CLAMP_MODE=soft +++ export FINAL_LOGITS_CLAMP_VALUE=30.0 +++ FINAL_LOGITS_CLAMP_VALUE=30.0 +++ export MOE_ROUTER_DTYPE=fp32 +++ MOE_ROUTER_DTYPE=fp32 +++ export TORCH_DTYPE=bfloat16 +++ TORCH_DTYPE=bfloat16 +++ export VOCAB_SIZE=99072 +++ VOCAB_SIZE=99072 ++ echo LINEAR_NUM_KEY_HEADS=8 +LINEAR_NUM_KEY_HEADS=8 ++ echo LINEAR_NUM_VALUE_HEADS=32 +LINEAR_NUM_VALUE_HEADS=32 ++ echo 'LINEAR_ATTN_FREQ=[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1]' +LINEAR_ATTN_FREQ=[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1] ++ echo 'NUM_LAYERS=56 HIDDEN_SIZE=2048' +NUM_LAYERS=56 HIDDEN_SIZE=2048 ++ echo 'NUM_ATTN_HEADS=32 NUM_QUERY_GROUPS=8' +NUM_ATTN_HEADS=32 NUM_QUERY_GROUPS=8 ++ echo 'MAX_POSITION_EMBEDDINGS=131072 KV_CHANNELS=64' +MAX_POSITION_EMBEDDINGS=131072 KV_CHANNELS=64 ++ echo 'FFN_HIDDEN_SIZE=4800 MOE_FFN_HIDDEN_SIZE=512' +FFN_HIDDEN_SIZE=4800 MOE_FFN_HIDDEN_SIZE=512 ++ echo 'NUM_EXPERTS=192 MOE_ROUTER_TOPK=4' +NUM_EXPERTS=192 MOE_ROUTER_TOPK=4 ++ echo MOE_ROUTER_SCORE_FUNCTION=softmax +MOE_ROUTER_SCORE_FUNCTION=softmax ++ echo 'MLP_ONLY_LAYERS=[0,1]' +MLP_ONLY_LAYERS=[0,1] ++ echo 'MOE_LAYER_FREQ=[0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]' +MOE_LAYER_FREQ=[0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1] ++ echo MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512 +MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512 ++ echo 'LINEAR_CONV_KERNEL_DIM=4 LINEAR_KEY_HEAD_DIM=64 LINEAR_VALUE_HEAD_DIM=64' +LINEAR_CONV_KERNEL_DIM=4 LINEAR_KEY_HEAD_DIM=64 LINEAR_VALUE_HEAD_DIM=64 ++ echo ATTENTION_OUTPUT_GATE=true +ATTENTION_OUTPUT_GATE=true ++ echo ROPE_THETA=10000.0 +ROPE_THETA=10000.0 ++ echo POSITION_EMBEDDING_TYPE=none +POSITION_EMBEDDING_TYPE=none ++ echo MOE_ROUTER_SQRT_GATE=true +MOE_ROUTER_SQRT_GATE=true +++ /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/utils/parse-mom-vars-from-mcore-config.py /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/load_view6855 +W1010 22:55:45.782000 624058 torch/utils/cpp_extension.py:2425] TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +W1010 22:55:45.782000 624058 torch/utils/cpp_extension.py:2425] If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'] to specific architectures. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. ++ eval 'export MOM_ENABLED=false +export MOM_DO_MASK_SCALE=true +export MOM_TARGET_FFN_HIDDEN_SIZE=512' +++ export MOM_ENABLED=false +++ MOM_ENABLED=false +++ export MOM_DO_MASK_SCALE=true +++ MOM_DO_MASK_SCALE=true +++ export MOM_TARGET_FFN_HIDDEN_SIZE=512 +++ MOM_TARGET_FFN_HIDDEN_SIZE=512 ++ echo 'MOM_ENABLED=false MOM_DO_MASK_SCALE=true MOM_TARGET_FFN_HIDDEN_SIZE=512' +MOM_ENABLED=false MOM_DO_MASK_SCALE=true MOM_TARGET_FFN_HIDDEN_SIZE=512 ++ '[' false = true ']' ++ '[' 192 -gt 0 ']' ++ '[' 512 -le 0 ']' ++ '[' 10000.0 = 0.0 ']' +++ /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python - ++ ROPE_THETA=10000 ++ '[' -z 131072 ']' ++ '[' 131072 = 0 ']' ++ FFN_TOKEN_SHIFT= ++ FFN_INTERMEDIATE_TOKEN_SHIFT= ++ ATTN_TOKEN_SHIFT= ++ ATTN_Q_TOKEN_SHIFT= ++ ATTN_K_TOKEN_SHIFT= ++ ATTN_V_TOKEN_SHIFT= ++ TOKEN_SHIFT_CONV_SIZE=4 ++ TOKEN_SHIFT_CONV_INIT=default ++ export FFN_TOKEN_SHIFT FFN_INTERMEDIATE_TOKEN_SHIFT ATTN_TOKEN_SHIFT ++ export ATTN_Q_TOKEN_SHIFT ATTN_K_TOKEN_SHIFT ATTN_V_TOKEN_SHIFT ++ export TOKEN_SHIFT_CONV_SIZE TOKEN_SHIFT_CONV_INIT ++ export ROPE_THETA MAX_POSITION_EMBEDDINGS POSITION_EMBEDDING_TYPE ATTENTION_OUTPUT_GATE MOE_ROUTER_SQRT_GATE ++ ATTENTION_OUTPUT_GATE=true ++ /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/utils/render_gdn_hf_config.py --checkpoint-path /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/load_view6855 --template-path /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/hf-configs/YuLan-Mini-for-gdn-convert --output-dir /tmp/mcore2hf_ws_6855/mcore2hf-config.RGQ3d0 +W1010 22:55:58.699000 624548 torch/utils/cpp_extension.py:2425] TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +W1010 22:55:58.699000 624548 torch/utils/cpp_extension.py:2425] If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'] to specific architectures. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. ++ echo '✅config.json: /tmp/mcore2hf_ws_6855/mcore2hf-config.RGQ3d0/config.json' +✅config.json: /tmp/mcore2hf_ws_6855/mcore2hf-config.RGQ3d0/config.json ++ /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python - /tmp/mcore2hf_ws_6855/mcore2hf-config.RGQ3d0/config.json +{ + "hidden_size": 2048, + "intermediate_size": 4800, + "num_hidden_layers": 56, + "num_attention_heads": 32, + "num_key_value_heads": 8, + "head_dim": 64, + "max_position_embeddings": 131072, + "rope_theta": 10000, + "attn_position_embedding_type": "nope", + "rnn_position_embedding_type": "nope", + "linear_num_key_heads": 8, + "linear_num_value_heads": 32, + "linear_key_head_dim": 64, + "linear_value_head_dim": 64, + "num_experts": 192, + "num_experts_per_tok": 4, + "attn_output_gate": true +} ++ sleep 5 ++ MODEL_SIZE=2.6b ++ LOAD_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/load_view6855 ++ MG2HF=true ++ USE_CUDA=false ++ PR=bf16 ++ HF_DIR=/tmp/mcore2hf_ws_6855/mcore2hf-config.RGQ3d0 ++ export DROP_MTP ++ PREPARED_LOAD_DIR= ++ prepare_load_dir_for_iter /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/load_view6855 6855 ++ local SOURCE_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/load_view6855 ++ local ITER_NUM=6855 ++ '[' -z 6855 ']' +++ printf %07d 6855 ++ local ITER_BASENAME=iter_0006855 ++ local SOURCE_ITER_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/load_view6855/iter_0006855 ++ '[' '!' -d /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/load_view6855/iter_0006855 ']' ++ local LATEST_VALUE= ++ '[' -f /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/load_view6855/latest_checkpointed_iteration.txt ']' +++ cat /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/load_view6855/latest_checkpointed_iteration.txt ++ LATEST_VALUE=6855 ++ '[' 6855 = 6855 ']' ++ PREPARED_LOAD_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/load_view6855 ++ return 0 ++ LOAD_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/load_view6855 ++ echo LOAD_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/load_view6855 +LOAD_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/load_view6855 ++ '[' false = true ']' ++ '[' false = true ']' ++ '[' '!' -f /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/load_view6855/latest_checkpointed_iteration.txt ']' +++ cat /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/load_view6855/latest_checkpointed_iteration.txt ++ LATEST_CHECKPOINT_ITERATION=6855 ++ echo -e '\033[32mLatest checkpoint iteration: 6855\033[0m\n' +Latest checkpoint iteration: 6855 + ++ I=6855 ++ [[ 6855 =~ ^[0-9]+$ ]] +++ printf %07d 6855 ++ ITER_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/load_view6855/iter_0006855 ++ '[' '!' -f /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/load_view6855/iter_0006855/common.pt ']' ++ SAVE_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/load_view6855/iter_6855-hf ++ '[' -n /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a ']' ++ SAVE_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/iter_6855-hf ++ '[' -n '' ']' ++ mkdir -p /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/iter_6855-hf ++ bash ./scripts/mcore_gdn_moe/convert.sh 2.6b /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/load_view6855 /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/iter_6855-hf true false bf16 /tmp/mcore2hf_ws_6855/mcore2hf-config.RGQ3d0 6855 ++ tee /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/iter_6855-hf/hf2mcore.log +✅Enable attention output gate +\[\e[33mWarning adding --moe-shared-expert-gate for YuLan model. If you are not YuLan, please check this flag.\e[0m\] +torchrun --nproc_per_node 1 --nnodes 1 --node_rank 0 --master_addr localhost --master_port 37595 /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/impl/convert.py --tokenizer-type HuggingFaceTokenizer --tokenizer-model /tmp/mcore2hf_ws_6855/mcore2hf-config.RGQ3d0 --hf-dir /tmp/mcore2hf_ws_6855/mcore2hf-config.RGQ3d0 --mcore2hf --bf16 --normalization RMSNorm --swiglu --disable-bias-linear --seq-length 1 --max-position-embeddings 131072 --attention-backend auto --position-embedding-type none --kv-channels 64 --group-query-attention --add-qkv-bias --num-layers 56 --hidden-size 2048 --ffn-hidden-size 4800 --num-attention-heads 32 --rotary-base 10000 --rotary-percent 1.00 --num-query-groups 8 --normalization RMSNorm --norm-epsilon 1e-6 --untie-embeddings-and-output-weights --attention-output-gate --linear-attention-type gated_delta_net --linear-attention-freq [1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1] --linear-conv-kernel-dim 4 --linear-key-head-dim 64 --linear-value-head-dim 64 --linear-num-key-heads 8 --linear-num-value-heads 32 --num-experts 192 --moe-router-score-function softmax --moe-token-dispatcher-type alltoall --moe-router-topk 4 --moe-ffn-hidden-size 512 --moe-layer-freq [0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1] --moe-router-dtype fp32 --moe-shared-expert-intermediate-size 512 --moe-shared-expert-gate --moe-router-score-function softmax --moe-router-sqrt-gate --micro-batch-size 1 --global-batch-size 1024 --train-iters 500000 --weight-decay 0.1 --adam-beta1 0.9 --adam-beta2 0.95 --init-method-std 0.006 --clip-grad 1.0 --lr 2.0e-5 --lr-decay-style cosine --min-lr 6.0e-6 --lr-warmup-fraction .001 --lr-decay-iters 430000 --no-gradient-accumulation-fusion --bf16 --tensor-model-parallel-size 1 --pipeline-model-parallel-size 1 --expert-tensor-parallel-size 1 --expert-model-parallel-size 1 --log-interval 100 --save-interval 10000 --eval-interval 1000 --eval-iters 10 --model-type GPT --load-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/load_view6855 --save-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/iter_6855-hf --dist-ckpt-optim-fully-reshardable --skip-train --use-cpu-initialization --no-load-optim --no-load-rng --logging-level 1 --attention-backend auto --debug --max-shard-size 20GB --synchronizer mcore_gdn_moe --pretrain-script mcore_gdn_moe.model_provider --ckpt-step 6855 --gdn-split-in-proj --gated-norm --gated-norm-lora-rank 16 --distributed-backend gloo +W1010 22:56:18.872000 625157 torch/utils/cpp_extension.py:2425] TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +W1010 22:56:18.872000 625157 torch/utils/cpp_extension.py:2425] If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'] to specific architectures. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +Warning: Pai-Megatron-Patch arguments not available, some arguments may not be recognized +using world size: 1, data-parallel size: 1, context-parallel size: 1, hierarchical context-parallel sizes: None, tensor-model-parallel size: 1, pipeline-model-parallel size: 1 +Number of virtual stages per pipeline stage: None +accumulate and all-reduce gradients in fp32 for bfloat16 data type. +using torch.bfloat16 for parameters ... +------------------------ arguments ------------------------ + account_for_embedding_in_pipeline_split ......... False + account_for_loss_in_pipeline_split .............. False + accumulate_allreduce_grads_in_fp32 .............. True + activation_func_clamp_mode ...................... hard + activation_func_clamp_value ..................... None + adam_beta1 ...................................... 0.9 + adam_beta2 ...................................... 0.95 + adam_bias_correction ............................ True + adam_eps ........................................ 1e-08 + adamh_eps ....................................... 1e-08 + adamh_radius .................................... None + adamh_scope ..................................... lm_head + adamw_lr_mup_scaler ............................. False + add_bias_linear ................................. False + add_position_embedding .......................... True + add_qkv_bias .................................... True + adlr_autoresume ................................. False + adlr_autoresume_interval ........................ 1000 + align_grad_reduce ............................... True + align_param_gather .............................. False + allow_ambiguous_pad_tokens ...................... False + app_tag_run_name ................................ None + app_tag_run_version ............................. 0.0.0 + apply_layernorm_1p .............................. False + apply_query_key_layer_scaling ................... False + apply_residual_connection_post_layernorm ........ False + apply_rope_fusion ............................... False + async_save ...................................... None + async_tensor_model_parallel_allreduce ........... True + attention_backend ............................... AttnBackend.auto + attention_dropout ............................... 0.1 + attention_output_gate ........................... True + attention_output_gate_upcycling ................. False + attention_softmax_in_fp32 ....................... False + attn_k_token_shift .............................. None + attn_output_gate ................................ None + attn_output_gate_rand_init ...................... False + attn_q_token_shift .............................. None + attn_token_shift ................................ None + attn_v_token_shift .............................. None + auto_detect_ckpt_format ......................... False + auto_generate_cu_seqlens ........................ False + auto_model ...................................... AutoModelForCausalLM + barrier_with_L1_time ............................ True + benchmark_eval .................................. False + benchmark_global_batch .......................... None + benchmark_interval .............................. None + benchmark_micro_batch ........................... None + benchmark_sequence_length ....................... None + benchmark_tasks ................................. None + bert_binary_head ................................ True + bert_embedder_type .............................. megatron + bert_load ....................................... None + bf16 ............................................ True + bias_dropout_fusion ............................. True + bias_gelu_fusion ................................ False + bias_swiglu_fusion .............................. True + biencoder_projection_dim ........................ 0 + biencoder_shared_query_context_model ............ False + block_data_path ................................. None + cache_mla_latents ............................... False + calc_ft_timeouts ................................ False + calculate_per_token_loss ........................ False + check_for_large_grads ........................... False + check_for_nan_in_loss_and_grad .................. True + check_for_spiky_loss ............................ False + check_weight_hash_across_dp_replicas_interval ... None + ckpt_assume_constant_structure .................. False + ckpt_convert_format ............................. None + ckpt_convert_save ............................... None + ckpt_convert_update_legacy_dist_opt_format ...... False + ckpt_format ..................................... torch_dist + ckpt_fully_parallel_load ........................ False + ckpt_fully_parallel_save ........................ True + ckpt_fully_parallel_save_deprecated ............. False + ckpt_step ....................................... 6855 + classes_fraction ................................ 1.0 + clip_grad ....................................... 1.0 + clone_scatter_output_in_embedding ............... True + config_logger_dir ............................... + consumed_train_samples .......................... 0 + consumed_valid_samples .......................... 0 + context_parallel_size ........................... 1 + continual_pretrain .............................. False + continual_pretrain_preserve_opt_param_scheduler . False + corrected_weight_decay .......................... False + cp_comm_type .................................... ['p2p'] + create_attention_mask_in_dataloader ............. True + cross_entropy_fusion_impl ....................... native + cross_entropy_loss_fusion ....................... False + cuda_graph_impl ................................. none + cuda_graph_scope ................................ [] + cuda_graph_warmup_steps ......................... 3 + data_args_path .................................. None + data_cache_path ................................. None + data_parallel_random_init ....................... False + data_parallel_sharding_strategy ................. no_shard + data_parallel_size .............................. 1 + data_path ....................................... None + data_per_class_fraction ......................... 1.0 + data_sharding ................................... True + dataloader_defer_npy_index_mmap ................. False + dataloader_fast_cache_load ...................... False + dataloader_type ................................. single + dc_shortconv .................................... False + dc_shortconv_optimizer_state .................... conservative + dc_shortconv_upcycling .......................... False + ddp_average_in_collective ....................... False + ddp_bucket_size ................................. None + ddp_num_buckets ................................. None + ddp_pad_buckets_for_high_nccl_busbw ............. False + debug ........................................... True + decode_only_cuda_graphs ......................... False + decoder_first_pipeline_num_layers ............... None + decoder_last_pipeline_num_layers ................ None + decoder_num_layers .............................. None + decoder_seq_length .............................. None + decoupled_lr .................................... None + decoupled_min_lr ................................ None + decrease_batch_size_if_needed ................... False + defer_embedding_wgrad_compute ................... False + defer_grad_checks ............................... False + delay_wgrad_compute ............................. False + deprecated_use_mcore_models ..................... False + deterministic_mode .............................. False + dino_bottleneck_size ............................ 256 + dino_freeze_last_layer .......................... 1 + dino_head_hidden_size ........................... 2048 + dino_local_crops_number ......................... 10 + dino_local_img_size ............................. 96 + dino_norm_last_layer ............................ False + dino_teacher_temp ............................... 0.07 + dino_warmup_teacher_temp ........................ 0.04 + dino_warmup_teacher_temp_epochs ................. 30 + disable_attn_output_gate ........................ False + disable_bf16_reduced_precision_matmul ........... False + disable_chunked_prefill ......................... False + disable_explicit_attention_mask ................. False + disable_mamba_mem_eff_path ...................... False + disable_straggler_on_startup .................... False + disable_symmetric_registration .................. False + dist_ckpt_format_deprecated ..................... None + dist_ckpt_optim_fully_reshardable ............... True + dist_ckpt_save_pre_mcore_014 .................... False + dist_ckpt_strictness ............................ assume_ok_unexpected + distrib_optim_fully_reshardable_mem_efficient ... False + distribute_saved_activations .................... False + distributed_backend ............................. gloo + distributed_timeout_minutes ..................... 10 + distributed_timeout_seconds_after_init .......... None + document_packing_algorithm ...................... ogbfd + document_packing_dp_size ........................ None + drop_mtp ........................................ False + dryrun .......................................... False + dump_param_to_param_group_map ................... None + dynamic_context_parallel ........................ False + dynamic_context_parallel_sizes .................. None + dynamic_cp_candidate_sizes ...................... None + dynamic_cp_gdn_policy ........................... None + dynamic_cp_moe_policy ........................... None + dynamic_cp_on_error ............................. error + dynamic_cp_scheduler ............................ default_dynamic_cp + dynamic_cp_wave_coexecution ..................... False + dynamic_cp_workspace_max_tokens ................. None + emb_deviation_loss_coeff ........................ 0 + emb_deviation_type .............................. None + embedding_init_method_std ....................... None + embedding_path .................................. None + empty_unused_memory_level ....................... 0 + enable_cuda_graph ............................... False + enable_debug_logging ............................ False + enable_experimental ............................. False + enable_ft_package ............................... False + enable_full_sharding_in_hsdp .................... False + enable_gloo_process_groups ...................... True + enable_msc ...................................... True + enable_one_logger ............................... True + encoder_num_layers .............................. 56 + encoder_seq_length .............................. 1 + end_weight_decay ................................ 0.1 + eod_mask_loss ................................... False + error_injection_rate ............................ 0 + error_injection_type ............................ transient_error + eval_interval ................................... 1000 + eval_iters ...................................... 10 + evidence_data_path .............................. None + exit_duration_in_mins ........................... None + exit_interval ................................... None + exit_on_missing_checkpoint ...................... False + exit_signal_handler ............................. False + exp_avg_dtype ................................... torch.float32 + exp_avg_sq_dtype ................................ torch.float32 + expert_model_parallel_size ...................... 1 + expert_tensor_parallel_size ..................... 1 + export_force_local_attention .................... False + export_kd_cfg ................................... None + export_kd_teacher_ckpt_format ................... None + export_kd_teacher_load .......................... None + export_kv_cache_quant ........................... False + export_legacy_megatron .......................... False + export_model_type ............................... GPTModel + export_moe_apply_probs_on_input ................. False + export_offline_model ............................ False + export_qk_l2_norm ............................... False + export_quant_cfg ................................ None + export_real_quant_cfg ........................... None + export_te_mcore_model ........................... False + external_cuda_graph ............................. False + fast_thd_a2a .................................... False + ffn_hidden_size ................................. 4800 + ffn_intermediate_token_shift .................... None + ffn_token_shift ................................. None + final_logits_clamp_value ........................ None + fine_grained_activation_offloading .............. False + finetune ........................................ False + finetune_data_split ............................. train + finetune_hf_dataset ............................. None + first_last_layers_bf16 .......................... False + flash_decode .................................... False + fp16 ............................................ False + fp16_lm_cross_entropy ........................... False + fp32_residual_connection ........................ False + fp4 ............................................. None + fp4_param ....................................... False + fp4_recipe ...................................... nvfp4 + fp8 ............................................. None + fp8_amax_compute_algo ........................... most_recent + fp8_amax_history_len ............................ 1 + fp8_interval .................................... 1 + fp8_margin ...................................... 0 + fp8_param_gather ................................ False + fp8_recipe ...................................... delayed + fp8_wgrad ....................................... True + freeze_layernorm_weight ......................... False + freeze_non_mamba ................................ False + fsdp_double_buffer .............................. False + full_validation ................................. False + gated_norm ...................................... True + gated_norm_lora_rank ............................ 16 + gated_norm_upcycling ............................ False + gdn_alpha_weight_scale .......................... 1.0 + gdn_alpha_weight_scale_mode ..................... matrix + gdn_backend ..................................... fla + gdn_fused_split_forward ......................... False + gdn_gated_norm_backend .......................... torch + gdn_lr_mult_alpha ............................... 1.0 + gdn_lr_mult_beta ................................ 1.0 + gdn_lr_mult_k ................................... 1.0 + gdn_lr_mult_q ................................... 1.0 + gdn_lr_mult_v ................................... 1.0 + gdn_postinit_alog_shift ......................... 0.0 + gdn_postinit_alpha_scale ........................ 1.0 + gdn_postinit_key_scale .......................... 1.0 + gdn_pre_gdr_fusion .............................. False + gdn_pre_gdr_fusion_deterministic_weight_gradient False + gdn_pre_gdr_fusion_high_precision_backward ...... False + gdn_shortconv_separate_gate ..................... False + gdn_shortconv_separate_gate_upcycling ........... False + gdn_split_in_proj ............................... True + geglu ........................................... False + global_batch_size ............................... 1024 + glu_linear_offset ............................... 0.0 + grad_reduce_in_bf16 ............................. False + gradient_accumulation_fusion .................... False + gradient_reduce_div_fusion ...................... True + group_query_attention ........................... True + grpo_clamp_eps_lower ............................ 0.01 + grpo_clamp_eps_upper ............................ 0.01 + grpo_default_temperature ........................ 1.0 + grpo_default_top_p .............................. 0 + grpo_entropy_term_weight ........................ 0.0 + grpo_filter_groups_with_same_reward ............. False + grpo_group_size ................................. 2 + grpo_iterations ................................. 2 + grpo_kl_beta .................................... 0.001 + grpo_prompts_per_step ........................... 32 + head_lr_mult .................................... 1.0 + heterogeneous_layers_config_encoded_json ........ None + heterogeneous_layers_config_path ................ None + hf_dir .......................................... /tmp/mcore2hf_ws_6855/mcore2hf-config.RGQ3d0 + hidden_dropout .................................. 0.1 + hidden_size ..................................... 2048 + hidden_states_shrink ............................ 1.0 + hierarchical_context_parallel_sizes ............. None + high_priority_stream_groups ..................... [] + hybrid_adamh .................................... False + hybrid_attention_ratio .......................... 0.0 + hybrid_context_parallel ......................... False + hybrid_mlp_ratio ................................ 0.0 + hybrid_override_pattern ......................... None + hysteresis ...................................... 2 + ict_head_size ................................... None + ict_load ........................................ None + img_h ........................................... 224 + img_w ........................................... 224 + increase_log_level_interval ..................... 1000 + increase_log_level_iters ........................ 5 + indexer_batch_size .............................. 128 + indexer_log_interval ............................ 1000 + inference_batch_times_seqlen_threshold .......... -1 + inference_dynamic_batching ...................... False + inference_dynamic_batching_block_size ........... 256 + inference_dynamic_batching_buffer_guaranteed_fraction 0.2 + inference_dynamic_batching_buffer_overflow_factor None + inference_dynamic_batching_buffer_size_gb ....... 40.0 + inference_dynamic_batching_max_requests_override None + inference_dynamic_batching_max_tokens_override .. None + inference_dynamic_batching_num_cuda_graphs ...... 16 + inference_dynamic_batching_track_paused_request_events False + inference_dynamic_batching_unified_memory_level . 0 + inference_max_batch_size ........................ 8 + inference_max_seq_length ........................ 2560 + inference_rng_tracker ........................... False + init_method_std ................................. 0.006 + init_method_xavier_uniform ...................... False + init_model_with_meta_device ..................... False + initial_loss_scale .............................. 4294967296 + inprocess_active_world_size ..................... 1 + inprocess_barrier_timeout ....................... 120 + inprocess_completion_timeout .................... 120 + inprocess_empty_cuda_cache ...................... False + inprocess_granularity ........................... node + inprocess_hard_timeout .......................... 90 + inprocess_heartbeat_interval .................... 30 + inprocess_heartbeat_timeout ..................... 60 + inprocess_last_call_wait ........................ 1 + inprocess_max_iterations ........................ None + inprocess_monitor_process_interval .............. 1.0 + inprocess_monitor_thread_interval ............... 1.0 + inprocess_progress_watchdog_interval ............ 1.0 + inprocess_restart ............................... False + inprocess_soft_timeout .......................... 60 + inprocess_termination_grace_time ................ 1 + is_hybrid_model ................................. False + iter_per_epoch .................................. 1250 + iterations_to_skip .............................. [] + keep_fp8_transpose_cache ........................ False + kitchen_config_file ............................. None + kitchen_recipe_number ........................... None + kv_channels ..................................... 64 + kv_lora_rank .................................... 32 + langrl_env_config ............................... None + langrl_external_server .......................... False + langrl_inference_server_conversation_template ... None + langrl_inference_server_type .................... inplace_megatron + lazy_mpu_init ................................... None + legacy_tokenizer ................................ False + linear_attention_freq ........................... [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 0, 0, 1, 1, 1, 1, 1, 1] + linear_attention_type ........................... gated_delta_net + linear_conv_kernel_dim .......................... 4 + linear_cross_entropy_accumulate_grad_in_fp32 .... True + linear_cross_entropy_num_chunks ................. 8 + linear_key_head_dim ............................. 64 + linear_num_key_heads ............................ 8 + linear_num_value_heads .......................... 32 + linear_rope_mode ................................ none + linear_value_head_dim ........................... 64 + lm_head_alpha ................................... 1.0 + lm_loss_normalization ........................... legacy + load ............................................ None + load_complemental_dataset ....................... None + load_dir ........................................ /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/load_view6855 + load_main_params_from_ckpt ...................... None + local_rank ...................................... 0 + log_attn_logits ................................. False + log_energy ...................................... False + log_hidden_states ............................... [] + log_interval .................................... 100 + log_logits_z_loss ............................... False + log_loss_scale_to_tensorboard ................... True + log_memory_to_tensorboard ....................... False + log_metrics_interval ............................ None + log_moe_activations ............................. False + log_moe_params .................................. False + log_moe_router_metrics .......................... False + log_num_zeros_in_grad ........................... False + log_padding_stats ............................... False + log_params ...................................... [] + log_params_norm ................................. False + log_per_layer_grad_rms .......................... False + log_per_module_grad_rms ......................... False + log_per_module_update_rms ....................... False + log_progress .................................... False + log_spike_diagnostics ........................... False + log_straggler ................................... False + log_throughput .................................. False + log_timers_to_tensorboard ....................... False + log_update_rms .................................. False + log_validation_ppl_to_tensorboard ............... False + log_world_size_to_tensorboard ................... False + logging_level ................................... 1 + loss_scale ...................................... None + loss_scale_window ............................... 1000 + lr .............................................. 2e-05 + lr_decay_iters .................................. 430000 + lr_decay_samples ................................ None + lr_decay_style .................................. cosine + lr_warmup_fraction .............................. 0.001 + lr_warmup_init .................................. 0.0 + lr_warmup_iters ................................. 0 + lr_warmup_samples ............................... 0 + lr_wsd_decay_iters .............................. None + lr_wsd_decay_samples ............................ None + lr_wsd_decay_style .............................. exponential + main_grads_dtype ................................ torch.float32 + main_params_dtype ............................... torch.float32 + make_vocab_size_divisible_by .................... 128 + mamba_disable_cp ................................ False + mamba_expand .................................... 2 + mamba_head_dim .................................. 64 + mamba_num_groups ................................ 8 + mamba_num_heads ................................. None + mamba_state_dim ................................. 128 + manual_gc ....................................... False + manual_gc_eval .................................. True + manual_gc_interval .............................. 0 + mask_factor ..................................... 1.0 + mask_prob ....................................... 0.15 + mask_type ....................................... random + masked_softmax_fusion ........................... True + max_position_embeddings ......................... 131072 + max_seqlen_per_cp_rank .......................... None + max_shard_size .................................. 20GB + max_tokens_to_oom ............................... 12000 + mcore2hf ........................................ True + memory_snapshot_path ............................ None + merge_file ...................................... None + micro_batch_size ................................ 1 + microbatch_group_size_per_vp_stage .............. None + mid_level_dataset_surplus ....................... 0.005 + min_loss_scale .................................. 1.0 + min_lr .......................................... 6e-06 + min_offloaded_tensor_size ....................... 1048576 + mlp_chunks_for_prefill .......................... 1 + mmap_bin_files .................................. True + mock_data ....................................... False + model_type ...................................... GPT + modelopt_enabled ................................ False + moe_apply_probs_on_input ........................ False + moe_aux_loss_coeff .............................. 0.0 + moe_deepep_num_sms .............................. 20 + moe_enable_deepep ............................... False + moe_expert_capacity_factor ...................... None + moe_extended_tp ................................. False + moe_ffn_hidden_size ............................. 512 + moe_flex_dispatcher_backend ..................... deepep + moe_gaux_overlap ................................ False + moe_grouped_gemm ................................ False + moe_hybridep_num_sms ............................ 16 + moe_input_jitter_eps ............................ None + moe_layer_freq .................................. [0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] + moe_layer_recompute ............................. False + moe_monitor_layers .............................. + moe_monitor_log_interval ........................ None + moe_pad_expert_input_to_capacity ................ False + moe_pad_experts_for_cuda_graph_inference ........ False + moe_per_layer_logging ........................... False + moe_permute_backend ............................. auto + moe_permute_fusion .............................. False + moe_router_aux_loss_fusion ...................... auto + moe_router_bias_update_method ................... sign + moe_router_bias_update_rate ..................... 0.001 + moe_router_dtype ................................ fp32 + moe_router_enable_expert_bias ................... False + moe_router_force_load_balancing ................. False + moe_router_fusion ............................... False + moe_router_gemm_backend ......................... torch + moe_router_group_topk ........................... None + moe_router_load_balancing_type .................. aux_loss + moe_router_num_groups ........................... None + moe_router_padding_for_fp8 ...................... False + moe_router_padding_for_quantization ............. False + moe_router_pre_softmax .......................... False + moe_router_score_function ....................... softmax + moe_router_sqrt_gate ............................ True + moe_router_topk ................................. 4 + moe_router_topk_scaling_factor .................. None + moe_shared_expert_gate .......................... True + moe_shared_expert_intermediate_size ............. 512 + moe_shared_expert_overlap ....................... False + moe_token_dispatcher_type ....................... alltoall + moe_token_drop_policy ........................... probs + moe_triton_dispatcher_exclude_padding_tokens .... False + moe_triton_mega_activation_sms .................. 4 + moe_triton_mega_combine_sms ..................... 32 + moe_triton_mega_compute_sms ..................... 24 + moe_triton_mega_dispatch_sms .................... 48 + moe_triton_mega_num_warps ....................... 16 + moe_triton_mega_reduce_sms ...................... 32 + moe_triton_mega_tail_sms ........................ 16 + moe_triton_shared_expert_overlap ................ False + moe_triton_workspace_expand ..................... 4.0 + moe_upcycling_granularity ....................... 1 + moe_use_legacy_grouped_gemm ..................... False + moe_use_upcycling ............................... False + moe_z_loss_coeff ................................ None + mrope_section ................................... None + mscale .......................................... 1.0 + mscale_all_dim .................................. 0.0 + mtp_linear_attention_type ....................... None + mtp_loss_scaling_factor ......................... 0.1 + mtp_num_layers .................................. None + multi_latent_attention .......................... False + multiple_validation_sets ........................ False + muon_ball_momentum .............................. 0.9 + muon_ball_msign_steps ........................... 5 + muon_ball_power_iteration_steps ................. 10 + muon_ball_qkv_split_mode ........................ component + muon_ball_radius_mode ........................... spectral_mup + muon_ball_retract_alpha ......................... 0.05 + muon_ball_retract_mode .......................... hard + muon_ball_scale_mode ............................ spectral_mup + muon_ball_split_fc1 ............................. True + muon_ball_split_moe_experts ..................... True + muon_ball_split_qkv ............................. True + muon_ball_use_nesterov .......................... True + muon_extra_scale_factor ......................... 1.0 + muon_fp32_matmul_prec ........................... medium + muon_momentum ................................... 0.9 + muon_num_ns_steps ............................... 5 + muon_qkv_split_mode ............................. component + muon_scale_mode ................................. spectral_mup + muon_scale_vectorized_mode ...................... full + muon_split_fc1 .................................. True + muon_split_moe_experts .......................... True + muon_split_qkv .................................. True + muon_tp_mode .................................... blockwise + muon_use_nesterov ............................... False + muon_vectorize .................................. [] + muon_vectorize_attn_dim ......................... hidden_size + mup_lr_mult ..................................... 1.0 + nccl_all_reduce_for_prefill ..................... False + nccl_communicator_config_path ................... None + nccl_ub ......................................... False + no_load_optim ................................... True + no_load_rng ..................................... True + no_load_scheduler ............................... None + no_persist_layer_norm ........................... False + no_rope_freq .................................... None + no_save_optim ................................... None + no_save_rng ..................................... None + no_save_step_one ................................ None + no_weight_decay_cond_type ....................... None + non_persistent_ckpt_type ........................ None + non_persistent_global_ckpt_dir .................. None + non_persistent_keep_last ........................ 1 + non_persistent_local_ckpt_algo .................. fully_parallel + non_persistent_local_ckpt_dir ................... None + non_persistent_save_interval .................... None + norm_epsilon .................................... 1e-06 + normalization ................................... RMSNorm + num_attention_heads ............................. 32 + num_channels .................................... 3 + num_classes ..................................... 1000 + num_dataset_builder_threads ..................... 1 + num_distributed_optimizer_instances ............. 1 + num_experts ..................................... 192 + num_hf_saver .................................... None + num_layers ...................................... 56 + num_layers_at_end_in_bf16 ....................... 1 + num_layers_at_start_in_bf16 ..................... 1 + num_layers_per_virtual_pipeline_stage ........... None + num_query_groups ................................ 8 + num_virtual_stages_per_pipeline_rank ............ None + num_workers ..................................... 2 + object_storage_cache_path ....................... None + offload_modules ................................. [] + one_logger_async ................................ False + one_logger_project .............................. megatron-lm + one_logger_run_name ............................. None + onnx_safe ....................................... None + openai_gelu ..................................... False + optimizer ....................................... adam + optimizer_cpu_offload ........................... False + optimizer_offload_fraction ...................... 1.0 + output_bert_embeddings .......................... False + output_logits_z_loss_coeff ...................... None + overlap_cpu_optimizer_d2h_h2d ................... False + overlap_grad_reduce ............................. False + overlap_moe_expert_parallel_comm ................ False + overlap_p2p_comm ................................ False + overlap_p2p_comm_warmup_flush ................... False + overlap_param_gather ............................ False + overlap_param_gather_with_optimizer_step ........ False + override_hf_eod_token_id ........................ None + override_opt_param_scheduler .................... False + packing_tokens_per_shard ........................ 100000000000 + padded_vocab_size ............................... None + params_dtype .................................... torch.bfloat16 + patch_dim ....................................... 16 + per_split_data_args_path ........................ None + perform_initialization .......................... True + perform_rl_step ................................. False + pin_cpu_grads ................................... True + pin_cpu_params .................................. True + pipeline_model_parallel_comm_backend ............ None + pipeline_model_parallel_layout .................. None + pipeline_model_parallel_size .................... 1 + position_embedding_type ......................... none + pp_aware_finegrained_recompute .................. False + pp_aware_finegrained_recompute_global_layers .... None + pp_aware_finegrained_recompute_pp_ranks ......... None + prefetch_factor ................................. None + pretrain_script ................................. mcore_gdn_moe.model_provider + pretrained_checkpoint ........................... None + previous_stage_seq_length ....................... None + profile ......................................... False + profile_microbatch_count ........................ 2 + profile_microbatch_dir .......................... None + profile_microbatch_iteration .................... None + profile_microbatch_start ........................ 0 + profile_ranks ................................... [0] + profile_step_end ................................ None + profile_step_start .............................. None + q_lora_rank ..................................... None + qk_head_dim ..................................... 128 + qk_l2_norm ...................................... False + qk_layernorm .................................... False + qk_pos_emb_head_dim ............................. 64 + query_in_block_prob ............................. 0.1 + quick_geglu ..................................... False + rampup_batch_size ............................... None + rank ............................................ 0 + recompute_granularity ........................... None + recompute_method ................................ None + recompute_modules ............................... None + recompute_num_layers ............................ None + record_memory_history ........................... False + relative_attention_max_distance ................. 128 + relative_attention_num_buckets .................. 32 + reparam_checkpoint .............................. None + reparam_fallback_value .......................... None + reparam_keys .................................... None + replication ..................................... False + replication_factor .............................. 2 + replication_jump ................................ None + rerun_mode ...................................... validate_results + reset_attention_mask ............................ False + reset_iteration_one_to_zero ..................... False + reset_position_ids .............................. False + result_rejected_tracker_filename ................ None + retriever_report_topk_accuracies ................ [] + retriever_score_scaling ......................... False + retriever_seq_length ............................ 256 + retro_add_retriever ............................. False + retro_attention_gate ............................ 1 + retro_cyclic_train_iters ........................ None + retro_encoder_attention_dropout ................. 0.1 + retro_encoder_hidden_dropout .................... 0.1 + retro_encoder_layers ............................ 2 + retro_num_neighbors ............................. 2 + retro_num_retrieved_chunks ...................... 2 + retro_project_dir ............................... None + retro_verify_neighbor_count ..................... True + reuse_grad_buf_for_mxfp8_param_ag ............... False + reuse_training_communicators .................... False + rl_calculate_intra_group_similarity ............. False + rl_importance_sampling_truncation_coef .......... None + rl_inference_logprobs_is_correction ............. False + rl_offload_kv_cache_during_training ............. False + rl_offload_optimizer_during_inference ........... False + rl_partial_rollouts ............................. False + rl_prompts_per_eval ............................. 32 + rl_remove_kv_cache_during_training .............. False + rl_reset_cuda_graphs ............................ False + rl_sequence_packing_algo ........................ fifo + rl_sequence_packing_bin_size .................... 8192 + rl_use_sequence_packing ......................... False + rope_scaling_factor ............................. 8.0 + rope_type ....................................... None + rotary_base ..................................... 10000 + rotary_interleaved .............................. False + rotary_percent .................................. 1.0 + rotary_scaling_factor ........................... 1.0 + rotary_seq_len_interpolation_factor ............. None + run_workload_inspector_server ................... False + runtime_audit_dir ............................... None + sample_rate ..................................... 1.0 + save ............................................ None + save_after_load ................................. False + save_dir ........................................ /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/iter_6855-hf + save_interval ................................... 10000 + save_retain_interval ............................ None + scale_emb ....................................... 1.0 + scatter_gather_tensors_in_pipeline .............. True + seed ............................................ 1234 + seq_length ...................................... 1 + sequence_packing_scheduler ...................... default_dynamic_cp + sequence_parallel ............................... False + sft ............................................. False + sft_tokenizer_prompt_format ..................... nemotron-h-aligned + sgd_momentum .................................... 0.9 + sharp_enabled_group ............................. None + short_seq_prob .................................. 0.1 + skip_train ...................................... True + skipped_train_samples ........................... 0 + softmax_type .................................... vanilla + spec ............................................ None + spectra_clip .................................... 1.0 + spectra_grad_clip ............................... None + spectra_hyperball ............................... False + spectra_ns_steps ................................ 10 + spectra_scope ................................... matrices + spectral_ball_momentum .......................... 0.9 + spectral_ball_msign_steps ....................... 8 + spectral_ball_power_iteration_steps ............. 20 + spectral_ball_qkv_split_mode .................... component + spectral_ball_radius_mode ....................... spectral_mup + spectral_ball_retract_alpha ..................... 0.05 + spectral_ball_retract_mode ...................... hard + spectral_ball_scale_mode ........................ spectral_mup + spectral_ball_solver ............................ bisection + spectral_ball_solver_max_iterations ............. 20 + spectral_ball_solver_tolerance_f ................ 1e-08 + spectral_ball_split_fc1 ......................... True + spectral_ball_split_moe_experts ................. True + spectral_ball_split_qkv ......................... True + spectral_ball_use_nesterov ...................... True + spectral_mup_init ............................... False + spike_diagnostics_dir ........................... None + spike_diagnostics_end ........................... 21 + spike_diagnostics_start ......................... 2 + split ........................................... None + split_expert_init ............................... True + split_fc1_init .................................. True + split_qkv_init .................................. True + split_qkv_init_mode ............................. group + sqreglu ......................................... False + squared_relu .................................... False + start_samples ................................... None + start_weight_decay .............................. 0.1 + straggler_ctrlr_port ............................ 65535 + straggler_minmax_count .......................... 1 + strict_fsdp_dtensor_load ........................ True + suggested_communication_unit_size ............... None + swanlab_exp_name ................................ + swanlab_project ................................. + swanlab_save_dir ................................ + swanlab_workspace ............................... + swiglu .......................................... True + swin_backbone_type .............................. tiny + symmetric_ar_type ............................... None + synchronizer .................................... mcore_gdn_moe + target_ckpt_format .............................. torch_dist + te_rng_tracker .................................. False + teacher_model_config ............................ None + tensor_model_parallel_size ...................... 1 + tensorboard_dir ................................. None + tensorboard_log_interval ........................ 1 + tensorboard_queue_size .......................... 1000 + test_data_path .................................. None + test_mode ....................................... False + tiktoken_num_special_tokens ..................... 1000 + tiktoken_pattern ................................ None + tiktoken_special_tokens ......................... None + timing_log_level ................................ 0 + timing_log_option ............................... minmax + titles_data_path ................................ None + token_shift_conv_init ........................... default + token_shift_conv_size ........................... 4 + tokenizer_metadata .............................. None + tokenizer_model ................................. /tmp/mcore2hf_ws_6855/mcore2hf-config.RGQ3d0 + tokenizer_type .................................. HuggingFaceTokenizer + torch_fsdp2_reshard_after_forward ............... True + tp_comm_bootstrap_backend ....................... nccl + tp_comm_bulk_dgrad .............................. True + tp_comm_bulk_wgrad .............................. True + tp_comm_overlap ................................. False + tp_comm_overlap_ag .............................. True + tp_comm_overlap_cfg ............................. None + tp_comm_overlap_rs .............................. True + tp_comm_overlap_rs_dgrad ........................ False + tp_comm_split_ag ................................ True + tp_comm_split_rs ................................ True + train_data_path ................................. None + train_iters ..................................... 500000 + train_samples ................................... None + train_sync_interval ............................. None + train_until_stopped ............................. False + transformer_impl ................................ transformer_engine + transformer_pipeline_model_parallel_size ........ 1 + trust_remote_code ............................... False + untie_embeddings_and_output_weights ............. True + use_checkpoint_args ............................. False + use_checkpoint_opt_param_scheduler .............. False + use_cpu_initialization .......................... True + use_dist_ckpt ................................... True + use_dist_ckpt_deprecated ........................ False + use_distributed_optimizer ....................... False + use_flash_attn .................................. False + use_fused_weighted_squared_relu ................. False + use_gpu ......................................... False + use_legacy_models ............................... False + use_megatron_fsdp ............................... False + use_mp_args_from_checkpoint_args ................ False + use_one_sent_docs ............................... False + use_persistent_ckpt_worker ...................... False + use_precision_aware_optimizer ................... False + use_pytorch_profiler ............................ False + use_ring_exchange_p2p ........................... False + use_rope_scaling ................................ False + use_rotary_position_embeddings .................. False + use_sharp ....................................... False + use_te_activation_func .......................... False + use_tokenizer_model_from_checkpoint_args ........ True + use_torch_fsdp2 ................................. False + use_torch_optimizer_for_cpu_offload ............. False + use_tp_pp_dp_mapping ............................ False + v_head_dim ...................................... 128 + valid_data_path ................................. None + variable_seq_lengths ............................ False + virtual_pipeline_model_parallel_size ............ None + vision_backbone_type ............................ vit + vision_pretraining .............................. False + vision_pretraining_type ......................... classify + vocab_extra_ids ................................. 0 + vocab_file ...................................... None + vocab_size ...................................... None + wandb_entity .................................... + wandb_exp_name .................................. + wandb_project ................................... + wandb_save_dir .................................. + weight_decay .................................... 0.1 + weight_decay_incr_style ......................... constant + wgrad_deferral_limit ............................ 0 + window_attn_skip_freq ........................... None + window_size ..................................... None + word_embedding_dropout_prob ..................... 0.0 + world_size ...................................... 1 + yaml_cfg ........................................ None +-------------------- end of arguments --------------------- +INFO:megatron.core.num_microbatches_calculator:setting number of microbatches to constant 1024 +> building HuggingFaceTokenizer tokenizer ... + > padded vocab (size: 99000) with 72 dummy tokens (new size: 99072) +WARNING: one_logger package is required to enable e2e metrics tracking. please go to https://confluence.nvidia.com/display/MLWFO/Package+Repositories for details to install it +INFO:megatron.training.initialize:Setting logging level to 1 +WARNING:megatron.core.rerun_state_machine:RerunStateMachine initialized in mode validate_results +> initializing torch distributed ... +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +> initialized tensor model parallel with size 1 +> initialized pipeline model parallel with size 1 +> setting random seeds to 1234 ... +> compiling dataset index builder ... +make: Entering directory '/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/datasets' +make: Nothing to be done for 'default'. +make: Leaving directory '/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/datasets' +>>> done with dataset index builder. Compilation time: 0.134 seconds +WARNING: constraints for invoking optimized fused softmax kernel are not met. We default back to unfused kernel invocations. +> compiling and loading fused kernels ... +>>> done with compiling and loading fused kernels. Compilation time: 0.005 seconds +WORLD_SIZE: 1, RANK: 0, LOCAL_RANK: 0 +[convert] creating synchronizer: mode=mcore2hf +building GPT model ... +[GPT_LAYER_SPEC] log_hidden_states=[] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=[] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=[] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=[] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/transformers/modeling_rope_utils.py:1036: FutureWarning: `rope_config_validation` is deprecated and has been removed. Its functionality has been moved to RotaryEmbeddingConfigMixin.validate_rope method. PreTrainedConfig inherits this class, so please call self.validate_rope() instead. Also, make sure to use the new rope_parameters syntax. You can call self.standardize_rope_params() in the meantime. + warnings.warn( +[transformers] `layer_type_validation` is deprecated and will be removed in v5.20. Use `PreTrainedConfig.validate_layer_type` instead +[transformers] `layer_type_validation` is deprecated and will be removed in v5.20. Use `PreTrainedConfig.validate_layer_type` instead +[transformers] `torch_dtype` is deprecated! Use `dtype` instead! +[ERROR] `cache_position` is part of Qwen3NextModel.forward's signature, but not documented. Make sure to add it to the docstring of the function in /mnt/ssd/cache_tmp/huggingface/modules/transformers_modules/mcore2hf_hyphen_config_dot_RGQ3d0/10c6759df797195a/modeling_qwen3_next.py. +[ERROR] `cache_position` is part of Qwen3NextForCausalLM.forward's signature, but not documented. Make sure to add it to the docstring of the function in /mnt/ssd/cache_tmp/huggingface/modules/transformers_modules/mcore2hf_hyphen_config_dot_RGQ3d0/10c6759df797195a/modeling_qwen3_next.py. +[transformers] `torch_dtype` is deprecated! Use `dtype` instead! +INFO:transformers_modules.mcore2hf_hyphen_config_dot_RGQ3d0.10c6759df797195a.modeling_qwen3_next:[Qwen3Next custom] attn_position_embedding_type=nope, rnn_position_embedding_type=nope, attn_logits_scaling=None +Qwen3NextForCausalLM( + (model): Qwen3NextModel( + (embed_tokens): Embedding(99072, 2048, padding_idx=102) + (layers): ModuleList( + (0-1): 2 x Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=4800, bias=False) + (up_proj): Linear(in_features=2048, out_features=4800, bias=False) + (down_proj): Linear(in_features=4800, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (2-11): 10 x Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (12): Qwen3NextDecoderLayer( + (self_attn): Qwen3NextAttention( + (q_proj): Linear(in_features=2048, out_features=4096, bias=True) + (k_proj): Linear(in_features=2048, out_features=512, bias=True) + (v_proj): Linear(in_features=2048, out_features=512, bias=True) + (o_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (13-19): 7 x Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (20-21): 2 x Qwen3NextDecoderLayer( + (self_attn): Qwen3NextAttention( + (q_proj): Linear(in_features=2048, out_features=4096, bias=True) + (k_proj): Linear(in_features=2048, out_features=512, bias=True) + (v_proj): Linear(in_features=2048, out_features=512, bias=True) + (o_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (22): Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (23): Qwen3NextDecoderLayer( + (self_attn): Qwen3NextAttention( + (q_proj): Linear(in_features=2048, out_features=4096, bias=True) + (k_proj): Linear(in_features=2048, out_features=512, bias=True) + (v_proj): Linear(in_features=2048, out_features=512, bias=True) + (o_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (24-45): 22 x Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (46): Qwen3NextDecoderLayer( + (self_attn): Qwen3NextAttention( + (q_proj): Linear(in_features=2048, out_features=4096, bias=True) + (k_proj): Linear(in_features=2048, out_features=512, bias=True) + (v_proj): Linear(in_features=2048, out_features=512, bias=True) + (o_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (47): Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (48-49): 2 x Qwen3NextDecoderLayer( + (self_attn): Qwen3NextAttention( + (q_proj): Linear(in_features=2048, out_features=4096, bias=True) + (k_proj): Linear(in_features=2048, out_features=512, bias=True) + (v_proj): Linear(in_features=2048, out_features=512, bias=True) + (o_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (50-55): 6 x Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + ) + (norm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (rotary_emb): Qwen3NextRotaryEmbedding() + ) + (lm_head): Linear(in_features=2048, out_features=99072, bias=False) +) +GPTModel( + (embedding): LanguageModelEmbedding( + (word_embeddings): VocabParallelEmbedding() + (embedding_dropout): Dropout(p=0.1, inplace=False) + ) + (decoder): TransformerBlock( + (layers): ModuleList( + (0-1): 2 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=9600, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=4800, out_features=2048, bias=False, TP=1) + ) + ) + (2-11): 10 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (12): TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): SelfAttention( + (core_attention): TEDotProductAttention( + (flash_attention): FlashAttention() + (fused_attention): FusedAttention() + (unfused_attention): UnfusedDotProductAttention( + (scale_mask_softmax): FusedScaleMaskSoftmax() + (attention_dropout): Dropout(p=0.1, inplace=False) + ) + ) + (linear_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (linear_qkv): TEColumnParallelLinear(in_features=2048, out_features=5120, bias=True, TP=1) + (q_layernorm): IdentityOp() + (k_layernorm): IdentityOp() + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (13-19): 7 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (20-21): 2 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): SelfAttention( + (core_attention): TEDotProductAttention( + (flash_attention): FlashAttention() + (fused_attention): FusedAttention() + (unfused_attention): UnfusedDotProductAttention( + (scale_mask_softmax): FusedScaleMaskSoftmax() + (attention_dropout): Dropout(p=0.1, inplace=False) + ) + ) + (linear_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (linear_qkv): TEColumnParallelLinear(in_features=2048, out_features=5120, bias=True, TP=1) + (q_layernorm): IdentityOp() + (k_layernorm): IdentityOp() + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (22): TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (23): TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): SelfAttention( + (core_attention): TEDotProductAttention( + (flash_attention): FlashAttention() + (fused_attention): FusedAttention() + (unfused_attention): UnfusedDotProductAttention( + (scale_mask_softmax): FusedScaleMaskSoftmax() + (attention_dropout): Dropout(p=0.1, inplace=False) + ) + ) + (linear_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (linear_qkv): TEColumnParallelLinear(in_features=2048, out_features=5120, bias=True, TP=1) + (q_layernorm): IdentityOp() + (k_layernorm): IdentityOp() + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (24-45): 22 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (46): TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): SelfAttention( + (core_attention): TEDotProductAttention( + (flash_attention): FlashAttention() + (fused_attention): FusedAttention() + (unfused_attention): UnfusedDotProductAttention( + (scale_mask_softmax): FusedScaleMaskSoftmax() + (attention_dropout): Dropout(p=0.1, inplace=False) + ) + ) + (linear_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (linear_qkv): TEColumnParallelLinear(in_features=2048, out_features=5120, bias=True, TP=1) + (q_layernorm): IdentityOp() + (k_layernorm): IdentityOp() + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (47): TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (48-49): 2 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): SelfAttention( + (core_attention): TEDotProductAttention( + (flash_attention): FlashAttention() + (fused_attention): FusedAttention() + (unfused_attention): UnfusedDotProductAttention( + (scale_mask_softmax): FusedScaleMaskSoftmax() + (attention_dropout): Dropout(p=0.1, inplace=False) + ) + ) + (linear_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (linear_qkv): TEColumnParallelLinear(in_features=2048, out_features=5120, bias=True, TP=1) + (q_layernorm): IdentityOp() + (k_layernorm): IdentityOp() + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (50-55): 6 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + ) + (final_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (output_layer): ColumnParallelLinear(in_features=2048, out_features=99072, bias=False, TP=1) +) +DEBUG:multistorageclient.config:Searching MSC config file paths: ['/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/msc/msc_config.yaml', '/mnt/ssd/cache_tmp/.config/msc/config.yaml', '/mnt/ssd/cache_tmp/.config/msc/config.json', '/root/.msc_config.yaml', '/root/.msc_config.json', '/root/.config/msc/config.yaml', '/root/.config/msc/config.json', '/etc/xdg/msc/config.yaml', '/etc/xdg/msc/config.json', '/etc/msc_config.yaml', '/etc/msc_config.json'] +DEBUG:multistorageclient.config:Using MSC config file: /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/msc/msc_config.yaml +DEBUG:multistorageclient.config:Searching MSC config file paths: ['/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/msc/msc_config.yaml', '/mnt/ssd/cache_tmp/.config/msc/config.yaml', '/mnt/ssd/cache_tmp/.config/msc/config.json', '/root/.msc_config.yaml', '/root/.msc_config.json', '/root/.config/msc/config.yaml', '/root/.config/msc/config.json', '/etc/xdg/msc/config.yaml', '/etc/xdg/msc/config.json', '/etc/msc_config.yaml', '/etc/msc_config.json'] +DEBUG:multistorageclient.config:Using MSC config file: /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/msc/msc_config.yaml +DEBUG:multistorageclient.client.single:No metadata provider configured, auto-commit will not be enabled +DEBUG:multistorageclient.client.client:StorageClient '__filesystem__' using SingleStorageClient +sharded_state_dict metadata loaded from the checkpoint: {'singleton_local_shards': True, 'distrib_optim_sharding_type': 'dp_reshardable', 'chained_optim_avoid_prefix': True} +Job sharding has changed: Rerun state will be ignored + loading distributed checkpoint from /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/load_view6855 at iteration 6855 +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/dist_checkpointing/strategies/torch.py:956: FutureWarning: `load_state_dict` is deprecated and will be removed in future versions. Please use `load` instead. + checkpoint.load_state_dict( + checkpoint version 3.0 + successfully loaded checkpoint from /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/load_view6855 [ t 1/1, p 1/1 ] at iteration 6855 +[convert] synchronizer created in 375.29s +[convert] starting sync_params +INFO:root:Converting layer 0 is_gdn=True is_not_moe=True +INFO:root:Converting layer 1 is_gdn=True is_not_moe=True +INFO:root:Converting layer 2 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.078125 mean=-0.000507354736328125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.10498046875 mean=0.0018157958984375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 3 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.08056640625 mean=-0.00017070770263671875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.1083984375 mean=-0.00011730194091796875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 4 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0869140625 mean=-0.000274658203125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.080078125 mean=0.000774383544921875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 5 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0908203125 mean=-0.00020885467529296875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.07958984375 mean=0.00135040283203125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 6 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0986328125 mean=-0.00024318695068359375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.07421875 mean=-0.000873565673828125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 7 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.09375 mean=-7.534027099609375e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0625 mean=-0.0001678466796875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 8 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.09619140625 mean=-8.106231689453125e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0625 mean=0.00011491775512695312 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 9 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.09033203125 mean=2.5153160095214844e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0556640625 mean=-0.00016498565673828125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 10 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.08642578125 mean=2.2530555725097656e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.053466796875 mean=-0.00045013427734375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 11 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.08447265625 mean=6.580352783203125e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0654296875 mean=-0.0003871917724609375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 12 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 12: args.attention_output_gate=True +WARNING:root:Layer 12: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.078125 mean=0.00022125244140625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.052490234375 mean=-0.00057220458984375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 13 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.07373046875 mean=0.00015544891357421875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.042236328125 mean=-0.00091552734375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 14 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.078125 mean=0.0001659393310546875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.04541015625 mean=-0.0005645751953125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 15 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.07568359375 mean=0.00023174285888671875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.048095703125 mean=-0.00083160400390625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 16 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.07421875 mean=0.0002899169921875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.047119140625 mean=-0.000659942626953125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 17 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.07177734375 mean=0.0002956390380859375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0458984375 mean=-0.00122833251953125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 18 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0673828125 mean=0.0002803802490234375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.046875 mean=-0.00225830078125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 19 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0703125 mean=0.000606536865234375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.049072265625 mean=-0.0015869140625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 20 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 20: args.attention_output_gate=True +WARNING:root:Layer 20: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.06884765625 mean=0.0003833770751953125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.04833984375 mean=-0.0009765625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 21 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 21: args.attention_output_gate=True +WARNING:root:Layer 21: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.061767578125 mean=0.0004215240478515625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.040283203125 mean=-0.0019378662109375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 22 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.056884765625 mean=0.0003604888916015625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.043212890625 mean=-0.0028839111328125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 23 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 23: args.attention_output_gate=True +WARNING:root:Layer 23: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.05517578125 mean=0.0004863739013671875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.03662109375 mean=-0.0020904541015625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 24 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.053955078125 mean=0.0004444122314453125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.034423828125 mean=-0.00225830078125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 25 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.05126953125 mean=0.0004119873046875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.03271484375 mean=-0.0015869140625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 26 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.05078125 mean=0.000331878662109375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.030517578125 mean=-0.00168609619140625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 27 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.052978515625 mean=0.0004596710205078125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.028564453125 mean=-0.00162506103515625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 28 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.04931640625 mean=0.0004119873046875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0264892578125 mean=-0.000946044921875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 29 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0498046875 mean=0.00054931640625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0230712890625 mean=-0.0010528564453125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 30 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.052978515625 mean=0.00084686279296875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0242919921875 mean=-0.000827789306640625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 31 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.050537109375 mean=0.000530242919921875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.025390625 mean=-0.000514984130859375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 32 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0478515625 mean=0.00013256072998046875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0245361328125 mean=-8.821487426757812e-05 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 33 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.05126953125 mean=9.870529174804688e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.027099609375 mean=1.8551945686340332e-06 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 34 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0537109375 mean=0.00018310546875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0267333984375 mean=0.00018596649169921875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 35 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.058837890625 mean=8.487701416015625e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.031494140625 mean=0.0004425048828125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 36 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.060791015625 mean=5.7697296142578125e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.04248046875 mean=0.0009918212890625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 37 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.060546875 mean=0.00011539459228515625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.04248046875 mean=0.0013275146484375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 38 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.059326171875 mean=0.000125885009765625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.036865234375 mean=0.00167083740234375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 39 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.057861328125 mean=8.487701416015625e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.036376953125 mean=0.0013275146484375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 40 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.05419921875 mean=0.0001354217529296875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.025634765625 mean=0.0004482269287109375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 41 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.05078125 mean=0.0001430511474609375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.02392578125 mean=0.0003108978271484375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 42 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.045654296875 mean=0.00037384033203125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.02001953125 mean=-0.00046539306640625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 43 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.039794921875 mean=0.00020599365234375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.019775390625 mean=-0.000278472900390625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 44 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.036865234375 mean=0.000431060791015625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.020751953125 mean=-0.0003643035888671875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 45 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.037109375 mean=0.00015735626220703125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.02294921875 mean=-0.000743865966796875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 46 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 46: args.attention_output_gate=True +WARNING:root:Layer 46: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.03759765625 mean=0.00035858154296875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.020263671875 mean=-0.000499725341796875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 47 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.037841796875 mean=0.0002269744873046875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.022705078125 mean=-0.00067901611328125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 48 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 48: args.attention_output_gate=True +WARNING:root:Layer 48: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.03515625 mean=-7.724761962890625e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0189208984375 mean=-0.000453948974609375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 49 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 49: args.attention_output_gate=True +WARNING:root:Layer 49: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.034912109375 mean=-9.72747802734375e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.020751953125 mean=-0.0009613037109375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 50 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0380859375 mean=-0.00010824203491210938 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.022216796875 mean=-0.000820159912109375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 51 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.038330078125 mean=2.193450927734375e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0223388671875 mean=-0.001068115234375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 52 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.04052734375 mean=0.000217437744140625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0286865234375 mean=-0.00121307373046875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 53 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0390625 mean=6.866455078125e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.04931640625 mean=-0.00213623046875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 54 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0400390625 mean=6.4849853515625e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.03759765625 mean=-0.00182342529296875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 55 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.04443359375 mean=-0.00016117095947265625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.041259765625 mean=0.0024566650390625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +[convert] finished sync_params in 0.92s +[convert] starting check_and_save -> /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/iter_6855-hf +DEBUG:root:[RANK 0] 0 send op & 0 recv op. +INFO:root:[Iters 0 RANK 0] starts synchronizing parameters with other ranks... +INFO:root:[Iters 0 RANK 0] finishes synchronizing +[Iters 0 RANK 0] model-00001-of-00004.safetensors is saved. +DEBUG:root:[Iters 0 RANK 0] joined +DEBUG:root:[RANK 0] 0 send op & 0 recv op. +INFO:root:[Iters 1 RANK 0] starts synchronizing parameters with other ranks... +INFO:root:[Iters 1 RANK 0] finishes synchronizing +[Iters 1 RANK 0] model-00002-of-00004.safetensors is saved. +DEBUG:root:[Iters 1 RANK 0] joined +DEBUG:root:[RANK 0] 0 send op & 0 recv op. +INFO:root:[Iters 2 RANK 0] starts synchronizing parameters with other ranks... +INFO:root:[Iters 2 RANK 0] finishes synchronizing +[Iters 2 RANK 0] model-00003-of-00004.safetensors is saved. +DEBUG:root:[Iters 2 RANK 0] joined +DEBUG:root:[RANK 0] 0 send op & 0 recv op. +INFO:root:[Iters 3 RANK 0] starts synchronizing parameters with other ranks... +INFO:root:[Iters 3 RANK 0] finishes synchronizing +[Iters 3 RANK 0] model-00004-of-00004.safetensors is saved. +DEBUG:root:[Iters 3 RANK 0] joined +[convert] finished check_and_save in 77.32s +Conversion finished in 454.7529239654541 seconds. ++ echo '✅Conversion completed: /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/iter_6855-hf' +✅Conversion completed: /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/iter_6855-hf ++ cleanup_real_config_path ++ rm -rf /tmp/mcore2hf_ws_6855/mcore2hf-config.RGQ3d0 ++ '[' -n '' ']' diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/hf_export_logs/convert_iter840.log b/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/hf_export_logs/convert_iter840.log new file mode 100644 index 0000000000000000000000000000000000000000..06ef98b0ac47d2c2ca0d9868b283f4dcbb8736b4 --- /dev/null +++ b/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/hf_export_logs/convert_iter840.log @@ -0,0 +1,2665 @@ +++ realpath /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/scripts/mcore_gdn_moe/../../../.. ++ MEGATRON_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher ++ CONVERTOR_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor ++ REPAIR_SCRIPT=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/utils/repair_gdn_checkpoint.py ++ RENDER_CONFIG_SCRIPT=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/utils/render_gdn_hf_config.py ++ export PYTHON_ENV_ROOT=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud ++ PYTHON_ENV_ROOT=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud ++ PYTHON_BIN=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python ++ '[' '!' -x /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python ']' ++ '[' '!' -x /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python ']' ++ '[' '!' -x /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python ']' ++ '[' '!' -x /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python ']' ++ export PYTHON_BIN ++ /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python -c 'import torch' ++ export PATH=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin:/root/.kimi-code/bin:/root/.local/bin:/mnt/hdd/minyingqian/mihomo/bin:/root/.grok/bin:/usr/local/bin:/root/miniconda3/bin:/root/.local/bin:/usr/local/bin:/root/miniconda3/bin:/root/miniconda3/bin:/mnt/ssd/liuzian/pandoc/current/bin:/mnt/ssd/cache_tmp/.local/share/../bin:/mnt/ssd/yulan/bin:/mnt/ssd/yulan/bin:/root/.local/bin:/usr/local/bin:/root/miniconda3/bin:/root/.local/bin:/usr/local/bin:/root/miniconda3/bin:/mnt/ssd/cache_tmp/.local/bin:/root/.local/bin:/root/.local/bin:/mnt/hdd/minyingqian/mihomo/bin:/root/.grok/bin:/usr/local/bin:/root/miniconda3/bin:/root/.local/bin:/usr/local/bin:/root/miniconda3/bin:/root/miniconda3/bin:/mnt/ssd/liuzian/pandoc/current/bin:/mnt/ssd/cache_tmp/.local/bin:/mnt/ssd/yulan/bin:/mnt/ssd/yulan/bin:/root/.local/bin:/root/miniconda3/bin:/root/miniconda3/condabin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/mnt/ssd/yulan/cache/installs/nsight-compute/2026.2:/mnt/ssd/yulan/cache/installs/nsight-systems/2026.3.1/bin:/mnt/ssd/yulan/cache/installs/nsight-compute/2026.2:/mnt/ssd/yulan/cache/installs/nsight-systems/2026.3.1/bin ++ PATH=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin:/root/.kimi-code/bin:/root/.local/bin:/mnt/hdd/minyingqian/mihomo/bin:/root/.grok/bin:/usr/local/bin:/root/miniconda3/bin:/root/.local/bin:/usr/local/bin:/root/miniconda3/bin:/root/miniconda3/bin:/mnt/ssd/liuzian/pandoc/current/bin:/mnt/ssd/cache_tmp/.local/share/../bin:/mnt/ssd/yulan/bin:/mnt/ssd/yulan/bin:/root/.local/bin:/usr/local/bin:/root/miniconda3/bin:/root/.local/bin:/usr/local/bin:/root/miniconda3/bin:/mnt/ssd/cache_tmp/.local/bin:/root/.local/bin:/root/.local/bin:/mnt/hdd/minyingqian/mihomo/bin:/root/.grok/bin:/usr/local/bin:/root/miniconda3/bin:/root/.local/bin:/usr/local/bin:/root/miniconda3/bin:/root/miniconda3/bin:/mnt/ssd/liuzian/pandoc/current/bin:/mnt/ssd/cache_tmp/.local/bin:/mnt/ssd/yulan/bin:/mnt/ssd/yulan/bin:/root/.local/bin:/root/miniconda3/bin:/root/miniconda3/condabin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/mnt/ssd/yulan/cache/installs/nsight-compute/2026.2:/mnt/ssd/yulan/cache/installs/nsight-systems/2026.3.1/bin:/mnt/ssd/yulan/cache/installs/nsight-compute/2026.2:/mnt/ssd/yulan/cache/installs/nsight-systems/2026.3.1/bin ++ export VIRTUAL_ENV=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud ++ VIRTUAL_ENV=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud ++ echo ✅MEGATRON_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher +✅MEGATRON_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher ++ echo ✅PYTHON_BIN=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python +✅PYTHON_BIN=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python ++ export CUDA_LAUNCH_BLOCKING=1 ++ CUDA_LAUNCH_BLOCKING=1 ++ export CUDA_VISIBLE_DEVICES=0 ++ CUDA_VISIBLE_DEVICES=0 ++ export KUBERNETES_CONTAINER_RESOURCE_GPU=1 ++ KUBERNETES_CONTAINER_RESOURCE_GPU=1 ++ export TP=1 ++ TP=1 ++ export PP=1 ++ PP=1 ++ export ETP=1 ++ ETP=1 ++ export EP=1 ++ EP=1 ++ export 'MODEL_PARALLEL_ARGS=--tensor-model-parallel-size 1 --pipeline-model-parallel-size 1 --expert-tensor-parallel-size 1 --expert-model-parallel-size 1' ++ MODEL_PARALLEL_ARGS='--tensor-model-parallel-size 1 --pipeline-model-parallel-size 1 --expert-tensor-parallel-size 1 --expert-model-parallel-size 1' ++ WORKSPACE_TMP_DIR=/tmp/mcore2hf_ws_840 ++ mkdir -p /tmp/mcore2hf_ws_840 ++ MEGATRON_CHECKPOINT_PATH=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/load_view840 ++ echo MEGATRON_CHECKPOINT_PATH=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/load_view840 +MEGATRON_CHECKPOINT_PATH=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/load_view840 ++ MANUAL_ITER= ++ CONVERT_ALL=false ++ FORCE_RECONVERT=false ++ DROP_MTP=false ++ POSITIONAL_ARGS=() ++ [[ 3 -gt 0 ]] ++ case $1 in ++ POSITIONAL_ARGS+=("$1") ++ shift ++ [[ 2 -gt 0 ]] ++ case $1 in ++ '[' -z 840 ']' ++ MANUAL_ITER=840 ++ shift 2 ++ [[ 0 -gt 0 ]] ++ set -- /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/load_view840 ++ '[' -e /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/load_view840 ']' +++ readlink -f /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/load_view840 ++ MEGATRON_CHECKPOINT_PATH=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/load_view840 ++ echo ✅MEGATRON_CHECKPOINT_PATH=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/load_view840 +✅MEGATRON_CHECKPOINT_PATH=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/load_view840 +++ basename /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/load_view840 ++ CHECKPOINT_BASENAME=load_view840 ++ [[ load_view840 =~ ^iter_([0-9]+)$ ]] ++ '[' false = true ']' ++ HF_CONFIG_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/hf-configs/YuLan-Mini-for-gdn-convert ++ echo HF_CONFIG_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/hf-configs/YuLan-Mini-for-gdn-convert +HF_CONFIG_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/hf-configs/YuLan-Mini-for-gdn-convert ++ '[' '!' -f /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/hf-configs/YuLan-Mini-for-gdn-convert/config.template.json ']' +++ mktemp -d /tmp/mcore2hf_ws_840/mcore2hf-config.XXXXXX ++ REAL_CONFIG_PATH=/tmp/mcore2hf_ws_840/mcore2hf-config.aB8A3A ++ LOAD_VIEW_DIR= ++ trap cleanup_real_config_path EXIT ++ export PYTHONPATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher: ++ PYTHONPATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher: ++ export PYTHONPATH=/tmp/mcore2hf_ws_840/mcore2hf-config.aB8A3A:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher: ++ PYTHONPATH=/tmp/mcore2hf_ws_840/mcore2hf-config.aB8A3A:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher: +++ /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/utils/parse-linear-vars-from-mcore-config.py /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/load_view840 +W1004 13:28:04.214000 698603 torch/utils/cpp_extension.py:2425] TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +W1004 13:28:04.214000 698603 torch/utils/cpp_extension.py:2425] If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'] to specific architectures. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. ++ eval 'export LINEAR_NUM_KEY_HEADS=8 +export LINEAR_NUM_VALUE_HEADS=32 +export LINEAR_ATTN_FREQ=[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1] +export FFN_HIDDEN_SIZE=4800 +export NUM_LAYERS=56 +export HIDDEN_SIZE=2048 +export NUM_ATTN_HEADS=32 +export NUM_QUERY_GROUPS=8 +export MAX_POSITION_EMBEDDINGS=131072 +export KV_CHANNELS=64 +export LINEAR_CONV_KERNEL_DIM=4 +export LINEAR_KEY_HEAD_DIM=64 +export LINEAR_VALUE_HEAD_DIM=64 +export MOE_FFN_HIDDEN_SIZE=512 +export NUM_EXPERTS=192 +export MOE_ROUTER_TOPK=4 +export MOE_ROUTER_SCORE_FUNCTION=softmax +export MOE_ROUTER_ENABLE_EXPERT_BIAS=false +export MOE_LAYER_FREQ=[0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1] +export MLP_ONLY_LAYERS=[0,1] +export MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512 +export ROPE_THETA=10000.0 +export POSITION_EMBEDDING_TYPE=none +export ATTENTION_OUTPUT_GATE=true +export MTP_NUM_LAYERS=0 +export MOE_ROUTER_SQRT_GATE=true +export GATED_NORM=true +export GATED_NORM_RANK=16 +export GATED_NORM_GATE_SCALE=1.0 +export GATED_NORM_UP_BIAS=true +export FINAL_GATED_NORM=true +export ACTIVATION_FUNC_CLAMP_VALUE=7.0 +export ACTIVATION_FUNC_CLAMP_MODE=soft +export FINAL_LOGITS_CLAMP_VALUE=30.0 +export MOE_ROUTER_DTYPE=fp32 +export TORCH_DTYPE=bfloat16 +export VOCAB_SIZE=99072' +++ export LINEAR_NUM_KEY_HEADS=8 +++ LINEAR_NUM_KEY_HEADS=8 +++ export LINEAR_NUM_VALUE_HEADS=32 +++ LINEAR_NUM_VALUE_HEADS=32 +++ export 'LINEAR_ATTN_FREQ=[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1]' +++ LINEAR_ATTN_FREQ='[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1]' +++ export FFN_HIDDEN_SIZE=4800 +++ FFN_HIDDEN_SIZE=4800 +++ export NUM_LAYERS=56 +++ NUM_LAYERS=56 +++ export HIDDEN_SIZE=2048 +++ HIDDEN_SIZE=2048 +++ export NUM_ATTN_HEADS=32 +++ NUM_ATTN_HEADS=32 +++ export NUM_QUERY_GROUPS=8 +++ NUM_QUERY_GROUPS=8 +++ export MAX_POSITION_EMBEDDINGS=131072 +++ MAX_POSITION_EMBEDDINGS=131072 +++ export KV_CHANNELS=64 +++ KV_CHANNELS=64 +++ export LINEAR_CONV_KERNEL_DIM=4 +++ LINEAR_CONV_KERNEL_DIM=4 +++ export LINEAR_KEY_HEAD_DIM=64 +++ LINEAR_KEY_HEAD_DIM=64 +++ export LINEAR_VALUE_HEAD_DIM=64 +++ LINEAR_VALUE_HEAD_DIM=64 +++ export MOE_FFN_HIDDEN_SIZE=512 +++ MOE_FFN_HIDDEN_SIZE=512 +++ export NUM_EXPERTS=192 +++ NUM_EXPERTS=192 +++ export MOE_ROUTER_TOPK=4 +++ MOE_ROUTER_TOPK=4 +++ export MOE_ROUTER_SCORE_FUNCTION=softmax +++ MOE_ROUTER_SCORE_FUNCTION=softmax +++ export MOE_ROUTER_ENABLE_EXPERT_BIAS=false +++ MOE_ROUTER_ENABLE_EXPERT_BIAS=false +++ export 'MOE_LAYER_FREQ=[0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]' +++ MOE_LAYER_FREQ='[0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]' +++ export 'MLP_ONLY_LAYERS=[0,1]' +++ MLP_ONLY_LAYERS='[0,1]' +++ export MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512 +++ MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512 +++ export ROPE_THETA=10000.0 +++ ROPE_THETA=10000.0 +++ export POSITION_EMBEDDING_TYPE=none +++ POSITION_EMBEDDING_TYPE=none +++ export ATTENTION_OUTPUT_GATE=true +++ ATTENTION_OUTPUT_GATE=true +++ export MTP_NUM_LAYERS=0 +++ MTP_NUM_LAYERS=0 +++ export MOE_ROUTER_SQRT_GATE=true +++ MOE_ROUTER_SQRT_GATE=true +++ export GATED_NORM=true +++ GATED_NORM=true +++ export GATED_NORM_RANK=16 +++ GATED_NORM_RANK=16 +++ export GATED_NORM_GATE_SCALE=1.0 +++ GATED_NORM_GATE_SCALE=1.0 +++ export GATED_NORM_UP_BIAS=true +++ GATED_NORM_UP_BIAS=true +++ export FINAL_GATED_NORM=true +++ FINAL_GATED_NORM=true +++ export ACTIVATION_FUNC_CLAMP_VALUE=7.0 +++ ACTIVATION_FUNC_CLAMP_VALUE=7.0 +++ export ACTIVATION_FUNC_CLAMP_MODE=soft +++ ACTIVATION_FUNC_CLAMP_MODE=soft +++ export FINAL_LOGITS_CLAMP_VALUE=30.0 +++ FINAL_LOGITS_CLAMP_VALUE=30.0 +++ export MOE_ROUTER_DTYPE=fp32 +++ MOE_ROUTER_DTYPE=fp32 +++ export TORCH_DTYPE=bfloat16 +++ TORCH_DTYPE=bfloat16 +++ export VOCAB_SIZE=99072 +++ VOCAB_SIZE=99072 ++ echo LINEAR_NUM_KEY_HEADS=8 +LINEAR_NUM_KEY_HEADS=8 ++ echo LINEAR_NUM_VALUE_HEADS=32 +LINEAR_NUM_VALUE_HEADS=32 ++ echo 'LINEAR_ATTN_FREQ=[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1]' +LINEAR_ATTN_FREQ=[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1] ++ echo 'NUM_LAYERS=56 HIDDEN_SIZE=2048' +NUM_LAYERS=56 HIDDEN_SIZE=2048 ++ echo 'NUM_ATTN_HEADS=32 NUM_QUERY_GROUPS=8' +NUM_ATTN_HEADS=32 NUM_QUERY_GROUPS=8 ++ echo 'MAX_POSITION_EMBEDDINGS=131072 KV_CHANNELS=64' +MAX_POSITION_EMBEDDINGS=131072 KV_CHANNELS=64 ++ echo 'FFN_HIDDEN_SIZE=4800 MOE_FFN_HIDDEN_SIZE=512' +FFN_HIDDEN_SIZE=4800 MOE_FFN_HIDDEN_SIZE=512 ++ echo 'NUM_EXPERTS=192 MOE_ROUTER_TOPK=4' +NUM_EXPERTS=192 MOE_ROUTER_TOPK=4 ++ echo MOE_ROUTER_SCORE_FUNCTION=softmax +MOE_ROUTER_SCORE_FUNCTION=softmax ++ echo 'MLP_ONLY_LAYERS=[0,1]' +MLP_ONLY_LAYERS=[0,1] ++ echo 'MOE_LAYER_FREQ=[0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]' +MOE_LAYER_FREQ=[0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1] ++ echo MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512 +MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512 ++ echo 'LINEAR_CONV_KERNEL_DIM=4 LINEAR_KEY_HEAD_DIM=64 LINEAR_VALUE_HEAD_DIM=64' +LINEAR_CONV_KERNEL_DIM=4 LINEAR_KEY_HEAD_DIM=64 LINEAR_VALUE_HEAD_DIM=64 ++ echo ATTENTION_OUTPUT_GATE=true +ATTENTION_OUTPUT_GATE=true ++ echo ROPE_THETA=10000.0 +ROPE_THETA=10000.0 ++ echo POSITION_EMBEDDING_TYPE=none +POSITION_EMBEDDING_TYPE=none ++ echo MOE_ROUTER_SQRT_GATE=true +MOE_ROUTER_SQRT_GATE=true +++ /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/utils/parse-mom-vars-from-mcore-config.py /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/load_view840 +W1004 13:28:19.075000 699032 torch/utils/cpp_extension.py:2425] TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +W1004 13:28:19.075000 699032 torch/utils/cpp_extension.py:2425] If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'] to specific architectures. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. ++ eval 'export MOM_ENABLED=false +export MOM_DO_MASK_SCALE=true +export MOM_TARGET_FFN_HIDDEN_SIZE=512' +++ export MOM_ENABLED=false +++ MOM_ENABLED=false +++ export MOM_DO_MASK_SCALE=true +++ MOM_DO_MASK_SCALE=true +++ export MOM_TARGET_FFN_HIDDEN_SIZE=512 +++ MOM_TARGET_FFN_HIDDEN_SIZE=512 ++ echo 'MOM_ENABLED=false MOM_DO_MASK_SCALE=true MOM_TARGET_FFN_HIDDEN_SIZE=512' +MOM_ENABLED=false MOM_DO_MASK_SCALE=true MOM_TARGET_FFN_HIDDEN_SIZE=512 ++ '[' false = true ']' ++ '[' 192 -gt 0 ']' ++ '[' 512 -le 0 ']' ++ '[' 10000.0 = 0.0 ']' +++ /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python - ++ ROPE_THETA=10000 ++ '[' -z 131072 ']' ++ '[' 131072 = 0 ']' ++ FFN_TOKEN_SHIFT= ++ FFN_INTERMEDIATE_TOKEN_SHIFT= ++ ATTN_TOKEN_SHIFT= ++ ATTN_Q_TOKEN_SHIFT= ++ ATTN_K_TOKEN_SHIFT= ++ ATTN_V_TOKEN_SHIFT= ++ TOKEN_SHIFT_CONV_SIZE=4 ++ TOKEN_SHIFT_CONV_INIT=default ++ export FFN_TOKEN_SHIFT FFN_INTERMEDIATE_TOKEN_SHIFT ATTN_TOKEN_SHIFT ++ export ATTN_Q_TOKEN_SHIFT ATTN_K_TOKEN_SHIFT ATTN_V_TOKEN_SHIFT ++ export TOKEN_SHIFT_CONV_SIZE TOKEN_SHIFT_CONV_INIT ++ export ROPE_THETA MAX_POSITION_EMBEDDINGS POSITION_EMBEDDING_TYPE ATTENTION_OUTPUT_GATE MOE_ROUTER_SQRT_GATE ++ ATTENTION_OUTPUT_GATE=true ++ /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/utils/render_gdn_hf_config.py --checkpoint-path /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/load_view840 --template-path /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/hf-configs/YuLan-Mini-for-gdn-convert --output-dir /tmp/mcore2hf_ws_840/mcore2hf-config.aB8A3A +W1004 13:28:32.422000 699417 torch/utils/cpp_extension.py:2425] TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +W1004 13:28:32.422000 699417 torch/utils/cpp_extension.py:2425] If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'] to specific architectures. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. ++ echo '✅config.json: /tmp/mcore2hf_ws_840/mcore2hf-config.aB8A3A/config.json' +✅config.json: /tmp/mcore2hf_ws_840/mcore2hf-config.aB8A3A/config.json ++ /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python - /tmp/mcore2hf_ws_840/mcore2hf-config.aB8A3A/config.json +{ + "hidden_size": 2048, + "intermediate_size": 4800, + "num_hidden_layers": 56, + "num_attention_heads": 32, + "num_key_value_heads": 8, + "head_dim": 64, + "max_position_embeddings": 131072, + "rope_theta": 10000, + "attn_position_embedding_type": "nope", + "rnn_position_embedding_type": "nope", + "linear_num_key_heads": 8, + "linear_num_value_heads": 32, + "linear_key_head_dim": 64, + "linear_value_head_dim": 64, + "num_experts": 192, + "num_experts_per_tok": 4, + "attn_output_gate": true +} ++ sleep 5 ++ MODEL_SIZE=2.6b ++ LOAD_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/load_view840 ++ MG2HF=true ++ USE_CUDA=false ++ PR=bf16 ++ HF_DIR=/tmp/mcore2hf_ws_840/mcore2hf-config.aB8A3A ++ export DROP_MTP ++ PREPARED_LOAD_DIR= ++ prepare_load_dir_for_iter /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/load_view840 840 ++ local SOURCE_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/load_view840 ++ local ITER_NUM=840 ++ '[' -z 840 ']' +++ printf %07d 840 ++ local ITER_BASENAME=iter_0000840 ++ local SOURCE_ITER_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/load_view840/iter_0000840 ++ '[' '!' -d /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/load_view840/iter_0000840 ']' ++ local LATEST_VALUE= ++ '[' -f /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/load_view840/latest_checkpointed_iteration.txt ']' +++ cat /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/load_view840/latest_checkpointed_iteration.txt ++ LATEST_VALUE=840 ++ '[' 840 = 840 ']' ++ PREPARED_LOAD_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/load_view840 ++ return 0 ++ LOAD_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/load_view840 ++ echo LOAD_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/load_view840 +LOAD_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/load_view840 ++ '[' false = true ']' ++ '[' false = true ']' ++ '[' '!' -f /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/load_view840/latest_checkpointed_iteration.txt ']' +++ cat /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/load_view840/latest_checkpointed_iteration.txt ++ LATEST_CHECKPOINT_ITERATION=840 ++ echo -e '\033[32mLatest checkpoint iteration: 840\033[0m\n' +Latest checkpoint iteration: 840 + ++ I=840 ++ [[ 840 =~ ^[0-9]+$ ]] +++ printf %07d 840 ++ ITER_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/load_view840/iter_0000840 ++ '[' '!' -f /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/load_view840/iter_0000840/common.pt ']' ++ SAVE_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/load_view840/iter_840-hf ++ '[' -n /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a ']' ++ SAVE_DIR=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/iter_840-hf ++ '[' -n '' ']' ++ mkdir -p /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/iter_840-hf ++ bash ./scripts/mcore_gdn_moe/convert.sh 2.6b /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/load_view840 /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/iter_840-hf true false bf16 /tmp/mcore2hf_ws_840/mcore2hf-config.aB8A3A 840 ++ tee /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/iter_840-hf/hf2mcore.log +✅Enable attention output gate +\[\e[33mWarning adding --moe-shared-expert-gate for YuLan model. If you are not YuLan, please check this flag.\e[0m\] +torchrun --nproc_per_node 1 --nnodes 1 --node_rank 0 --master_addr localhost --master_port 42639 /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/scripts/distributed_checkpoints_convertor/impl/convert.py --tokenizer-type HuggingFaceTokenizer --tokenizer-model /tmp/mcore2hf_ws_840/mcore2hf-config.aB8A3A --hf-dir /tmp/mcore2hf_ws_840/mcore2hf-config.aB8A3A --mcore2hf --bf16 --normalization RMSNorm --swiglu --disable-bias-linear --seq-length 1 --max-position-embeddings 131072 --attention-backend auto --position-embedding-type none --kv-channels 64 --group-query-attention --add-qkv-bias --num-layers 56 --hidden-size 2048 --ffn-hidden-size 4800 --num-attention-heads 32 --rotary-base 10000 --rotary-percent 1.00 --num-query-groups 8 --normalization RMSNorm --norm-epsilon 1e-6 --untie-embeddings-and-output-weights --attention-output-gate --linear-attention-type gated_delta_net --linear-attention-freq [1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1] --linear-conv-kernel-dim 4 --linear-key-head-dim 64 --linear-value-head-dim 64 --linear-num-key-heads 8 --linear-num-value-heads 32 --num-experts 192 --moe-router-score-function softmax --moe-token-dispatcher-type alltoall --moe-router-topk 4 --moe-ffn-hidden-size 512 --moe-layer-freq [0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1] --moe-router-dtype fp32 --moe-shared-expert-intermediate-size 512 --moe-shared-expert-gate --moe-router-score-function softmax --moe-router-sqrt-gate --micro-batch-size 1 --global-batch-size 1024 --train-iters 500000 --weight-decay 0.1 --adam-beta1 0.9 --adam-beta2 0.95 --init-method-std 0.006 --clip-grad 1.0 --lr 2.0e-5 --lr-decay-style cosine --min-lr 6.0e-6 --lr-warmup-fraction .001 --lr-decay-iters 430000 --no-gradient-accumulation-fusion --bf16 --tensor-model-parallel-size 1 --pipeline-model-parallel-size 1 --expert-tensor-parallel-size 1 --expert-model-parallel-size 1 --log-interval 100 --save-interval 10000 --eval-interval 1000 --eval-iters 10 --model-type GPT --load-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/load_view840 --save-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/iter_840-hf --dist-ckpt-optim-fully-reshardable --skip-train --use-cpu-initialization --no-load-optim --no-load-rng --logging-level 1 --attention-backend auto --debug --max-shard-size 20GB --synchronizer mcore_gdn_moe --pretrain-script mcore_gdn_moe.model_provider --ckpt-step 840 --gdn-split-in-proj --gated-norm --gated-norm-lora-rank 16 --distributed-backend gloo +W1004 13:28:52.780000 699985 torch/utils/cpp_extension.py:2425] TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +W1004 13:28:52.780000 699985 torch/utils/cpp_extension.py:2425] If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'] to specific architectures. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +Warning: Pai-Megatron-Patch arguments not available, some arguments may not be recognized +using world size: 1, data-parallel size: 1, context-parallel size: 1, hierarchical context-parallel sizes: None, tensor-model-parallel size: 1, pipeline-model-parallel size: 1 +Number of virtual stages per pipeline stage: None +accumulate and all-reduce gradients in fp32 for bfloat16 data type. +using torch.bfloat16 for parameters ... +------------------------ arguments ------------------------ + account_for_embedding_in_pipeline_split ......... False + account_for_loss_in_pipeline_split .............. False + accumulate_allreduce_grads_in_fp32 .............. True + activation_func_clamp_mode ...................... hard + activation_func_clamp_value ..................... None + adam_beta1 ...................................... 0.9 + adam_beta2 ...................................... 0.95 + adam_bias_correction ............................ True + adam_eps ........................................ 1e-08 + adamh_eps ....................................... 1e-08 + adamh_radius .................................... None + adamh_scope ..................................... lm_head + adamw_lr_mup_scaler ............................. False + add_bias_linear ................................. False + add_position_embedding .......................... True + add_qkv_bias .................................... True + adlr_autoresume ................................. False + adlr_autoresume_interval ........................ 1000 + align_grad_reduce ............................... True + align_param_gather .............................. False + allow_ambiguous_pad_tokens ...................... False + app_tag_run_name ................................ None + app_tag_run_version ............................. 0.0.0 + apply_layernorm_1p .............................. False + apply_query_key_layer_scaling ................... False + apply_residual_connection_post_layernorm ........ False + apply_rope_fusion ............................... False + async_save ...................................... None + async_tensor_model_parallel_allreduce ........... True + attention_backend ............................... AttnBackend.auto + attention_dropout ............................... 0.1 + attention_output_gate ........................... True + attention_output_gate_upcycling ................. False + attention_softmax_in_fp32 ....................... False + attn_k_token_shift .............................. None + attn_output_gate ................................ None + attn_output_gate_rand_init ...................... False + attn_q_token_shift .............................. None + attn_token_shift ................................ None + attn_v_token_shift .............................. None + auto_detect_ckpt_format ......................... False + auto_generate_cu_seqlens ........................ False + auto_model ...................................... AutoModelForCausalLM + barrier_with_L1_time ............................ True + benchmark_eval .................................. False + benchmark_global_batch .......................... None + benchmark_interval .............................. None + benchmark_micro_batch ........................... None + benchmark_sequence_length ....................... None + benchmark_tasks ................................. None + bert_binary_head ................................ True + bert_embedder_type .............................. megatron + bert_load ....................................... None + bf16 ............................................ True + bias_dropout_fusion ............................. True + bias_gelu_fusion ................................ False + bias_swiglu_fusion .............................. True + biencoder_projection_dim ........................ 0 + biencoder_shared_query_context_model ............ False + block_data_path ................................. None + cache_mla_latents ............................... False + calc_ft_timeouts ................................ False + calculate_per_token_loss ........................ False + check_for_large_grads ........................... False + check_for_nan_in_loss_and_grad .................. True + check_for_spiky_loss ............................ False + check_weight_hash_across_dp_replicas_interval ... None + ckpt_assume_constant_structure .................. False + ckpt_convert_format ............................. None + ckpt_convert_save ............................... None + ckpt_convert_update_legacy_dist_opt_format ...... False + ckpt_format ..................................... torch_dist + ckpt_fully_parallel_load ........................ False + ckpt_fully_parallel_save ........................ True + ckpt_fully_parallel_save_deprecated ............. False + ckpt_step ....................................... 840 + classes_fraction ................................ 1.0 + clip_grad ....................................... 1.0 + clone_scatter_output_in_embedding ............... True + config_logger_dir ............................... + consumed_train_samples .......................... 0 + consumed_valid_samples .......................... 0 + context_parallel_size ........................... 1 + continual_pretrain .............................. False + continual_pretrain_preserve_opt_param_scheduler . False + corrected_weight_decay .......................... False + cp_comm_type .................................... ['p2p'] + create_attention_mask_in_dataloader ............. True + cross_entropy_fusion_impl ....................... native + cross_entropy_loss_fusion ....................... False + cuda_graph_impl ................................. none + cuda_graph_scope ................................ [] + cuda_graph_warmup_steps ......................... 3 + data_args_path .................................. None + data_cache_path ................................. None + data_parallel_random_init ....................... False + data_parallel_sharding_strategy ................. no_shard + data_parallel_size .............................. 1 + data_path ....................................... None + data_per_class_fraction ......................... 1.0 + data_sharding ................................... True + dataloader_defer_npy_index_mmap ................. False + dataloader_fast_cache_load ...................... False + dataloader_type ................................. single + dc_shortconv .................................... False + dc_shortconv_optimizer_state .................... conservative + dc_shortconv_upcycling .......................... False + ddp_average_in_collective ....................... False + ddp_bucket_size ................................. None + ddp_num_buckets ................................. None + ddp_pad_buckets_for_high_nccl_busbw ............. False + debug ........................................... True + decode_only_cuda_graphs ......................... False + decoder_first_pipeline_num_layers ............... None + decoder_last_pipeline_num_layers ................ None + decoder_num_layers .............................. None + decoder_seq_length .............................. None + decoupled_lr .................................... None + decoupled_min_lr ................................ None + decrease_batch_size_if_needed ................... False + defer_embedding_wgrad_compute ................... False + defer_grad_checks ............................... False + delay_wgrad_compute ............................. False + deprecated_use_mcore_models ..................... False + deterministic_mode .............................. False + dino_bottleneck_size ............................ 256 + dino_freeze_last_layer .......................... 1 + dino_head_hidden_size ........................... 2048 + dino_local_crops_number ......................... 10 + dino_local_img_size ............................. 96 + dino_norm_last_layer ............................ False + dino_teacher_temp ............................... 0.07 + dino_warmup_teacher_temp ........................ 0.04 + dino_warmup_teacher_temp_epochs ................. 30 + disable_attn_output_gate ........................ False + disable_bf16_reduced_precision_matmul ........... False + disable_chunked_prefill ......................... False + disable_explicit_attention_mask ................. False + disable_mamba_mem_eff_path ...................... False + disable_straggler_on_startup .................... False + disable_symmetric_registration .................. False + dist_ckpt_format_deprecated ..................... None + dist_ckpt_optim_fully_reshardable ............... True + dist_ckpt_save_pre_mcore_014 .................... False + dist_ckpt_strictness ............................ assume_ok_unexpected + distrib_optim_fully_reshardable_mem_efficient ... False + distribute_saved_activations .................... False + distributed_backend ............................. gloo + distributed_timeout_minutes ..................... 10 + distributed_timeout_seconds_after_init .......... None + document_packing_algorithm ...................... ogbfd + document_packing_dp_size ........................ None + drop_mtp ........................................ False + dryrun .......................................... False + dump_param_to_param_group_map ................... None + dynamic_context_parallel ........................ False + dynamic_context_parallel_sizes .................. None + dynamic_cp_candidate_sizes ...................... None + dynamic_cp_gdn_policy ........................... None + dynamic_cp_moe_policy ........................... None + dynamic_cp_on_error ............................. error + dynamic_cp_scheduler ............................ default_dynamic_cp + dynamic_cp_wave_coexecution ..................... False + dynamic_cp_workspace_max_tokens ................. None + emb_deviation_loss_coeff ........................ 0 + emb_deviation_type .............................. None + embedding_init_method_std ....................... None + embedding_path .................................. None + empty_unused_memory_level ....................... 0 + enable_cuda_graph ............................... False + enable_debug_logging ............................ False + enable_experimental ............................. False + enable_ft_package ............................... False + enable_full_sharding_in_hsdp .................... False + enable_gloo_process_groups ...................... True + enable_msc ...................................... True + enable_one_logger ............................... True + encoder_num_layers .............................. 56 + encoder_seq_length .............................. 1 + end_weight_decay ................................ 0.1 + eod_mask_loss ................................... False + error_injection_rate ............................ 0 + error_injection_type ............................ transient_error + eval_interval ................................... 1000 + eval_iters ...................................... 10 + evidence_data_path .............................. None + exit_duration_in_mins ........................... None + exit_interval ................................... None + exit_on_missing_checkpoint ...................... False + exit_signal_handler ............................. False + exp_avg_dtype ................................... torch.float32 + exp_avg_sq_dtype ................................ torch.float32 + expert_model_parallel_size ...................... 1 + expert_tensor_parallel_size ..................... 1 + export_force_local_attention .................... False + export_kd_cfg ................................... None + export_kd_teacher_ckpt_format ................... None + export_kd_teacher_load .......................... None + export_kv_cache_quant ........................... False + export_legacy_megatron .......................... False + export_model_type ............................... GPTModel + export_moe_apply_probs_on_input ................. False + export_offline_model ............................ False + export_qk_l2_norm ............................... False + export_quant_cfg ................................ None + export_real_quant_cfg ........................... None + export_te_mcore_model ........................... False + external_cuda_graph ............................. False + fast_thd_a2a .................................... False + ffn_hidden_size ................................. 4800 + ffn_intermediate_token_shift .................... None + ffn_token_shift ................................. None + final_logits_clamp_value ........................ None + fine_grained_activation_offloading .............. False + finetune ........................................ False + finetune_data_split ............................. train + finetune_hf_dataset ............................. None + first_last_layers_bf16 .......................... False + flash_decode .................................... False + fp16 ............................................ False + fp16_lm_cross_entropy ........................... False + fp32_residual_connection ........................ False + fp4 ............................................. None + fp4_param ....................................... False + fp4_recipe ...................................... nvfp4 + fp8 ............................................. None + fp8_amax_compute_algo ........................... most_recent + fp8_amax_history_len ............................ 1 + fp8_interval .................................... 1 + fp8_margin ...................................... 0 + fp8_param_gather ................................ False + fp8_recipe ...................................... delayed + fp8_wgrad ....................................... True + freeze_layernorm_weight ......................... False + freeze_non_mamba ................................ False + fsdp_double_buffer .............................. False + full_validation ................................. False + gated_norm ...................................... True + gated_norm_lora_rank ............................ 16 + gated_norm_upcycling ............................ False + gdn_alpha_weight_scale .......................... 1.0 + gdn_alpha_weight_scale_mode ..................... matrix + gdn_backend ..................................... fla + gdn_fused_split_forward ......................... False + gdn_gated_norm_backend .......................... torch + gdn_lr_mult_alpha ............................... 1.0 + gdn_lr_mult_beta ................................ 1.0 + gdn_lr_mult_k ................................... 1.0 + gdn_lr_mult_q ................................... 1.0 + gdn_lr_mult_v ................................... 1.0 + gdn_postinit_alog_shift ......................... 0.0 + gdn_postinit_alpha_scale ........................ 1.0 + gdn_postinit_key_scale .......................... 1.0 + gdn_pre_gdr_fusion .............................. False + gdn_pre_gdr_fusion_deterministic_weight_gradient False + gdn_pre_gdr_fusion_high_precision_backward ...... False + gdn_shortconv_separate_gate ..................... False + gdn_shortconv_separate_gate_upcycling ........... False + gdn_split_in_proj ............................... True + geglu ........................................... False + global_batch_size ............................... 1024 + glu_linear_offset ............................... 0.0 + grad_reduce_in_bf16 ............................. False + gradient_accumulation_fusion .................... False + gradient_reduce_div_fusion ...................... True + group_query_attention ........................... True + grpo_clamp_eps_lower ............................ 0.01 + grpo_clamp_eps_upper ............................ 0.01 + grpo_default_temperature ........................ 1.0 + grpo_default_top_p .............................. 0 + grpo_entropy_term_weight ........................ 0.0 + grpo_filter_groups_with_same_reward ............. False + grpo_group_size ................................. 2 + grpo_iterations ................................. 2 + grpo_kl_beta .................................... 0.001 + grpo_prompts_per_step ........................... 32 + head_lr_mult .................................... 1.0 + heterogeneous_layers_config_encoded_json ........ None + heterogeneous_layers_config_path ................ None + hf_dir .......................................... /tmp/mcore2hf_ws_840/mcore2hf-config.aB8A3A + hidden_dropout .................................. 0.1 + hidden_size ..................................... 2048 + hidden_states_shrink ............................ 1.0 + hierarchical_context_parallel_sizes ............. None + high_priority_stream_groups ..................... [] + hybrid_adamh .................................... False + hybrid_attention_ratio .......................... 0.0 + hybrid_context_parallel ......................... False + hybrid_mlp_ratio ................................ 0.0 + hybrid_override_pattern ......................... None + hysteresis ...................................... 2 + ict_head_size ................................... None + ict_load ........................................ None + img_h ........................................... 224 + img_w ........................................... 224 + increase_log_level_interval ..................... 1000 + increase_log_level_iters ........................ 5 + indexer_batch_size .............................. 128 + indexer_log_interval ............................ 1000 + inference_batch_times_seqlen_threshold .......... -1 + inference_dynamic_batching ...................... False + inference_dynamic_batching_block_size ........... 256 + inference_dynamic_batching_buffer_guaranteed_fraction 0.2 + inference_dynamic_batching_buffer_overflow_factor None + inference_dynamic_batching_buffer_size_gb ....... 40.0 + inference_dynamic_batching_max_requests_override None + inference_dynamic_batching_max_tokens_override .. None + inference_dynamic_batching_num_cuda_graphs ...... 16 + inference_dynamic_batching_track_paused_request_events False + inference_dynamic_batching_unified_memory_level . 0 + inference_max_batch_size ........................ 8 + inference_max_seq_length ........................ 2560 + inference_rng_tracker ........................... False + init_method_std ................................. 0.006 + init_method_xavier_uniform ...................... False + init_model_with_meta_device ..................... False + initial_loss_scale .............................. 4294967296 + inprocess_active_world_size ..................... 1 + inprocess_barrier_timeout ....................... 120 + inprocess_completion_timeout .................... 120 + inprocess_empty_cuda_cache ...................... False + inprocess_granularity ........................... node + inprocess_hard_timeout .......................... 90 + inprocess_heartbeat_interval .................... 30 + inprocess_heartbeat_timeout ..................... 60 + inprocess_last_call_wait ........................ 1 + inprocess_max_iterations ........................ None + inprocess_monitor_process_interval .............. 1.0 + inprocess_monitor_thread_interval ............... 1.0 + inprocess_progress_watchdog_interval ............ 1.0 + inprocess_restart ............................... False + inprocess_soft_timeout .......................... 60 + inprocess_termination_grace_time ................ 1 + is_hybrid_model ................................. False + iter_per_epoch .................................. 1250 + iterations_to_skip .............................. [] + keep_fp8_transpose_cache ........................ False + kitchen_config_file ............................. None + kitchen_recipe_number ........................... None + kv_channels ..................................... 64 + kv_lora_rank .................................... 32 + langrl_env_config ............................... None + langrl_external_server .......................... False + langrl_inference_server_conversation_template ... None + langrl_inference_server_type .................... inplace_megatron + lazy_mpu_init ................................... None + legacy_tokenizer ................................ False + linear_attention_freq ........................... [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 0, 0, 1, 1, 1, 1, 1, 1] + linear_attention_type ........................... gated_delta_net + linear_conv_kernel_dim .......................... 4 + linear_cross_entropy_accumulate_grad_in_fp32 .... True + linear_cross_entropy_num_chunks ................. 8 + linear_key_head_dim ............................. 64 + linear_num_key_heads ............................ 8 + linear_num_value_heads .......................... 32 + linear_rope_mode ................................ none + linear_value_head_dim ........................... 64 + lm_head_alpha ................................... 1.0 + lm_loss_normalization ........................... legacy + load ............................................ None + load_complemental_dataset ....................... None + load_dir ........................................ /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/load_view840 + load_main_params_from_ckpt ...................... None + local_rank ...................................... 0 + log_attn_logits ................................. False + log_energy ...................................... False + log_hidden_states ............................... [] + log_interval .................................... 100 + log_logits_z_loss ............................... False + log_loss_scale_to_tensorboard ................... True + log_memory_to_tensorboard ....................... False + log_metrics_interval ............................ None + log_moe_activations ............................. False + log_moe_params .................................. False + log_moe_router_metrics .......................... False + log_num_zeros_in_grad ........................... False + log_padding_stats ............................... False + log_params ...................................... [] + log_params_norm ................................. False + log_per_layer_grad_rms .......................... False + log_per_module_grad_rms ......................... False + log_per_module_update_rms ....................... False + log_progress .................................... False + log_spike_diagnostics ........................... False + log_straggler ................................... False + log_throughput .................................. False + log_timers_to_tensorboard ....................... False + log_update_rms .................................. False + log_validation_ppl_to_tensorboard ............... False + log_world_size_to_tensorboard ................... False + logging_level ................................... 1 + loss_scale ...................................... None + loss_scale_window ............................... 1000 + lr .............................................. 2e-05 + lr_decay_iters .................................. 430000 + lr_decay_samples ................................ None + lr_decay_style .................................. cosine + lr_warmup_fraction .............................. 0.001 + lr_warmup_init .................................. 0.0 + lr_warmup_iters ................................. 0 + lr_warmup_samples ............................... 0 + lr_wsd_decay_iters .............................. None + lr_wsd_decay_samples ............................ None + lr_wsd_decay_style .............................. exponential + main_grads_dtype ................................ torch.float32 + main_params_dtype ............................... torch.float32 + make_vocab_size_divisible_by .................... 128 + mamba_disable_cp ................................ False + mamba_expand .................................... 2 + mamba_head_dim .................................. 64 + mamba_num_groups ................................ 8 + mamba_num_heads ................................. None + mamba_state_dim ................................. 128 + manual_gc ....................................... False + manual_gc_eval .................................. True + manual_gc_interval .............................. 0 + mask_factor ..................................... 1.0 + mask_prob ....................................... 0.15 + mask_type ....................................... random + masked_softmax_fusion ........................... True + max_position_embeddings ......................... 131072 + max_seqlen_per_cp_rank .......................... None + max_shard_size .................................. 20GB + max_tokens_to_oom ............................... 12000 + mcore2hf ........................................ True + memory_snapshot_path ............................ None + merge_file ...................................... None + micro_batch_size ................................ 1 + microbatch_group_size_per_vp_stage .............. None + mid_level_dataset_surplus ....................... 0.005 + min_loss_scale .................................. 1.0 + min_lr .......................................... 6e-06 + min_offloaded_tensor_size ....................... 1048576 + mlp_chunks_for_prefill .......................... 1 + mmap_bin_files .................................. True + mock_data ....................................... False + model_type ...................................... GPT + modelopt_enabled ................................ False + moe_apply_probs_on_input ........................ False + moe_aux_loss_coeff .............................. 0.0 + moe_deepep_num_sms .............................. 20 + moe_enable_deepep ............................... False + moe_expert_capacity_factor ...................... None + moe_extended_tp ................................. False + moe_ffn_hidden_size ............................. 512 + moe_flex_dispatcher_backend ..................... deepep + moe_gaux_overlap ................................ False + moe_grouped_gemm ................................ False + moe_hybridep_num_sms ............................ 16 + moe_input_jitter_eps ............................ None + moe_layer_freq .................................. [0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] + moe_layer_recompute ............................. False + moe_monitor_layers .............................. + moe_monitor_log_interval ........................ None + moe_pad_expert_input_to_capacity ................ False + moe_pad_experts_for_cuda_graph_inference ........ False + moe_per_layer_logging ........................... False + moe_permute_backend ............................. auto + moe_permute_fusion .............................. False + moe_router_aux_loss_fusion ...................... auto + moe_router_bias_update_method ................... sign + moe_router_bias_update_rate ..................... 0.001 + moe_router_dtype ................................ fp32 + moe_router_enable_expert_bias ................... False + moe_router_force_load_balancing ................. False + moe_router_fusion ............................... False + moe_router_gemm_backend ......................... torch + moe_router_group_topk ........................... None + moe_router_load_balancing_type .................. aux_loss + moe_router_num_groups ........................... None + moe_router_padding_for_fp8 ...................... False + moe_router_padding_for_quantization ............. False + moe_router_pre_softmax .......................... False + moe_router_score_function ....................... softmax + moe_router_sqrt_gate ............................ True + moe_router_topk ................................. 4 + moe_router_topk_scaling_factor .................. None + moe_shared_expert_gate .......................... True + moe_shared_expert_intermediate_size ............. 512 + moe_shared_expert_overlap ....................... False + moe_token_dispatcher_type ....................... alltoall + moe_token_drop_policy ........................... probs + moe_triton_dispatcher_exclude_padding_tokens .... False + moe_triton_mega_activation_sms .................. 4 + moe_triton_mega_combine_sms ..................... 32 + moe_triton_mega_compute_sms ..................... 24 + moe_triton_mega_dispatch_sms .................... 48 + moe_triton_mega_num_warps ....................... 16 + moe_triton_mega_reduce_sms ...................... 32 + moe_triton_mega_tail_sms ........................ 16 + moe_triton_shared_expert_overlap ................ False + moe_triton_workspace_expand ..................... 4.0 + moe_upcycling_granularity ....................... 1 + moe_use_legacy_grouped_gemm ..................... False + moe_use_upcycling ............................... False + moe_z_loss_coeff ................................ None + mrope_section ................................... None + mscale .......................................... 1.0 + mscale_all_dim .................................. 0.0 + mtp_linear_attention_type ....................... None + mtp_loss_scaling_factor ......................... 0.1 + mtp_num_layers .................................. None + multi_latent_attention .......................... False + multiple_validation_sets ........................ False + muon_ball_momentum .............................. 0.9 + muon_ball_msign_steps ........................... 5 + muon_ball_power_iteration_steps ................. 10 + muon_ball_qkv_split_mode ........................ component + muon_ball_radius_mode ........................... spectral_mup + muon_ball_retract_alpha ......................... 0.05 + muon_ball_retract_mode .......................... hard + muon_ball_scale_mode ............................ spectral_mup + muon_ball_split_fc1 ............................. True + muon_ball_split_moe_experts ..................... True + muon_ball_split_qkv ............................. True + muon_ball_use_nesterov .......................... True + muon_extra_scale_factor ......................... 1.0 + muon_fp32_matmul_prec ........................... medium + muon_momentum ................................... 0.9 + muon_num_ns_steps ............................... 5 + muon_qkv_split_mode ............................. component + muon_scale_mode ................................. spectral_mup + muon_scale_vectorized_mode ...................... full + muon_split_fc1 .................................. True + muon_split_moe_experts .......................... True + muon_split_qkv .................................. True + muon_tp_mode .................................... blockwise + muon_use_nesterov ............................... False + muon_vectorize .................................. [] + muon_vectorize_attn_dim ......................... hidden_size + mup_lr_mult ..................................... 1.0 + nccl_all_reduce_for_prefill ..................... False + nccl_communicator_config_path ................... None + nccl_ub ......................................... False + no_load_optim ................................... True + no_load_rng ..................................... True + no_load_scheduler ............................... None + no_persist_layer_norm ........................... False + no_rope_freq .................................... None + no_save_optim ................................... None + no_save_rng ..................................... None + no_save_step_one ................................ None + no_weight_decay_cond_type ....................... None + non_persistent_ckpt_type ........................ None + non_persistent_global_ckpt_dir .................. None + non_persistent_keep_last ........................ 1 + non_persistent_local_ckpt_algo .................. fully_parallel + non_persistent_local_ckpt_dir ................... None + non_persistent_save_interval .................... None + norm_epsilon .................................... 1e-06 + normalization ................................... RMSNorm + num_attention_heads ............................. 32 + num_channels .................................... 3 + num_classes ..................................... 1000 + num_dataset_builder_threads ..................... 1 + num_distributed_optimizer_instances ............. 1 + num_experts ..................................... 192 + num_hf_saver .................................... None + num_layers ...................................... 56 + num_layers_at_end_in_bf16 ....................... 1 + num_layers_at_start_in_bf16 ..................... 1 + num_layers_per_virtual_pipeline_stage ........... None + num_query_groups ................................ 8 + num_virtual_stages_per_pipeline_rank ............ None + num_workers ..................................... 2 + object_storage_cache_path ....................... None + offload_modules ................................. [] + one_logger_async ................................ False + one_logger_project .............................. megatron-lm + one_logger_run_name ............................. None + onnx_safe ....................................... None + openai_gelu ..................................... False + optimizer ....................................... adam + optimizer_cpu_offload ........................... False + optimizer_offload_fraction ...................... 1.0 + output_bert_embeddings .......................... False + output_logits_z_loss_coeff ...................... None + overlap_cpu_optimizer_d2h_h2d ................... False + overlap_grad_reduce ............................. False + overlap_moe_expert_parallel_comm ................ False + overlap_p2p_comm ................................ False + overlap_p2p_comm_warmup_flush ................... False + overlap_param_gather ............................ False + overlap_param_gather_with_optimizer_step ........ False + override_hf_eod_token_id ........................ None + override_opt_param_scheduler .................... False + packing_tokens_per_shard ........................ 100000000000 + padded_vocab_size ............................... None + params_dtype .................................... torch.bfloat16 + patch_dim ....................................... 16 + per_split_data_args_path ........................ None + perform_initialization .......................... True + perform_rl_step ................................. False + pin_cpu_grads ................................... True + pin_cpu_params .................................. True + pipeline_model_parallel_comm_backend ............ None + pipeline_model_parallel_layout .................. None + pipeline_model_parallel_size .................... 1 + position_embedding_type ......................... none + pp_aware_finegrained_recompute .................. False + pp_aware_finegrained_recompute_global_layers .... None + pp_aware_finegrained_recompute_pp_ranks ......... None + prefetch_factor ................................. None + pretrain_script ................................. mcore_gdn_moe.model_provider + pretrained_checkpoint ........................... None + previous_stage_seq_length ....................... None + profile ......................................... False + profile_microbatch_count ........................ 2 + profile_microbatch_dir .......................... None + profile_microbatch_iteration .................... None + profile_microbatch_start ........................ 0 + profile_ranks ................................... [0] + profile_step_end ................................ None + profile_step_start .............................. None + q_lora_rank ..................................... None + qk_head_dim ..................................... 128 + qk_l2_norm ...................................... False + qk_layernorm .................................... False + qk_pos_emb_head_dim ............................. 64 + query_in_block_prob ............................. 0.1 + quick_geglu ..................................... False + rampup_batch_size ............................... None + rank ............................................ 0 + recompute_granularity ........................... None + recompute_method ................................ None + recompute_modules ............................... None + recompute_num_layers ............................ None + record_memory_history ........................... False + relative_attention_max_distance ................. 128 + relative_attention_num_buckets .................. 32 + reparam_checkpoint .............................. None + reparam_fallback_value .......................... None + reparam_keys .................................... None + replication ..................................... False + replication_factor .............................. 2 + replication_jump ................................ None + rerun_mode ...................................... validate_results + reset_attention_mask ............................ False + reset_iteration_one_to_zero ..................... False + reset_position_ids .............................. False + result_rejected_tracker_filename ................ None + retriever_report_topk_accuracies ................ [] + retriever_score_scaling ......................... False + retriever_seq_length ............................ 256 + retro_add_retriever ............................. False + retro_attention_gate ............................ 1 + retro_cyclic_train_iters ........................ None + retro_encoder_attention_dropout ................. 0.1 + retro_encoder_hidden_dropout .................... 0.1 + retro_encoder_layers ............................ 2 + retro_num_neighbors ............................. 2 + retro_num_retrieved_chunks ...................... 2 + retro_project_dir ............................... None + retro_verify_neighbor_count ..................... True + reuse_grad_buf_for_mxfp8_param_ag ............... False + reuse_training_communicators .................... False + rl_calculate_intra_group_similarity ............. False + rl_importance_sampling_truncation_coef .......... None + rl_inference_logprobs_is_correction ............. False + rl_offload_kv_cache_during_training ............. False + rl_offload_optimizer_during_inference ........... False + rl_partial_rollouts ............................. False + rl_prompts_per_eval ............................. 32 + rl_remove_kv_cache_during_training .............. False + rl_reset_cuda_graphs ............................ False + rl_sequence_packing_algo ........................ fifo + rl_sequence_packing_bin_size .................... 8192 + rl_use_sequence_packing ......................... False + rope_scaling_factor ............................. 8.0 + rope_type ....................................... None + rotary_base ..................................... 10000 + rotary_interleaved .............................. False + rotary_percent .................................. 1.0 + rotary_scaling_factor ........................... 1.0 + rotary_seq_len_interpolation_factor ............. None + run_workload_inspector_server ................... False + runtime_audit_dir ............................... None + sample_rate ..................................... 1.0 + save ............................................ None + save_after_load ................................. False + save_dir ........................................ /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/iter_840-hf + save_interval ................................... 10000 + save_retain_interval ............................ None + scale_emb ....................................... 1.0 + scatter_gather_tensors_in_pipeline .............. True + seed ............................................ 1234 + seq_length ...................................... 1 + sequence_packing_scheduler ...................... default_dynamic_cp + sequence_parallel ............................... False + sft ............................................. False + sft_tokenizer_prompt_format ..................... nemotron-h-aligned + sgd_momentum .................................... 0.9 + sharp_enabled_group ............................. None + short_seq_prob .................................. 0.1 + skip_train ...................................... True + skipped_train_samples ........................... 0 + softmax_type .................................... vanilla + spec ............................................ None + spectra_clip .................................... 1.0 + spectra_grad_clip ............................... None + spectra_hyperball ............................... False + spectra_ns_steps ................................ 10 + spectra_scope ................................... matrices + spectral_ball_momentum .......................... 0.9 + spectral_ball_msign_steps ....................... 8 + spectral_ball_power_iteration_steps ............. 20 + spectral_ball_qkv_split_mode .................... component + spectral_ball_radius_mode ....................... spectral_mup + spectral_ball_retract_alpha ..................... 0.05 + spectral_ball_retract_mode ...................... hard + spectral_ball_scale_mode ........................ spectral_mup + spectral_ball_solver ............................ bisection + spectral_ball_solver_max_iterations ............. 20 + spectral_ball_solver_tolerance_f ................ 1e-08 + spectral_ball_split_fc1 ......................... True + spectral_ball_split_moe_experts ................. True + spectral_ball_split_qkv ......................... True + spectral_ball_use_nesterov ...................... True + spectral_mup_init ............................... False + spike_diagnostics_dir ........................... None + spike_diagnostics_end ........................... 21 + spike_diagnostics_start ......................... 2 + split ........................................... None + split_expert_init ............................... True + split_fc1_init .................................. True + split_qkv_init .................................. True + split_qkv_init_mode ............................. group + sqreglu ......................................... False + squared_relu .................................... False + start_samples ................................... None + start_weight_decay .............................. 0.1 + straggler_ctrlr_port ............................ 65535 + straggler_minmax_count .......................... 1 + strict_fsdp_dtensor_load ........................ True + suggested_communication_unit_size ............... None + swanlab_exp_name ................................ + swanlab_project ................................. + swanlab_save_dir ................................ + swanlab_workspace ............................... + swiglu .......................................... True + swin_backbone_type .............................. tiny + symmetric_ar_type ............................... None + synchronizer .................................... mcore_gdn_moe + target_ckpt_format .............................. torch_dist + te_rng_tracker .................................. False + teacher_model_config ............................ None + tensor_model_parallel_size ...................... 1 + tensorboard_dir ................................. None + tensorboard_log_interval ........................ 1 + tensorboard_queue_size .......................... 1000 + test_data_path .................................. None + test_mode ....................................... False + tiktoken_num_special_tokens ..................... 1000 + tiktoken_pattern ................................ None + tiktoken_special_tokens ......................... None + timing_log_level ................................ 0 + timing_log_option ............................... minmax + titles_data_path ................................ None + token_shift_conv_init ........................... default + token_shift_conv_size ........................... 4 + tokenizer_metadata .............................. None + tokenizer_model ................................. /tmp/mcore2hf_ws_840/mcore2hf-config.aB8A3A + tokenizer_type .................................. HuggingFaceTokenizer + torch_fsdp2_reshard_after_forward ............... True + tp_comm_bootstrap_backend ....................... nccl + tp_comm_bulk_dgrad .............................. True + tp_comm_bulk_wgrad .............................. True + tp_comm_overlap ................................. False + tp_comm_overlap_ag .............................. True + tp_comm_overlap_cfg ............................. None + tp_comm_overlap_rs .............................. True + tp_comm_overlap_rs_dgrad ........................ False + tp_comm_split_ag ................................ True + tp_comm_split_rs ................................ True + train_data_path ................................. None + train_iters ..................................... 500000 + train_samples ................................... None + train_sync_interval ............................. None + train_until_stopped ............................. False + transformer_impl ................................ transformer_engine + transformer_pipeline_model_parallel_size ........ 1 + trust_remote_code ............................... False + untie_embeddings_and_output_weights ............. True + use_checkpoint_args ............................. False + use_checkpoint_opt_param_scheduler .............. False + use_cpu_initialization .......................... True + use_dist_ckpt ................................... True + use_dist_ckpt_deprecated ........................ False + use_distributed_optimizer ....................... False + use_flash_attn .................................. False + use_fused_weighted_squared_relu ................. False + use_gpu ......................................... False + use_legacy_models ............................... False + use_megatron_fsdp ............................... False + use_mp_args_from_checkpoint_args ................ False + use_one_sent_docs ............................... False + use_persistent_ckpt_worker ...................... False + use_precision_aware_optimizer ................... False + use_pytorch_profiler ............................ False + use_ring_exchange_p2p ........................... False + use_rope_scaling ................................ False + use_rotary_position_embeddings .................. False + use_sharp ....................................... False + use_te_activation_func .......................... False + use_tokenizer_model_from_checkpoint_args ........ True + use_torch_fsdp2 ................................. False + use_torch_optimizer_for_cpu_offload ............. False + use_tp_pp_dp_mapping ............................ False + v_head_dim ...................................... 128 + valid_data_path ................................. None + variable_seq_lengths ............................ False + virtual_pipeline_model_parallel_size ............ None + vision_backbone_type ............................ vit + vision_pretraining .............................. False + vision_pretraining_type ......................... classify + vocab_extra_ids ................................. 0 + vocab_file ...................................... None + vocab_size ...................................... None + wandb_entity .................................... + wandb_exp_name .................................. + wandb_project ................................... + wandb_save_dir .................................. + weight_decay .................................... 0.1 + weight_decay_incr_style ......................... constant + wgrad_deferral_limit ............................ 0 + window_attn_skip_freq ........................... None + window_size ..................................... None + word_embedding_dropout_prob ..................... 0.0 + world_size ...................................... 1 + yaml_cfg ........................................ None +-------------------- end of arguments --------------------- +INFO:megatron.core.num_microbatches_calculator:setting number of microbatches to constant 1024 +> building HuggingFaceTokenizer tokenizer ... + > padded vocab (size: 99000) with 72 dummy tokens (new size: 99072) +WARNING: one_logger package is required to enable e2e metrics tracking. please go to https://confluence.nvidia.com/display/MLWFO/Package+Repositories for details to install it +INFO:megatron.training.initialize:Setting logging level to 1 +WARNING:megatron.core.rerun_state_machine:RerunStateMachine initialized in mode validate_results +> initializing torch distributed ... +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +> initialized tensor model parallel with size 1 +> initialized pipeline model parallel with size 1 +> setting random seeds to 1234 ... +> compiling dataset index builder ... +make: Entering directory '/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/datasets' +make: Nothing to be done for 'default'. +make: Leaving directory '/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/datasets' +>>> done with dataset index builder. Compilation time: 0.134 seconds +WARNING: constraints for invoking optimized fused softmax kernel are not met. We default back to unfused kernel invocations. +> compiling and loading fused kernels ... +>>> done with compiling and loading fused kernels. Compilation time: 0.003 seconds +WORLD_SIZE: 1, RANK: 0, LOCAL_RANK: 0 +[convert] creating synchronizer: mode=mcore2hf +building GPT model ... +[GPT_LAYER_SPEC] log_hidden_states=[] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=[] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=[] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=[] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/transformers/modeling_rope_utils.py:1036: FutureWarning: `rope_config_validation` is deprecated and has been removed. Its functionality has been moved to RotaryEmbeddingConfigMixin.validate_rope method. PreTrainedConfig inherits this class, so please call self.validate_rope() instead. Also, make sure to use the new rope_parameters syntax. You can call self.standardize_rope_params() in the meantime. + warnings.warn( +[transformers] `layer_type_validation` is deprecated and will be removed in v5.20. Use `PreTrainedConfig.validate_layer_type` instead +[transformers] `layer_type_validation` is deprecated and will be removed in v5.20. Use `PreTrainedConfig.validate_layer_type` instead +[transformers] `torch_dtype` is deprecated! Use `dtype` instead! +[ERROR] `cache_position` is part of Qwen3NextModel.forward's signature, but not documented. Make sure to add it to the docstring of the function in /mnt/ssd/cache_tmp/huggingface/modules/transformers_modules/mcore2hf_hyphen_config_dot_aB8A3A/10c6759df797195a/modeling_qwen3_next.py. +[ERROR] `cache_position` is part of Qwen3NextForCausalLM.forward's signature, but not documented. Make sure to add it to the docstring of the function in /mnt/ssd/cache_tmp/huggingface/modules/transformers_modules/mcore2hf_hyphen_config_dot_aB8A3A/10c6759df797195a/modeling_qwen3_next.py. +[transformers] `torch_dtype` is deprecated! Use `dtype` instead! +INFO:transformers_modules.mcore2hf_hyphen_config_dot_aB8A3A.10c6759df797195a.modeling_qwen3_next:[Qwen3Next custom] attn_position_embedding_type=nope, rnn_position_embedding_type=nope, attn_logits_scaling=None +Qwen3NextForCausalLM( + (model): Qwen3NextModel( + (embed_tokens): Embedding(99072, 2048, padding_idx=102) + (layers): ModuleList( + (0-1): 2 x Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=4800, bias=False) + (up_proj): Linear(in_features=2048, out_features=4800, bias=False) + (down_proj): Linear(in_features=4800, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (2-11): 10 x Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (12): Qwen3NextDecoderLayer( + (self_attn): Qwen3NextAttention( + (q_proj): Linear(in_features=2048, out_features=4096, bias=True) + (k_proj): Linear(in_features=2048, out_features=512, bias=True) + (v_proj): Linear(in_features=2048, out_features=512, bias=True) + (o_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (13-19): 7 x Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (20-21): 2 x Qwen3NextDecoderLayer( + (self_attn): Qwen3NextAttention( + (q_proj): Linear(in_features=2048, out_features=4096, bias=True) + (k_proj): Linear(in_features=2048, out_features=512, bias=True) + (v_proj): Linear(in_features=2048, out_features=512, bias=True) + (o_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (22): Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (23): Qwen3NextDecoderLayer( + (self_attn): Qwen3NextAttention( + (q_proj): Linear(in_features=2048, out_features=4096, bias=True) + (k_proj): Linear(in_features=2048, out_features=512, bias=True) + (v_proj): Linear(in_features=2048, out_features=512, bias=True) + (o_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (24-45): 22 x Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (46): Qwen3NextDecoderLayer( + (self_attn): Qwen3NextAttention( + (q_proj): Linear(in_features=2048, out_features=4096, bias=True) + (k_proj): Linear(in_features=2048, out_features=512, bias=True) + (v_proj): Linear(in_features=2048, out_features=512, bias=True) + (o_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (47): Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (48-49): 2 x Qwen3NextDecoderLayer( + (self_attn): Qwen3NextAttention( + (q_proj): Linear(in_features=2048, out_features=4096, bias=True) + (k_proj): Linear(in_features=2048, out_features=512, bias=True) + (v_proj): Linear(in_features=2048, out_features=512, bias=True) + (o_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (50-55): 6 x Qwen3NextDecoderLayer( + (linear_attn): Qwen3NextGatedDeltaNet( + (act): SiLUActivation() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (in_proj_qkvz): Linear(in_features=2048, out_features=5120, bias=False) + (in_proj_ba): Linear(in_features=2048, out_features=64, bias=False) + (norm): FusedRMSNormGated(64, eps=1e-06, activation=silu) + (out_proj): Linear(in_features=2048, out_features=2048, bias=False) + ) + (mlp): Qwen3NextSparseMoeBlock( + (gate): Linear(in_features=2048, out_features=192, bias=False) + (experts): ModuleList( + (0-191): 192 x Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + ) + (shared_expert): Qwen3NextMLP( + (gate_proj): Linear(in_features=2048, out_features=512, bias=False) + (up_proj): Linear(in_features=2048, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=2048, bias=False) + (act_fn): SiLUActivation() + ) + (shared_expert_gate): Linear(in_features=2048, out_features=1, bias=False) + ) + (input_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (post_attention_layernorm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + ) + (norm): TrainingGatedNorm( + (rms_norm): LlamaRMSNorm() + (w_down): Linear(in_features=2048, out_features=16, bias=False) + (w_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (rotary_emb): Qwen3NextRotaryEmbedding() + ) + (lm_head): Linear(in_features=2048, out_features=99072, bias=False) +) +GPTModel( + (embedding): LanguageModelEmbedding( + (word_embeddings): VocabParallelEmbedding() + (embedding_dropout): Dropout(p=0.1, inplace=False) + ) + (decoder): TransformerBlock( + (layers): ModuleList( + (0-1): 2 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=9600, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=4800, out_features=2048, bias=False, TP=1) + ) + ) + (2-11): 10 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (12): TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): SelfAttention( + (core_attention): TEDotProductAttention( + (flash_attention): FlashAttention() + (fused_attention): FusedAttention() + (unfused_attention): UnfusedDotProductAttention( + (scale_mask_softmax): FusedScaleMaskSoftmax() + (attention_dropout): Dropout(p=0.1, inplace=False) + ) + ) + (linear_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (linear_qkv): TEColumnParallelLinear(in_features=2048, out_features=5120, bias=True, TP=1) + (q_layernorm): IdentityOp() + (k_layernorm): IdentityOp() + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (13-19): 7 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (20-21): 2 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): SelfAttention( + (core_attention): TEDotProductAttention( + (flash_attention): FlashAttention() + (fused_attention): FusedAttention() + (unfused_attention): UnfusedDotProductAttention( + (scale_mask_softmax): FusedScaleMaskSoftmax() + (attention_dropout): Dropout(p=0.1, inplace=False) + ) + ) + (linear_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (linear_qkv): TEColumnParallelLinear(in_features=2048, out_features=5120, bias=True, TP=1) + (q_layernorm): IdentityOp() + (k_layernorm): IdentityOp() + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (22): TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (23): TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): SelfAttention( + (core_attention): TEDotProductAttention( + (flash_attention): FlashAttention() + (fused_attention): FusedAttention() + (unfused_attention): UnfusedDotProductAttention( + (scale_mask_softmax): FusedScaleMaskSoftmax() + (attention_dropout): Dropout(p=0.1, inplace=False) + ) + ) + (linear_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (linear_qkv): TEColumnParallelLinear(in_features=2048, out_features=5120, bias=True, TP=1) + (q_layernorm): IdentityOp() + (k_layernorm): IdentityOp() + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (24-45): 22 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (46): TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): SelfAttention( + (core_attention): TEDotProductAttention( + (flash_attention): FlashAttention() + (fused_attention): FusedAttention() + (unfused_attention): UnfusedDotProductAttention( + (scale_mask_softmax): FusedScaleMaskSoftmax() + (attention_dropout): Dropout(p=0.1, inplace=False) + ) + ) + (linear_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (linear_qkv): TEColumnParallelLinear(in_features=2048, out_features=5120, bias=True, TP=1) + (q_layernorm): IdentityOp() + (k_layernorm): IdentityOp() + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (47): TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (48-49): 2 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): SelfAttention( + (core_attention): TEDotProductAttention( + (flash_attention): FlashAttention() + (fused_attention): FusedAttention() + (unfused_attention): UnfusedDotProductAttention( + (scale_mask_softmax): FusedScaleMaskSoftmax() + (attention_dropout): Dropout(p=0.1, inplace=False) + ) + ) + (linear_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (linear_qkv): TEColumnParallelLinear(in_features=2048, out_features=5120, bias=True, TP=1) + (q_layernorm): IdentityOp() + (k_layernorm): IdentityOp() + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (50-55): 6 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=aux_loss, score_function=softmax, enable_expert_bias=False, fused=False) + (experts): SequentialMLP( + (local_experts): ModuleList( + (0-191): 192 x MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + ) + (final_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (output_layer): ColumnParallelLinear(in_features=2048, out_features=99072, bias=False, TP=1) +) +DEBUG:multistorageclient.config:Searching MSC config file paths: ['/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/msc/msc_config.yaml', '/mnt/ssd/cache_tmp/.config/msc/config.yaml', '/mnt/ssd/cache_tmp/.config/msc/config.json', '/root/.msc_config.yaml', '/root/.msc_config.json', '/root/.config/msc/config.yaml', '/root/.config/msc/config.json', '/etc/xdg/msc/config.yaml', '/etc/xdg/msc/config.json', '/etc/msc_config.yaml', '/etc/msc_config.json'] +DEBUG:multistorageclient.config:Using MSC config file: /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/msc/msc_config.yaml +DEBUG:multistorageclient.config:Searching MSC config file paths: ['/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/msc/msc_config.yaml', '/mnt/ssd/cache_tmp/.config/msc/config.yaml', '/mnt/ssd/cache_tmp/.config/msc/config.json', '/root/.msc_config.yaml', '/root/.msc_config.json', '/root/.config/msc/config.yaml', '/root/.config/msc/config.json', '/etc/xdg/msc/config.yaml', '/etc/xdg/msc/config.json', '/etc/msc_config.yaml', '/etc/msc_config.json'] +DEBUG:multistorageclient.config:Using MSC config file: /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/msc/msc_config.yaml +DEBUG:multistorageclient.client.single:No metadata provider configured, auto-commit will not be enabled +DEBUG:multistorageclient.client.client:StorageClient '__filesystem__' using SingleStorageClient +sharded_state_dict metadata loaded from the checkpoint: {'singleton_local_shards': True, 'distrib_optim_sharding_type': 'dp_reshardable', 'chained_optim_avoid_prefix': True} +Job sharding has changed: Rerun state will be ignored + loading distributed checkpoint from /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/load_view840 at iteration 840 +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/dist_checkpointing/strategies/torch.py:956: FutureWarning: `load_state_dict` is deprecated and will be removed in future versions. Please use `load` instead. + checkpoint.load_state_dict( + checkpoint version 3.0 + successfully loaded checkpoint from /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/load_view840 [ t 1/1, p 1/1 ] at iteration 840 +[convert] synchronizer created in 373.42s +[convert] starting sync_params +INFO:root:Converting layer 0 is_gdn=True is_not_moe=True +INFO:root:Converting layer 1 is_gdn=True is_not_moe=True +INFO:root:Converting layer 2 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.076171875 mean=-0.00048828125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.11767578125 mean=0.002349853515625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 3 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0771484375 mean=-0.000179290771484375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.12451171875 mean=0.00010728836059570312 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 4 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.083984375 mean=-0.000263214111328125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.09228515625 mean=0.0013885498046875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 5 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.087890625 mean=-0.0002460479736328125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0947265625 mean=0.0023651123046875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 6 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.095703125 mean=-0.0002155303955078125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.08837890625 mean=-0.001373291015625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 7 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.091796875 mean=-4.410743713378906e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.07275390625 mean=-8.249282836914062e-05 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 8 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.09375 mean=-7.295608520507812e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.072265625 mean=0.0001201629638671875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 9 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.08837890625 mean=1.9669532775878906e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.06640625 mean=9.441375732421875e-05 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 10 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.08447265625 mean=5.650520324707031e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.064453125 mean=-0.000957489013671875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 11 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.08203125 mean=7.867813110351562e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0751953125 mean=-0.000324249267578125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 12 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 12: args.attention_output_gate=True +WARNING:root:Layer 12: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.076171875 mean=0.0002288818359375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.06298828125 mean=-0.000942230224609375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 13 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.07177734375 mean=0.0001850128173828125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.050537109375 mean=-0.00136566162109375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 14 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.076171875 mean=0.00019359588623046875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.054931640625 mean=-0.00136566162109375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 15 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.07373046875 mean=0.0002651214599609375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0595703125 mean=-0.001312255859375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 16 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.072265625 mean=0.000347137451171875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0595703125 mean=-0.00135040283203125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 17 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0693359375 mean=0.00028228759765625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.057373046875 mean=-0.0019683837890625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 18 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0654296875 mean=0.0002765655517578125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.060546875 mean=-0.0025177001953125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 19 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.068359375 mean=0.00061798095703125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.05859375 mean=-0.0021209716796875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 20 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 20: args.attention_output_gate=True +WARNING:root:Layer 20: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.06689453125 mean=0.000408172607421875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.06103515625 mean=-0.001556396484375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 21 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 21: args.attention_output_gate=True +WARNING:root:Layer 21: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.059814453125 mean=0.000446319580078125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.052490234375 mean=-0.0025634765625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 22 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.054931640625 mean=0.0003185272216796875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.05322265625 mean=-0.003448486328125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 23 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 23: args.attention_output_gate=True +WARNING:root:Layer 23: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.053466796875 mean=0.00048065185546875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.046142578125 mean=-0.002777099609375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 24 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.05224609375 mean=0.000385284423828125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0439453125 mean=-0.00311279296875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 25 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.049560546875 mean=0.0003070831298828125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.041748046875 mean=-0.002288818359375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 26 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.04931640625 mean=0.0002613067626953125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.039306640625 mean=-0.0028839111328125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 27 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.051513671875 mean=0.0004100799560546875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.036865234375 mean=-0.002471923828125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 28 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0478515625 mean=0.00034332275390625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.032470703125 mean=-0.00102996826171875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 29 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.04833984375 mean=0.00045013427734375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.028564453125 mean=-0.00131988525390625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 30 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.051025390625 mean=0.00069427490234375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.030517578125 mean=-0.0013275146484375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 31 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.048583984375 mean=0.0004425048828125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0301513671875 mean=-0.000843048095703125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 32 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.046630859375 mean=9.250640869140625e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0283203125 mean=-0.00060272216796875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 33 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.050048828125 mean=7.724761962890625e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0306396484375 mean=-0.0003261566162109375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 34 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.05224609375 mean=0.00014400482177734375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0301513671875 mean=-0.00018405914306640625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 35 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.057373046875 mean=0.00010728836059570312 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.03466796875 mean=8.296966552734375e-05 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 36 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.059326171875 mean=7.05718994140625e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.044677734375 mean=0.000896453857421875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 37 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.059326171875 mean=0.00012063980102539062 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.04345703125 mean=0.00101470947265625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 38 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.057861328125 mean=0.00013065338134765625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.038818359375 mean=0.00144195556640625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 39 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.056640625 mean=0.00010061264038085938 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.037841796875 mean=0.0009002685546875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 40 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.052978515625 mean=0.00014781951904296875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.028564453125 mean=1.6927719116210938e-05 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 41 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.049560546875 mean=0.00013828277587890625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0264892578125 mean=-4.26173210144043e-06 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 42 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.044189453125 mean=0.0003376007080078125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0235595703125 mean=-0.000766754150390625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 43 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.038330078125 mean=0.0001621246337890625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0228271484375 mean=-0.000316619873046875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 44 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.034912109375 mean=0.000274658203125 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0234375 mean=-0.0003528594970703125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 45 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.035400390625 mean=0.0001964569091796875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.026123046875 mean=-0.0009002685546875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 46 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 46: args.attention_output_gate=True +WARNING:root:Layer 46: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.035400390625 mean=0.00034332275390625 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.022705078125 mean=-0.00077056884765625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 47 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.03662109375 mean=0.00017833709716796875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0272216796875 mean=-0.000957489013671875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 48 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 48: args.attention_output_gate=True +WARNING:root:Layer 48: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0341796875 mean=-8.249282836914062e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0223388671875 mean=-0.000576019287109375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 49 is_gdn=False is_not_moe=False +INFO:root:[DEBUG] Layer 49: args.attention_output_gate=True +WARNING:root:Layer 49: attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: args.attention_output_gate=True +WARNING:root:attention_output_gate=True but linear_qgkv not found, using linear_qkv instead (Megatron may have forgotten to rename) +INFO:root:[DEBUG] set_gated_selfattn_state: attention_output_gate=True, linear_layer=TEColumnParallelLinear +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.03466796875 mean=9.298324584960938e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.027099609375 mean=-0.00119781494140625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 50 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.037109375 mean=-2.4437904357910156e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.027587890625 mean=-0.0009765625 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 51 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.036376953125 mean=4.5299530029296875e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.0269775390625 mean=-0.00135040283203125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 52 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0419921875 mean=0.00022792816162109375 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.034912109375 mean=-0.0016326904296875 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 53 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.0380859375 mean=7.963180541992188e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.05078125 mean=-0.0023040771484375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 54 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.03857421875 mean=6.198883056640625e-05 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.042724609375 mean=-0.0023040771484375 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +INFO:root:Converting layer 55 is_gdn=True is_not_moe=False +DEBUG:root:moe.router.weight shape=torch.Size([192, 2048]) std=0.042236328125 mean=-0.00014781951904296875 +INFO:root:copying shared experts moe.shared_experts.use_shared_expert_gate=True Linear(in_features=2048, out_features=1, bias=False) +DEBUG:root:moe.shared_experts.gate_weight shape=torch.Size([1, 2048]) std=0.04443359375 mean=0.0025177001953125 +DEBUG:root:hf_shared_expert.gate_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.up_proj.weight shape=torch.Size([512, 2048]) std=N/A (meta tensor) mean=N/A (meta tensor) +DEBUG:root:hf_shared_expert.down_proj.weight shape=torch.Size([2048, 512]) std=N/A (meta tensor) mean=N/A (meta tensor) +[convert] finished sync_params in 0.85s +[convert] starting check_and_save -> /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/iter_840-hf +DEBUG:root:[RANK 0] 0 send op & 0 recv op. +INFO:root:[Iters 0 RANK 0] starts synchronizing parameters with other ranks... +INFO:root:[Iters 0 RANK 0] finishes synchronizing +[Iters 0 RANK 0] model-00001-of-00004.safetensors is saved. +DEBUG:root:[Iters 0 RANK 0] joined +DEBUG:root:[RANK 0] 0 send op & 0 recv op. +INFO:root:[Iters 1 RANK 0] starts synchronizing parameters with other ranks... +INFO:root:[Iters 1 RANK 0] finishes synchronizing +[Iters 1 RANK 0] model-00002-of-00004.safetensors is saved. +DEBUG:root:[Iters 1 RANK 0] joined +DEBUG:root:[RANK 0] 0 send op & 0 recv op. +INFO:root:[Iters 2 RANK 0] starts synchronizing parameters with other ranks... +INFO:root:[Iters 2 RANK 0] finishes synchronizing +[Iters 2 RANK 0] model-00003-of-00004.safetensors is saved. +DEBUG:root:[Iters 2 RANK 0] joined +DEBUG:root:[RANK 0] 0 send op & 0 recv op. +INFO:root:[Iters 3 RANK 0] starts synchronizing parameters with other ranks... +INFO:root:[Iters 3 RANK 0] finishes synchronizing +[Iters 3 RANK 0] model-00004-of-00004.safetensors is saved. +DEBUG:root:[Iters 3 RANK 0] joined +[convert] finished check_and_save in 79.89s +Conversion finished in 455.5373373031616 seconds. ++ echo '✅Conversion completed: /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/iter_840-hf' +✅Conversion completed: /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/hf_exports/stage15_gdnlralpha_off_20261003a/iter_840-hf ++ cleanup_real_config_path ++ rm -rf /tmp/mcore2hf_ws_840/mcore2hf-config.aB8A3A ++ '[' -n '' ']' diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/launch_node0.sh b/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/launch_node0.sh new file mode 100644 index 0000000000000000000000000000000000000000..6e0598b478fe1dc6a2c019b5c54cae16464df4d3 --- /dev/null +++ b/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/launch_node0.sh @@ -0,0 +1,68 @@ +#!/bin/bash +# Stage15 launch: loads stage14 final weights (--continual-pretrain resets iteration to 0). +# Token budget 877440 samples x 131072 seq = 115.0B tokens (dataset has 115.59B). +# Command identical to stage14 resume except: data-path/cache, train-samples, load+continual-pretrain, master-port 35871. +ulimit -n 1048576 +# Env replicated from tools/launch_training.py + configs/training/formal3_128k_adamh.json +# (config environment applied first; explicit exports below win via setdefault semantics). +# MOE_AMPERE_GROUPED_MLP=1 is REQUIRED: without it experts fall back to SequentialMLP +# and the optimizer checkpoint shape-mismatches (ckpt saved with A100GroupedMLP). +export MOE_ROUTER_SQRT_GATE=1 +export MOE_ROUTER_SCORE_FUNCTION=softmax +export TORCH_NCCL_TRACE_BUFFER_SIZE=0 +export MOE_AMPERE_GROUPED_MLP=1 +export NCCL_IB_PCI_RELAXED_ORDERING=1 +export NCCL_MIN_NCHANNELS=8 +export GDN_ALPHA_WEIGHT_SCALE=1.0 +export NCCL_RAS_ENABLE=0 +export TORCH_NCCL_TRACE_CPP_STACK=0 +export NCCL_SOCKET_IFNAME=eth0 +export GDN_EPS=1e-6 +export NCCL_NET_GDR_LEVEL=3 +export GDN_PRE_GDR_FUSION=0 +export MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512 +export NCCL_DEBUG=WARN +export MOE_TOKEN_DISPATCHER_TYPE=alltoall +export NCCL_TIMEOUT=18000 +export NCCL_IB_HCA=mlx5 +export GDN_CP_DISABLE_LAYOUT_BARRIER=0 +export NCCL_IB_GID_INDEX=3 +export TORCH_PROFILER_WITH_STACK=1 +export MOE_TRITON_SHARED_EXPERT_OVERLAP=0 +export MOE_EXPERT_CAPACITY_FACTOR=0 +export TORCH_FORCE_NO_WEIGHTS_ONLY_LOAD=true +export GDN_ALPHA_PROFILE_NAME=a05_lr +export NCCL_IB_QPS_PER_CONNECTION=4 +export GDN_PRE_GDR_FUSION_OVERRIDE=0 +export TORCH_NCCL_DUMP_ON_TIMEOUT=0 +export MOE_FFN_HIDDEN_SIZE=512 +export MOE_ROUTER_NORM_TOPK_PROB=true +export FLA_GDN_CP_STATE_PRECISION=bf16 +export NCCL_IB_TIMEOUT=23 +export MOE_ROUTER_TOPK=4 +export MOE_AUX_LOSS_COEFF=0.05 +export GDN_SPLIT_IN_PROJ=1 +export TORCH_FR_CPP_STACK=0 +export OMP_NUM_THREADS=4 +export TORCH_FR_BUFFER_SIZE=0 +export FLA_GDN_NUMERICAL_CP_SIZE=0 +export GDN_FUSED_SPLIT_FORWARD=1 +export GDN_ALPHA_WEIGHT_SCALE_MODE=matrix +export NVTE_ALLOW_NONDETERMINISTIC_ALGO=0 +export MOE_LAYER_FREQ=([0]*2+[1]*54) +export NCCL_CROSS_NIC=1 +export OPENBLAS_NUM_THREADS=1 +export MKL_NUM_THREADS=1 +export PATH=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin +export PYTHONPATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/flash-linear-attention:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/triton-distributed:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/linghe +export FLA_SOURCE_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/flash-linear-attention +export TRITON_DIST_PYTHON=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/triton-distributed +export TRITON_CACHE_DIR=/mnt/ssd/cpt/training_runtime/$(hostname)/triton +export TORCH_EXTENSIONS_DIR=/mnt/ssd/cpt/training_runtime/$(hostname)/torch_extensions +export TORCHINDUCTOR_CACHE_DIR=/mnt/ssd/cpt/training_runtime/$(hostname)/inductor +export PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True +export GDN_LR_MULT_ALPHA=1.0 +export MSC_CONFIG=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/msc/msc_config.yaml +export CUDA_DEVICE_MAX_CONNECTIONS=1 +mkdir -p "$TRITON_CACHE_DIR" "$TORCH_EXTENSIONS_DIR" "$TORCHINDUCTOR_CACHE_DIR" "$NUMA_BIND_LOG_DIR" 2>/dev/null || true +exec /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python -m torch.distributed.run --nnodes 8 --nproc-per-node 8 --node-rank 0 --master-addr 10.239.252.55 --master-port 35871 /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/pretrain_gpt.py --use-mcore-models --hybrid-attention-ratio 0.0625 --hybrid-mlp-ratio 0.5 --is-hybrid-model --linear-attention-type gated_delta_net --linear-conv-kernel-dim 4 --linear-key-head-dim 64 --linear-value-head-dim 64 --linear-num-key-heads 8 --linear-num-value-heads 32 --disable-bias-linear --seq-length 131072 --max-position-embeddings 131072 --num-layers 56 --hidden-size 2048 --ffn-hidden-size 4800 --num-attention-heads 32 --init-method-std 0.02 --attention-dropout 0.0 --hidden-dropout 0.0 --normalization RMSNorm --swiglu --group-query-attention --num-query-groups 8 --no-masked-softmax-fusion --use-flash-attn --add-qkv-bias --mtp-num-layers 0 --mtp-loss-scaling-factor 0.1 --norm-epsilon 1e-06 --mtp-linear-attention-type gated_delta_net --position-embedding-type none --linear-attention-freq '[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1]' --optimizer adam --adam-beta1 0.9 --adam-beta2 0.95 --adam-eps 1e-15 --weight-decay 0.1 --clip-grad 1.0 --data-path '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_{0..78}_input_ids_tmp' --split 999,1,0 --data-cache-path /mnt/hdd/cpt/binidx_cache_stage15_final_20261001_v2 --no-create-attention-mask-in-dataloader --num-experts 192 --moe-grouped-gemm --moe-router-topk 4 --moe-router-score-function softmax --overlap-param-gather --overlap-grad-reduce --moe-expert-capacity-factor 0 --moe-shared-expert-intermediate-size 512 --moe-router-dtype fp32 --moe-permute-fusion --moe-ffn-hidden-size 512 --moe-shared-expert-gate --moe-shared-expert-overlap --moe-layer-freq '([0]*2+[1]*54)' --moe-aux-loss-coeff 0.05 --moe-router-load-balancing-type global_aux_loss --micro-batch-size 1 --global-batch-size 128 --lr 5e-4 --train-samples 877440 --lr-warmup-samples 0 --lr-decay-samples 877440 --lr-decay-style constant --min-lr 0.0005 --bf16 --async-save --ckpt-fully-parallel-load --auto-detect-ckpt-format --load-complemental-dataset loss_mask --tensor-model-parallel-size 1 --pipeline-model-parallel-size 1 --context-parallel-size 8 --expert-model-parallel-size 8 --sequence-parallel --use-distributed-optimizer --log-interval 1 --log-throughput --save-interval 40 --eval-interval 99999999 --eval-iters 10 --increase-log-level-iters 5 --timing-log-level 0 --wandb-project qwen2.5_3b_mom_cpt --tokenizer-type HuggingFaceTokenizer --tokenizer-model /mnt/ssd/yulan/pretrain-linear-moe/preprocess/modify_tokenizer/YuLan-Mini-Qwen-Template_new_sp --no-adam-bias-correction --corrected-weight-decay --auto-generate-cu-seqlens --cp-comm-type a2a --log-moe-activations --final-logits-clamp-value 30 --attention-output-gate --gated-norm --gated-norm-lora-rank 16 --dataloader-type cyclic --lm-loss-normalization global-token --prefetch-factor 8 --activation-func-clamp-value 7 --activation-func-clamp-mode soft --gdn-split-in-proj --gdn-fused-split-forward --gdn-alpha-weight-scale 1.0 --gdn-alpha-weight-scale-mode matrix --gdn-lr-mult-alpha 0.5 --start-samples 0 --ckpt-format torch_dist --rerun-mode validate_results --distributed-timeout-minutes 120 --dist-ckpt-strictness assume_ok_unexpected --gdn-backend fla --gdn-gated-norm-backend linghe --moe-router-gemm-backend te --log-params-norm --timing-log-option all --no-barrier-with-level-1-timing --log-metrics-interval 10 --cross-entropy-loss-fusion --cross-entropy-fusion-impl linear_triton --linear-cross-entropy-num-chunks 8 --num-workers 4 --moe-router-sqrt-gate --moe-router-fusion --moe-token-dispatcher-type alltoall --log-hidden-states embeddings residual_attn input_layernorm attention::q_layernorm attention::k_layernorm attention::core_attention attention::o_proj residual_mlp pre_mlp_layernorm mlp moe::router moe::shared_experts moe::routed_experts gated_delta::in_proj gated_delta::conv1d gated_delta::query gated_delta::key gated_delta::value gated_delta::gate gated_delta::beta gated_delta::alpha gated_delta::g gated_delta::core_attention gated_delta::out_norm gated_delta::out_proj final_layernorm --log-params attention::linear_qkv attention::o_proj mlp::linear_fc1 mlp::linear_fc2 input_layernorm pre_mlp_layernorm embedding lm_head gated_delta::in_proj gated_delta::in_proj_bias gated_delta::conv1d gated_delta::conv1d_bias gated_delta::dt_bias gated_delta::A_log gated_delta::out_norm gated_delta::out_norm_bias gated_delta::out_proj gated_delta::out_proj_bias --log-update-rms --log-per-module-grad-rms --log-per-layer-grad-rms --document-packing-algorithm obfd --exit-on-missing-checkpoint --no-initialization --untie-embeddings-and-output-weights --recompute-granularity full --recompute-method block --recompute-num-layers 56 --no-save-step-one --save-retain-interval 1000000000 --exit-signal-handler --fast-thd-a2a --hybrid-adamh --reuse-training-communicators --ddp-bucket-size 40000000 --load /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage14_120B_8node_gdnlralpha_off_20260925a/checkpoints --continual-pretrain --save /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints --tensorboard-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/tensorboard --wandb-save-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/wandb --wandb-exp-name stage14_120B_8node_gdnlralpha_off_20260925a --runtime-audit-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/runtime_audit diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/launch_node1.sh b/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/launch_node1.sh new file mode 100644 index 0000000000000000000000000000000000000000..d9274c621e0e9f46b095a8766e3bd0c6e9229e45 --- /dev/null +++ b/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/launch_node1.sh @@ -0,0 +1,68 @@ +#!/bin/bash +# Stage15 launch: loads stage14 final weights (--continual-pretrain resets iteration to 0). +# Token budget 877440 samples x 131072 seq = 115.0B tokens (dataset has 115.59B). +# Command identical to stage14 resume except: data-path/cache, train-samples, load+continual-pretrain, master-port 35871. +ulimit -n 1048576 +# Env replicated from tools/launch_training.py + configs/training/formal3_128k_adamh.json +# (config environment applied first; explicit exports below win via setdefault semantics). +# MOE_AMPERE_GROUPED_MLP=1 is REQUIRED: without it experts fall back to SequentialMLP +# and the optimizer checkpoint shape-mismatches (ckpt saved with A100GroupedMLP). +export MOE_ROUTER_SQRT_GATE=1 +export MOE_ROUTER_SCORE_FUNCTION=softmax +export TORCH_NCCL_TRACE_BUFFER_SIZE=0 +export MOE_AMPERE_GROUPED_MLP=1 +export NCCL_IB_PCI_RELAXED_ORDERING=1 +export NCCL_MIN_NCHANNELS=8 +export GDN_ALPHA_WEIGHT_SCALE=1.0 +export NCCL_RAS_ENABLE=0 +export TORCH_NCCL_TRACE_CPP_STACK=0 +export NCCL_SOCKET_IFNAME=eth0 +export GDN_EPS=1e-6 +export NCCL_NET_GDR_LEVEL=3 +export GDN_PRE_GDR_FUSION=0 +export MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512 +export NCCL_DEBUG=WARN +export MOE_TOKEN_DISPATCHER_TYPE=alltoall +export NCCL_TIMEOUT=18000 +export NCCL_IB_HCA=mlx5 +export GDN_CP_DISABLE_LAYOUT_BARRIER=0 +export NCCL_IB_GID_INDEX=3 +export TORCH_PROFILER_WITH_STACK=1 +export MOE_TRITON_SHARED_EXPERT_OVERLAP=0 +export MOE_EXPERT_CAPACITY_FACTOR=0 +export TORCH_FORCE_NO_WEIGHTS_ONLY_LOAD=true +export GDN_ALPHA_PROFILE_NAME=a05_lr +export NCCL_IB_QPS_PER_CONNECTION=4 +export GDN_PRE_GDR_FUSION_OVERRIDE=0 +export TORCH_NCCL_DUMP_ON_TIMEOUT=0 +export MOE_FFN_HIDDEN_SIZE=512 +export MOE_ROUTER_NORM_TOPK_PROB=true +export FLA_GDN_CP_STATE_PRECISION=bf16 +export NCCL_IB_TIMEOUT=23 +export MOE_ROUTER_TOPK=4 +export MOE_AUX_LOSS_COEFF=0.05 +export GDN_SPLIT_IN_PROJ=1 +export TORCH_FR_CPP_STACK=0 +export OMP_NUM_THREADS=4 +export TORCH_FR_BUFFER_SIZE=0 +export FLA_GDN_NUMERICAL_CP_SIZE=0 +export GDN_FUSED_SPLIT_FORWARD=1 +export GDN_ALPHA_WEIGHT_SCALE_MODE=matrix +export NVTE_ALLOW_NONDETERMINISTIC_ALGO=0 +export MOE_LAYER_FREQ=([0]*2+[1]*54) +export NCCL_CROSS_NIC=1 +export OPENBLAS_NUM_THREADS=1 +export MKL_NUM_THREADS=1 +export PATH=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin +export PYTHONPATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/flash-linear-attention:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/triton-distributed:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/linghe +export FLA_SOURCE_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/flash-linear-attention +export TRITON_DIST_PYTHON=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/triton-distributed +export TRITON_CACHE_DIR=/mnt/ssd/cpt/training_runtime/$(hostname)/triton +export TORCH_EXTENSIONS_DIR=/mnt/ssd/cpt/training_runtime/$(hostname)/torch_extensions +export TORCHINDUCTOR_CACHE_DIR=/mnt/ssd/cpt/training_runtime/$(hostname)/inductor +export PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True +export GDN_LR_MULT_ALPHA=1.0 +export MSC_CONFIG=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/msc/msc_config.yaml +export CUDA_DEVICE_MAX_CONNECTIONS=1 +mkdir -p "$TRITON_CACHE_DIR" "$TORCH_EXTENSIONS_DIR" "$TORCHINDUCTOR_CACHE_DIR" "$NUMA_BIND_LOG_DIR" 2>/dev/null || true +exec /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python -m torch.distributed.run --nnodes 8 --nproc-per-node 8 --node-rank 1 --master-addr 10.239.252.55 --master-port 35871 /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/pretrain_gpt.py --use-mcore-models --hybrid-attention-ratio 0.0625 --hybrid-mlp-ratio 0.5 --is-hybrid-model --linear-attention-type gated_delta_net --linear-conv-kernel-dim 4 --linear-key-head-dim 64 --linear-value-head-dim 64 --linear-num-key-heads 8 --linear-num-value-heads 32 --disable-bias-linear --seq-length 131072 --max-position-embeddings 131072 --num-layers 56 --hidden-size 2048 --ffn-hidden-size 4800 --num-attention-heads 32 --init-method-std 0.02 --attention-dropout 0.0 --hidden-dropout 0.0 --normalization RMSNorm --swiglu --group-query-attention --num-query-groups 8 --no-masked-softmax-fusion --use-flash-attn --add-qkv-bias --mtp-num-layers 0 --mtp-loss-scaling-factor 0.1 --norm-epsilon 1e-06 --mtp-linear-attention-type gated_delta_net --position-embedding-type none --linear-attention-freq '[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1]' --optimizer adam --adam-beta1 0.9 --adam-beta2 0.95 --adam-eps 1e-15 --weight-decay 0.1 --clip-grad 1.0 --data-path '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_{0..78}_input_ids_tmp' --split 999,1,0 --data-cache-path /mnt/hdd/cpt/binidx_cache_stage15_final_20261001_v2 --no-create-attention-mask-in-dataloader --num-experts 192 --moe-grouped-gemm --moe-router-topk 4 --moe-router-score-function softmax --overlap-param-gather --overlap-grad-reduce --moe-expert-capacity-factor 0 --moe-shared-expert-intermediate-size 512 --moe-router-dtype fp32 --moe-permute-fusion --moe-ffn-hidden-size 512 --moe-shared-expert-gate --moe-shared-expert-overlap --moe-layer-freq '([0]*2+[1]*54)' --moe-aux-loss-coeff 0.05 --moe-router-load-balancing-type global_aux_loss --micro-batch-size 1 --global-batch-size 128 --lr 5e-4 --train-samples 877440 --lr-warmup-samples 0 --lr-decay-samples 877440 --lr-decay-style constant --min-lr 0.0005 --bf16 --async-save --ckpt-fully-parallel-load --auto-detect-ckpt-format --load-complemental-dataset loss_mask --tensor-model-parallel-size 1 --pipeline-model-parallel-size 1 --context-parallel-size 8 --expert-model-parallel-size 8 --sequence-parallel --use-distributed-optimizer --log-interval 1 --log-throughput --save-interval 40 --eval-interval 99999999 --eval-iters 10 --increase-log-level-iters 5 --timing-log-level 0 --wandb-project qwen2.5_3b_mom_cpt --tokenizer-type HuggingFaceTokenizer --tokenizer-model /mnt/ssd/yulan/pretrain-linear-moe/preprocess/modify_tokenizer/YuLan-Mini-Qwen-Template_new_sp --no-adam-bias-correction --corrected-weight-decay --auto-generate-cu-seqlens --cp-comm-type a2a --log-moe-activations --final-logits-clamp-value 30 --attention-output-gate --gated-norm --gated-norm-lora-rank 16 --dataloader-type cyclic --lm-loss-normalization global-token --prefetch-factor 8 --activation-func-clamp-value 7 --activation-func-clamp-mode soft --gdn-split-in-proj --gdn-fused-split-forward --gdn-alpha-weight-scale 1.0 --gdn-alpha-weight-scale-mode matrix --gdn-lr-mult-alpha 0.5 --start-samples 0 --ckpt-format torch_dist --rerun-mode validate_results --distributed-timeout-minutes 120 --dist-ckpt-strictness assume_ok_unexpected --gdn-backend fla --gdn-gated-norm-backend linghe --moe-router-gemm-backend te --log-params-norm --timing-log-option all --no-barrier-with-level-1-timing --log-metrics-interval 10 --cross-entropy-loss-fusion --cross-entropy-fusion-impl linear_triton --linear-cross-entropy-num-chunks 8 --num-workers 4 --moe-router-sqrt-gate --moe-router-fusion --moe-token-dispatcher-type alltoall --log-hidden-states embeddings residual_attn input_layernorm attention::q_layernorm attention::k_layernorm attention::core_attention attention::o_proj residual_mlp pre_mlp_layernorm mlp moe::router moe::shared_experts moe::routed_experts gated_delta::in_proj gated_delta::conv1d gated_delta::query gated_delta::key gated_delta::value gated_delta::gate gated_delta::beta gated_delta::alpha gated_delta::g gated_delta::core_attention gated_delta::out_norm gated_delta::out_proj final_layernorm --log-params attention::linear_qkv attention::o_proj mlp::linear_fc1 mlp::linear_fc2 input_layernorm pre_mlp_layernorm embedding lm_head gated_delta::in_proj gated_delta::in_proj_bias gated_delta::conv1d gated_delta::conv1d_bias gated_delta::dt_bias gated_delta::A_log gated_delta::out_norm gated_delta::out_norm_bias gated_delta::out_proj gated_delta::out_proj_bias --log-update-rms --log-per-module-grad-rms --log-per-layer-grad-rms --document-packing-algorithm obfd --exit-on-missing-checkpoint --no-initialization --untie-embeddings-and-output-weights --recompute-granularity full --recompute-method block --recompute-num-layers 56 --no-save-step-one --save-retain-interval 1000000000 --exit-signal-handler --fast-thd-a2a --hybrid-adamh --reuse-training-communicators --ddp-bucket-size 40000000 --load /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage14_120B_8node_gdnlralpha_off_20260925a/checkpoints --continual-pretrain --save /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints --tensorboard-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/tensorboard --wandb-save-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/wandb --wandb-exp-name stage14_120B_8node_gdnlralpha_off_20260925a --runtime-audit-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/runtime_audit diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/launch_node2.sh b/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/launch_node2.sh new file mode 100644 index 0000000000000000000000000000000000000000..5f2963f4f3a61b3de68422f802079f04dae77014 --- /dev/null +++ b/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/launch_node2.sh @@ -0,0 +1,68 @@ +#!/bin/bash +# Stage15 launch: loads stage14 final weights (--continual-pretrain resets iteration to 0). +# Token budget 877440 samples x 131072 seq = 115.0B tokens (dataset has 115.59B). +# Command identical to stage14 resume except: data-path/cache, train-samples, load+continual-pretrain, master-port 35871. +ulimit -n 1048576 +# Env replicated from tools/launch_training.py + configs/training/formal3_128k_adamh.json +# (config environment applied first; explicit exports below win via setdefault semantics). +# MOE_AMPERE_GROUPED_MLP=1 is REQUIRED: without it experts fall back to SequentialMLP +# and the optimizer checkpoint shape-mismatches (ckpt saved with A100GroupedMLP). +export MOE_ROUTER_SQRT_GATE=1 +export MOE_ROUTER_SCORE_FUNCTION=softmax +export TORCH_NCCL_TRACE_BUFFER_SIZE=0 +export MOE_AMPERE_GROUPED_MLP=1 +export NCCL_IB_PCI_RELAXED_ORDERING=1 +export NCCL_MIN_NCHANNELS=8 +export GDN_ALPHA_WEIGHT_SCALE=1.0 +export NCCL_RAS_ENABLE=0 +export TORCH_NCCL_TRACE_CPP_STACK=0 +export NCCL_SOCKET_IFNAME=eth0 +export GDN_EPS=1e-6 +export NCCL_NET_GDR_LEVEL=3 +export GDN_PRE_GDR_FUSION=0 +export MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512 +export NCCL_DEBUG=WARN +export MOE_TOKEN_DISPATCHER_TYPE=alltoall +export NCCL_TIMEOUT=18000 +export NCCL_IB_HCA=mlx5 +export GDN_CP_DISABLE_LAYOUT_BARRIER=0 +export NCCL_IB_GID_INDEX=3 +export TORCH_PROFILER_WITH_STACK=1 +export MOE_TRITON_SHARED_EXPERT_OVERLAP=0 +export MOE_EXPERT_CAPACITY_FACTOR=0 +export TORCH_FORCE_NO_WEIGHTS_ONLY_LOAD=true +export GDN_ALPHA_PROFILE_NAME=a05_lr +export NCCL_IB_QPS_PER_CONNECTION=4 +export GDN_PRE_GDR_FUSION_OVERRIDE=0 +export TORCH_NCCL_DUMP_ON_TIMEOUT=0 +export MOE_FFN_HIDDEN_SIZE=512 +export MOE_ROUTER_NORM_TOPK_PROB=true +export FLA_GDN_CP_STATE_PRECISION=bf16 +export NCCL_IB_TIMEOUT=23 +export MOE_ROUTER_TOPK=4 +export MOE_AUX_LOSS_COEFF=0.05 +export GDN_SPLIT_IN_PROJ=1 +export TORCH_FR_CPP_STACK=0 +export OMP_NUM_THREADS=4 +export TORCH_FR_BUFFER_SIZE=0 +export FLA_GDN_NUMERICAL_CP_SIZE=0 +export GDN_FUSED_SPLIT_FORWARD=1 +export GDN_ALPHA_WEIGHT_SCALE_MODE=matrix +export NVTE_ALLOW_NONDETERMINISTIC_ALGO=0 +export MOE_LAYER_FREQ=([0]*2+[1]*54) +export NCCL_CROSS_NIC=1 +export OPENBLAS_NUM_THREADS=1 +export MKL_NUM_THREADS=1 +export PATH=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin +export PYTHONPATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/flash-linear-attention:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/triton-distributed:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/linghe +export FLA_SOURCE_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/flash-linear-attention +export TRITON_DIST_PYTHON=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/triton-distributed +export TRITON_CACHE_DIR=/mnt/ssd/cpt/training_runtime/$(hostname)/triton +export TORCH_EXTENSIONS_DIR=/mnt/ssd/cpt/training_runtime/$(hostname)/torch_extensions +export TORCHINDUCTOR_CACHE_DIR=/mnt/ssd/cpt/training_runtime/$(hostname)/inductor +export PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True +export GDN_LR_MULT_ALPHA=1.0 +export MSC_CONFIG=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/msc/msc_config.yaml +export CUDA_DEVICE_MAX_CONNECTIONS=1 +mkdir -p "$TRITON_CACHE_DIR" "$TORCH_EXTENSIONS_DIR" "$TORCHINDUCTOR_CACHE_DIR" "$NUMA_BIND_LOG_DIR" 2>/dev/null || true +exec /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python -m torch.distributed.run --nnodes 8 --nproc-per-node 8 --node-rank 2 --master-addr 10.239.252.55 --master-port 35871 /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/pretrain_gpt.py --use-mcore-models --hybrid-attention-ratio 0.0625 --hybrid-mlp-ratio 0.5 --is-hybrid-model --linear-attention-type gated_delta_net --linear-conv-kernel-dim 4 --linear-key-head-dim 64 --linear-value-head-dim 64 --linear-num-key-heads 8 --linear-num-value-heads 32 --disable-bias-linear --seq-length 131072 --max-position-embeddings 131072 --num-layers 56 --hidden-size 2048 --ffn-hidden-size 4800 --num-attention-heads 32 --init-method-std 0.02 --attention-dropout 0.0 --hidden-dropout 0.0 --normalization RMSNorm --swiglu --group-query-attention --num-query-groups 8 --no-masked-softmax-fusion --use-flash-attn --add-qkv-bias --mtp-num-layers 0 --mtp-loss-scaling-factor 0.1 --norm-epsilon 1e-06 --mtp-linear-attention-type gated_delta_net --position-embedding-type none --linear-attention-freq '[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1]' --optimizer adam --adam-beta1 0.9 --adam-beta2 0.95 --adam-eps 1e-15 --weight-decay 0.1 --clip-grad 1.0 --data-path '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_{0..78}_input_ids_tmp' --split 999,1,0 --data-cache-path /mnt/hdd/cpt/binidx_cache_stage15_final_20261001_v2 --no-create-attention-mask-in-dataloader --num-experts 192 --moe-grouped-gemm --moe-router-topk 4 --moe-router-score-function softmax --overlap-param-gather --overlap-grad-reduce --moe-expert-capacity-factor 0 --moe-shared-expert-intermediate-size 512 --moe-router-dtype fp32 --moe-permute-fusion --moe-ffn-hidden-size 512 --moe-shared-expert-gate --moe-shared-expert-overlap --moe-layer-freq '([0]*2+[1]*54)' --moe-aux-loss-coeff 0.05 --moe-router-load-balancing-type global_aux_loss --micro-batch-size 1 --global-batch-size 128 --lr 5e-4 --train-samples 877440 --lr-warmup-samples 0 --lr-decay-samples 877440 --lr-decay-style constant --min-lr 0.0005 --bf16 --async-save --ckpt-fully-parallel-load --auto-detect-ckpt-format --load-complemental-dataset loss_mask --tensor-model-parallel-size 1 --pipeline-model-parallel-size 1 --context-parallel-size 8 --expert-model-parallel-size 8 --sequence-parallel --use-distributed-optimizer --log-interval 1 --log-throughput --save-interval 40 --eval-interval 99999999 --eval-iters 10 --increase-log-level-iters 5 --timing-log-level 0 --wandb-project qwen2.5_3b_mom_cpt --tokenizer-type HuggingFaceTokenizer --tokenizer-model /mnt/ssd/yulan/pretrain-linear-moe/preprocess/modify_tokenizer/YuLan-Mini-Qwen-Template_new_sp --no-adam-bias-correction --corrected-weight-decay --auto-generate-cu-seqlens --cp-comm-type a2a --log-moe-activations --final-logits-clamp-value 30 --attention-output-gate --gated-norm --gated-norm-lora-rank 16 --dataloader-type cyclic --lm-loss-normalization global-token --prefetch-factor 8 --activation-func-clamp-value 7 --activation-func-clamp-mode soft --gdn-split-in-proj --gdn-fused-split-forward --gdn-alpha-weight-scale 1.0 --gdn-alpha-weight-scale-mode matrix --gdn-lr-mult-alpha 0.5 --start-samples 0 --ckpt-format torch_dist --rerun-mode validate_results --distributed-timeout-minutes 120 --dist-ckpt-strictness assume_ok_unexpected --gdn-backend fla --gdn-gated-norm-backend linghe --moe-router-gemm-backend te --log-params-norm --timing-log-option all --no-barrier-with-level-1-timing --log-metrics-interval 10 --cross-entropy-loss-fusion --cross-entropy-fusion-impl linear_triton --linear-cross-entropy-num-chunks 8 --num-workers 4 --moe-router-sqrt-gate --moe-router-fusion --moe-token-dispatcher-type alltoall --log-hidden-states embeddings residual_attn input_layernorm attention::q_layernorm attention::k_layernorm attention::core_attention attention::o_proj residual_mlp pre_mlp_layernorm mlp moe::router moe::shared_experts moe::routed_experts gated_delta::in_proj gated_delta::conv1d gated_delta::query gated_delta::key gated_delta::value gated_delta::gate gated_delta::beta gated_delta::alpha gated_delta::g gated_delta::core_attention gated_delta::out_norm gated_delta::out_proj final_layernorm --log-params attention::linear_qkv attention::o_proj mlp::linear_fc1 mlp::linear_fc2 input_layernorm pre_mlp_layernorm embedding lm_head gated_delta::in_proj gated_delta::in_proj_bias gated_delta::conv1d gated_delta::conv1d_bias gated_delta::dt_bias gated_delta::A_log gated_delta::out_norm gated_delta::out_norm_bias gated_delta::out_proj gated_delta::out_proj_bias --log-update-rms --log-per-module-grad-rms --log-per-layer-grad-rms --document-packing-algorithm obfd --exit-on-missing-checkpoint --no-initialization --untie-embeddings-and-output-weights --recompute-granularity full --recompute-method block --recompute-num-layers 56 --no-save-step-one --save-retain-interval 1000000000 --exit-signal-handler --fast-thd-a2a --hybrid-adamh --reuse-training-communicators --ddp-bucket-size 40000000 --load /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage14_120B_8node_gdnlralpha_off_20260925a/checkpoints --continual-pretrain --save /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints --tensorboard-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/tensorboard --wandb-save-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/wandb --wandb-exp-name stage14_120B_8node_gdnlralpha_off_20260925a --runtime-audit-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/runtime_audit diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/launch_node3.sh b/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/launch_node3.sh new file mode 100644 index 0000000000000000000000000000000000000000..8a205c86b09b9ba753c80a2128fac7e5234e2036 --- /dev/null +++ b/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/launch_node3.sh @@ -0,0 +1,68 @@ +#!/bin/bash +# Stage15 launch: loads stage14 final weights (--continual-pretrain resets iteration to 0). +# Token budget 877440 samples x 131072 seq = 115.0B tokens (dataset has 115.59B). +# Command identical to stage14 resume except: data-path/cache, train-samples, load+continual-pretrain, master-port 35871. +ulimit -n 1048576 +# Env replicated from tools/launch_training.py + configs/training/formal3_128k_adamh.json +# (config environment applied first; explicit exports below win via setdefault semantics). +# MOE_AMPERE_GROUPED_MLP=1 is REQUIRED: without it experts fall back to SequentialMLP +# and the optimizer checkpoint shape-mismatches (ckpt saved with A100GroupedMLP). +export MOE_ROUTER_SQRT_GATE=1 +export MOE_ROUTER_SCORE_FUNCTION=softmax +export TORCH_NCCL_TRACE_BUFFER_SIZE=0 +export MOE_AMPERE_GROUPED_MLP=1 +export NCCL_IB_PCI_RELAXED_ORDERING=1 +export NCCL_MIN_NCHANNELS=8 +export GDN_ALPHA_WEIGHT_SCALE=1.0 +export NCCL_RAS_ENABLE=0 +export TORCH_NCCL_TRACE_CPP_STACK=0 +export NCCL_SOCKET_IFNAME=eth0 +export GDN_EPS=1e-6 +export NCCL_NET_GDR_LEVEL=3 +export GDN_PRE_GDR_FUSION=0 +export MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512 +export NCCL_DEBUG=WARN +export MOE_TOKEN_DISPATCHER_TYPE=alltoall +export NCCL_TIMEOUT=18000 +export NCCL_IB_HCA=mlx5 +export GDN_CP_DISABLE_LAYOUT_BARRIER=0 +export NCCL_IB_GID_INDEX=3 +export TORCH_PROFILER_WITH_STACK=1 +export MOE_TRITON_SHARED_EXPERT_OVERLAP=0 +export MOE_EXPERT_CAPACITY_FACTOR=0 +export TORCH_FORCE_NO_WEIGHTS_ONLY_LOAD=true +export GDN_ALPHA_PROFILE_NAME=a05_lr +export NCCL_IB_QPS_PER_CONNECTION=4 +export GDN_PRE_GDR_FUSION_OVERRIDE=0 +export TORCH_NCCL_DUMP_ON_TIMEOUT=0 +export MOE_FFN_HIDDEN_SIZE=512 +export MOE_ROUTER_NORM_TOPK_PROB=true +export FLA_GDN_CP_STATE_PRECISION=bf16 +export NCCL_IB_TIMEOUT=23 +export MOE_ROUTER_TOPK=4 +export MOE_AUX_LOSS_COEFF=0.05 +export GDN_SPLIT_IN_PROJ=1 +export TORCH_FR_CPP_STACK=0 +export OMP_NUM_THREADS=4 +export TORCH_FR_BUFFER_SIZE=0 +export FLA_GDN_NUMERICAL_CP_SIZE=0 +export GDN_FUSED_SPLIT_FORWARD=1 +export GDN_ALPHA_WEIGHT_SCALE_MODE=matrix +export NVTE_ALLOW_NONDETERMINISTIC_ALGO=0 +export MOE_LAYER_FREQ=([0]*2+[1]*54) +export NCCL_CROSS_NIC=1 +export OPENBLAS_NUM_THREADS=1 +export MKL_NUM_THREADS=1 +export PATH=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin +export PYTHONPATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/flash-linear-attention:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/triton-distributed:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/linghe +export FLA_SOURCE_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/flash-linear-attention +export TRITON_DIST_PYTHON=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/triton-distributed +export TRITON_CACHE_DIR=/mnt/ssd/cpt/training_runtime/$(hostname)/triton +export TORCH_EXTENSIONS_DIR=/mnt/ssd/cpt/training_runtime/$(hostname)/torch_extensions +export TORCHINDUCTOR_CACHE_DIR=/mnt/ssd/cpt/training_runtime/$(hostname)/inductor +export PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True +export GDN_LR_MULT_ALPHA=1.0 +export MSC_CONFIG=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/msc/msc_config.yaml +export CUDA_DEVICE_MAX_CONNECTIONS=1 +mkdir -p "$TRITON_CACHE_DIR" "$TORCH_EXTENSIONS_DIR" "$TORCHINDUCTOR_CACHE_DIR" "$NUMA_BIND_LOG_DIR" 2>/dev/null || true +exec /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python -m torch.distributed.run --nnodes 8 --nproc-per-node 8 --node-rank 3 --master-addr 10.239.252.55 --master-port 35871 /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/pretrain_gpt.py --use-mcore-models --hybrid-attention-ratio 0.0625 --hybrid-mlp-ratio 0.5 --is-hybrid-model --linear-attention-type gated_delta_net --linear-conv-kernel-dim 4 --linear-key-head-dim 64 --linear-value-head-dim 64 --linear-num-key-heads 8 --linear-num-value-heads 32 --disable-bias-linear --seq-length 131072 --max-position-embeddings 131072 --num-layers 56 --hidden-size 2048 --ffn-hidden-size 4800 --num-attention-heads 32 --init-method-std 0.02 --attention-dropout 0.0 --hidden-dropout 0.0 --normalization RMSNorm --swiglu --group-query-attention --num-query-groups 8 --no-masked-softmax-fusion --use-flash-attn --add-qkv-bias --mtp-num-layers 0 --mtp-loss-scaling-factor 0.1 --norm-epsilon 1e-06 --mtp-linear-attention-type gated_delta_net --position-embedding-type none --linear-attention-freq '[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1]' --optimizer adam --adam-beta1 0.9 --adam-beta2 0.95 --adam-eps 1e-15 --weight-decay 0.1 --clip-grad 1.0 --data-path '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_{0..78}_input_ids_tmp' --split 999,1,0 --data-cache-path /mnt/hdd/cpt/binidx_cache_stage15_final_20261001_v2 --no-create-attention-mask-in-dataloader --num-experts 192 --moe-grouped-gemm --moe-router-topk 4 --moe-router-score-function softmax --overlap-param-gather --overlap-grad-reduce --moe-expert-capacity-factor 0 --moe-shared-expert-intermediate-size 512 --moe-router-dtype fp32 --moe-permute-fusion --moe-ffn-hidden-size 512 --moe-shared-expert-gate --moe-shared-expert-overlap --moe-layer-freq '([0]*2+[1]*54)' --moe-aux-loss-coeff 0.05 --moe-router-load-balancing-type global_aux_loss --micro-batch-size 1 --global-batch-size 128 --lr 5e-4 --train-samples 877440 --lr-warmup-samples 0 --lr-decay-samples 877440 --lr-decay-style constant --min-lr 0.0005 --bf16 --async-save --ckpt-fully-parallel-load --auto-detect-ckpt-format --load-complemental-dataset loss_mask --tensor-model-parallel-size 1 --pipeline-model-parallel-size 1 --context-parallel-size 8 --expert-model-parallel-size 8 --sequence-parallel --use-distributed-optimizer --log-interval 1 --log-throughput --save-interval 40 --eval-interval 99999999 --eval-iters 10 --increase-log-level-iters 5 --timing-log-level 0 --wandb-project qwen2.5_3b_mom_cpt --tokenizer-type HuggingFaceTokenizer --tokenizer-model /mnt/ssd/yulan/pretrain-linear-moe/preprocess/modify_tokenizer/YuLan-Mini-Qwen-Template_new_sp --no-adam-bias-correction --corrected-weight-decay --auto-generate-cu-seqlens --cp-comm-type a2a --log-moe-activations --final-logits-clamp-value 30 --attention-output-gate --gated-norm --gated-norm-lora-rank 16 --dataloader-type cyclic --lm-loss-normalization global-token --prefetch-factor 8 --activation-func-clamp-value 7 --activation-func-clamp-mode soft --gdn-split-in-proj --gdn-fused-split-forward --gdn-alpha-weight-scale 1.0 --gdn-alpha-weight-scale-mode matrix --gdn-lr-mult-alpha 0.5 --start-samples 0 --ckpt-format torch_dist --rerun-mode validate_results --distributed-timeout-minutes 120 --dist-ckpt-strictness assume_ok_unexpected --gdn-backend fla --gdn-gated-norm-backend linghe --moe-router-gemm-backend te --log-params-norm --timing-log-option all --no-barrier-with-level-1-timing --log-metrics-interval 10 --cross-entropy-loss-fusion --cross-entropy-fusion-impl linear_triton --linear-cross-entropy-num-chunks 8 --num-workers 4 --moe-router-sqrt-gate --moe-router-fusion --moe-token-dispatcher-type alltoall --log-hidden-states embeddings residual_attn input_layernorm attention::q_layernorm attention::k_layernorm attention::core_attention attention::o_proj residual_mlp pre_mlp_layernorm mlp moe::router moe::shared_experts moe::routed_experts gated_delta::in_proj gated_delta::conv1d gated_delta::query gated_delta::key gated_delta::value gated_delta::gate gated_delta::beta gated_delta::alpha gated_delta::g gated_delta::core_attention gated_delta::out_norm gated_delta::out_proj final_layernorm --log-params attention::linear_qkv attention::o_proj mlp::linear_fc1 mlp::linear_fc2 input_layernorm pre_mlp_layernorm embedding lm_head gated_delta::in_proj gated_delta::in_proj_bias gated_delta::conv1d gated_delta::conv1d_bias gated_delta::dt_bias gated_delta::A_log gated_delta::out_norm gated_delta::out_norm_bias gated_delta::out_proj gated_delta::out_proj_bias --log-update-rms --log-per-module-grad-rms --log-per-layer-grad-rms --document-packing-algorithm obfd --exit-on-missing-checkpoint --no-initialization --untie-embeddings-and-output-weights --recompute-granularity full --recompute-method block --recompute-num-layers 56 --no-save-step-one --save-retain-interval 1000000000 --exit-signal-handler --fast-thd-a2a --hybrid-adamh --reuse-training-communicators --ddp-bucket-size 40000000 --load /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage14_120B_8node_gdnlralpha_off_20260925a/checkpoints --continual-pretrain --save /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints --tensorboard-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/tensorboard --wandb-save-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/wandb --wandb-exp-name stage14_120B_8node_gdnlralpha_off_20260925a --runtime-audit-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/runtime_audit diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/launch_node4.sh b/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/launch_node4.sh new file mode 100644 index 0000000000000000000000000000000000000000..adc7318a81a7d03cfb3f579cbc56427ea7d235c6 --- /dev/null +++ b/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/launch_node4.sh @@ -0,0 +1,68 @@ +#!/bin/bash +# Stage15 launch: loads stage14 final weights (--continual-pretrain resets iteration to 0). +# Token budget 877440 samples x 131072 seq = 115.0B tokens (dataset has 115.59B). +# Command identical to stage14 resume except: data-path/cache, train-samples, load+continual-pretrain, master-port 35871. +ulimit -n 1048576 +# Env replicated from tools/launch_training.py + configs/training/formal3_128k_adamh.json +# (config environment applied first; explicit exports below win via setdefault semantics). +# MOE_AMPERE_GROUPED_MLP=1 is REQUIRED: without it experts fall back to SequentialMLP +# and the optimizer checkpoint shape-mismatches (ckpt saved with A100GroupedMLP). +export MOE_ROUTER_SQRT_GATE=1 +export MOE_ROUTER_SCORE_FUNCTION=softmax +export TORCH_NCCL_TRACE_BUFFER_SIZE=0 +export MOE_AMPERE_GROUPED_MLP=1 +export NCCL_IB_PCI_RELAXED_ORDERING=1 +export NCCL_MIN_NCHANNELS=8 +export GDN_ALPHA_WEIGHT_SCALE=1.0 +export NCCL_RAS_ENABLE=0 +export TORCH_NCCL_TRACE_CPP_STACK=0 +export NCCL_SOCKET_IFNAME=eth0 +export GDN_EPS=1e-6 +export NCCL_NET_GDR_LEVEL=3 +export GDN_PRE_GDR_FUSION=0 +export MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512 +export NCCL_DEBUG=WARN +export MOE_TOKEN_DISPATCHER_TYPE=alltoall +export NCCL_TIMEOUT=18000 +export NCCL_IB_HCA=mlx5 +export GDN_CP_DISABLE_LAYOUT_BARRIER=0 +export NCCL_IB_GID_INDEX=3 +export TORCH_PROFILER_WITH_STACK=1 +export MOE_TRITON_SHARED_EXPERT_OVERLAP=0 +export MOE_EXPERT_CAPACITY_FACTOR=0 +export TORCH_FORCE_NO_WEIGHTS_ONLY_LOAD=true +export GDN_ALPHA_PROFILE_NAME=a05_lr +export NCCL_IB_QPS_PER_CONNECTION=4 +export GDN_PRE_GDR_FUSION_OVERRIDE=0 +export TORCH_NCCL_DUMP_ON_TIMEOUT=0 +export MOE_FFN_HIDDEN_SIZE=512 +export MOE_ROUTER_NORM_TOPK_PROB=true +export FLA_GDN_CP_STATE_PRECISION=bf16 +export NCCL_IB_TIMEOUT=23 +export MOE_ROUTER_TOPK=4 +export MOE_AUX_LOSS_COEFF=0.05 +export GDN_SPLIT_IN_PROJ=1 +export TORCH_FR_CPP_STACK=0 +export OMP_NUM_THREADS=4 +export TORCH_FR_BUFFER_SIZE=0 +export FLA_GDN_NUMERICAL_CP_SIZE=0 +export GDN_FUSED_SPLIT_FORWARD=1 +export GDN_ALPHA_WEIGHT_SCALE_MODE=matrix +export NVTE_ALLOW_NONDETERMINISTIC_ALGO=0 +export MOE_LAYER_FREQ=([0]*2+[1]*54) +export NCCL_CROSS_NIC=1 +export OPENBLAS_NUM_THREADS=1 +export MKL_NUM_THREADS=1 +export PATH=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin +export PYTHONPATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/flash-linear-attention:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/triton-distributed:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/linghe +export FLA_SOURCE_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/flash-linear-attention +export TRITON_DIST_PYTHON=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/triton-distributed +export TRITON_CACHE_DIR=/mnt/ssd/cpt/training_runtime/$(hostname)/triton +export TORCH_EXTENSIONS_DIR=/mnt/ssd/cpt/training_runtime/$(hostname)/torch_extensions +export TORCHINDUCTOR_CACHE_DIR=/mnt/ssd/cpt/training_runtime/$(hostname)/inductor +export PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True +export GDN_LR_MULT_ALPHA=1.0 +export MSC_CONFIG=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/msc/msc_config.yaml +export CUDA_DEVICE_MAX_CONNECTIONS=1 +mkdir -p "$TRITON_CACHE_DIR" "$TORCH_EXTENSIONS_DIR" "$TORCHINDUCTOR_CACHE_DIR" "$NUMA_BIND_LOG_DIR" 2>/dev/null || true +exec /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python -m torch.distributed.run --nnodes 8 --nproc-per-node 8 --node-rank 4 --master-addr 10.239.252.55 --master-port 35871 /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/pretrain_gpt.py --use-mcore-models --hybrid-attention-ratio 0.0625 --hybrid-mlp-ratio 0.5 --is-hybrid-model --linear-attention-type gated_delta_net --linear-conv-kernel-dim 4 --linear-key-head-dim 64 --linear-value-head-dim 64 --linear-num-key-heads 8 --linear-num-value-heads 32 --disable-bias-linear --seq-length 131072 --max-position-embeddings 131072 --num-layers 56 --hidden-size 2048 --ffn-hidden-size 4800 --num-attention-heads 32 --init-method-std 0.02 --attention-dropout 0.0 --hidden-dropout 0.0 --normalization RMSNorm --swiglu --group-query-attention --num-query-groups 8 --no-masked-softmax-fusion --use-flash-attn --add-qkv-bias --mtp-num-layers 0 --mtp-loss-scaling-factor 0.1 --norm-epsilon 1e-06 --mtp-linear-attention-type gated_delta_net --position-embedding-type none --linear-attention-freq '[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1]' --optimizer adam --adam-beta1 0.9 --adam-beta2 0.95 --adam-eps 1e-15 --weight-decay 0.1 --clip-grad 1.0 --data-path '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_{0..78}_input_ids_tmp' --split 999,1,0 --data-cache-path /mnt/hdd/cpt/binidx_cache_stage15_final_20261001_v2 --no-create-attention-mask-in-dataloader --num-experts 192 --moe-grouped-gemm --moe-router-topk 4 --moe-router-score-function softmax --overlap-param-gather --overlap-grad-reduce --moe-expert-capacity-factor 0 --moe-shared-expert-intermediate-size 512 --moe-router-dtype fp32 --moe-permute-fusion --moe-ffn-hidden-size 512 --moe-shared-expert-gate --moe-shared-expert-overlap --moe-layer-freq '([0]*2+[1]*54)' --moe-aux-loss-coeff 0.05 --moe-router-load-balancing-type global_aux_loss --micro-batch-size 1 --global-batch-size 128 --lr 5e-4 --train-samples 877440 --lr-warmup-samples 0 --lr-decay-samples 877440 --lr-decay-style constant --min-lr 0.0005 --bf16 --async-save --ckpt-fully-parallel-load --auto-detect-ckpt-format --load-complemental-dataset loss_mask --tensor-model-parallel-size 1 --pipeline-model-parallel-size 1 --context-parallel-size 8 --expert-model-parallel-size 8 --sequence-parallel --use-distributed-optimizer --log-interval 1 --log-throughput --save-interval 40 --eval-interval 99999999 --eval-iters 10 --increase-log-level-iters 5 --timing-log-level 0 --wandb-project qwen2.5_3b_mom_cpt --tokenizer-type HuggingFaceTokenizer --tokenizer-model /mnt/ssd/yulan/pretrain-linear-moe/preprocess/modify_tokenizer/YuLan-Mini-Qwen-Template_new_sp --no-adam-bias-correction --corrected-weight-decay --auto-generate-cu-seqlens --cp-comm-type a2a --log-moe-activations --final-logits-clamp-value 30 --attention-output-gate --gated-norm --gated-norm-lora-rank 16 --dataloader-type cyclic --lm-loss-normalization global-token --prefetch-factor 8 --activation-func-clamp-value 7 --activation-func-clamp-mode soft --gdn-split-in-proj --gdn-fused-split-forward --gdn-alpha-weight-scale 1.0 --gdn-alpha-weight-scale-mode matrix --gdn-lr-mult-alpha 0.5 --start-samples 0 --ckpt-format torch_dist --rerun-mode validate_results --distributed-timeout-minutes 120 --dist-ckpt-strictness assume_ok_unexpected --gdn-backend fla --gdn-gated-norm-backend linghe --moe-router-gemm-backend te --log-params-norm --timing-log-option all --no-barrier-with-level-1-timing --log-metrics-interval 10 --cross-entropy-loss-fusion --cross-entropy-fusion-impl linear_triton --linear-cross-entropy-num-chunks 8 --num-workers 4 --moe-router-sqrt-gate --moe-router-fusion --moe-token-dispatcher-type alltoall --log-hidden-states embeddings residual_attn input_layernorm attention::q_layernorm attention::k_layernorm attention::core_attention attention::o_proj residual_mlp pre_mlp_layernorm mlp moe::router moe::shared_experts moe::routed_experts gated_delta::in_proj gated_delta::conv1d gated_delta::query gated_delta::key gated_delta::value gated_delta::gate gated_delta::beta gated_delta::alpha gated_delta::g gated_delta::core_attention gated_delta::out_norm gated_delta::out_proj final_layernorm --log-params attention::linear_qkv attention::o_proj mlp::linear_fc1 mlp::linear_fc2 input_layernorm pre_mlp_layernorm embedding lm_head gated_delta::in_proj gated_delta::in_proj_bias gated_delta::conv1d gated_delta::conv1d_bias gated_delta::dt_bias gated_delta::A_log gated_delta::out_norm gated_delta::out_norm_bias gated_delta::out_proj gated_delta::out_proj_bias --log-update-rms --log-per-module-grad-rms --log-per-layer-grad-rms --document-packing-algorithm obfd --exit-on-missing-checkpoint --no-initialization --untie-embeddings-and-output-weights --recompute-granularity full --recompute-method block --recompute-num-layers 56 --no-save-step-one --save-retain-interval 1000000000 --exit-signal-handler --fast-thd-a2a --hybrid-adamh --reuse-training-communicators --ddp-bucket-size 40000000 --load /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage14_120B_8node_gdnlralpha_off_20260925a/checkpoints --continual-pretrain --save /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints --tensorboard-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/tensorboard --wandb-save-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/wandb --wandb-exp-name stage14_120B_8node_gdnlralpha_off_20260925a --runtime-audit-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/runtime_audit diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/launch_node5.sh b/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/launch_node5.sh new file mode 100644 index 0000000000000000000000000000000000000000..4ef8910d8dd40b04012573ba89916af8f5c3b4c5 --- /dev/null +++ b/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/launch_node5.sh @@ -0,0 +1,68 @@ +#!/bin/bash +# Stage15 launch: loads stage14 final weights (--continual-pretrain resets iteration to 0). +# Token budget 877440 samples x 131072 seq = 115.0B tokens (dataset has 115.59B). +# Command identical to stage14 resume except: data-path/cache, train-samples, load+continual-pretrain, master-port 35871. +ulimit -n 1048576 +# Env replicated from tools/launch_training.py + configs/training/formal3_128k_adamh.json +# (config environment applied first; explicit exports below win via setdefault semantics). +# MOE_AMPERE_GROUPED_MLP=1 is REQUIRED: without it experts fall back to SequentialMLP +# and the optimizer checkpoint shape-mismatches (ckpt saved with A100GroupedMLP). +export MOE_ROUTER_SQRT_GATE=1 +export MOE_ROUTER_SCORE_FUNCTION=softmax +export TORCH_NCCL_TRACE_BUFFER_SIZE=0 +export MOE_AMPERE_GROUPED_MLP=1 +export NCCL_IB_PCI_RELAXED_ORDERING=1 +export NCCL_MIN_NCHANNELS=8 +export GDN_ALPHA_WEIGHT_SCALE=1.0 +export NCCL_RAS_ENABLE=0 +export TORCH_NCCL_TRACE_CPP_STACK=0 +export NCCL_SOCKET_IFNAME=eth0 +export GDN_EPS=1e-6 +export NCCL_NET_GDR_LEVEL=3 +export GDN_PRE_GDR_FUSION=0 +export MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512 +export NCCL_DEBUG=WARN +export MOE_TOKEN_DISPATCHER_TYPE=alltoall +export NCCL_TIMEOUT=18000 +export NCCL_IB_HCA=mlx5 +export GDN_CP_DISABLE_LAYOUT_BARRIER=0 +export NCCL_IB_GID_INDEX=3 +export TORCH_PROFILER_WITH_STACK=1 +export MOE_TRITON_SHARED_EXPERT_OVERLAP=0 +export MOE_EXPERT_CAPACITY_FACTOR=0 +export TORCH_FORCE_NO_WEIGHTS_ONLY_LOAD=true +export GDN_ALPHA_PROFILE_NAME=a05_lr +export NCCL_IB_QPS_PER_CONNECTION=4 +export GDN_PRE_GDR_FUSION_OVERRIDE=0 +export TORCH_NCCL_DUMP_ON_TIMEOUT=0 +export MOE_FFN_HIDDEN_SIZE=512 +export MOE_ROUTER_NORM_TOPK_PROB=true +export FLA_GDN_CP_STATE_PRECISION=bf16 +export NCCL_IB_TIMEOUT=23 +export MOE_ROUTER_TOPK=4 +export MOE_AUX_LOSS_COEFF=0.05 +export GDN_SPLIT_IN_PROJ=1 +export TORCH_FR_CPP_STACK=0 +export OMP_NUM_THREADS=4 +export TORCH_FR_BUFFER_SIZE=0 +export FLA_GDN_NUMERICAL_CP_SIZE=0 +export GDN_FUSED_SPLIT_FORWARD=1 +export GDN_ALPHA_WEIGHT_SCALE_MODE=matrix +export NVTE_ALLOW_NONDETERMINISTIC_ALGO=0 +export MOE_LAYER_FREQ=([0]*2+[1]*54) +export NCCL_CROSS_NIC=1 +export OPENBLAS_NUM_THREADS=1 +export MKL_NUM_THREADS=1 +export PATH=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin +export PYTHONPATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/flash-linear-attention:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/triton-distributed:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/linghe +export FLA_SOURCE_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/flash-linear-attention +export TRITON_DIST_PYTHON=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/triton-distributed +export TRITON_CACHE_DIR=/mnt/ssd/cpt/training_runtime/$(hostname)/triton +export TORCH_EXTENSIONS_DIR=/mnt/ssd/cpt/training_runtime/$(hostname)/torch_extensions +export TORCHINDUCTOR_CACHE_DIR=/mnt/ssd/cpt/training_runtime/$(hostname)/inductor +export PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True +export GDN_LR_MULT_ALPHA=1.0 +export MSC_CONFIG=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/msc/msc_config.yaml +export CUDA_DEVICE_MAX_CONNECTIONS=1 +mkdir -p "$TRITON_CACHE_DIR" "$TORCH_EXTENSIONS_DIR" "$TORCHINDUCTOR_CACHE_DIR" "$NUMA_BIND_LOG_DIR" 2>/dev/null || true +exec /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python -m torch.distributed.run --nnodes 8 --nproc-per-node 8 --node-rank 5 --master-addr 10.239.252.55 --master-port 35871 /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/pretrain_gpt.py --use-mcore-models --hybrid-attention-ratio 0.0625 --hybrid-mlp-ratio 0.5 --is-hybrid-model --linear-attention-type gated_delta_net --linear-conv-kernel-dim 4 --linear-key-head-dim 64 --linear-value-head-dim 64 --linear-num-key-heads 8 --linear-num-value-heads 32 --disable-bias-linear --seq-length 131072 --max-position-embeddings 131072 --num-layers 56 --hidden-size 2048 --ffn-hidden-size 4800 --num-attention-heads 32 --init-method-std 0.02 --attention-dropout 0.0 --hidden-dropout 0.0 --normalization RMSNorm --swiglu --group-query-attention --num-query-groups 8 --no-masked-softmax-fusion --use-flash-attn --add-qkv-bias --mtp-num-layers 0 --mtp-loss-scaling-factor 0.1 --norm-epsilon 1e-06 --mtp-linear-attention-type gated_delta_net --position-embedding-type none --linear-attention-freq '[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1]' --optimizer adam --adam-beta1 0.9 --adam-beta2 0.95 --adam-eps 1e-15 --weight-decay 0.1 --clip-grad 1.0 --data-path '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_{0..78}_input_ids_tmp' --split 999,1,0 --data-cache-path /mnt/hdd/cpt/binidx_cache_stage15_final_20261001_v2 --no-create-attention-mask-in-dataloader --num-experts 192 --moe-grouped-gemm --moe-router-topk 4 --moe-router-score-function softmax --overlap-param-gather --overlap-grad-reduce --moe-expert-capacity-factor 0 --moe-shared-expert-intermediate-size 512 --moe-router-dtype fp32 --moe-permute-fusion --moe-ffn-hidden-size 512 --moe-shared-expert-gate --moe-shared-expert-overlap --moe-layer-freq '([0]*2+[1]*54)' --moe-aux-loss-coeff 0.05 --moe-router-load-balancing-type global_aux_loss --micro-batch-size 1 --global-batch-size 128 --lr 5e-4 --train-samples 877440 --lr-warmup-samples 0 --lr-decay-samples 877440 --lr-decay-style constant --min-lr 0.0005 --bf16 --async-save --ckpt-fully-parallel-load --auto-detect-ckpt-format --load-complemental-dataset loss_mask --tensor-model-parallel-size 1 --pipeline-model-parallel-size 1 --context-parallel-size 8 --expert-model-parallel-size 8 --sequence-parallel --use-distributed-optimizer --log-interval 1 --log-throughput --save-interval 40 --eval-interval 99999999 --eval-iters 10 --increase-log-level-iters 5 --timing-log-level 0 --wandb-project qwen2.5_3b_mom_cpt --tokenizer-type HuggingFaceTokenizer --tokenizer-model /mnt/ssd/yulan/pretrain-linear-moe/preprocess/modify_tokenizer/YuLan-Mini-Qwen-Template_new_sp --no-adam-bias-correction --corrected-weight-decay --auto-generate-cu-seqlens --cp-comm-type a2a --log-moe-activations --final-logits-clamp-value 30 --attention-output-gate --gated-norm --gated-norm-lora-rank 16 --dataloader-type cyclic --lm-loss-normalization global-token --prefetch-factor 8 --activation-func-clamp-value 7 --activation-func-clamp-mode soft --gdn-split-in-proj --gdn-fused-split-forward --gdn-alpha-weight-scale 1.0 --gdn-alpha-weight-scale-mode matrix --gdn-lr-mult-alpha 0.5 --start-samples 0 --ckpt-format torch_dist --rerun-mode validate_results --distributed-timeout-minutes 120 --dist-ckpt-strictness assume_ok_unexpected --gdn-backend fla --gdn-gated-norm-backend linghe --moe-router-gemm-backend te --log-params-norm --timing-log-option all --no-barrier-with-level-1-timing --log-metrics-interval 10 --cross-entropy-loss-fusion --cross-entropy-fusion-impl linear_triton --linear-cross-entropy-num-chunks 8 --num-workers 4 --moe-router-sqrt-gate --moe-router-fusion --moe-token-dispatcher-type alltoall --log-hidden-states embeddings residual_attn input_layernorm attention::q_layernorm attention::k_layernorm attention::core_attention attention::o_proj residual_mlp pre_mlp_layernorm mlp moe::router moe::shared_experts moe::routed_experts gated_delta::in_proj gated_delta::conv1d gated_delta::query gated_delta::key gated_delta::value gated_delta::gate gated_delta::beta gated_delta::alpha gated_delta::g gated_delta::core_attention gated_delta::out_norm gated_delta::out_proj final_layernorm --log-params attention::linear_qkv attention::o_proj mlp::linear_fc1 mlp::linear_fc2 input_layernorm pre_mlp_layernorm embedding lm_head gated_delta::in_proj gated_delta::in_proj_bias gated_delta::conv1d gated_delta::conv1d_bias gated_delta::dt_bias gated_delta::A_log gated_delta::out_norm gated_delta::out_norm_bias gated_delta::out_proj gated_delta::out_proj_bias --log-update-rms --log-per-module-grad-rms --log-per-layer-grad-rms --document-packing-algorithm obfd --exit-on-missing-checkpoint --no-initialization --untie-embeddings-and-output-weights --recompute-granularity full --recompute-method block --recompute-num-layers 56 --no-save-step-one --save-retain-interval 1000000000 --exit-signal-handler --fast-thd-a2a --hybrid-adamh --reuse-training-communicators --ddp-bucket-size 40000000 --load /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage14_120B_8node_gdnlralpha_off_20260925a/checkpoints --continual-pretrain --save /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints --tensorboard-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/tensorboard --wandb-save-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/wandb --wandb-exp-name stage14_120B_8node_gdnlralpha_off_20260925a --runtime-audit-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/runtime_audit diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/launch_node6.sh b/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/launch_node6.sh new file mode 100644 index 0000000000000000000000000000000000000000..add668aa75becfa0aa907bd58226531038ea2817 --- /dev/null +++ b/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/launch_node6.sh @@ -0,0 +1,68 @@ +#!/bin/bash +# Stage15 launch: loads stage14 final weights (--continual-pretrain resets iteration to 0). +# Token budget 877440 samples x 131072 seq = 115.0B tokens (dataset has 115.59B). +# Command identical to stage14 resume except: data-path/cache, train-samples, load+continual-pretrain, master-port 35871. +ulimit -n 1048576 +# Env replicated from tools/launch_training.py + configs/training/formal3_128k_adamh.json +# (config environment applied first; explicit exports below win via setdefault semantics). +# MOE_AMPERE_GROUPED_MLP=1 is REQUIRED: without it experts fall back to SequentialMLP +# and the optimizer checkpoint shape-mismatches (ckpt saved with A100GroupedMLP). +export MOE_ROUTER_SQRT_GATE=1 +export MOE_ROUTER_SCORE_FUNCTION=softmax +export TORCH_NCCL_TRACE_BUFFER_SIZE=0 +export MOE_AMPERE_GROUPED_MLP=1 +export NCCL_IB_PCI_RELAXED_ORDERING=1 +export NCCL_MIN_NCHANNELS=8 +export GDN_ALPHA_WEIGHT_SCALE=1.0 +export NCCL_RAS_ENABLE=0 +export TORCH_NCCL_TRACE_CPP_STACK=0 +export NCCL_SOCKET_IFNAME=eth0 +export GDN_EPS=1e-6 +export NCCL_NET_GDR_LEVEL=3 +export GDN_PRE_GDR_FUSION=0 +export MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512 +export NCCL_DEBUG=WARN +export MOE_TOKEN_DISPATCHER_TYPE=alltoall +export NCCL_TIMEOUT=18000 +export NCCL_IB_HCA=mlx5 +export GDN_CP_DISABLE_LAYOUT_BARRIER=0 +export NCCL_IB_GID_INDEX=3 +export TORCH_PROFILER_WITH_STACK=1 +export MOE_TRITON_SHARED_EXPERT_OVERLAP=0 +export MOE_EXPERT_CAPACITY_FACTOR=0 +export TORCH_FORCE_NO_WEIGHTS_ONLY_LOAD=true +export GDN_ALPHA_PROFILE_NAME=a05_lr +export NCCL_IB_QPS_PER_CONNECTION=4 +export GDN_PRE_GDR_FUSION_OVERRIDE=0 +export TORCH_NCCL_DUMP_ON_TIMEOUT=0 +export MOE_FFN_HIDDEN_SIZE=512 +export MOE_ROUTER_NORM_TOPK_PROB=true +export FLA_GDN_CP_STATE_PRECISION=bf16 +export NCCL_IB_TIMEOUT=23 +export MOE_ROUTER_TOPK=4 +export MOE_AUX_LOSS_COEFF=0.05 +export GDN_SPLIT_IN_PROJ=1 +export TORCH_FR_CPP_STACK=0 +export OMP_NUM_THREADS=4 +export TORCH_FR_BUFFER_SIZE=0 +export FLA_GDN_NUMERICAL_CP_SIZE=0 +export GDN_FUSED_SPLIT_FORWARD=1 +export GDN_ALPHA_WEIGHT_SCALE_MODE=matrix +export NVTE_ALLOW_NONDETERMINISTIC_ALGO=0 +export MOE_LAYER_FREQ=([0]*2+[1]*54) +export NCCL_CROSS_NIC=1 +export OPENBLAS_NUM_THREADS=1 +export MKL_NUM_THREADS=1 +export PATH=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin +export PYTHONPATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/flash-linear-attention:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/triton-distributed:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/linghe +export FLA_SOURCE_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/flash-linear-attention +export TRITON_DIST_PYTHON=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/triton-distributed +export TRITON_CACHE_DIR=/mnt/ssd/cpt/training_runtime/$(hostname)/triton +export TORCH_EXTENSIONS_DIR=/mnt/ssd/cpt/training_runtime/$(hostname)/torch_extensions +export TORCHINDUCTOR_CACHE_DIR=/mnt/ssd/cpt/training_runtime/$(hostname)/inductor +export PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True +export GDN_LR_MULT_ALPHA=1.0 +export MSC_CONFIG=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/msc/msc_config.yaml +export CUDA_DEVICE_MAX_CONNECTIONS=1 +mkdir -p "$TRITON_CACHE_DIR" "$TORCH_EXTENSIONS_DIR" "$TORCHINDUCTOR_CACHE_DIR" "$NUMA_BIND_LOG_DIR" 2>/dev/null || true +exec /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python -m torch.distributed.run --nnodes 8 --nproc-per-node 8 --node-rank 6 --master-addr 10.239.252.55 --master-port 35871 /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/pretrain_gpt.py --use-mcore-models --hybrid-attention-ratio 0.0625 --hybrid-mlp-ratio 0.5 --is-hybrid-model --linear-attention-type gated_delta_net --linear-conv-kernel-dim 4 --linear-key-head-dim 64 --linear-value-head-dim 64 --linear-num-key-heads 8 --linear-num-value-heads 32 --disable-bias-linear --seq-length 131072 --max-position-embeddings 131072 --num-layers 56 --hidden-size 2048 --ffn-hidden-size 4800 --num-attention-heads 32 --init-method-std 0.02 --attention-dropout 0.0 --hidden-dropout 0.0 --normalization RMSNorm --swiglu --group-query-attention --num-query-groups 8 --no-masked-softmax-fusion --use-flash-attn --add-qkv-bias --mtp-num-layers 0 --mtp-loss-scaling-factor 0.1 --norm-epsilon 1e-06 --mtp-linear-attention-type gated_delta_net --position-embedding-type none --linear-attention-freq '[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1]' --optimizer adam --adam-beta1 0.9 --adam-beta2 0.95 --adam-eps 1e-15 --weight-decay 0.1 --clip-grad 1.0 --data-path '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_{0..78}_input_ids_tmp' --split 999,1,0 --data-cache-path /mnt/hdd/cpt/binidx_cache_stage15_final_20261001_v2 --no-create-attention-mask-in-dataloader --num-experts 192 --moe-grouped-gemm --moe-router-topk 4 --moe-router-score-function softmax --overlap-param-gather --overlap-grad-reduce --moe-expert-capacity-factor 0 --moe-shared-expert-intermediate-size 512 --moe-router-dtype fp32 --moe-permute-fusion --moe-ffn-hidden-size 512 --moe-shared-expert-gate --moe-shared-expert-overlap --moe-layer-freq '([0]*2+[1]*54)' --moe-aux-loss-coeff 0.05 --moe-router-load-balancing-type global_aux_loss --micro-batch-size 1 --global-batch-size 128 --lr 5e-4 --train-samples 877440 --lr-warmup-samples 0 --lr-decay-samples 877440 --lr-decay-style constant --min-lr 0.0005 --bf16 --async-save --ckpt-fully-parallel-load --auto-detect-ckpt-format --load-complemental-dataset loss_mask --tensor-model-parallel-size 1 --pipeline-model-parallel-size 1 --context-parallel-size 8 --expert-model-parallel-size 8 --sequence-parallel --use-distributed-optimizer --log-interval 1 --log-throughput --save-interval 40 --eval-interval 99999999 --eval-iters 10 --increase-log-level-iters 5 --timing-log-level 0 --wandb-project qwen2.5_3b_mom_cpt --tokenizer-type HuggingFaceTokenizer --tokenizer-model /mnt/ssd/yulan/pretrain-linear-moe/preprocess/modify_tokenizer/YuLan-Mini-Qwen-Template_new_sp --no-adam-bias-correction --corrected-weight-decay --auto-generate-cu-seqlens --cp-comm-type a2a --log-moe-activations --final-logits-clamp-value 30 --attention-output-gate --gated-norm --gated-norm-lora-rank 16 --dataloader-type cyclic --lm-loss-normalization global-token --prefetch-factor 8 --activation-func-clamp-value 7 --activation-func-clamp-mode soft --gdn-split-in-proj --gdn-fused-split-forward --gdn-alpha-weight-scale 1.0 --gdn-alpha-weight-scale-mode matrix --gdn-lr-mult-alpha 0.5 --start-samples 0 --ckpt-format torch_dist --rerun-mode validate_results --distributed-timeout-minutes 120 --dist-ckpt-strictness assume_ok_unexpected --gdn-backend fla --gdn-gated-norm-backend linghe --moe-router-gemm-backend te --log-params-norm --timing-log-option all --no-barrier-with-level-1-timing --log-metrics-interval 10 --cross-entropy-loss-fusion --cross-entropy-fusion-impl linear_triton --linear-cross-entropy-num-chunks 8 --num-workers 4 --moe-router-sqrt-gate --moe-router-fusion --moe-token-dispatcher-type alltoall --log-hidden-states embeddings residual_attn input_layernorm attention::q_layernorm attention::k_layernorm attention::core_attention attention::o_proj residual_mlp pre_mlp_layernorm mlp moe::router moe::shared_experts moe::routed_experts gated_delta::in_proj gated_delta::conv1d gated_delta::query gated_delta::key gated_delta::value gated_delta::gate gated_delta::beta gated_delta::alpha gated_delta::g gated_delta::core_attention gated_delta::out_norm gated_delta::out_proj final_layernorm --log-params attention::linear_qkv attention::o_proj mlp::linear_fc1 mlp::linear_fc2 input_layernorm pre_mlp_layernorm embedding lm_head gated_delta::in_proj gated_delta::in_proj_bias gated_delta::conv1d gated_delta::conv1d_bias gated_delta::dt_bias gated_delta::A_log gated_delta::out_norm gated_delta::out_norm_bias gated_delta::out_proj gated_delta::out_proj_bias --log-update-rms --log-per-module-grad-rms --log-per-layer-grad-rms --document-packing-algorithm obfd --exit-on-missing-checkpoint --no-initialization --untie-embeddings-and-output-weights --recompute-granularity full --recompute-method block --recompute-num-layers 56 --no-save-step-one --save-retain-interval 1000000000 --exit-signal-handler --fast-thd-a2a --hybrid-adamh --reuse-training-communicators --ddp-bucket-size 40000000 --load /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage14_120B_8node_gdnlralpha_off_20260925a/checkpoints --continual-pretrain --save /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints --tensorboard-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/tensorboard --wandb-save-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/wandb --wandb-exp-name stage14_120B_8node_gdnlralpha_off_20260925a --runtime-audit-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/runtime_audit diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/launch_node7.sh b/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/launch_node7.sh new file mode 100644 index 0000000000000000000000000000000000000000..e0b11b9ac4f9f4773d6d69dd40464dd8bfcf5b3d --- /dev/null +++ b/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/launch_node7.sh @@ -0,0 +1,68 @@ +#!/bin/bash +# Stage15 launch: loads stage14 final weights (--continual-pretrain resets iteration to 0). +# Token budget 877440 samples x 131072 seq = 115.0B tokens (dataset has 115.59B). +# Command identical to stage14 resume except: data-path/cache, train-samples, load+continual-pretrain, master-port 35871. +ulimit -n 1048576 +# Env replicated from tools/launch_training.py + configs/training/formal3_128k_adamh.json +# (config environment applied first; explicit exports below win via setdefault semantics). +# MOE_AMPERE_GROUPED_MLP=1 is REQUIRED: without it experts fall back to SequentialMLP +# and the optimizer checkpoint shape-mismatches (ckpt saved with A100GroupedMLP). +export MOE_ROUTER_SQRT_GATE=1 +export MOE_ROUTER_SCORE_FUNCTION=softmax +export TORCH_NCCL_TRACE_BUFFER_SIZE=0 +export MOE_AMPERE_GROUPED_MLP=1 +export NCCL_IB_PCI_RELAXED_ORDERING=1 +export NCCL_MIN_NCHANNELS=8 +export GDN_ALPHA_WEIGHT_SCALE=1.0 +export NCCL_RAS_ENABLE=0 +export TORCH_NCCL_TRACE_CPP_STACK=0 +export NCCL_SOCKET_IFNAME=eth0 +export GDN_EPS=1e-6 +export NCCL_NET_GDR_LEVEL=3 +export GDN_PRE_GDR_FUSION=0 +export MOE_SHARED_EXPERT_INTERMEDIATE_SIZE=512 +export NCCL_DEBUG=WARN +export MOE_TOKEN_DISPATCHER_TYPE=alltoall +export NCCL_TIMEOUT=18000 +export NCCL_IB_HCA=mlx5 +export GDN_CP_DISABLE_LAYOUT_BARRIER=0 +export NCCL_IB_GID_INDEX=3 +export TORCH_PROFILER_WITH_STACK=1 +export MOE_TRITON_SHARED_EXPERT_OVERLAP=0 +export MOE_EXPERT_CAPACITY_FACTOR=0 +export TORCH_FORCE_NO_WEIGHTS_ONLY_LOAD=true +export GDN_ALPHA_PROFILE_NAME=a05_lr +export NCCL_IB_QPS_PER_CONNECTION=4 +export GDN_PRE_GDR_FUSION_OVERRIDE=0 +export TORCH_NCCL_DUMP_ON_TIMEOUT=0 +export MOE_FFN_HIDDEN_SIZE=512 +export MOE_ROUTER_NORM_TOPK_PROB=true +export FLA_GDN_CP_STATE_PRECISION=bf16 +export NCCL_IB_TIMEOUT=23 +export MOE_ROUTER_TOPK=4 +export MOE_AUX_LOSS_COEFF=0.05 +export GDN_SPLIT_IN_PROJ=1 +export TORCH_FR_CPP_STACK=0 +export OMP_NUM_THREADS=4 +export TORCH_FR_BUFFER_SIZE=0 +export FLA_GDN_NUMERICAL_CP_SIZE=0 +export GDN_FUSED_SPLIT_FORWARD=1 +export GDN_ALPHA_WEIGHT_SCALE_MODE=matrix +export NVTE_ALLOW_NONDETERMINISTIC_ALGO=0 +export MOE_LAYER_FREQ=([0]*2+[1]*54) +export NCCL_CROSS_NIC=1 +export OPENBLAS_NUM_THREADS=1 +export MKL_NUM_THREADS=1 +export PATH=/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin +export PYTHONPATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/flash-linear-attention:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/triton-distributed:/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/linghe +export FLA_SOURCE_PATH=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/flash-linear-attention +export TRITON_DIST_PYTHON=/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/third_party/triton-distributed +export TRITON_CACHE_DIR=/mnt/ssd/cpt/training_runtime/$(hostname)/triton +export TORCH_EXTENSIONS_DIR=/mnt/ssd/cpt/training_runtime/$(hostname)/torch_extensions +export TORCHINDUCTOR_CACHE_DIR=/mnt/ssd/cpt/training_runtime/$(hostname)/inductor +export PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True +export GDN_LR_MULT_ALPHA=1.0 +export MSC_CONFIG=/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/msc/msc_config.yaml +export CUDA_DEVICE_MAX_CONNECTIONS=1 +mkdir -p "$TRITON_CACHE_DIR" "$TORCH_EXTENSIONS_DIR" "$TORCHINDUCTOR_CACHE_DIR" "$NUMA_BIND_LOG_DIR" 2>/dev/null || true +exec /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python -m torch.distributed.run --nnodes 8 --nproc-per-node 8 --node-rank 7 --master-addr 10.239.252.55 --master-port 35871 /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/pretrain_gpt.py --use-mcore-models --hybrid-attention-ratio 0.0625 --hybrid-mlp-ratio 0.5 --is-hybrid-model --linear-attention-type gated_delta_net --linear-conv-kernel-dim 4 --linear-key-head-dim 64 --linear-value-head-dim 64 --linear-num-key-heads 8 --linear-num-value-heads 32 --disable-bias-linear --seq-length 131072 --max-position-embeddings 131072 --num-layers 56 --hidden-size 2048 --ffn-hidden-size 4800 --num-attention-heads 32 --init-method-std 0.02 --attention-dropout 0.0 --hidden-dropout 0.0 --normalization RMSNorm --swiglu --group-query-attention --num-query-groups 8 --no-masked-softmax-fusion --use-flash-attn --add-qkv-bias --mtp-num-layers 0 --mtp-loss-scaling-factor 0.1 --norm-epsilon 1e-06 --mtp-linear-attention-type gated_delta_net --position-embedding-type none --linear-attention-freq '[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1]' --optimizer adam --adam-beta1 0.9 --adam-beta2 0.95 --adam-eps 1e-15 --weight-decay 0.1 --clip-grad 1.0 --data-path '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_{0..78}_input_ids_tmp' --split 999,1,0 --data-cache-path /mnt/hdd/cpt/binidx_cache_stage15_final_20261001_v2 --no-create-attention-mask-in-dataloader --num-experts 192 --moe-grouped-gemm --moe-router-topk 4 --moe-router-score-function softmax --overlap-param-gather --overlap-grad-reduce --moe-expert-capacity-factor 0 --moe-shared-expert-intermediate-size 512 --moe-router-dtype fp32 --moe-permute-fusion --moe-ffn-hidden-size 512 --moe-shared-expert-gate --moe-shared-expert-overlap --moe-layer-freq '([0]*2+[1]*54)' --moe-aux-loss-coeff 0.05 --moe-router-load-balancing-type global_aux_loss --micro-batch-size 1 --global-batch-size 128 --lr 5e-4 --train-samples 877440 --lr-warmup-samples 0 --lr-decay-samples 877440 --lr-decay-style constant --min-lr 0.0005 --bf16 --async-save --ckpt-fully-parallel-load --auto-detect-ckpt-format --load-complemental-dataset loss_mask --tensor-model-parallel-size 1 --pipeline-model-parallel-size 1 --context-parallel-size 8 --expert-model-parallel-size 8 --sequence-parallel --use-distributed-optimizer --log-interval 1 --log-throughput --save-interval 40 --eval-interval 99999999 --eval-iters 10 --increase-log-level-iters 5 --timing-log-level 0 --wandb-project qwen2.5_3b_mom_cpt --tokenizer-type HuggingFaceTokenizer --tokenizer-model /mnt/ssd/yulan/pretrain-linear-moe/preprocess/modify_tokenizer/YuLan-Mini-Qwen-Template_new_sp --no-adam-bias-correction --corrected-weight-decay --auto-generate-cu-seqlens --cp-comm-type a2a --log-moe-activations --final-logits-clamp-value 30 --attention-output-gate --gated-norm --gated-norm-lora-rank 16 --dataloader-type cyclic --lm-loss-normalization global-token --prefetch-factor 8 --activation-func-clamp-value 7 --activation-func-clamp-mode soft --gdn-split-in-proj --gdn-fused-split-forward --gdn-alpha-weight-scale 1.0 --gdn-alpha-weight-scale-mode matrix --gdn-lr-mult-alpha 0.5 --start-samples 0 --ckpt-format torch_dist --rerun-mode validate_results --distributed-timeout-minutes 120 --dist-ckpt-strictness assume_ok_unexpected --gdn-backend fla --gdn-gated-norm-backend linghe --moe-router-gemm-backend te --log-params-norm --timing-log-option all --no-barrier-with-level-1-timing --log-metrics-interval 10 --cross-entropy-loss-fusion --cross-entropy-fusion-impl linear_triton --linear-cross-entropy-num-chunks 8 --num-workers 4 --moe-router-sqrt-gate --moe-router-fusion --moe-token-dispatcher-type alltoall --log-hidden-states embeddings residual_attn input_layernorm attention::q_layernorm attention::k_layernorm attention::core_attention attention::o_proj residual_mlp pre_mlp_layernorm mlp moe::router moe::shared_experts moe::routed_experts gated_delta::in_proj gated_delta::conv1d gated_delta::query gated_delta::key gated_delta::value gated_delta::gate gated_delta::beta gated_delta::alpha gated_delta::g gated_delta::core_attention gated_delta::out_norm gated_delta::out_proj final_layernorm --log-params attention::linear_qkv attention::o_proj mlp::linear_fc1 mlp::linear_fc2 input_layernorm pre_mlp_layernorm embedding lm_head gated_delta::in_proj gated_delta::in_proj_bias gated_delta::conv1d gated_delta::conv1d_bias gated_delta::dt_bias gated_delta::A_log gated_delta::out_norm gated_delta::out_norm_bias gated_delta::out_proj gated_delta::out_proj_bias --log-update-rms --log-per-module-grad-rms --log-per-layer-grad-rms --document-packing-algorithm obfd --exit-on-missing-checkpoint --no-initialization --untie-embeddings-and-output-weights --recompute-granularity full --recompute-method block --recompute-num-layers 56 --no-save-step-one --save-retain-interval 1000000000 --exit-signal-handler --fast-thd-a2a --hybrid-adamh --reuse-training-communicators --ddp-bucket-size 40000000 --load /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage14_120B_8node_gdnlralpha_off_20260925a/checkpoints --continual-pretrain --save /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints --tensorboard-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/tensorboard --wandb-save-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/wandb --wandb-exp-name stage14_120B_8node_gdnlralpha_off_20260925a --runtime-audit-dir /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/runtime_audit diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/node0.log b/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/node0.log new file mode 100644 index 0000000000000000000000000000000000000000..d8765c6b7e84bc66cdde132a41cd6a9433a710a2 --- /dev/null +++ b/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/node0.log @@ -0,0 +1,5515 @@ +W1003 15:36:45.288000 313157 torch/utils/cpp_extension.py:2425] TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +W1003 15:36:45.288000 313157 torch/utils/cpp_extension.py:2425] If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'] to specific architectures. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +using world size: 64, data-parallel size: 8, context-parallel size: 8, hierarchical context-parallel sizes: None, tensor-model-parallel size: 1, pipeline-model-parallel size: 1 +WARNING: overriding default arguments for tokenizer_type:GPT2BPETokenizer with tokenizer_type:HuggingFaceTokenizer +Number of virtual stages per pipeline stage: None +[gdn-fused-split-forward] optimizer overlap enabled: param-gather=ddp-extra-param-bridge grad-reduce=standard-autograd-hooks +accumulate and all-reduce gradients in fp32 for bfloat16 data type. +using torch.bfloat16 for parameters ... +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/training/utils.py:548: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn(message) +------------------------ arguments ------------------------ + account_for_embedding_in_pipeline_split ......... False + account_for_loss_in_pipeline_split .............. False + accumulate_allreduce_grads_in_fp32 .............. True + activation_func_clamp_mode ...................... soft + activation_func_clamp_value ..................... 7.0 + adam_beta1 ...................................... 0.9 + adam_beta2 ...................................... 0.95 + adam_bias_correction ............................ False + adam_eps ........................................ 1e-15 + adamh_eps ....................................... 1e-08 + adamh_radius .................................... None + adamh_scope ..................................... lm_head + adamw_lr_mup_scaler ............................. False + add_bias_linear ................................. False + add_position_embedding .......................... True + add_qkv_bias .................................... True + adlr_autoresume ................................. False + adlr_autoresume_interval ........................ 1000 + align_grad_reduce ............................... True + align_param_gather .............................. False + allow_ambiguous_pad_tokens ...................... False + app_tag_run_name ................................ None + app_tag_run_version ............................. 0.0.0 + apply_layernorm_1p .............................. False + apply_query_key_layer_scaling ................... False + apply_residual_connection_post_layernorm ........ False + apply_rope_fusion ............................... False + async_save ...................................... True + async_tensor_model_parallel_allreduce ........... True + attention_backend ............................... AttnBackend.auto + attention_dropout ............................... 0.0 + attention_output_gate ........................... True + attention_output_gate_upcycling ................. False + attention_softmax_in_fp32 ....................... False + attn_k_token_shift .............................. None + attn_output_gate ................................ None + attn_q_token_shift .............................. None + attn_token_shift ................................ None + attn_v_token_shift .............................. None + auto_detect_ckpt_format ......................... True + auto_generate_cu_seqlens ........................ True + barrier_with_L1_time ............................ False + benchmark_eval .................................. False + benchmark_global_batch .......................... None + benchmark_interval .............................. None + benchmark_micro_batch ........................... None + benchmark_sequence_length ....................... None + benchmark_tasks ................................. None + bert_binary_head ................................ True + bert_embedder_type .............................. megatron + bert_load ....................................... None + bf16 ............................................ True + bias_dropout_fusion ............................. True + bias_gelu_fusion ................................ False + bias_swiglu_fusion .............................. True + biencoder_projection_dim ........................ 0 + biencoder_shared_query_context_model ............ False + block_data_path ................................. None + cache_mla_latents ............................... False + calc_ft_timeouts ................................ False + calculate_per_token_loss ........................ False + check_for_large_grads ........................... False + check_for_nan_in_loss_and_grad .................. True + check_for_spiky_loss ............................ False + check_weight_hash_across_dp_replicas_interval ... None + ckpt_assume_constant_structure .................. False + ckpt_convert_format ............................. None + ckpt_convert_save ............................... None + ckpt_convert_update_legacy_dist_opt_format ...... False + ckpt_format ..................................... torch_dist + ckpt_fully_parallel_load ........................ True + ckpt_fully_parallel_save ........................ True + ckpt_fully_parallel_save_deprecated ............. False + ckpt_step ....................................... None + classes_fraction ................................ 1.0 + clip_grad ....................................... 1.0 + clone_scatter_output_in_embedding ............... True + config_logger_dir ............................... + consumed_train_samples .......................... 0 + consumed_valid_samples .......................... 0 + context_parallel_size ........................... 8 + continual_pretrain .............................. True + continual_pretrain_preserve_opt_param_scheduler . False + corrected_weight_decay .......................... True + cp_comm_type .................................... ['a2a'] + create_attention_mask_in_dataloader ............. False + cross_entropy_fusion_impl ....................... linear_triton + cross_entropy_loss_fusion ....................... True + cuda_graph_impl ................................. none + cuda_graph_scope ................................ [] + cuda_graph_warmup_steps ......................... 3 + data_args_path .................................. None + data_cache_path ................................. /mnt/hdd/cpt/binidx_cache_stage15_final_20261001_v2 + data_parallel_random_init ....................... False + data_parallel_sharding_strategy ................. no_shard + data_parallel_size .............................. 8 + data_path ....................................... ['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_{0..78}_input_ids_tmp'] + data_per_class_fraction ......................... 1.0 + data_sharding ................................... True + dataloader_defer_npy_index_mmap ................. False + dataloader_fast_cache_load ...................... False + dataloader_type ................................. cyclic + dc_shortconv .................................... False + dc_shortconv_optimizer_state .................... conservative + dc_shortconv_upcycling .......................... False + ddp_average_in_collective ....................... False + ddp_bucket_size ................................. 40000000 + ddp_num_buckets ................................. None + ddp_pad_buckets_for_high_nccl_busbw ............. False + decode_only_cuda_graphs ......................... False + decoder_first_pipeline_num_layers ............... None + decoder_last_pipeline_num_layers ................ None + decoder_num_layers .............................. None + decoder_seq_length .............................. None + decoupled_lr .................................... None + decoupled_min_lr ................................ None + decrease_batch_size_if_needed ................... False + defer_embedding_wgrad_compute ................... False + defer_grad_checks ............................... False + delay_wgrad_compute ............................. False + deprecated_use_mcore_models ..................... True + deterministic_mode .............................. False + dino_bottleneck_size ............................ 256 + dino_freeze_last_layer .......................... 1 + dino_head_hidden_size ........................... 2048 + dino_local_crops_number ......................... 10 + dino_local_img_size ............................. 96 + dino_norm_last_layer ............................ False + dino_teacher_temp ............................... 0.07 + dino_warmup_teacher_temp ........................ 0.04 + dino_warmup_teacher_temp_epochs ................. 30 + disable_bf16_reduced_precision_matmul ........... False + disable_chunked_prefill ......................... False + disable_explicit_attention_mask ................. False + disable_mamba_mem_eff_path ...................... False + disable_straggler_on_startup .................... False + disable_symmetric_registration .................. False + dist_ckpt_format_deprecated ..................... None + dist_ckpt_optim_fully_reshardable ............... False + dist_ckpt_save_pre_mcore_014 .................... False + dist_ckpt_strictness ............................ assume_ok_unexpected + distrib_optim_fully_reshardable_mem_efficient ... False + distribute_saved_activations .................... False + distributed_backend ............................. nccl + distributed_timeout_minutes ..................... 120 + distributed_timeout_seconds_after_init .......... None + document_packing_algorithm ...................... obfd + document_packing_dp_size ........................ None + dump_param_to_param_group_map ................... None + dynamic_context_parallel ........................ False + dynamic_context_parallel_sizes .................. None + dynamic_cp_candidate_sizes ...................... None + dynamic_cp_gdn_policy ........................... None + dynamic_cp_moe_policy ........................... None + dynamic_cp_on_error ............................. error + dynamic_cp_scheduler ............................ default_dynamic_cp + dynamic_cp_wave_coexecution ..................... False + dynamic_cp_workspace_max_tokens ................. None + emb_deviation_loss_coeff ........................ 0 + emb_deviation_type .............................. None + embedding_init_method_std ....................... None + embedding_path .................................. None + empty_unused_memory_level ....................... 0 + enable_cuda_graph ............................... False + enable_debug_logging ............................ False + enable_experimental ............................. False + enable_ft_package ............................... False + enable_full_sharding_in_hsdp .................... False + enable_gloo_process_groups ...................... True + enable_msc ...................................... True + enable_one_logger ............................... True + encoder_num_layers .............................. 56 + encoder_seq_length .............................. 131072 + end_weight_decay ................................ 0.1 + eod_mask_loss ................................... False + error_injection_rate ............................ 0 + error_injection_type ............................ transient_error + eval_interval ................................... 99999999 + eval_iters ...................................... 10 + evidence_data_path .............................. None + exit_duration_in_mins ........................... None + exit_interval ................................... None + exit_on_missing_checkpoint ...................... True + exit_signal_handler ............................. True + exp_avg_dtype ................................... torch.float32 + exp_avg_sq_dtype ................................ torch.float32 + expert_model_parallel_size ...................... 8 + expert_tensor_parallel_size ..................... 1 + export_force_local_attention .................... False + export_kd_cfg ................................... None + export_kd_teacher_ckpt_format ................... None + export_kd_teacher_load .......................... None + export_kv_cache_quant ........................... False + export_legacy_megatron .......................... False + export_model_type ............................... GPTModel + export_moe_apply_probs_on_input ................. False + export_offline_model ............................ False + export_qk_l2_norm ............................... False + export_quant_cfg ................................ None + export_real_quant_cfg ........................... None + export_te_mcore_model ........................... False + external_cuda_graph ............................. False + fast_thd_a2a .................................... True + ffn_hidden_size ................................. 4800 + ffn_intermediate_token_shift .................... None + ffn_token_shift ................................. None + final_logits_clamp_value ........................ 30.0 + fine_grained_activation_offloading .............. False + finetune ........................................ False + finetune_data_split ............................. train + finetune_hf_dataset ............................. None + first_last_layers_bf16 .......................... False + flash_decode .................................... False + fp16 ............................................ False + fp16_lm_cross_entropy ........................... False + fp32_residual_connection ........................ False + fp4 ............................................. None + fp4_param ....................................... False + fp4_recipe ...................................... nvfp4 + fp8 ............................................. None + fp8_amax_compute_algo ........................... most_recent + fp8_amax_history_len ............................ 1 + fp8_interval .................................... 1 + fp8_margin ...................................... 0 + fp8_param_gather ................................ False + fp8_recipe ...................................... delayed + fp8_wgrad ....................................... True + freeze_layernorm_weight ......................... False + freeze_non_mamba ................................ False + fsdp_double_buffer .............................. False + full_validation ................................. False + gated_norm ...................................... True + gated_norm_lora_rank ............................ 16 + gated_norm_upcycling ............................ False + gdn_alpha_weight_scale .......................... 1.0 + gdn_alpha_weight_scale_mode ..................... matrix + gdn_backend ..................................... fla + gdn_fused_split_forward ......................... True + gdn_gated_norm_backend .......................... linghe + gdn_lr_mult_alpha ............................... 0.5 + gdn_lr_mult_beta ................................ 1.0 + gdn_lr_mult_k ................................... 1.0 + gdn_lr_mult_q ................................... 1.0 + gdn_lr_mult_v ................................... 1.0 + gdn_postinit_alog_shift ......................... 0.0 + gdn_postinit_alpha_scale ........................ 1.0 + gdn_postinit_key_scale .......................... 1.0 + gdn_pre_gdr_fusion .............................. False + gdn_pre_gdr_fusion_deterministic_weight_gradient False + gdn_pre_gdr_fusion_high_precision_backward ...... False + gdn_shortconv_separate_gate ..................... False + gdn_shortconv_separate_gate_upcycling ........... False + gdn_split_in_proj ............................... True + geglu ........................................... False + global_batch_size ............................... 128 + glu_linear_offset ............................... 0.0 + grad_reduce_in_bf16 ............................. False + gradient_accumulation_fusion .................... True + gradient_reduce_div_fusion ...................... True + group_query_attention ........................... True + grpo_clamp_eps_lower ............................ 0.01 + grpo_clamp_eps_upper ............................ 0.01 + grpo_default_temperature ........................ 1.0 + grpo_default_top_p .............................. 0 + grpo_entropy_term_weight ........................ 0.0 + grpo_filter_groups_with_same_reward ............. False + grpo_group_size ................................. 2 + grpo_iterations ................................. 2 + grpo_kl_beta .................................... 0.001 + grpo_prompts_per_step ........................... 32 + head_lr_mult .................................... 1.0 + heterogeneous_layers_config_encoded_json ........ None + heterogeneous_layers_config_path ................ None + hidden_dropout .................................. 0.0 + hidden_size ..................................... 2048 + hidden_states_shrink ............................ 1.0 + hierarchical_context_parallel_sizes ............. None + high_priority_stream_groups ..................... [] + hybrid_adamh .................................... True + hybrid_attention_ratio .......................... 0.0625 + hybrid_context_parallel ......................... False + hybrid_mlp_ratio ................................ 0.5 + hybrid_override_pattern ......................... None + hysteresis ...................................... 2 + ict_head_size ................................... None + ict_load ........................................ None + img_h ........................................... 224 + img_w ........................................... 224 + increase_log_level_interval ..................... 1000 + increase_log_level_iters ........................ 5 + indexer_batch_size .............................. 128 + indexer_log_interval ............................ 1000 + inference_batch_times_seqlen_threshold .......... -1 + inference_dynamic_batching ...................... False + inference_dynamic_batching_block_size ........... 256 + inference_dynamic_batching_buffer_guaranteed_fraction 0.2 + inference_dynamic_batching_buffer_overflow_factor None + inference_dynamic_batching_buffer_size_gb ....... 40.0 + inference_dynamic_batching_max_requests_override None + inference_dynamic_batching_max_tokens_override .. None + inference_dynamic_batching_num_cuda_graphs ...... 16 + inference_dynamic_batching_track_paused_request_events False + inference_dynamic_batching_unified_memory_level . 0 + inference_max_batch_size ........................ 8 + inference_max_seq_length ........................ 2560 + inference_rng_tracker ........................... False + init_method_std ................................. 0.02 + init_method_xavier_uniform ...................... False + init_model_with_meta_device ..................... False + initial_loss_scale .............................. 4294967296 + inprocess_active_world_size ..................... 64 + inprocess_barrier_timeout ....................... 120 + inprocess_completion_timeout .................... 120 + inprocess_empty_cuda_cache ...................... False + inprocess_granularity ........................... node + inprocess_hard_timeout .......................... 90 + inprocess_heartbeat_interval .................... 30 + inprocess_heartbeat_timeout ..................... 60 + inprocess_last_call_wait ........................ 1 + inprocess_max_iterations ........................ None + inprocess_monitor_process_interval .............. 1.0 + inprocess_monitor_thread_interval ............... 1.0 + inprocess_progress_watchdog_interval ............ 1.0 + inprocess_restart ............................... False + inprocess_soft_timeout .......................... 60 + inprocess_termination_grace_time ................ 1 + is_hybrid_model ................................. True + iter_per_epoch .................................. 1250 + iterations_to_skip .............................. [] + keep_fp8_transpose_cache ........................ False + kitchen_config_file ............................. None + kitchen_recipe_number ........................... None + kv_channels ..................................... 64 + kv_lora_rank .................................... 32 + langrl_env_config ............................... None + langrl_external_server .......................... False + langrl_inference_server_conversation_template ... None + langrl_inference_server_type .................... inplace_megatron + lazy_mpu_init ................................... None + legacy_tokenizer ................................ False + linear_attention_freq ........................... [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 0, 0, 1, 1, 1, 1, 1, 1] + linear_attention_type ........................... gated_delta_net + linear_conv_kernel_dim .......................... 4 + linear_cross_entropy_accumulate_grad_in_fp32 .... True + linear_cross_entropy_num_chunks ................. 8 + linear_key_head_dim ............................. 64 + linear_num_key_heads ............................ 8 + linear_num_value_heads .......................... 32 + linear_rope_mode ................................ none + linear_value_head_dim ........................... 64 + lm_head_alpha ................................... 1.0 + lm_loss_normalization ........................... global-token + load ............................................ /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage14_120B_8node_gdnlralpha_off_20260925a/checkpoints + load_complemental_dataset ....................... ['loss_mask'] + load_main_params_from_ckpt ...................... None + local_rank ...................................... 0 + log_attn_logits ................................. False + log_energy ...................................... False + log_hidden_states ............................... ['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] + log_interval .................................... 1 + log_logits_z_loss ............................... False + log_loss_scale_to_tensorboard ................... True + log_memory_to_tensorboard ....................... False + log_metrics_interval ............................ 10 + log_moe_activations ............................. True + log_moe_params .................................. False + log_moe_router_metrics .......................... False + log_num_zeros_in_grad ........................... False + log_padding_stats ............................... False + log_params ...................................... ['attention::linear_qkv', 'attention::o_proj', 'mlp::linear_fc1', 'mlp::linear_fc2', 'input_layernorm', 'pre_mlp_layernorm', 'embedding', 'lm_head', 'gated_delta::in_proj', 'gated_delta::in_proj_bias', 'gated_delta::conv1d', 'gated_delta::conv1d_bias', 'gated_delta::dt_bias', 'gated_delta::A_log', 'gated_delta::out_norm', 'gated_delta::out_norm_bias', 'gated_delta::out_proj', 'gated_delta::out_proj_bias'] + log_params_norm ................................. True + log_per_layer_grad_rms .......................... True + log_per_module_grad_rms ......................... True + log_per_module_update_rms ....................... False + log_progress .................................... False + log_spike_diagnostics ........................... False + log_straggler ................................... False + log_throughput .................................. True + log_timers_to_tensorboard ....................... False + log_update_rms .................................. True + log_validation_ppl_to_tensorboard ............... False + log_world_size_to_tensorboard ................... False + logging_level ................................... None + loss_scale ...................................... None + loss_scale_window ............................... 1000 + lr .............................................. 0.0005 + lr_decay_iters .................................. None + lr_decay_samples ................................ 877440 + lr_decay_style .................................. constant + lr_warmup_fraction .............................. None + lr_warmup_init .................................. 0.0 + lr_warmup_iters ................................. 0 + lr_warmup_samples ............................... 0 + lr_wsd_decay_iters .............................. None + lr_wsd_decay_samples ............................ None + lr_wsd_decay_style .............................. exponential + main_grads_dtype ................................ torch.float32 + main_params_dtype ............................... torch.float32 + make_vocab_size_divisible_by .................... 128 + mamba_disable_cp ................................ False + mamba_expand .................................... 2 + mamba_head_dim .................................. 64 + mamba_num_groups ................................ 8 + mamba_num_heads ................................. None + mamba_state_dim ................................. 128 + manual_gc ....................................... False + manual_gc_eval .................................. True + manual_gc_interval .............................. 0 + mask_factor ..................................... 1.0 + mask_prob ....................................... 0.15 + mask_type ....................................... random + masked_softmax_fusion ........................... False + max_position_embeddings ......................... 131072 + max_seqlen_per_cp_rank .......................... None + max_tokens_to_oom ............................... 12000 + memory_snapshot_path ............................ None + merge_file ...................................... None + micro_batch_size ................................ 1 + microbatch_group_size_per_vp_stage .............. None + mid_level_dataset_surplus ....................... 0.005 + min_loss_scale .................................. 1.0 + min_lr .......................................... 0.0005 + min_offloaded_tensor_size ....................... 1048576 + mlp_chunks_for_prefill .......................... 1 + mmap_bin_files .................................. True + mock_data ....................................... False + modelopt_enabled ................................ False + moe_apply_probs_on_input ........................ False + moe_aux_loss_coeff .............................. 0.05 + moe_deepep_num_sms .............................. 20 + moe_enable_deepep ............................... False + moe_expert_capacity_factor ...................... None + moe_extended_tp ................................. False + moe_ffn_hidden_size ............................. 512 + moe_flex_dispatcher_backend ..................... deepep + moe_gaux_overlap ................................ False + moe_grouped_gemm ................................ True + moe_hybridep_num_sms ............................ 16 + moe_input_jitter_eps ............................ None + moe_layer_freq .................................. [0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] + moe_layer_recompute ............................. False + moe_monitor_layers .............................. + moe_monitor_log_interval ........................ None + moe_pad_expert_input_to_capacity ................ False + moe_pad_experts_for_cuda_graph_inference ........ False + moe_per_layer_logging ........................... False + moe_permute_backend ............................. auto + moe_permute_fusion .............................. True + moe_router_aux_loss_fusion ...................... auto + moe_router_bias_update_method ................... sign + moe_router_bias_update_rate ..................... 0.001 + moe_router_dtype ................................ fp32 + moe_router_enable_expert_bias ................... False + moe_router_force_load_balancing ................. False + moe_router_fusion ............................... True + moe_router_gemm_backend ......................... te + moe_router_group_topk ........................... None + moe_router_load_balancing_type .................. global_aux_loss + moe_router_num_groups ........................... None + moe_router_padding_for_fp8 ...................... False + moe_router_padding_for_quantization ............. False + moe_router_pre_softmax .......................... False + moe_router_score_function ....................... softmax + moe_router_sqrt_gate ............................ True + moe_router_topk ................................. 4 + moe_router_topk_scaling_factor .................. None + moe_shared_expert_gate .......................... True + moe_shared_expert_intermediate_size ............. 512 + moe_shared_expert_overlap ....................... True + moe_token_dispatcher_type ....................... alltoall + moe_token_drop_policy ........................... probs + moe_triton_dispatcher_exclude_padding_tokens .... False + moe_triton_mega_activation_sms .................. 4 + moe_triton_mega_combine_sms ..................... 32 + moe_triton_mega_compute_sms ..................... 24 + moe_triton_mega_dispatch_sms .................... 48 + moe_triton_mega_num_warps ....................... 16 + moe_triton_mega_reduce_sms ...................... 32 + moe_triton_mega_tail_sms ........................ 16 + moe_triton_shared_expert_overlap ................ False + moe_triton_workspace_expand ..................... 4.0 + moe_upcycling_granularity ....................... 1 + moe_use_legacy_grouped_gemm ..................... False + moe_use_upcycling ............................... False + moe_z_loss_coeff ................................ None + mrope_section ................................... None + mscale .......................................... 1.0 + mscale_all_dim .................................. 0.0 + mtp_linear_attention_type ....................... gated_delta_net + mtp_loss_scaling_factor ......................... 0.1 + mtp_num_layers .................................. 0 + multi_latent_attention .......................... False + multiple_validation_sets ........................ False + muon_ball_momentum .............................. 0.9 + muon_ball_msign_steps ........................... 5 + muon_ball_power_iteration_steps ................. 10 + muon_ball_qkv_split_mode ........................ component + muon_ball_radius_mode ........................... spectral_mup + muon_ball_retract_alpha ......................... 0.05 + muon_ball_retract_mode .......................... hard + muon_ball_scale_mode ............................ spectral_mup + muon_ball_split_fc1 ............................. True + muon_ball_split_moe_experts ..................... True + muon_ball_split_qkv ............................. True + muon_ball_use_nesterov .......................... True + muon_extra_scale_factor ......................... 1.0 + muon_fp32_matmul_prec ........................... medium + muon_momentum ................................... 0.9 + muon_num_ns_steps ............................... 5 + muon_qkv_split_mode ............................. component + muon_scale_mode ................................. spectral_mup + muon_scale_vectorized_mode ...................... full + muon_split_fc1 .................................. True + muon_split_moe_experts .......................... True + muon_split_qkv .................................. True + muon_tp_mode .................................... blockwise + muon_use_nesterov ............................... False + muon_vectorize .................................. [] + muon_vectorize_attn_dim ......................... hidden_size + mup_lr_mult ..................................... 1.0 + nccl_all_reduce_for_prefill ..................... False + nccl_communicator_config_path ................... None + nccl_ub ......................................... False + no_load_optim ................................... None + no_load_rng ..................................... True + no_load_scheduler ............................... None + no_persist_layer_norm ........................... False + no_rope_freq .................................... None + no_save_optim ................................... None + no_save_rng ..................................... None + no_save_step_one ................................ True + no_weight_decay_cond_type ....................... None + non_persistent_ckpt_type ........................ None + non_persistent_global_ckpt_dir .................. None + non_persistent_keep_last ........................ 1 + non_persistent_local_ckpt_algo .................. fully_parallel + non_persistent_local_ckpt_dir ................... None + non_persistent_save_interval .................... None + norm_epsilon .................................... 1e-06 + normalization ................................... RMSNorm + num_attention_heads ............................. 32 + num_channels .................................... 3 + num_classes ..................................... 1000 + num_dataset_builder_threads ..................... 1 + num_distributed_optimizer_instances ............. 1 + num_experts ..................................... 192 + num_layers ...................................... 56 + num_layers_at_end_in_bf16 ....................... 1 + num_layers_at_start_in_bf16 ..................... 1 + num_layers_per_virtual_pipeline_stage ........... None + num_query_groups ................................ 8 + num_virtual_stages_per_pipeline_rank ............ None + num_workers ..................................... 4 + object_storage_cache_path ....................... None + offload_modules ................................. [] + one_logger_async ................................ False + one_logger_project .............................. megatron-lm + one_logger_run_name ............................. None + onnx_safe ....................................... None + openai_gelu ..................................... False + optimizer ....................................... adam + optimizer_cpu_offload ........................... False + optimizer_offload_fraction ...................... 1.0 + output_bert_embeddings .......................... False + output_logits_z_loss_coeff ...................... None + overlap_cpu_optimizer_d2h_h2d ................... False + overlap_grad_reduce ............................. True + overlap_moe_expert_parallel_comm ................ False + overlap_p2p_comm ................................ False + overlap_p2p_comm_warmup_flush ................... False + overlap_param_gather ............................ True + overlap_param_gather_with_optimizer_step ........ False + override_hf_eod_token_id ........................ None + override_opt_param_scheduler .................... True + packing_tokens_per_shard ........................ 100000000000 + padded_vocab_size ............................... None + params_dtype .................................... torch.bfloat16 + patch_dim ....................................... 16 + per_split_data_args_path ........................ None + perform_initialization .......................... False + perform_rl_step ................................. False + pin_cpu_grads ................................... True + pin_cpu_params .................................. True + pipeline_model_parallel_comm_backend ............ None + pipeline_model_parallel_layout .................. None + pipeline_model_parallel_size .................... 1 + position_embedding_type ......................... none + pp_aware_finegrained_recompute .................. False + pp_aware_finegrained_recompute_global_layers .... None + pp_aware_finegrained_recompute_pp_ranks ......... None + prefetch_factor ................................. 8 + pretrained_checkpoint ........................... None + previous_stage_seq_length ....................... None + profile ......................................... False + profile_microbatch_count ........................ 2 + profile_microbatch_dir .......................... None + profile_microbatch_iteration .................... None + profile_microbatch_start ........................ 0 + profile_ranks ................................... [0] + profile_step_end ................................ None + profile_step_start .............................. None + q_lora_rank ..................................... None + qk_head_dim ..................................... 128 + qk_l2_norm ...................................... False + qk_layernorm .................................... False + qk_pos_emb_head_dim ............................. 64 + query_in_block_prob ............................. 0.1 + quick_geglu ..................................... False + rampup_batch_size ............................... None + rank ............................................ 0 + recompute_granularity ........................... full + recompute_method ................................ block + recompute_modules ............................... None + recompute_num_layers ............................ 56 + record_memory_history ........................... False + relative_attention_max_distance ................. 128 + relative_attention_num_buckets .................. 32 + reparam_keys .................................... None + replication ..................................... False + replication_factor .............................. 2 + replication_jump ................................ None + rerun_mode ...................................... validate_results + reset_attention_mask ............................ False + reset_iteration_one_to_zero ..................... False + reset_position_ids .............................. False + result_rejected_tracker_filename ................ None + retriever_report_topk_accuracies ................ [] + retriever_score_scaling ......................... False + retriever_seq_length ............................ 256 + retro_add_retriever ............................. False + retro_attention_gate ............................ 1 + retro_cyclic_train_iters ........................ None + retro_encoder_attention_dropout ................. 0.1 + retro_encoder_hidden_dropout .................... 0.1 + retro_encoder_layers ............................ 2 + retro_num_neighbors ............................. 2 + retro_num_retrieved_chunks ...................... 2 + retro_project_dir ............................... None + retro_verify_neighbor_count ..................... True + reuse_grad_buf_for_mxfp8_param_ag ............... False + reuse_training_communicators .................... True + rl_calculate_intra_group_similarity ............. False + rl_importance_sampling_truncation_coef .......... None + rl_inference_logprobs_is_correction ............. False + rl_offload_kv_cache_during_training ............. False + rl_offload_optimizer_during_inference ........... False + rl_partial_rollouts ............................. False + rl_prompts_per_eval ............................. 32 + rl_remove_kv_cache_during_training .............. False + rl_reset_cuda_graphs ............................ False + rl_sequence_packing_algo ........................ fifo + rl_sequence_packing_bin_size .................... 8192 + rl_use_sequence_packing ......................... False + rope_scaling_factor ............................. 8.0 + rope_type ....................................... None + rotary_base ..................................... 10000 + rotary_interleaved .............................. False + rotary_percent .................................. 1.0 + rotary_scaling_factor ........................... 1.0 + rotary_seq_len_interpolation_factor ............. None + run_workload_inspector_server ................... False + runtime_audit_dir ............................... /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/runtime_audit + sample_rate ..................................... 1.0 + save ............................................ /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints + save_after_load ................................. False + save_interval ................................... 40 + save_retain_interval ............................ 1000000000 + scale_emb ....................................... 1.0 + scatter_gather_tensors_in_pipeline .............. True + seed ............................................ 1234 + seq_length ...................................... 131072 + sequence_packing_scheduler ...................... default_dynamic_cp + sequence_parallel ............................... False + sft ............................................. False + sft_tokenizer_prompt_format ..................... nemotron-h-aligned + sgd_momentum .................................... 0.9 + sharp_enabled_group ............................. None + short_seq_prob .................................. 0.1 + skip_train ...................................... False + skipped_train_samples ........................... 0 + softmax_type .................................... vanilla + spec ............................................ None + spectra_clip .................................... 1.0 + spectra_grad_clip ............................... None + spectra_hyperball ............................... False + spectra_ns_steps ................................ 10 + spectra_scope ................................... matrices + spectral_ball_momentum .......................... 0.9 + spectral_ball_msign_steps ....................... 8 + spectral_ball_power_iteration_steps ............. 20 + spectral_ball_qkv_split_mode .................... component + spectral_ball_radius_mode ....................... spectral_mup + spectral_ball_retract_alpha ..................... 0.05 + spectral_ball_retract_mode ...................... hard + spectral_ball_scale_mode ........................ spectral_mup + spectral_ball_solver ............................ bisection + spectral_ball_solver_max_iterations ............. 20 + spectral_ball_solver_tolerance_f ................ 1e-08 + spectral_ball_split_fc1 ......................... True + spectral_ball_split_moe_experts ................. True + spectral_ball_split_qkv ......................... True + spectral_ball_use_nesterov ...................... True + spectral_mup_init ............................... False + spike_diagnostics_dir ........................... None + spike_diagnostics_end ........................... 21 + spike_diagnostics_start ......................... 2 + split ........................................... 999,1,0 + split_expert_init ............................... True + split_fc1_init .................................. True + split_qkv_init .................................. True + split_qkv_init_mode ............................. group + sqreglu ......................................... False + squared_relu .................................... False + start_samples ................................... 0 + start_weight_decay .............................. 0.1 + straggler_ctrlr_port ............................ 65535 + straggler_minmax_count .......................... 1 + strict_fsdp_dtensor_load ........................ True + suggested_communication_unit_size ............... None + swanlab_exp_name ................................ + swanlab_project ................................. + swanlab_save_dir ................................ + swanlab_workspace ............................... + swiglu .......................................... True + swin_backbone_type .............................. tiny + symmetric_ar_type ............................... None + te_rng_tracker .................................. False + teacher_model_config ............................ None + tensor_model_parallel_size ...................... 1 + tensorboard_dir ................................. /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/tensorboard + tensorboard_log_interval ........................ 1 + tensorboard_queue_size .......................... 1000 + test_data_path .................................. None + test_mode ....................................... False + tiktoken_num_special_tokens ..................... 1000 + tiktoken_pattern ................................ None + tiktoken_special_tokens ......................... None + timing_log_level ................................ 0 + timing_log_option ............................... all + titles_data_path ................................ None + token_shift_conv_init ........................... default + token_shift_conv_size ........................... 4 + tokenizer_metadata .............................. None + tokenizer_model ................................. /mnt/ssd/yulan/pretrain-linear-moe/preprocess/modify_tokenizer/YuLan-Mini-Qwen-Template_new_sp + tokenizer_type .................................. HuggingFaceTokenizer + torch_fsdp2_reshard_after_forward ............... True + tp_comm_bootstrap_backend ....................... nccl + tp_comm_bulk_dgrad .............................. True + tp_comm_bulk_wgrad .............................. True + tp_comm_overlap ................................. False + tp_comm_overlap_ag .............................. True + tp_comm_overlap_cfg ............................. None + tp_comm_overlap_rs .............................. True + tp_comm_overlap_rs_dgrad ........................ False + tp_comm_split_ag ................................ True + tp_comm_split_rs ................................ True + train_data_path ................................. None + train_iters ..................................... None + train_samples ................................... 877440 + train_sync_interval ............................. None + train_until_stopped ............................. False + transformer_impl ................................ transformer_engine + transformer_pipeline_model_parallel_size ........ 1 + trust_remote_code ............................... False + untie_embeddings_and_output_weights ............. True + use_checkpoint_args ............................. False + use_checkpoint_opt_param_scheduler .............. False + use_cpu_initialization .......................... None + use_dist_ckpt ................................... True + use_dist_ckpt_deprecated ........................ False + use_distributed_optimizer ....................... True + use_flash_attn .................................. True + use_fused_weighted_squared_relu ................. False + use_legacy_models ............................... False + use_megatron_fsdp ............................... False + use_mp_args_from_checkpoint_args ................ False + use_one_sent_docs ............................... False + use_persistent_ckpt_worker ...................... False + use_precision_aware_optimizer ................... False + use_pytorch_profiler ............................ False + use_ring_exchange_p2p ........................... False + use_rope_scaling ................................ False + use_rotary_position_embeddings .................. False + use_sharp ....................................... False + use_te_activation_func .......................... False + use_tokenizer_model_from_checkpoint_args ........ True + use_torch_fsdp2 ................................. False + use_torch_optimizer_for_cpu_offload ............. False + use_tp_pp_dp_mapping ............................ False + v_head_dim ...................................... 128 + valid_data_path ................................. None + variable_seq_lengths ............................ False + virtual_pipeline_model_parallel_size ............ None + vision_backbone_type ............................ vit + vision_pretraining .............................. False + vision_pretraining_type ......................... classify + vocab_extra_ids ................................. 0 + vocab_file ...................................... None + vocab_size ...................................... None + wandb_entity .................................... + wandb_exp_name .................................. stage14_120B_8node_gdnlralpha_off_20260925a + wandb_project ................................... qwen2.5_3b_mom_cpt + wandb_save_dir .................................. /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/wandb + weight_decay .................................... 0.1 + weight_decay_incr_style ......................... constant + wgrad_deferral_limit ............................ 0 + window_attn_skip_freq ........................... None + window_size ..................................... None + word_embedding_dropout_prob ..................... 0.0 + world_size ...................................... 64 + yaml_cfg ........................................ None +-------------------- end of arguments --------------------- +INFO:megatron.core.num_microbatches_calculator:setting number of microbatches to constant 16 +> building HuggingFaceTokenizer tokenizer ... + > padded vocab (size: 99000) with 72 dummy tokens (new size: 99072) +WARNING:megatron.core.rerun_state_machine:RerunStateMachine initialized in mode validate_results +> initializing torch distributed ... +[Gloo] Rank 6 is connected to 63 peer ranks. Expected number of connected peer ranks is : 63 +[Gloo] Rank 5 is connected to 63 peer ranks. Expected number of connected peer ranks is : 63 +[rank5]:[W1003 15:37:33.988492870 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank5]:[W1003 15:37:33.988512169 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[Gloo] Rank 2 is connected to 63 peer ranks. Expected number of connected peer ranks is : 63 +[Gloo] Rank 1 is connected to 63 peer ranks. Expected number of connected peer ranks is : 63 +[rank2]:[W1003 15:37:33.988745022 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank2]:[W1003 15:37:33.988763926 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[Gloo] Rank 7 is connected to 63[rank1]:[W1003 15:37:33.988881013 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) + peer ranks. [rank1]:[W1003 15:37:33.988903815 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +Expected number of connected peer ranks is : 63 +[Gloo] Rank [Gloo] Rank 40[Gloo] Rank is connected to is connected to [rank7]:[W1003 15:37:33.989046573 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +36363[rank7]:[W1003 15:37:33.989101383 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) + is connected to peer ranks. peer ranks. 63Expected number of connected peer ranks is : Expected number of connected peer ranks is : peer ranks. 6363Expected number of connected peer ranks is : + +63 +[rank4]:[W1003 15:37:33.989312683 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank0]:[W1003 15:37:33.989318634 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank4]:[W1003 15:37:33.989330326 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank3]:[W1003 15:37:33.989329447 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank0]:[W1003 15:37:33.989341671 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank3]:[W1003 15:37:33.989358722 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank6]:[W1003 15:37:33.989453694 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank6]:[W1003 15:37:33.989491035 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7[Gloo] Rank +0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank [Gloo] Rank 00 is connected to is connected to 77 peer ranks. peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : 77 + +[Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +> initialized tensor model parallel with size 1 +> initialized pipeline model parallel with size 1 +> setting random seeds to 1234 ... +[Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. + warnings.warn( # warn only once +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. + warnings.warn( # warn only once +[rank6]:[W1003 15:37:33.159817582 ProcessGroupNCCL.cpp:5023] [PG ID 0 PG GUID 0 Rank 6] using GPU 6 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank5]:[W1003 15:37:33.159819370 ProcessGroupNCCL.cpp:5023] [PG ID 0 PG GUID 0 Rank 5] using GPU 5 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. + warnings.warn( # warn only once +[rank1]:[W1003 15:37:33.160174768 ProcessGroupNCCL.cpp:5023] [PG ID 0 PG GUID 0 Rank 1] using GPU 1 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +> compiling dataset index builder .../mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. + warnings.warn( # warn only once + +[rank7]:[W1003 15:37:33.160520399 ProcessGroupNCCL.cpp:5023] [PG ID 0 PG GUID 0 Rank 7] using GPU 7 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. + warnings.warn( # warn only once +[rank4]:[W1003 15:37:33.161275181 ProcessGroupNCCL.cpp:5023] [PG ID 0 PG GUID 0 Rank 4] using GPU 4 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. + warnings.warn( # warn only once +[rank3]:[W1003 15:37:33.161765027 ProcessGroupNCCL.cpp:5023] [PG ID 0 PG GUID 0 Rank 3] using GPU 3 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. + warnings.warn( # warn only once +[rank2]:[W1003 15:37:33.162054560 ProcessGroupNCCL.cpp:5023] [PG ID 0 PG GUID 0 Rank 2] using GPU 2 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +make: Entering directory '/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/datasets' +make: Nothing to be done for 'default'. +make: Leaving directory '/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/datasets' +>>> done with dataset index builder. Compilation time: 0.129 seconds +WARNING: constraints for invoking optimized fused softmax kernel are not met. We default back to unfused kernel invocations. +> compiling and loading fused kernels ... +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. + warnings.warn( # warn only once +[rank0]:[W1003 15:37:33.293192114 ProcessGroupNCCL.cpp:5023] [PG ID 0 PG GUID 0 Rank 0] using GPU 0 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +NCCL version 2.27.3+cuda12.9 +[rank6]:[W1003 15:37:34.154667743 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank6]:[W1003 15:37:34.154721295 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank1]:[W1003 15:37:34.154799064 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank3]:[W1003 15:37:34.154828854 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank1]:[W1003 15:37:34.154836488 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank3]:[W1003 15:37:34.154881758 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank5]:[W1003 15:37:34.154952965 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank5]:[W1003 15:37:34.154993473 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank0]:[W1003 15:37:34.155056844 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank4]:[W1003 15:37:34.155067892 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank2]:[W1003 15:37:34.155108718 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank0]:[W1003 15:37:34.155100351 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank7]:[W1003 15:37:34.155105009 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank4]:[W1003 15:37:34.155123384 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank2]:[W1003 15:37:34.155146721 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank7]:[W1003 15:37:34.155184150 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +>>> done with compiling and loading fused kernels. Compilation time: 1.884 seconds +time to initialize megatron (seconds): 358.846 +[after megatron is initialized] datetime: 2026-10-03 15:42:51 +building GPT model ... +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + + + + + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + > number of parameters on (tensor, pipeline) model parallel rank (0, 0): 5,569,877,120 + > number of trainable parameters on (tensor, pipeline) model parallel rank (0, 0): 5,569,877,120 +INFO:megatron.core.distributed.distributed_data_parallel:Setting up DistributedDataParallel with config DistributedDataParallelConfig(grad_reduce_in_fp32=True, overlap_grad_reduce=True, overlap_param_gather=True, align_param_gather=False, use_distributed_optimizer=True, num_distributed_optimizer_instances=1, check_for_nan_in_grad=True, defer_grad_checks=False, check_for_large_grads=False, bucket_size=40000000, pad_buckets_for_high_nccl_busbw=False, average_in_collective=False, fp8_param_gather=False, reuse_grad_buf_for_mxfp8_param_ag=False, use_megatron_fsdp=False, use_custom_fsdp=False, data_parallel_sharding_strategy='no_shard', gradient_reduce_div_fusion=True, suggested_communication_unit_size=None, preserve_fp32_weights=True, keep_fp8_transpose_cache=False, nccl_ub=False, fsdp_double_buffer=False, outer_dp_sharding_strategy='no_shard', disable_symmetric_registration=False, delay_wgrad_compute=False) +INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 28 +Params for bucket 1 (202899456 elements, 202899456 padded size): + module.output_layer.weight torch.Size([99072, 2048]) +Params for bucket 2 (40222848 elements, 40222848 padded size): + module.decoder.final_layernorm.norm.weight torch.Size([2048]) + module.decoder.final_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.55.pre_mlp_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.55.self_attention.k_proj.weight torch.Size([512, 2048]) + module.decoder.layers.54.pre_mlp_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.54.input_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.55.self_attention.conv1d.weight torch.Size([3072, 1, 4]) + module.decoder.layers.55.self_attention.q_proj.weight torch.Size([512, 2048]) + module.decoder.layers.55.input_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.55.input_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.54.mlp.shared_experts.gate_weight torch.Size([1, 2048]) + module.decoder.layers.54.self_attention.out_norm.weight torch.Size([64]) + module.decoder.layers.54.self_attention.v_proj.weight torch.Size([2048, 2048]) + module.decoder.final_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.55.mlp.shared_experts.gate_weight torch.Size([1, 2048]) + module.decoder.layers.55.pre_mlp_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.55.input_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.54.self_attention.a_proj.weight torch.Size([32, 2048]) + module.decoder.layers.54.self_attention.k_proj.weight torch.Size([512, 2048]) + module.decoder.layers.54.input_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.55.self_attention.out_norm.weight torch.Size([64]) + module.decoder.layers.54.self_attention.z_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.54.self_attention.q_proj.weight torch.Size([512, 2048]) + module.decoder.layers.54.input_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.55.self_attention.a_proj.weight torch.Size([32, 2048]) + module.decoder.layers.55.input_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.54.mlp.shared_experts.linear_fc2.weight torch.Size([2048, 512]) + module.decoder.layers.54.mlp.router.weight torch.Size([192, 2048]) + module.decoder.layers.54.self_attention.out_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.55.self_attention.b_proj.weight torch.Size([32, 2048]) + module.decoder.layers.54.mlp.shared_experts.linear_fc1.weight torch.Size([1024, 2048]) + module.decoder.layers.54.pre_mlp_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.54.pre_mlp_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.54.self_attention.b_proj.weight torch.Size([32, 2048]) + module.decoder.layers.55.mlp.shared_experts.linear_fc2.weight torch.Size([2048, 512]) + module.decoder.layers.55.mlp.router.weight torch.Size([192, 2048]) + module.decoder.layers.55.self_attention.out_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.55.self_attention.z_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.54.pre_mlp_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.53.mlp.shared_experts.linear_fc2.weight torch.Size([2048, 512]) + module.decoder.final_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.55.mlp.shared_experts.linear_fc1.weight torch.Size([1024, 2048]) + module.decoder.layers.55.pre_mlp_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.55.pre_mlp_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.55.self_attention.v_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.54.self_attention.conv1d.weight torch.Size([3072, 1, 4]) + module.decoder.layers.54.input_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.53.mlp.shared_experts.linear_fc1.weight torch.Size([1024, 2048]) +Params for bucket 3 (41666688 elements, 41666688 padded size): + module.decoder.layers.53.mlp.router.weight torch.Size([192, 2048]) + module.decoder.layers.53.self_attention.out_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.53.self_attention.z_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.52.pre_mlp_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.52.self_attention.out_norm.weight torch.Size([64]) + module.decoder.layers.51.mlp.shared_experts.gate_weight torch.Size([1, 2048]) + module.decoder.layers.53.pre_mlp_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.53.pre_mlp_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.53.self_attention.v_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.52.self_attention.a_proj.weight torch.Size([32, 2048]) + module.decoder.layers.52.input_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.51.mlp.shared_experts.linear_fc2.weight torch.Size([2048, 512]) + module.decoder.layers.51.mlp.router.weight torch.Size([192, 2048]) + module.decoder.layers.53.mlp.shared_experts.gate_weight torch.Size([1, 2048]) + module.decoder.layers.53.pre_mlp_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.53.self_attention.k_proj.weight torch.Size([512, 2048]) + module.decoder.layers.52.mlp.shared_experts.gate_weight torch.Size([1, 2048]) + module.decoder.layers.52.pre_mlp_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.52.self_attention.b_proj.weight torch.Size([32, 2048]) + module.decoder.layers.52.input_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.51.pre_mlp_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.53.self_attention.conv1d.weight torch.Size([3072, 1, 4]) + module.decoder.layers.53.self_attention.q_proj.weight torch.Size([512, 2048]) + module.decoder.layers.53.input_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.53.input_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.52.self_attention.out_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.52.self_attention.z_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.51.pre_mlp_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.53.pre_mlp_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.53.input_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.52.pre_mlp_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.52.self_attention.v_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.52.input_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.53.self_attention.out_norm.weight torch.Size([64]) + module.decoder.layers.52.self_attention.k_proj.weight torch.Size([512, 2048]) + module.decoder.layers.51.pre_mlp_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.51.self_attention.out_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.53.self_attention.a_proj.weight torch.Size([32, 2048]) + module.decoder.layers.53.input_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.52.mlp.shared_experts.linear_fc2.weight torch.Size([2048, 512]) + module.decoder.layers.52.mlp.router.weight torch.Size([192, 2048]) + module.decoder.layers.52.self_attention.conv1d.weight torch.Size([3072, 1, 4]) + module.decoder.layers.52.self_attention.q_proj.weight torch.Size([512, 2048]) + module.decoder.layers.52.input_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.51.pre_mlp_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.53.self_attention.b_proj.weight torch.Size([32, 2048]) + module.decoder.layers.52.mlp.shared_experts.linear_fc1.weight torch.Size([1024, 2048]) + module.decoder.layers.52.pre_mlp_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.51.mlp.shared_experts.linear_fc1.weight torch.Size([1024, 2048]) +Params for bucket 4 (43304064 elements, 43304064 padded size): + module.decoder.layers.51.self_attention.conv1d.weight torch.Size([3072, 1, 4]) + module.decoder.layers.51.self_attention.q_proj.weight torch.Size([512, 2048]) + module.decoder.layers.51.input_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.50.self_attention.out_norm.weight torch.Size([64]) + module.decoder.layers.50.self_attention.z_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.49.pre_mlp_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.49.pre_mlp_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.50.pre_mlp_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.50.self_attention.v_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.50.input_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.51.self_attention.out_norm.weight torch.Size([64]) + module.decoder.layers.50.self_attention.k_proj.weight torch.Size([512, 2048]) + module.decoder.layers.49.mlp.shared_experts.gate_weight torch.Size([1, 2048]) + module.decoder.layers.49.pre_mlp_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.51.self_attention.a_proj.weight torch.Size([32, 2048]) + module.decoder.layers.50.mlp.shared_experts.linear_fc2.weight torch.Size([2048, 512]) + module.decoder.layers.50.mlp.router.weight torch.Size([192, 2048]) + module.decoder.layers.50.self_attention.out_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.50.self_attention.q_proj.weight torch.Size([512, 2048]) + module.decoder.layers.50.input_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.51.self_attention.b_proj.weight torch.Size([32, 2048]) + module.decoder.layers.51.input_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.50.mlp.shared_experts.linear_fc1.weight torch.Size([1024, 2048]) + module.decoder.layers.50.pre_mlp_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.49.self_attention.linear_qkv.bias torch.Size([5120]) + module.decoder.layers.51.self_attention.z_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.51.input_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.50.pre_mlp_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.51.self_attention.v_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.50.self_attention.a_proj.weight torch.Size([32, 2048]) + module.decoder.layers.50.input_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.49.mlp.shared_experts.linear_fc2.weight torch.Size([2048, 512]) + module.decoder.layers.49.mlp.router.weight torch.Size([192, 2048]) + module.decoder.layers.49.self_attention.linear_qkv.weight torch.Size([5120, 2048]) + module.decoder.layers.51.self_attention.k_proj.weight torch.Size([512, 2048]) + module.decoder.layers.51.input_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.50.mlp.shared_experts.gate_weight torch.Size([1, 2048]) + module.decoder.layers.50.pre_mlp_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.50.self_attention.conv1d.weight torch.Size([3072, 1, 4]) + module.decoder.layers.50.self_attention.b_proj.weight torch.Size([32, 2048]) + module.decoder.layers.50.input_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.49.mlp.shared_experts.linear_fc1.weight torch.Size([1024, 2048]) + module.decoder.layers.49.pre_mlp_layernorm.gate_up.bias torch.Size([2048]) +Params for bucket 5 (40014912 elements, 40014976 padded size): + module.decoder.layers.48.pre_mlp_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.47.self_attention.out_norm.weight torch.Size([64]) + module.decoder.layers.49.input_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.48.mlp.shared_experts.linear_fc2.weight torch.Size([2048, 512]) + module.decoder.layers.47.self_attention.a_proj.weight torch.Size([32, 2048]) + module.decoder.layers.49.input_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.48.mlp.shared_experts.linear_fc1.weight torch.Size([1024, 2048]) + module.decoder.layers.48.input_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.47.mlp.shared_experts.gate_weight torch.Size([1, 2048]) + module.decoder.layers.47.self_attention.b_proj.weight torch.Size([32, 2048]) + module.decoder.layers.48.self_attention.linear_qkv.weight torch.Size([5120, 2048]) + module.decoder.layers.48.input_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.47.mlp.router.weight torch.Size([192, 2048]) + module.decoder.layers.47.self_attention.out_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.47.self_attention.z_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.49.input_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.48.mlp.router.weight torch.Size([192, 2048]) + module.decoder.layers.47.pre_mlp_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.47.pre_mlp_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.47.self_attention.v_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.49.self_attention.linear_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.48.mlp.shared_experts.gate_weight torch.Size([1, 2048]) + module.decoder.layers.48.pre_mlp_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.48.input_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.47.pre_mlp_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.47.self_attention.k_proj.weight torch.Size([512, 2048]) + module.decoder.layers.49.input_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.48.pre_mlp_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.48.pre_mlp_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.48.self_attention.linear_qkv.bias torch.Size([5120]) + module.decoder.layers.47.mlp.shared_experts.linear_fc2.weight torch.Size([2048, 512]) + module.decoder.layers.47.self_attention.conv1d.weight torch.Size([3072, 1, 4]) + module.decoder.layers.48.self_attention.linear_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.48.input_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.47.mlp.shared_experts.linear_fc1.weight torch.Size([1024, 2048]) + module.decoder.layers.47.pre_mlp_layernorm.gate_up.weight torch.Size([2048, 16]) +Params for bucket 6 (41133120 elements, 41133184 padded size): + module.decoder.layers.46.pre_mlp_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.46.self_attention.linear_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.46.input_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.45.mlp.shared_experts.gate_weight torch.Size([1, 2048]) + module.decoder.layers.45.pre_mlp_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.44.mlp.shared_experts.linear_fc2.weight torch.Size([2048, 512]) + module.decoder.layers.47.input_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.46.mlp.shared_experts.linear_fc2.weight torch.Size([2048, 512]) + module.decoder.layers.46.pre_mlp_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.46.input_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.45.self_attention.conv1d.weight torch.Size([3072, 1, 4]) + module.decoder.layers.45.self_attention.z_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.45.input_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.46.mlp.shared_experts.linear_fc1.weight torch.Size([1024, 2048]) + module.decoder.layers.46.self_attention.linear_qkv.bias torch.Size([5120]) + module.decoder.layers.46.input_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.45.pre_mlp_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.45.input_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.45.self_attention.out_norm.weight torch.Size([64]) + module.decoder.layers.45.self_attention.v_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.46.mlp.router.weight torch.Size([192, 2048]) + module.decoder.layers.45.mlp.shared_experts.linear_fc2.weight torch.Size([2048, 512]) + module.decoder.layers.45.self_attention.a_proj.weight torch.Size([32, 2048]) + module.decoder.layers.45.self_attention.k_proj.weight torch.Size([512, 2048]) + module.decoder.layers.45.input_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.46.mlp.shared_experts.gate_weight torch.Size([1, 2048]) + module.decoder.layers.46.pre_mlp_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.45.mlp.shared_experts.linear_fc1.weight torch.Size([1024, 2048]) + module.decoder.layers.45.self_attention.b_proj.weight torch.Size([32, 2048]) + module.decoder.layers.45.self_attention.q_proj.weight torch.Size([512, 2048]) + module.decoder.layers.45.input_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.47.self_attention.q_proj.weight torch.Size([512, 2048]) + module.decoder.layers.47.input_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.47.input_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.46.pre_mlp_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.45.mlp.router.weight torch.Size([192, 2048]) + module.decoder.layers.45.self_attention.out_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.44.mlp.shared_experts.linear_fc1.weight torch.Size([1024, 2048]) + module.decoder.layers.47.input_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.46.self_attention.linear_qkv.weight torch.Size([5120, 2048]) + module.decoder.layers.46.input_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.45.pre_mlp_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.45.pre_mlp_layernorm.norm.weight torch.Size([2048]) +Params for bucket 7 (41666688 elements, 41666688 padded size): + module.decoder.layers.44.mlp.router.weight torch.Size([192, 2048]) + module.decoder.layers.44.self_attention.out_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.44.self_attention.z_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.43.self_attention.b_proj.weight torch.Size([32, 2048]) + module.decoder.layers.42.mlp.shared_experts.gate_weight torch.Size([1, 2048]) + module.decoder.layers.42.pre_mlp_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.44.mlp.shared_experts.gate_weight torch.Size([1, 2048]) + module.decoder.layers.44.pre_mlp_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.44.pre_mlp_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.44.self_attention.v_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.43.mlp.router.weight torch.Size([192, 2048]) + module.decoder.layers.43.self_attention.out_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.43.self_attention.z_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.42.pre_mlp_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.44.pre_mlp_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.44.self_attention.k_proj.weight torch.Size([512, 2048]) + module.decoder.layers.43.mlp.shared_experts.gate_weight torch.Size([1, 2048]) + module.decoder.layers.43.pre_mlp_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.43.pre_mlp_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.43.self_attention.v_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.44.self_attention.conv1d.weight torch.Size([3072, 1, 4]) + module.decoder.layers.44.self_attention.q_proj.weight torch.Size([512, 2048]) + module.decoder.layers.44.input_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.44.input_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.43.pre_mlp_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.43.self_attention.k_proj.weight torch.Size([512, 2048]) + module.decoder.layers.42.pre_mlp_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.42.self_attention.out_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.44.pre_mlp_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.44.input_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.43.self_attention.conv1d.weight torch.Size([3072, 1, 4]) + module.decoder.layers.43.self_attention.q_proj.weight torch.Size([512, 2048]) + module.decoder.layers.43.input_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.43.input_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.42.pre_mlp_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.44.self_attention.out_norm.weight torch.Size([64]) + module.decoder.layers.43.pre_mlp_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.43.input_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.44.self_attention.a_proj.weight torch.Size([32, 2048]) + module.decoder.layers.44.input_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.43.mlp.shared_experts.linear_fc2.weight torch.Size([2048, 512]) + module.decoder.layers.43.self_attention.out_norm.weight torch.Size([64]) + module.decoder.layers.42.mlp.shared_experts.linear_fc1.weight torch.Size([1024, 2048]) + module.decoder.layers.44.self_attention.b_proj.weight torch.Size([32, 2048]) + module.decoder.layers.43.mlp.shared_experts.linear_fc1.weight torch.Size([1024, 2048]) + module.decoder.layers.43.self_attention.a_proj.weight torch.Size([32, 2048]) + module.decoder.layers.43.input_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.42.mlp.shared_experts.linear_fc2.weight torch.Size([2048, 512]) + module.decoder.layers.42.mlp.router.weight torch.Size([192, 2048]) +Params for bucket 8 (41345216 elements, 41345280 padded size): + module.decoder.layers.42.self_attention.out_norm.weight torch.Size([64]) + module.decoder.layers.41.self_attention.b_proj.weight torch.Size([32, 2048]) + module.decoder.layers.40.mlp.shared_experts.linear_fc1.weight torch.Size([1024, 2048]) + module.decoder.layers.40.pre_mlp_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.42.self_attention.a_proj.weight torch.Size([32, 2048]) + module.decoder.layers.41.mlp.shared_experts.linear_fc2.weight torch.Size([2048, 512]) + module.decoder.layers.41.mlp.router.weight torch.Size([192, 2048]) + module.decoder.layers.41.self_attention.out_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.41.self_attention.z_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.40.pre_mlp_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.42.self_attention.b_proj.weight torch.Size([32, 2048]) + module.decoder.layers.42.input_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.41.mlp.shared_experts.linear_fc1.weight torch.Size([1024, 2048]) + module.decoder.layers.41.pre_mlp_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.41.pre_mlp_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.41.self_attention.v_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.40.self_attention.a_proj.weight torch.Size([32, 2048]) + module.decoder.layers.42.self_attention.z_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.42.input_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.41.pre_mlp_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.41.self_attention.k_proj.weight torch.Size([512, 2048]) + module.decoder.layers.40.mlp.shared_experts.gate_weight torch.Size([1, 2048]) + module.decoder.layers.40.pre_mlp_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.40.self_attention.b_proj.weight torch.Size([32, 2048]) + module.decoder.layers.42.self_attention.v_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.41.self_attention.conv1d.weight torch.Size([3072, 1, 4]) + module.decoder.layers.41.self_attention.q_proj.weight torch.Size([512, 2048]) + module.decoder.layers.41.input_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.41.input_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.40.self_attention.out_norm.weight torch.Size([64]) + module.decoder.layers.40.self_attention.z_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.42.self_attention.k_proj.weight torch.Size([512, 2048]) + module.decoder.layers.42.input_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.41.mlp.shared_experts.gate_weight torch.Size([1, 2048]) + module.decoder.layers.41.pre_mlp_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.41.input_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.40.pre_mlp_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.42.self_attention.conv1d.weight torch.Size([3072, 1, 4]) + module.decoder.layers.42.self_attention.q_proj.weight torch.Size([512, 2048]) + module.decoder.layers.42.input_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.41.self_attention.out_norm.weight torch.Size([64]) + module.decoder.layers.41.self_attention.a_proj.weight torch.Size([32, 2048]) + module.decoder.layers.41.input_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.40.mlp.shared_experts.linear_fc2.weight torch.Size([2048, 512]) + module.decoder.layers.40.mlp.router.weight torch.Size([192, 2048]) + module.decoder.layers.40.self_attention.out_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.40.self_attention.conv1d.weight torch.Size([3072, 1, 4]) +Params for bucket 9 (41201792 elements, 41201792 padded size): + module.decoder.layers.39.self_attention.out_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.39.self_attention.z_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.38.self_attention.b_proj.weight torch.Size([32, 2048]) + module.decoder.layers.39.pre_mlp_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.39.self_attention.v_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.38.mlp.router.weight torch.Size([192, 2048]) + module.decoder.layers.38.self_attention.out_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.38.self_attention.z_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.40.input_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.39.mlp.shared_experts.linear_fc2.weight torch.Size([2048, 512]) + module.decoder.layers.39.self_attention.k_proj.weight torch.Size([512, 2048]) + module.decoder.layers.38.mlp.shared_experts.gate_weight torch.Size([1, 2048]) + module.decoder.layers.38.pre_mlp_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.38.pre_mlp_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.38.self_attention.v_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.40.input_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.39.mlp.shared_experts.linear_fc1.weight torch.Size([1024, 2048]) + module.decoder.layers.39.mlp.shared_experts.gate_weight torch.Size([1, 2048]) + module.decoder.layers.39.self_attention.conv1d.weight torch.Size([3072, 1, 4]) + module.decoder.layers.39.self_attention.q_proj.weight torch.Size([512, 2048]) + module.decoder.layers.39.input_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.39.input_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.38.pre_mlp_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.39.pre_mlp_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.39.input_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.38.self_attention.conv1d.weight torch.Size([3072, 1, 4]) + module.decoder.layers.39.self_attention.out_norm.weight torch.Size([64]) + module.decoder.layers.40.self_attention.v_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.40.input_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.39.pre_mlp_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.38.pre_mlp_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.40.self_attention.k_proj.weight torch.Size([512, 2048]) + module.decoder.layers.39.mlp.router.weight torch.Size([192, 2048]) + module.decoder.layers.39.self_attention.a_proj.weight torch.Size([32, 2048]) + module.decoder.layers.39.input_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.38.mlp.shared_experts.linear_fc2.weight torch.Size([2048, 512]) + module.decoder.layers.38.self_attention.out_norm.weight torch.Size([64]) + module.decoder.layers.40.self_attention.q_proj.weight torch.Size([512, 2048]) + module.decoder.layers.40.input_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.39.pre_mlp_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.39.self_attention.b_proj.weight torch.Size([32, 2048]) + module.decoder.layers.38.mlp.shared_experts.linear_fc1.weight torch.Size([1024, 2048]) + module.decoder.layers.38.self_attention.a_proj.weight torch.Size([32, 2048]) +Params for bucket 10 (40222848 elements, 40222848 padded size): + module.decoder.layers.37.mlp.shared_experts.gate_weight torch.Size([1, 2048]) + module.decoder.layers.37.pre_mlp_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.37.input_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.36.self_attention.b_proj.weight torch.Size([32, 2048]) + module.decoder.layers.37.self_attention.out_norm.weight torch.Size([64]) + module.decoder.layers.36.mlp.router.weight torch.Size([192, 2048]) + module.decoder.layers.36.self_attention.out_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.36.self_attention.z_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.36.input_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.37.self_attention.a_proj.weight torch.Size([32, 2048]) + module.decoder.layers.37.input_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.36.mlp.shared_experts.linear_fc2.weight torch.Size([2048, 512]) + module.decoder.layers.36.pre_mlp_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.36.pre_mlp_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.36.self_attention.v_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.38.self_attention.k_proj.weight torch.Size([512, 2048]) + module.decoder.layers.37.self_attention.b_proj.weight torch.Size([32, 2048]) + module.decoder.layers.36.mlp.shared_experts.gate_weight torch.Size([1, 2048]) + module.decoder.layers.36.pre_mlp_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.36.self_attention.k_proj.weight torch.Size([512, 2048]) + module.decoder.layers.38.input_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.38.self_attention.q_proj.weight torch.Size([512, 2048]) + module.decoder.layers.37.mlp.shared_experts.linear_fc2.weight torch.Size([2048, 512]) + module.decoder.layers.37.mlp.router.weight torch.Size([192, 2048]) + module.decoder.layers.37.self_attention.out_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.37.self_attention.z_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.36.mlp.shared_experts.linear_fc1.weight torch.Size([1024, 2048]) + module.decoder.layers.36.self_attention.conv1d.weight torch.Size([3072, 1, 4]) + module.decoder.layers.36.self_attention.q_proj.weight torch.Size([512, 2048]) + module.decoder.layers.36.input_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.38.input_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.37.mlp.shared_experts.linear_fc1.weight torch.Size([1024, 2048]) + module.decoder.layers.37.pre_mlp_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.37.pre_mlp_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.37.self_attention.v_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.36.pre_mlp_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.36.input_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.38.input_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.37.pre_mlp_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.37.self_attention.k_proj.weight torch.Size([512, 2048]) + module.decoder.layers.36.self_attention.out_norm.weight torch.Size([64]) + module.decoder.layers.38.input_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.37.self_attention.conv1d.weight torch.Size([3072, 1, 4]) + module.decoder.layers.37.self_attention.q_proj.weight torch.Size([512, 2048]) + module.decoder.layers.37.input_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.37.input_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.36.self_attention.a_proj.weight torch.Size([32, 2048]) + module.decoder.layers.36.input_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.35.mlp.shared_experts.linear_fc2.weight torch.Size([2048, 512]) +Params for bucket 11 (43763840 elements, 43763840 padded size): + module.decoder.layers.35.mlp.shared_experts.linear_fc1.weight torch.Size([1024, 2048]) + module.decoder.layers.35.pre_mlp_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.35.pre_mlp_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.35.self_attention.v_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.34.mlp.router.weight torch.Size([192, 2048]) + module.decoder.layers.34.self_attention.out_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.34.self_attention.z_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.35.pre_mlp_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.35.self_attention.k_proj.weight torch.Size([512, 2048]) + module.decoder.layers.34.mlp.shared_experts.gate_weight torch.Size([1, 2048]) + module.decoder.layers.34.pre_mlp_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.34.pre_mlp_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.34.self_attention.v_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.35.self_attention.conv1d.weight torch.Size([3072, 1, 4]) + module.decoder.layers.35.self_attention.q_proj.weight torch.Size([512, 2048]) + module.decoder.layers.35.input_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.35.input_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.34.pre_mlp_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.33.mlp.shared_experts.linear_fc2.weight torch.Size([2048, 512]) + module.decoder.layers.33.mlp.router.weight torch.Size([192, 2048]) + module.decoder.layers.33.self_attention.out_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.35.mlp.shared_experts.gate_weight torch.Size([1, 2048]) + module.decoder.layers.35.pre_mlp_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.35.input_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.34.self_attention.conv1d.weight torch.Size([3072, 1, 4]) + module.decoder.layers.34.input_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.33.mlp.shared_experts.linear_fc1.weight torch.Size([1024, 2048]) + module.decoder.layers.33.mlp.shared_experts.gate_weight torch.Size([1, 2048]) + module.decoder.layers.33.pre_mlp_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.33.pre_mlp_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.35.self_attention.out_norm.weight torch.Size([64]) + module.decoder.layers.34.pre_mlp_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.34.input_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.33.pre_mlp_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.35.self_attention.a_proj.weight torch.Size([32, 2048]) + module.decoder.layers.35.input_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.34.mlp.shared_experts.linear_fc2.weight torch.Size([2048, 512]) + module.decoder.layers.34.self_attention.out_norm.weight torch.Size([64]) + module.decoder.layers.35.self_attention.b_proj.weight torch.Size([32, 2048]) + module.decoder.layers.34.mlp.shared_experts.linear_fc1.weight torch.Size([1024, 2048]) + module.decoder.layers.34.self_attention.a_proj.weight torch.Size([32, 2048]) + module.decoder.layers.34.self_attention.k_proj.weight torch.Size([512, 2048]) + module.decoder.layers.34.input_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.33.pre_mlp_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.35.mlp.router.weight torch.Size([192, 2048]) + module.decoder.layers.35.self_attention.out_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.35.self_attention.z_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.34.self_attention.b_proj.weight torch.Size([32, 2048]) + module.decoder.layers.34.self_attention.q_proj.weight torch.Size([512, 2048]) + module.decoder.layers.34.input_layernorm.norm.weight torch.Size([2048]) +Params for bucket 12 (41345216 elements, 41345280 padded size): + module.decoder.layers.33.self_attention.a_proj.weight torch.Size([32, 2048]) + module.decoder.layers.33.input_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.32.mlp.shared_experts.linear_fc2.weight torch.Size([2048, 512]) + module.decoder.layers.32.mlp.router.weight torch.Size([192, 2048]) + module.decoder.layers.32.self_attention.out_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.32.self_attention.z_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.31.pre_mlp_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.31.self_attention.a_proj.weight torch.Size([32, 2048]) + module.decoder.layers.33.self_attention.b_proj.weight torch.Size([32, 2048]) + module.decoder.layers.32.mlp.shared_experts.linear_fc1.weight torch.Size([1024, 2048]) + module.decoder.layers.32.pre_mlp_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.32.pre_mlp_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.32.self_attention.v_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.31.self_attention.b_proj.weight torch.Size([32, 2048]) + module.decoder.layers.33.self_attention.z_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.32.pre_mlp_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.32.self_attention.k_proj.weight torch.Size([512, 2048]) + module.decoder.layers.31.self_attention.out_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.31.self_attention.z_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.33.self_attention.v_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.32.self_attention.conv1d.weight torch.Size([3072, 1, 4]) + module.decoder.layers.32.self_attention.q_proj.weight torch.Size([512, 2048]) + module.decoder.layers.32.input_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.32.input_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.31.mlp.router.weight torch.Size([192, 2048]) + module.decoder.layers.31.pre_mlp_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.33.self_attention.k_proj.weight torch.Size([512, 2048]) + module.decoder.layers.32.mlp.shared_experts.gate_weight torch.Size([1, 2048]) + module.decoder.layers.32.pre_mlp_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.32.input_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.31.mlp.shared_experts.linear_fc1.weight torch.Size([1024, 2048]) + module.decoder.layers.31.pre_mlp_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.33.self_attention.conv1d.weight torch.Size([3072, 1, 4]) + module.decoder.layers.33.self_attention.q_proj.weight torch.Size([512, 2048]) + module.decoder.layers.33.input_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.33.input_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.32.self_attention.out_norm.weight torch.Size([64]) + module.decoder.layers.31.mlp.shared_experts.gate_weight torch.Size([1, 2048]) + module.decoder.layers.31.self_attention.conv1d.weight torch.Size([3072, 1, 4]) + module.decoder.layers.33.input_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.32.self_attention.a_proj.weight torch.Size([32, 2048]) + module.decoder.layers.32.input_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.31.mlp.shared_experts.linear_fc2.weight torch.Size([2048, 512]) + module.decoder.layers.33.self_attention.out_norm.weight torch.Size([64]) + module.decoder.layers.32.self_attention.b_proj.weight torch.Size([32, 2048]) + module.decoder.layers.31.pre_mlp_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.31.self_attention.out_norm.weight torch.Size([64]) +Params for bucket 13 (41201792 elements, 41201792 padded size): + module.decoder.layers.30.mlp.shared_experts.linear_fc2.weight torch.Size([2048, 512]) + module.decoder.layers.30.self_attention.out_norm.weight torch.Size([64]) + module.decoder.layers.31.input_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.30.mlp.shared_experts.linear_fc1.weight torch.Size([1024, 2048]) + module.decoder.layers.30.self_attention.a_proj.weight torch.Size([32, 2048]) + module.decoder.layers.30.input_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.29.mlp.shared_experts.linear_fc2.weight torch.Size([2048, 512]) + module.decoder.layers.29.mlp.router.weight torch.Size([192, 2048]) + module.decoder.layers.29.self_attention.a_proj.weight torch.Size([32, 2048]) + module.decoder.layers.31.input_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.30.self_attention.b_proj.weight torch.Size([32, 2048]) + module.decoder.layers.29.mlp.shared_experts.linear_fc1.weight torch.Size([1024, 2048]) + module.decoder.layers.29.pre_mlp_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.29.self_attention.b_proj.weight torch.Size([32, 2048]) + module.decoder.layers.31.self_attention.v_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.30.mlp.router.weight torch.Size([192, 2048]) + module.decoder.layers.30.self_attention.out_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.30.self_attention.z_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.29.pre_mlp_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.29.self_attention.out_norm.weight torch.Size([64]) + module.decoder.layers.29.self_attention.z_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.31.input_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.31.self_attention.k_proj.weight torch.Size([512, 2048]) + module.decoder.layers.30.mlp.shared_experts.gate_weight torch.Size([1, 2048]) + module.decoder.layers.30.pre_mlp_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.30.pre_mlp_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.30.self_attention.v_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.29.pre_mlp_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.29.self_attention.v_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.31.self_attention.q_proj.weight torch.Size([512, 2048]) + module.decoder.layers.31.input_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.30.pre_mlp_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.30.self_attention.k_proj.weight torch.Size([512, 2048]) + module.decoder.layers.29.mlp.shared_experts.gate_weight torch.Size([1, 2048]) + module.decoder.layers.29.pre_mlp_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.30.self_attention.conv1d.weight torch.Size([3072, 1, 4]) + module.decoder.layers.30.self_attention.q_proj.weight torch.Size([512, 2048]) + module.decoder.layers.30.input_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.30.input_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.29.self_attention.out_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.29.self_attention.conv1d.weight torch.Size([3072, 1, 4]) + module.decoder.layers.30.pre_mlp_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.30.input_layernorm.gate_down.weight torch.Size([16, 2048]) +Params for bucket 14 (40222848 elements, 40222848 padded size): + module.decoder.layers.28.mlp.shared_experts.linear_fc1.weight torch.Size([1024, 2048]) + module.decoder.layers.28.self_attention.conv1d.weight torch.Size([3072, 1, 4]) + module.decoder.layers.28.self_attention.q_proj.weight torch.Size([512, 2048]) + module.decoder.layers.28.input_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.28.input_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.27.self_attention.out_norm.weight torch.Size([64]) + module.decoder.layers.27.input_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.28.mlp.shared_experts.linear_fc2.weight torch.Size([2048, 512]) + module.decoder.layers.28.pre_mlp_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.28.input_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.27.self_attention.a_proj.weight torch.Size([32, 2048]) + module.decoder.layers.27.input_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.28.mlp.shared_experts.gate_weight torch.Size([1, 2048]) + module.decoder.layers.28.pre_mlp_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.28.self_attention.out_norm.weight torch.Size([64]) + module.decoder.layers.27.self_attention.b_proj.weight torch.Size([32, 2048]) + module.decoder.layers.28.self_attention.a_proj.weight torch.Size([32, 2048]) + module.decoder.layers.28.input_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.27.mlp.shared_experts.linear_fc2.weight torch.Size([2048, 512]) + module.decoder.layers.27.mlp.router.weight torch.Size([192, 2048]) + module.decoder.layers.27.self_attention.out_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.27.self_attention.z_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.29.input_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.28.mlp.router.weight torch.Size([192, 2048]) + module.decoder.layers.28.pre_mlp_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.28.self_attention.b_proj.weight torch.Size([32, 2048]) + module.decoder.layers.27.mlp.shared_experts.linear_fc1.weight torch.Size([1024, 2048]) + module.decoder.layers.27.pre_mlp_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.27.pre_mlp_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.27.self_attention.v_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.29.self_attention.k_proj.weight torch.Size([512, 2048]) + module.decoder.layers.29.input_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.28.self_attention.out_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.28.self_attention.z_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.27.pre_mlp_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.27.self_attention.k_proj.weight torch.Size([512, 2048]) + module.decoder.layers.29.self_attention.q_proj.weight torch.Size([512, 2048]) + module.decoder.layers.29.input_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.28.pre_mlp_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.28.self_attention.v_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.27.self_attention.conv1d.weight torch.Size([3072, 1, 4]) + module.decoder.layers.27.self_attention.q_proj.weight torch.Size([512, 2048]) + module.decoder.layers.27.input_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.26.mlp.shared_experts.linear_fc2.weight torch.Size([2048, 512]) + module.decoder.layers.29.input_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.28.self_attention.k_proj.weight torch.Size([512, 2048]) + module.decoder.layers.27.mlp.shared_experts.gate_weight torch.Size([1, 2048]) + module.decoder.layers.27.pre_mlp_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.27.input_layernorm.gate_down.weight torch.Size([16, 2048]) +Params for bucket 15 (43763840 elements, 43763840 padded size): + module.decoder.layers.26.self_attention.b_proj.weight torch.Size([32, 2048]) + module.decoder.layers.25.mlp.shared_experts.gate_weight torch.Size([1, 2048]) + module.decoder.layers.25.self_attention.out_norm.weight torch.Size([64]) + module.decoder.layers.26.mlp.router.weight torch.Size([192, 2048]) + module.decoder.layers.26.self_attention.out_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.26.self_attention.z_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.25.self_attention.a_proj.weight torch.Size([32, 2048]) + module.decoder.layers.25.input_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.24.mlp.shared_experts.linear_fc2.weight torch.Size([2048, 512]) + module.decoder.layers.24.mlp.router.weight torch.Size([192, 2048]) + module.decoder.layers.24.self_attention.out_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.26.mlp.shared_experts.gate_weight torch.Size([1, 2048]) + module.decoder.layers.26.pre_mlp_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.26.pre_mlp_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.26.self_attention.v_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.25.self_attention.b_proj.weight torch.Size([32, 2048]) + module.decoder.layers.24.mlp.shared_experts.linear_fc1.weight torch.Size([1024, 2048]) + module.decoder.layers.24.pre_mlp_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.24.pre_mlp_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.26.pre_mlp_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.26.self_attention.k_proj.weight torch.Size([512, 2048]) + module.decoder.layers.25.mlp.router.weight torch.Size([192, 2048]) + module.decoder.layers.25.self_attention.out_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.25.self_attention.z_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.24.pre_mlp_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.26.self_attention.conv1d.weight torch.Size([3072, 1, 4]) + module.decoder.layers.26.self_attention.q_proj.weight torch.Size([512, 2048]) + module.decoder.layers.26.input_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.26.input_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.25.pre_mlp_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.25.pre_mlp_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.25.self_attention.v_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.26.pre_mlp_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.26.input_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.25.pre_mlp_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.25.self_attention.k_proj.weight torch.Size([512, 2048]) + module.decoder.layers.24.mlp.shared_experts.gate_weight torch.Size([1, 2048]) + module.decoder.layers.24.pre_mlp_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.26.self_attention.out_norm.weight torch.Size([64]) + module.decoder.layers.25.mlp.shared_experts.linear_fc1.weight torch.Size([1024, 2048]) + module.decoder.layers.25.self_attention.conv1d.weight torch.Size([3072, 1, 4]) + module.decoder.layers.25.self_attention.q_proj.weight torch.Size([512, 2048]) + module.decoder.layers.25.input_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.25.input_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.26.mlp.shared_experts.linear_fc1.weight torch.Size([1024, 2048]) + module.decoder.layers.26.self_attention.a_proj.weight torch.Size([32, 2048]) + module.decoder.layers.26.input_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.25.mlp.shared_experts.linear_fc2.weight torch.Size([2048, 512]) + module.decoder.layers.25.pre_mlp_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.25.input_layernorm.gate_down.weight torch.Size([16, 2048]) +Params for bucket 16 (41206912 elements, 41206912 padded size): + module.decoder.layers.24.self_attention.b_proj.weight torch.Size([32, 2048]) + module.decoder.layers.23.mlp.shared_experts.linear_fc1.weight torch.Size([1024, 2048]) + module.decoder.layers.23.input_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.23.input_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.22.self_attention.a_proj.weight torch.Size([32, 2048]) + module.decoder.layers.24.self_attention.z_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.22.self_attention.b_proj.weight torch.Size([32, 2048]) + module.decoder.layers.24.self_attention.v_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.23.mlp.router.weight torch.Size([192, 2048]) + module.decoder.layers.22.mlp.router.weight torch.Size([192, 2048]) + module.decoder.layers.22.self_attention.out_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.22.self_attention.z_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.24.self_attention.k_proj.weight torch.Size([512, 2048]) + module.decoder.layers.23.mlp.shared_experts.gate_weight torch.Size([1, 2048]) + module.decoder.layers.23.pre_mlp_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.23.self_attention.linear_qkv.bias torch.Size([5120]) + module.decoder.layers.22.mlp.shared_experts.linear_fc1.weight torch.Size([1024, 2048]) + module.decoder.layers.22.mlp.shared_experts.gate_weight torch.Size([1, 2048]) + module.decoder.layers.22.pre_mlp_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.22.pre_mlp_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.24.self_attention.conv1d.weight torch.Size([3072, 1, 4]) + module.decoder.layers.24.self_attention.q_proj.weight torch.Size([512, 2048]) + module.decoder.layers.24.input_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.24.input_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.23.pre_mlp_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.23.self_attention.linear_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.22.mlp.shared_experts.linear_fc2.weight torch.Size([2048, 512]) + module.decoder.layers.22.pre_mlp_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.24.input_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.23.self_attention.linear_qkv.weight torch.Size([5120, 2048]) + module.decoder.layers.23.input_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.22.self_attention.conv1d.weight torch.Size([3072, 1, 4]) + module.decoder.layers.24.self_attention.out_norm.weight torch.Size([64]) + module.decoder.layers.23.pre_mlp_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.23.input_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.22.pre_mlp_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.24.self_attention.a_proj.weight torch.Size([32, 2048]) + module.decoder.layers.24.input_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.23.mlp.shared_experts.linear_fc2.weight torch.Size([2048, 512]) + module.decoder.layers.23.pre_mlp_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.22.self_attention.out_norm.weight torch.Size([64]) +Params for bucket 17 (43022336 elements, 43022336 padded size): + module.decoder.layers.22.input_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.21.mlp.shared_experts.linear_fc2.weight torch.Size([2048, 512]) + module.decoder.layers.21.mlp.router.weight torch.Size([192, 2048]) + module.decoder.layers.21.mlp.shared_experts.linear_fc1.weight torch.Size([1024, 2048]) + module.decoder.layers.21.pre_mlp_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.21.self_attention.linear_qkv.bias torch.Size([5120]) + module.decoder.layers.20.mlp.shared_experts.linear_fc2.weight torch.Size([2048, 512]) + module.decoder.layers.20.mlp.router.weight torch.Size([192, 2048]) + module.decoder.layers.21.pre_mlp_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.21.self_attention.linear_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.20.mlp.shared_experts.linear_fc1.weight torch.Size([1024, 2048]) + module.decoder.layers.20.pre_mlp_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.20.self_attention.linear_qkv.bias torch.Size([5120]) + module.decoder.layers.22.self_attention.v_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.21.self_attention.linear_qkv.weight torch.Size([5120, 2048]) + module.decoder.layers.21.input_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.20.pre_mlp_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.20.self_attention.linear_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.22.self_attention.k_proj.weight torch.Size([512, 2048]) + module.decoder.layers.21.mlp.shared_experts.gate_weight torch.Size([1, 2048]) + module.decoder.layers.21.pre_mlp_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.21.input_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.20.self_attention.linear_qkv.weight torch.Size([5120, 2048]) + module.decoder.layers.22.self_attention.q_proj.weight torch.Size([512, 2048]) + module.decoder.layers.22.input_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.22.input_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.21.pre_mlp_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.20.mlp.shared_experts.gate_weight torch.Size([1, 2048]) + module.decoder.layers.20.pre_mlp_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.22.input_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.21.input_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.21.input_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.20.pre_mlp_layernorm.norm.weight torch.Size([2048]) +Params for bucket 18 (40222848 elements, 40222848 padded size): + module.decoder.layers.19.mlp.shared_experts.gate_weight torch.Size([1, 2048]) + module.decoder.layers.19.self_attention.b_proj.weight torch.Size([32, 2048]) + module.decoder.layers.18.self_attention.b_proj.weight torch.Size([32, 2048]) + module.decoder.layers.17.mlp.shared_experts.linear_fc1.weight torch.Size([1024, 2048]) + module.decoder.layers.19.self_attention.out_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.19.self_attention.z_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.18.mlp.router.weight torch.Size([192, 2048]) + module.decoder.layers.18.self_attention.out_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.18.self_attention.z_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.19.pre_mlp_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.19.pre_mlp_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.19.self_attention.v_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.18.pre_mlp_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.18.pre_mlp_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.18.self_attention.v_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.19.pre_mlp_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.19.self_attention.k_proj.weight torch.Size([512, 2048]) + module.decoder.layers.18.pre_mlp_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.18.self_attention.k_proj.weight torch.Size([512, 2048]) + module.decoder.layers.20.input_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.20.input_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.19.mlp.shared_experts.linear_fc1.weight torch.Size([1024, 2048]) + module.decoder.layers.19.self_attention.conv1d.weight torch.Size([3072, 1, 4]) + module.decoder.layers.19.self_attention.q_proj.weight torch.Size([512, 2048]) + module.decoder.layers.19.input_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.18.mlp.shared_experts.linear_fc2.weight torch.Size([2048, 512]) + module.decoder.layers.18.self_attention.conv1d.weight torch.Size([3072, 1, 4]) + module.decoder.layers.18.self_attention.q_proj.weight torch.Size([512, 2048]) + module.decoder.layers.18.input_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.20.input_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.19.pre_mlp_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.19.input_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.18.mlp.shared_experts.linear_fc1.weight torch.Size([1024, 2048]) + module.decoder.layers.18.pre_mlp_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.18.input_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.19.mlp.router.weight torch.Size([192, 2048]) + module.decoder.layers.19.self_attention.out_norm.weight torch.Size([64]) + module.decoder.layers.19.input_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.18.mlp.shared_experts.gate_weight torch.Size([1, 2048]) + module.decoder.layers.18.self_attention.out_norm.weight torch.Size([64]) + module.decoder.layers.19.mlp.shared_experts.linear_fc2.weight torch.Size([2048, 512]) + module.decoder.layers.20.input_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.19.self_attention.a_proj.weight torch.Size([32, 2048]) + module.decoder.layers.19.input_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.18.self_attention.a_proj.weight torch.Size([32, 2048]) + module.decoder.layers.18.input_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.17.mlp.shared_experts.linear_fc2.weight torch.Size([2048, 512]) + module.decoder.layers.18.input_layernorm.norm.weight torch.Size([2048]) +Params for bucket 19 (41666688 elements, 41666688 padded size): + module.decoder.layers.17.pre_mlp_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.17.pre_mlp_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.17.self_attention.v_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.16.pre_mlp_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.16.input_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.17.pre_mlp_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.17.self_attention.k_proj.weight torch.Size([512, 2048]) + module.decoder.layers.16.mlp.shared_experts.linear_fc2.weight torch.Size([2048, 512]) + module.decoder.layers.16.self_attention.out_norm.weight torch.Size([64]) + module.decoder.layers.15.pre_mlp_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.17.self_attention.conv1d.weight torch.Size([3072, 1, 4]) + module.decoder.layers.17.input_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.16.mlp.shared_experts.linear_fc1.weight torch.Size([1024, 2048]) + module.decoder.layers.16.self_attention.a_proj.weight torch.Size([32, 2048]) + module.decoder.layers.16.input_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.15.mlp.shared_experts.linear_fc2.weight torch.Size([2048, 512]) + module.decoder.layers.17.mlp.shared_experts.gate_weight torch.Size([1, 2048]) + module.decoder.layers.17.pre_mlp_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.17.input_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.16.self_attention.b_proj.weight torch.Size([32, 2048]) + module.decoder.layers.15.mlp.shared_experts.linear_fc1.weight torch.Size([1024, 2048]) + module.decoder.layers.17.self_attention.out_norm.weight torch.Size([64]) + module.decoder.layers.17.self_attention.q_proj.weight torch.Size([512, 2048]) + module.decoder.layers.16.mlp.router.weight torch.Size([192, 2048]) + module.decoder.layers.16.self_attention.out_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.16.self_attention.z_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.17.self_attention.a_proj.weight torch.Size([32, 2048]) + module.decoder.layers.17.input_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.16.mlp.shared_experts.gate_weight torch.Size([1, 2048]) + module.decoder.layers.16.pre_mlp_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.16.pre_mlp_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.16.self_attention.v_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.15.mlp.router.weight torch.Size([192, 2048]) + module.decoder.layers.15.self_attention.out_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.17.self_attention.b_proj.weight torch.Size([32, 2048]) + module.decoder.layers.17.input_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.16.pre_mlp_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.16.self_attention.k_proj.weight torch.Size([512, 2048]) + module.decoder.layers.15.mlp.shared_experts.gate_weight torch.Size([1, 2048]) + module.decoder.layers.15.pre_mlp_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.15.pre_mlp_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.17.mlp.router.weight torch.Size([192, 2048]) + module.decoder.layers.17.self_attention.out_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.17.self_attention.z_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.16.self_attention.conv1d.weight torch.Size([3072, 1, 4]) + module.decoder.layers.16.self_attention.q_proj.weight torch.Size([512, 2048]) + module.decoder.layers.16.input_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.16.input_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.15.pre_mlp_layernorm.gate_down.weight torch.Size([16, 2048]) +Params for bucket 20 (41345216 elements, 41345280 padded size): + module.decoder.layers.15.self_attention.conv1d.weight torch.Size([3072, 1, 4]) + module.decoder.layers.15.self_attention.q_proj.weight torch.Size([512, 2048]) + module.decoder.layers.15.input_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.14.mlp.shared_experts.linear_fc2.weight torch.Size([2048, 512]) + module.decoder.layers.14.mlp.router.weight torch.Size([192, 2048]) + module.decoder.layers.14.self_attention.out_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.14.self_attention.v_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.13.self_attention.conv1d.weight torch.Size([3072, 1, 4]) + module.decoder.layers.15.input_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.14.mlp.shared_experts.linear_fc1.weight torch.Size([1024, 2048]) + module.decoder.layers.14.pre_mlp_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.14.pre_mlp_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.14.self_attention.k_proj.weight torch.Size([512, 2048]) + module.decoder.layers.13.mlp.shared_experts.gate_weight torch.Size([1, 2048]) + module.decoder.layers.13.pre_mlp_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.15.self_attention.out_norm.weight torch.Size([64]) + module.decoder.layers.15.input_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.14.pre_mlp_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.14.self_attention.q_proj.weight torch.Size([512, 2048]) + module.decoder.layers.14.input_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.13.self_attention.out_norm.weight torch.Size([64]) + module.decoder.layers.15.self_attention.a_proj.weight torch.Size([32, 2048]) + module.decoder.layers.15.input_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.14.self_attention.b_proj.weight torch.Size([32, 2048]) + module.decoder.layers.14.input_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.13.self_attention.a_proj.weight torch.Size([32, 2048]) + module.decoder.layers.15.self_attention.b_proj.weight torch.Size([32, 2048]) + module.decoder.layers.14.mlp.shared_experts.gate_weight torch.Size([1, 2048]) + module.decoder.layers.14.pre_mlp_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.14.self_attention.a_proj.weight torch.Size([32, 2048]) + module.decoder.layers.14.input_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.13.self_attention.b_proj.weight torch.Size([32, 2048]) + module.decoder.layers.15.self_attention.z_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.14.self_attention.out_norm.weight torch.Size([64]) + module.decoder.layers.13.mlp.shared_experts.linear_fc2.weight torch.Size([2048, 512]) + module.decoder.layers.13.mlp.router.weight torch.Size([192, 2048]) + module.decoder.layers.13.self_attention.out_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.13.self_attention.z_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.15.self_attention.v_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.14.self_attention.conv1d.weight torch.Size([3072, 1, 4]) + module.decoder.layers.14.input_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.13.mlp.shared_experts.linear_fc1.weight torch.Size([1024, 2048]) + module.decoder.layers.13.pre_mlp_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.13.pre_mlp_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.15.self_attention.k_proj.weight torch.Size([512, 2048]) + module.decoder.layers.14.self_attention.z_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.13.pre_mlp_layernorm.gate_down.weight torch.Size([16, 2048]) +Params for bucket 21 (41063488 elements, 41063552 padded size): + module.decoder.layers.13.input_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.13.self_attention.q_proj.weight torch.Size([512, 2048]) + module.decoder.layers.13.input_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.12.self_attention.linear_qkv.bias torch.Size([5120]) + module.decoder.layers.11.mlp.shared_experts.linear_fc2.weight torch.Size([2048, 512]) + module.decoder.layers.11.pre_mlp_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.11.self_attention.out_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.11.self_attention.v_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.13.input_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.12.mlp.router.weight torch.Size([192, 2048]) + module.decoder.layers.11.mlp.shared_experts.linear_fc1.weight torch.Size([1024, 2048]) + module.decoder.layers.12.mlp.shared_experts.gate_weight torch.Size([1, 2048]) + module.decoder.layers.12.pre_mlp_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.12.self_attention.linear_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.11.self_attention.conv1d.weight torch.Size([3072, 1, 4]) + module.decoder.layers.13.input_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.12.mlp.shared_experts.linear_fc2.weight torch.Size([2048, 512]) + module.decoder.layers.12.pre_mlp_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.11.mlp.router.weight torch.Size([192, 2048]) + module.decoder.layers.12.mlp.shared_experts.linear_fc1.weight torch.Size([1024, 2048]) + module.decoder.layers.12.input_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.11.mlp.shared_experts.gate_weight torch.Size([1, 2048]) + module.decoder.layers.11.pre_mlp_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.11.self_attention.out_norm.weight torch.Size([64]) + module.decoder.layers.12.pre_mlp_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.12.pre_mlp_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.12.input_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.12.input_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.11.pre_mlp_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.11.self_attention.a_proj.weight torch.Size([32, 2048]) + module.decoder.layers.13.self_attention.v_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.12.self_attention.linear_qkv.weight torch.Size([5120, 2048]) + module.decoder.layers.11.self_attention.b_proj.weight torch.Size([32, 2048]) + module.decoder.layers.13.self_attention.k_proj.weight torch.Size([512, 2048]) + module.decoder.layers.12.input_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.11.pre_mlp_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.11.self_attention.z_proj.weight torch.Size([2048, 2048]) +Params for bucket 22 (40222848 elements, 40222848 padded size): + module.decoder.layers.10.self_attention.conv1d.weight torch.Size([3072, 1, 4]) + module.decoder.layers.10.self_attention.q_proj.weight torch.Size([512, 2048]) + module.decoder.layers.10.input_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.10.input_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.9.self_attention.out_norm.weight torch.Size([64]) + module.decoder.layers.9.self_attention.conv1d.weight torch.Size([3072, 1, 4]) + module.decoder.layers.9.self_attention.k_proj.weight torch.Size([512, 2048]) + module.decoder.layers.11.self_attention.k_proj.weight torch.Size([512, 2048]) + module.decoder.layers.11.input_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.10.mlp.shared_experts.gate_weight torch.Size([1, 2048]) + module.decoder.layers.10.pre_mlp_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.10.input_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.9.self_attention.q_proj.weight torch.Size([512, 2048]) + module.decoder.layers.9.input_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.8.mlp.shared_experts.linear_fc2.weight torch.Size([2048, 512]) + module.decoder.layers.11.self_attention.q_proj.weight torch.Size([512, 2048]) + module.decoder.layers.11.input_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.10.self_attention.out_norm.weight torch.Size([64]) + module.decoder.layers.10.self_attention.a_proj.weight torch.Size([32, 2048]) + module.decoder.layers.10.input_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.9.mlp.shared_experts.linear_fc2.weight torch.Size([2048, 512]) + module.decoder.layers.9.mlp.router.weight torch.Size([192, 2048]) + module.decoder.layers.9.self_attention.out_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.10.self_attention.b_proj.weight torch.Size([32, 2048]) + module.decoder.layers.9.mlp.shared_experts.linear_fc1.weight torch.Size([1024, 2048]) + module.decoder.layers.9.pre_mlp_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.9.self_attention.a_proj.weight torch.Size([32, 2048]) + module.decoder.layers.10.mlp.shared_experts.linear_fc2.weight torch.Size([2048, 512]) + module.decoder.layers.10.mlp.router.weight torch.Size([192, 2048]) + module.decoder.layers.10.self_attention.out_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.10.self_attention.z_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.9.pre_mlp_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.9.self_attention.b_proj.weight torch.Size([32, 2048]) + module.decoder.layers.11.input_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.10.mlp.shared_experts.linear_fc1.weight torch.Size([1024, 2048]) + module.decoder.layers.10.pre_mlp_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.10.pre_mlp_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.10.self_attention.v_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.9.self_attention.z_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.9.input_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.9.input_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.11.input_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.10.pre_mlp_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.10.self_attention.k_proj.weight torch.Size([512, 2048]) + module.decoder.layers.9.mlp.shared_experts.gate_weight torch.Size([1, 2048]) + module.decoder.layers.9.pre_mlp_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.9.pre_mlp_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.9.self_attention.v_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.9.input_layernorm.gate_down.weight torch.Size([16, 2048]) +Params for bucket 23 (43763840 elements, 43763840 padded size): + module.decoder.layers.8.self_attention.out_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.8.self_attention.z_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.8.input_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.7.pre_mlp_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.7.self_attention.k_proj.weight torch.Size([512, 2048]) + module.decoder.layers.6.pre_mlp_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.8.pre_mlp_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.8.pre_mlp_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.8.self_attention.v_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.7.self_attention.conv1d.weight torch.Size([3072, 1, 4]) + module.decoder.layers.7.self_attention.q_proj.weight torch.Size([512, 2048]) + module.decoder.layers.7.input_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.6.mlp.shared_experts.linear_fc2.weight torch.Size([2048, 512]) + module.decoder.layers.8.mlp.shared_experts.gate_weight torch.Size([1, 2048]) + module.decoder.layers.8.pre_mlp_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.8.self_attention.k_proj.weight torch.Size([512, 2048]) + module.decoder.layers.7.pre_mlp_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.7.input_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.6.mlp.shared_experts.linear_fc1.weight torch.Size([1024, 2048]) + module.decoder.layers.8.mlp.shared_experts.linear_fc1.weight torch.Size([1024, 2048]) + module.decoder.layers.8.self_attention.conv1d.weight torch.Size([3072, 1, 4]) + module.decoder.layers.8.self_attention.q_proj.weight torch.Size([512, 2048]) + module.decoder.layers.8.input_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.7.mlp.shared_experts.linear_fc2.weight torch.Size([2048, 512]) + module.decoder.layers.7.self_attention.out_norm.weight torch.Size([64]) + module.decoder.layers.7.input_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.8.mlp.router.weight torch.Size([192, 2048]) + module.decoder.layers.8.pre_mlp_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.8.input_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.7.mlp.shared_experts.linear_fc1.weight torch.Size([1024, 2048]) + module.decoder.layers.7.self_attention.a_proj.weight torch.Size([32, 2048]) + module.decoder.layers.7.input_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.6.mlp.router.weight torch.Size([192, 2048]) + module.decoder.layers.6.self_attention.out_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.8.self_attention.out_norm.weight torch.Size([64]) + module.decoder.layers.7.mlp.shared_experts.gate_weight torch.Size([1, 2048]) + module.decoder.layers.7.self_attention.b_proj.weight torch.Size([32, 2048]) + module.decoder.layers.6.mlp.shared_experts.gate_weight torch.Size([1, 2048]) + module.decoder.layers.6.pre_mlp_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.6.pre_mlp_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.8.self_attention.a_proj.weight torch.Size([32, 2048]) + module.decoder.layers.8.input_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.7.mlp.router.weight torch.Size([192, 2048]) + module.decoder.layers.7.self_attention.out_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.7.self_attention.z_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.6.pre_mlp_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.8.self_attention.b_proj.weight torch.Size([32, 2048]) + module.decoder.layers.7.pre_mlp_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.7.pre_mlp_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.7.self_attention.v_proj.weight torch.Size([2048, 2048]) +Params for bucket 24 (41345216 elements, 41345280 padded size): + module.decoder.layers.6.input_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.5.pre_mlp_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.5.self_attention.k_proj.weight torch.Size([512, 2048]) + module.decoder.layers.4.mlp.shared_experts.gate_weight torch.Size([1, 2048]) + module.decoder.layers.4.pre_mlp_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.6.self_attention.out_norm.weight torch.Size([64]) + module.decoder.layers.5.mlp.shared_experts.linear_fc1.weight torch.Size([1024, 2048]) + module.decoder.layers.5.self_attention.conv1d.weight torch.Size([3072, 1, 4]) + module.decoder.layers.5.self_attention.q_proj.weight torch.Size([512, 2048]) + module.decoder.layers.5.input_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.5.input_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.4.self_attention.out_norm.weight torch.Size([64]) + module.decoder.layers.6.self_attention.a_proj.weight torch.Size([32, 2048]) + module.decoder.layers.6.input_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.5.mlp.shared_experts.linear_fc2.weight torch.Size([2048, 512]) + module.decoder.layers.5.pre_mlp_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.5.input_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.4.self_attention.a_proj.weight torch.Size([32, 2048]) + module.decoder.layers.6.self_attention.b_proj.weight torch.Size([32, 2048]) + module.decoder.layers.5.mlp.shared_experts.gate_weight torch.Size([1, 2048]) + module.decoder.layers.5.self_attention.out_norm.weight torch.Size([64]) + module.decoder.layers.4.self_attention.b_proj.weight torch.Size([32, 2048]) + module.decoder.layers.6.self_attention.z_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.5.self_attention.a_proj.weight torch.Size([32, 2048]) + module.decoder.layers.5.input_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.4.mlp.shared_experts.linear_fc2.weight torch.Size([2048, 512]) + module.decoder.layers.4.mlp.router.weight torch.Size([192, 2048]) + module.decoder.layers.4.self_attention.out_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.4.self_attention.z_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.6.self_attention.v_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.5.self_attention.b_proj.weight torch.Size([32, 2048]) + module.decoder.layers.4.mlp.shared_experts.linear_fc1.weight torch.Size([1024, 2048]) + module.decoder.layers.4.pre_mlp_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.4.pre_mlp_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.6.self_attention.k_proj.weight torch.Size([512, 2048]) + module.decoder.layers.5.mlp.router.weight torch.Size([192, 2048]) + module.decoder.layers.5.self_attention.out_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.5.self_attention.z_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.4.pre_mlp_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.6.self_attention.conv1d.weight torch.Size([3072, 1, 4]) + module.decoder.layers.6.self_attention.q_proj.weight torch.Size([512, 2048]) + module.decoder.layers.6.input_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.6.input_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.5.pre_mlp_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.5.pre_mlp_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.5.self_attention.v_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.4.self_attention.conv1d.weight torch.Size([3072, 1, 4]) +Params for bucket 25 (41201792 elements, 41201792 padded size): + module.decoder.layers.4.input_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.3.self_attention.a_proj.weight torch.Size([32, 2048]) + module.decoder.layers.3.input_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.2.self_attention.a_proj.weight torch.Size([32, 2048]) + module.decoder.layers.3.self_attention.b_proj.weight torch.Size([32, 2048]) + module.decoder.layers.3.self_attention.v_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.3.self_attention.q_proj.weight torch.Size([512, 2048]) + module.decoder.layers.2.self_attention.b_proj.weight torch.Size([32, 2048]) + module.decoder.layers.4.input_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.3.mlp.shared_experts.linear_fc2.weight torch.Size([2048, 512]) + module.decoder.layers.3.mlp.router.weight torch.Size([192, 2048]) + module.decoder.layers.3.self_attention.out_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.3.self_attention.z_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.2.self_attention.out_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.2.self_attention.z_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.3.mlp.shared_experts.linear_fc1.weight torch.Size([1024, 2048]) + module.decoder.layers.3.pre_mlp_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.3.pre_mlp_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.3.self_attention.k_proj.weight torch.Size([512, 2048]) + module.decoder.layers.2.mlp.shared_experts.linear_fc2.weight torch.Size([2048, 512]) + module.decoder.layers.2.pre_mlp_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.2.pre_mlp_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.2.self_attention.v_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.3.pre_mlp_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.2.mlp.shared_experts.linear_fc1.weight torch.Size([1024, 2048]) + module.decoder.layers.2.pre_mlp_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.4.self_attention.v_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.3.self_attention.conv1d.weight torch.Size([3072, 1, 4]) + module.decoder.layers.3.input_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.3.input_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.2.mlp.router.weight torch.Size([192, 2048]) + module.decoder.layers.2.self_attention.conv1d.weight torch.Size([3072, 1, 4]) + module.decoder.layers.4.self_attention.k_proj.weight torch.Size([512, 2048]) + module.decoder.layers.3.mlp.shared_experts.gate_weight torch.Size([1, 2048]) + module.decoder.layers.3.pre_mlp_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.3.input_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.2.pre_mlp_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.4.self_attention.q_proj.weight torch.Size([512, 2048]) + module.decoder.layers.4.input_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.4.input_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.3.self_attention.out_norm.weight torch.Size([64]) + module.decoder.layers.2.mlp.shared_experts.gate_weight torch.Size([1, 2048]) + module.decoder.layers.2.self_attention.out_norm.weight torch.Size([64]) +Params for bucket 26 (40259648 elements, 40259712 padded size): + module.decoder.layers.2.self_attention.k_proj.weight torch.Size([512, 2048]) + module.decoder.layers.2.input_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.2.input_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.2.input_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.1.mlp.linear_fc1.weight torch.Size([9600, 2048]) + module.decoder.layers.1.pre_mlp_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.1.pre_mlp_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.1.self_attention.out_norm.weight torch.Size([64]) + module.decoder.layers.1.self_attention.conv1d.weight torch.Size([3072, 1, 4]) + module.decoder.layers.1.self_attention.b_proj.weight torch.Size([32, 2048]) + module.decoder.layers.2.input_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.2.self_attention.q_proj.weight torch.Size([512, 2048]) + module.decoder.layers.1.mlp.linear_fc2.weight torch.Size([2048, 4800]) + module.decoder.layers.1.pre_mlp_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.1.pre_mlp_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.1.self_attention.out_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.1.self_attention.a_proj.weight torch.Size([32, 2048]) + module.decoder.layers.1.self_attention.z_proj.weight torch.Size([2048, 2048]) +Params for bucket 27 (40116224 elements, 40116224 padded size): + module.decoder.layers.1.self_attention.v_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.1.self_attention.q_proj.weight torch.Size([512, 2048]) + module.decoder.layers.1.input_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.0.mlp.linear_fc1.weight torch.Size([9600, 2048]) + module.decoder.layers.0.pre_mlp_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.0.pre_mlp_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.0.pre_mlp_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.0.self_attention.out_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.1.input_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.1.self_attention.k_proj.weight torch.Size([512, 2048]) + module.decoder.layers.1.input_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.1.input_layernorm.norm.weight torch.Size([2048]) + module.decoder.layers.0.mlp.linear_fc2.weight torch.Size([2048, 4800]) + module.decoder.layers.0.pre_mlp_layernorm.gate_up.weight torch.Size([2048, 16]) +Params for bucket 28 (213598272 elements, 213598336 padded size): + module.decoder.layers.0.self_attention.z_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.0.self_attention.conv1d.weight torch.Size([3072, 1, 4]) + module.decoder.layers.0.self_attention.q_proj.weight torch.Size([512, 2048]) + module.decoder.layers.0.input_layernorm.gate_up.weight torch.Size([2048, 16]) + module.decoder.layers.0.self_attention.out_norm.weight torch.Size([64]) + module.decoder.layers.0.self_attention.b_proj.weight torch.Size([32, 2048]) + module.decoder.layers.0.self_attention.a_proj.weight torch.Size([32, 2048]) + module.decoder.layers.0.self_attention.v_proj.weight torch.Size([2048, 2048]) + module.decoder.layers.0.self_attention.k_proj.weight torch.Size([512, 2048]) + module.decoder.layers.0.input_layernorm.gate_up.bias torch.Size([2048]) + module.decoder.layers.0.input_layernorm.gate_down.weight torch.Size([16, 2048]) + module.decoder.layers.0.input_layernorm.norm.weight torch.Size([2048]) + module.embedding.word_embeddings.weight torch.Size([99072, 2048]) +INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 1 +Params for bucket 1 (3136 elements, 6272 padded size): + module.decoder.layers.43.self_attention.A_log torch.Size([32]) + module.decoder.layers.40.self_attention.dt_bias torch.Size([32]) + module.decoder.layers.36.self_attention.A_log torch.Size([32]) + module.decoder.layers.26.self_attention.A_log torch.Size([32]) + module.decoder.layers.18.self_attention.A_log torch.Size([32]) + module.decoder.layers.33.self_attention.A_log torch.Size([32]) + module.decoder.layers.10.self_attention.dt_bias torch.Size([32]) + module.decoder.layers.1.self_attention.dt_bias torch.Size([32]) + module.decoder.layers.45.self_attention.dt_bias torch.Size([32]) + module.decoder.layers.30.self_attention.A_log torch.Size([32]) + module.decoder.layers.14.self_attention.dt_bias torch.Size([32]) + module.decoder.layers.7.self_attention.dt_bias torch.Size([32]) + module.decoder.layers.35.self_attention.dt_bias torch.Size([32]) + module.decoder.layers.17.self_attention.dt_bias torch.Size([32]) + module.decoder.layers.32.self_attention.dt_bias torch.Size([32]) + module.decoder.layers.28.self_attention.A_log torch.Size([32]) + module.decoder.layers.50.self_attention.dt_bias torch.Size([32]) + module.decoder.layers.5.self_attention.A_log torch.Size([32]) + module.decoder.layers.22.self_attention.dt_bias torch.Size([32]) + module.decoder.layers.47.self_attention.dt_bias torch.Size([32]) + module.decoder.layers.27.self_attention.dt_bias torch.Size([32]) + module.decoder.layers.38.self_attention.A_log torch.Size([32]) + module.decoder.layers.19.self_attention.A_log torch.Size([32]) + module.decoder.layers.16.self_attention.dt_bias torch.Size([32]) + module.decoder.layers.4.self_attention.dt_bias torch.Size([32]) + module.decoder.layers.51.self_attention.dt_bias torch.Size([32]) + module.decoder.layers.34.self_attention.A_log torch.Size([32]) + module.decoder.layers.13.self_attention.dt_bias torch.Size([32]) + module.decoder.layers.2.self_attention.A_log torch.Size([32]) + module.decoder.layers.55.self_attention.dt_bias torch.Size([32]) + module.decoder.layers.52.self_attention.A_log torch.Size([32]) + module.decoder.layers.42.self_attention.A_log torch.Size([32]) + module.decoder.layers.25.self_attention.A_log torch.Size([32]) + module.decoder.layers.53.self_attention.dt_bias torch.Size([32]) + module.decoder.layers.44.self_attention.dt_bias torch.Size([32]) + module.decoder.layers.34.self_attention.dt_bias torch.Size([32]) + module.decoder.layers.10.self_attention.A_log torch.Size([32]) + module.decoder.layers.1.self_attention.A_log torch.Size([32]) + module.decoder.layers.41.self_attention.dt_bias torch.Size([32]) + module.decoder.layers.24.self_attention.dt_bias torch.Size([32]) + module.decoder.layers.7.self_attention.A_log torch.Size([32]) + module.decoder.layers.54.self_attention.A_log torch.Size([32]) + module.decoder.layers.35.self_attention.A_log torch.Size([32]) + module.decoder.layers.31.self_attention.dt_bias torch.Size([32]) + module.decoder.layers.17.self_attention.A_log torch.Size([32]) + module.decoder.layers.0.self_attention.A_log torch.Size([32]) + module.decoder.layers.32.self_attention.A_log torch.Size([32]) + module.decoder.layers.29.self_attention.dt_bias torch.Size([32]) + module.decoder.layers.37.self_attention.dt_bias torch.Size([32]) + module.decoder.layers.6.self_attention.dt_bias torch.Size([32]) + module.decoder.layers.22.self_attention.A_log torch.Size([32]) + module.decoder.layers.15.self_attention.dt_bias torch.Size([32]) + module.decoder.layers.47.self_attention.A_log torch.Size([32]) + module.decoder.layers.27.self_attention.A_log torch.Size([32]) + module.decoder.layers.40.self_attention.A_log torch.Size([32]) + module.decoder.layers.29.self_attention.A_log torch.Size([32]) + module.decoder.layers.16.self_attention.A_log torch.Size([32]) + module.decoder.layers.14.self_attention.A_log torch.Size([32]) + module.decoder.layers.11.self_attention.dt_bias torch.Size([32]) + module.decoder.layers.4.self_attention.A_log torch.Size([32]) + module.decoder.layers.51.self_attention.A_log torch.Size([32]) + module.decoder.layers.39.self_attention.dt_bias torch.Size([32]) + module.decoder.layers.13.self_attention.A_log torch.Size([32]) + module.decoder.layers.8.self_attention.dt_bias torch.Size([32]) + module.decoder.layers.55.self_attention.A_log torch.Size([32]) + module.decoder.layers.43.self_attention.dt_bias torch.Size([32]) + module.decoder.layers.36.self_attention.dt_bias torch.Size([32]) + module.decoder.layers.26.self_attention.dt_bias torch.Size([32]) + module.decoder.layers.18.self_attention.dt_bias torch.Size([32]) + module.decoder.layers.9.self_attention.dt_bias torch.Size([32]) + module.decoder.layers.33.self_attention.dt_bias torch.Size([32]) + module.decoder.layers.3.self_attention.dt_bias torch.Size([32]) + module.decoder.layers.53.self_attention.A_log torch.Size([32]) + module.decoder.layers.44.self_attention.A_log torch.Size([32]) + module.decoder.layers.30.self_attention.dt_bias torch.Size([32]) + module.decoder.layers.54.self_attention.dt_bias torch.Size([32]) + module.decoder.layers.45.self_attention.A_log torch.Size([32]) + module.decoder.layers.41.self_attention.A_log torch.Size([32]) + module.decoder.layers.24.self_attention.A_log torch.Size([32]) + module.decoder.layers.3.self_attention.A_log torch.Size([32]) + module.decoder.layers.31.self_attention.A_log torch.Size([32]) + module.decoder.layers.28.self_attention.dt_bias torch.Size([32]) + module.decoder.layers.9.self_attention.A_log torch.Size([32]) + module.decoder.layers.5.self_attention.dt_bias torch.Size([32]) + module.decoder.layers.37.self_attention.A_log torch.Size([32]) + module.decoder.layers.6.self_attention.A_log torch.Size([32]) + module.decoder.layers.50.self_attention.A_log torch.Size([32]) + module.decoder.layers.15.self_attention.A_log torch.Size([32]) + module.decoder.layers.0.self_attention.dt_bias torch.Size([32]) + module.decoder.layers.38.self_attention.dt_bias torch.Size([32]) + module.decoder.layers.19.self_attention.dt_bias torch.Size([32]) + module.decoder.layers.11.self_attention.A_log torch.Size([32]) + module.decoder.layers.2.self_attention.dt_bias torch.Size([32]) + module.decoder.layers.25.self_attention.dt_bias torch.Size([32]) + module.decoder.layers.52.self_attention.dt_bias torch.Size([32]) + module.decoder.layers.42.self_attention.dt_bias torch.Size([32]) + module.decoder.layers.39.self_attention.A_log torch.Size([32]) + module.decoder.layers.8.self_attention.A_log torch.Size([32]) +INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 54 +Params for bucket 1 (75497472 elements, 75497472 padded size): + module.decoder.layers.55.mlp.experts.fc2_weight torch.Size([24, 2048, 512]) + module.decoder.layers.55.mlp.experts.fc1_weight torch.Size([24, 1024, 2048]) +Params for bucket 2 (75497472 elements, 75497472 padded size): + module.decoder.layers.54.mlp.experts.fc2_weight torch.Size([24, 2048, 512]) + module.decoder.layers.54.mlp.experts.fc1_weight torch.Size([24, 1024, 2048]) +Params for bucket 3 (75497472 elements, 75497472 padded size): + module.decoder.layers.53.mlp.experts.fc2_weight torch.Size([24, 2048, 512]) + module.decoder.layers.53.mlp.experts.fc1_weight torch.Size([24, 1024, 2048]) +Params for bucket 4 (75497472 elements, 75497472 padded size): + module.decoder.layers.52.mlp.experts.fc2_weight torch.Size([24, 2048, 512]) + module.decoder.layers.52.mlp.experts.fc1_weight torch.Size([24, 1024, 2048]) +Params for bucket 5 (75497472 elements, 75497472 padded size): + module.decoder.layers.51.mlp.experts.fc2_weight torch.Size([24, 2048, 512]) + module.decoder.layers.51.mlp.experts.fc1_weight torch.Size([24, 1024, 2048]) +Params for bucket 6 (75497472 elements, 75497472 padded size): + module.decoder.layers.50.mlp.experts.fc2_weight torch.Size([24, 2048, 512]) + module.decoder.layers.50.mlp.experts.fc1_weight torch.Size([24, 1024, 2048]) +Params for bucket 7 (75497472 elements, 75497472 padded size): + module.decoder.layers.49.mlp.experts.fc2_weight torch.Size([24, 2048, 512]) + module.decoder.layers.49.mlp.experts.fc1_weight torch.Size([24, 1024, 2048]) +Params for bucket 8 (75497472 elements, 75497472 padded size): + module.decoder.layers.48.mlp.experts.fc2_weight torch.Size([24, 2048, 512]) + module.decoder.layers.48.mlp.experts.fc1_weight torch.Size([24, 1024, 2048]) +Params for bucket 9 (75497472 elements, 75497472 padded size): + module.decoder.layers.47.mlp.experts.fc2_weight torch.Size([24, 2048, 512]) + module.decoder.layers.47.mlp.experts.fc1_weight torch.Size([24, 1024, 2048]) +Params for bucket 10 (75497472 elements, 75497472 padded size): + module.decoder.layers.46.mlp.experts.fc2_weight torch.Size([24, 2048, 512]) + module.decoder.layers.46.mlp.experts.fc1_weight torch.Size([24, 1024, 2048]) +Params for bucket 11 (75497472 elements, 75497472 padded size): + module.decoder.layers.45.mlp.experts.fc2_weight torch.Size([24, 2048, 512]) + module.decoder.layers.45.mlp.experts.fc1_weight torch.Size([24, 1024, 2048]) +Params for bucket 12 (75497472 elements, 75497472 padded size): + module.decoder.layers.44.mlp.experts.fc2_weight torch.Size([24, 2048, 512]) + module.decoder.layers.44.mlp.experts.fc1_weight torch.Size([24, 1024, 2048]) +Params for bucket 13 (75497472 elements, 75497472 padded size): + module.decoder.layers.43.mlp.experts.fc2_weight torch.Size([24, 2048, 512]) + module.decoder.layers.43.mlp.experts.fc1_weight torch.Size([24, 1024, 2048]) +Params for bucket 14 (75497472 elements, 75497472 padded size): + module.decoder.layers.42.mlp.experts.fc2_weight torch.Size([24, 2048, 512]) + module.decoder.layers.42.mlp.experts.fc1_weight torch.Size([24, 1024, 2048]) +Params for bucket 15 (75497472 elements, 75497472 padded size): + module.decoder.layers.41.mlp.experts.fc2_weight torch.Size([24, 2048, 512]) + module.decoder.layers.41.mlp.experts.fc1_weight torch.Size([24, 1024, 2048]) +Params for bucket 16 (75497472 elements, 75497472 padded size): + module.decoder.layers.40.mlp.experts.fc2_weight torch.Size([24, 2048, 512]) + module.decoder.layers.40.mlp.experts.fc1_weight torch.Size([24, 1024, 2048]) +Params for bucket 17 (75497472 elements, 75497472 padded size): + module.decoder.layers.39.mlp.experts.fc2_weight torch.Size([24, 2048, 512]) + module.decoder.layers.39.mlp.experts.fc1_weight torch.Size([24, 1024, 2048]) +Params for bucket 18 (75497472 elements, 75497472 padded size): + module.decoder.layers.38.mlp.experts.fc2_weight torch.Size([24, 2048, 512]) + module.decoder.layers.38.mlp.experts.fc1_weight torch.Size([24, 1024, 2048]) +Params for bucket 19 (75497472 elements, 75497472 padded size): + module.decoder.layers.37.mlp.experts.fc2_weight torch.Size([24, 2048, 512]) + module.decoder.layers.37.mlp.experts.fc1_weight torch.Size([24, 1024, 2048]) +Params for bucket 20 (75497472 elements, 75497472 padded size): + module.decoder.layers.36.mlp.experts.fc2_weight torch.Size([24, 2048, 512]) + module.decoder.layers.36.mlp.experts.fc1_weight torch.Size([24, 1024, 2048]) +Params for bucket 21 (75497472 elements, 75497472 padded size): + module.decoder.layers.35.mlp.experts.fc2_weight torch.Size([24, 2048, 512]) + module.decoder.layers.35.mlp.experts.fc1_weight torch.Size([24, 1024, 2048]) +Params for bucket 22 (75497472 elements, 75497472 padded size): + module.decoder.layers.34.mlp.experts.fc2_weight torch.Size([24, 2048, 512]) + module.decoder.layers.34.mlp.experts.fc1_weight torch.Size([24, 1024, 2048]) +Params for bucket 23 (75497472 elements, 75497472 padded size): + module.decoder.layers.33.mlp.experts.fc2_weight torch.Size([24, 2048, 512]) + module.decoder.layers.33.mlp.experts.fc1_weight torch.Size([24, 1024, 2048]) +Params for bucket 24 (75497472 elements, 75497472 padded size): + module.decoder.layers.32.mlp.experts.fc2_weight torch.Size([24, 2048, 512]) + module.decoder.layers.32.mlp.experts.fc1_weight torch.Size([24, 1024, 2048]) +Params for bucket 25 (75497472 elements, 75497472 padded size): + module.decoder.layers.31.mlp.experts.fc2_weight torch.Size([24, 2048, 512]) + module.decoder.layers.31.mlp.experts.fc1_weight torch.Size([24, 1024, 2048]) +Params for bucket 26 (75497472 elements, 75497472 padded size): + module.decoder.layers.30.mlp.experts.fc2_weight torch.Size([24, 2048, 512]) + module.decoder.layers.30.mlp.experts.fc1_weight torch.Size([24, 1024, 2048]) +Params for bucket 27 (75497472 elements, 75497472 padded size): + module.decoder.layers.29.mlp.experts.fc2_weight torch.Size([24, 2048, 512]) + module.decoder.layers.29.mlp.experts.fc1_weight torch.Size([24, 1024, 2048]) +Params for bucket 28 (75497472 elements, 75497472 padded size): + module.decoder.layers.28.mlp.experts.fc2_weight torch.Size([24, 2048, 512]) + module.decoder.layers.28.mlp.experts.fc1_weight torch.Size([24, 1024, 2048]) +Params for bucket 29 (75497472 elements, 75497472 padded size): + module.decoder.layers.27.mlp.experts.fc2_weight torch.Size([24, 2048, 512]) + module.decoder.layers.27.mlp.experts.fc1_weight torch.Size([24, 1024, 2048]) +Params for bucket 30 (75497472 elements, 75497472 padded size): + module.decoder.layers.26.mlp.experts.fc2_weight torch.Size([24, 2048, 512]) + module.decoder.layers.26.mlp.experts.fc1_weight torch.Size([24, 1024, 2048]) +Params for bucket 31 (75497472 elements, 75497472 padded size): + module.decoder.layers.25.mlp.experts.fc2_weight torch.Size([24, 2048, 512]) + module.decoder.layers.25.mlp.experts.fc1_weight torch.Size([24, 1024, 2048]) +Params for bucket 32 (75497472 elements, 75497472 padded size): + module.decoder.layers.24.mlp.experts.fc2_weight torch.Size([24, 2048, 512]) + module.decoder.layers.24.mlp.experts.fc1_weight torch.Size([24, 1024, 2048]) +Params for bucket 33 (75497472 elements, 75497472 padded size): + module.decoder.layers.23.mlp.experts.fc2_weight torch.Size([24, 2048, 512]) + module.decoder.layers.23.mlp.experts.fc1_weight torch.Size([24, 1024, 2048]) +Params for bucket 34 (75497472 elements, 75497472 padded size): + module.decoder.layers.22.mlp.experts.fc2_weight torch.Size([24, 2048, 512]) + module.decoder.layers.22.mlp.experts.fc1_weight torch.Size([24, 1024, 2048]) +Params for bucket 35 (75497472 elements, 75497472 padded size): + module.decoder.layers.21.mlp.experts.fc2_weight torch.Size([24, 2048, 512]) + module.decoder.layers.21.mlp.experts.fc1_weight torch.Size([24, 1024, 2048]) +Params for bucket 36 (75497472 elements, 75497472 padded size): + module.decoder.layers.20.mlp.experts.fc2_weight torch.Size([24, 2048, 512]) + module.decoder.layers.20.mlp.experts.fc1_weight torch.Size([24, 1024, 2048]) +Params for bucket 37 (75497472 elements, 75497472 padded size): + module.decoder.layers.19.mlp.experts.fc2_weight torch.Size([24, 2048, 512]) + module.decoder.layers.19.mlp.experts.fc1_weight torch.Size([24, 1024, 2048]) +Params for bucket 38 (75497472 elements, 75497472 padded size): + module.decoder.layers.18.mlp.experts.fc2_weight torch.Size([24, 2048, 512]) + module.decoder.layers.18.mlp.experts.fc1_weight torch.Size([24, 1024, 2048]) +Params for bucket 39 (75497472 elements, 75497472 padded size): + module.decoder.layers.17.mlp.experts.fc2_weight torch.Size([24, 2048, 512]) + module.decoder.layers.17.mlp.experts.fc1_weight torch.Size([24, 1024, 2048]) +Params for bucket 40 (75497472 elements, 75497472 padded size): + module.decoder.layers.16.mlp.experts.fc2_weight torch.Size([24, 2048, 512]) + module.decoder.layers.16.mlp.experts.fc1_weight torch.Size([24, 1024, 2048]) +Params for bucket 41 (75497472 elements, 75497472 padded size): + module.decoder.layers.15.mlp.experts.fc2_weight torch.Size([24, 2048, 512]) + module.decoder.layers.15.mlp.experts.fc1_weight torch.Size([24, 1024, 2048]) +Params for bucket 42 (75497472 elements, 75497472 padded size): + module.decoder.layers.14.mlp.experts.fc2_weight torch.Size([24, 2048, 512]) + module.decoder.layers.14.mlp.experts.fc1_weight torch.Size([24, 1024, 2048]) +Params for bucket 43 (75497472 elements, 75497472 padded size): + module.decoder.layers.13.mlp.experts.fc2_weight torch.Size([24, 2048, 512]) + module.decoder.layers.13.mlp.experts.fc1_weight torch.Size([24, 1024, 2048]) +Params for bucket 44 (75497472 elements, 75497472 padded size): + module.decoder.layers.12.mlp.experts.fc2_weight torch.Size([24, 2048, 512]) + module.decoder.layers.12.mlp.experts.fc1_weight torch.Size([24, 1024, 2048]) +Params for bucket 45 (75497472 elements, 75497472 padded size): + module.decoder.layers.11.mlp.experts.fc2_weight torch.Size([24, 2048, 512]) + module.decoder.layers.11.mlp.experts.fc1_weight torch.Size([24, 1024, 2048]) +Params for bucket 46 (75497472 elements, 75497472 padded size): + module.decoder.layers.10.mlp.experts.fc2_weight torch.Size([24, 2048, 512]) + module.decoder.layers.10.mlp.experts.fc1_weight torch.Size([24, 1024, 2048]) +Params for bucket 47 (75497472 elements, 75497472 padded size): + module.decoder.layers.9.mlp.experts.fc2_weight torch.Size([24, 2048, 512]) + module.decoder.layers.9.mlp.experts.fc1_weight torch.Size([24, 1024, 2048]) +Params for bucket 48 (75497472 elements, 75497472 padded size): + module.decoder.layers.8.mlp.experts.fc2_weight torch.Size([24, 2048, 512]) + module.decoder.layers.8.mlp.experts.fc1_weight torch.Size([24, 1024, 2048]) +Params for bucket 49 (75497472 elements, 75497472 padded size): + module.decoder.layers.7.mlp.experts.fc2_weight torch.Size([24, 2048, 512]) + module.decoder.layers.7.mlp.experts.fc1_weight torch.Size([24, 1024, 2048]) +Params for bucket 50 (75497472 elements, 75497472 padded size): + module.decoder.layers.6.mlp.experts.fc2_weight torch.Size([24, 2048, 512]) + module.decoder.layers.6.mlp.experts.fc1_weight torch.Size([24, 1024, 2048]) +Params for bucket 51 (75497472 elements, 75497472 padded size): + module.decoder.layers.5.mlp.experts.fc2_weight torch.Size([24, 2048, 512]) + module.decoder.layers.5.mlp.experts.fc1_weight torch.Size([24, 1024, 2048]) +Params for bucket 52 (75497472 elements, 75497472 padded size): + module.decoder.layers.4.mlp.experts.fc2_weight torch.Size([24, 2048, 512]) + module.decoder.layers.4.mlp.experts.fc1_weight torch.Size([24, 1024, 2048]) +Params for bucket 53 (75497472 elements, 75497472 padded size): + module.decoder.layers.3.mlp.experts.fc2_weight torch.Size([24, 2048, 512]) + module.decoder.layers.3.mlp.experts.fc1_weight torch.Size([24, 1024, 2048]) +Params for bucket 54 (75497472 elements, 75497472 padded size): + module.decoder.layers.2.mlp.experts.fc2_weight torch.Size([24, 2048, 512]) + module.decoder.layers.2.mlp.experts.fc1_weight torch.Size([24, 1024, 2048]) +[GPTModel( + (embedding): LanguageModelEmbedding( + (word_embeddings): VocabParallelEmbedding() + (embedding_dropout): Dropout(p=0.0, inplace=False) + ) + (decoder): TransformerBlock( + (layers): ModuleList( + (0-1): 2 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (_fused_split_linear): _FusedSplitLinearBoundary() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=9600, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=4800, out_features=2048, bias=False, TP=1) + ) + ) + (2-11): 10 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (_fused_split_linear): _FusedSplitLinearBoundary() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=global_aux_loss, score_function=softmax, enable_expert_bias=False, fused=True) + (experts): A100GroupedMLP() + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (12): TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): SelfAttention( + (core_attention): TEDotProductAttention( + (flash_attention): FlashAttention() + (fused_attention): FusedAttention() + (unfused_attention): UnfusedDotProductAttention( + (scale_mask_softmax): FusedScaleMaskSoftmax() + (attention_dropout): Dropout(p=0.0, inplace=False) + ) + ) + (linear_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (linear_qkv): TEColumnParallelLinear(in_features=2048, out_features=5120, bias=True, TP=1) + (q_layernorm): IdentityOp() + (k_layernorm): IdentityOp() + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=global_aux_loss, score_function=softmax, enable_expert_bias=False, fused=True) + (experts): A100GroupedMLP() + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (13-19): 7 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (_fused_split_linear): _FusedSplitLinearBoundary() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=global_aux_loss, score_function=softmax, enable_expert_bias=False, fused=True) + (experts): A100GroupedMLP() + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (20-21): 2 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): SelfAttention( + (core_attention): TEDotProductAttention( + (flash_attention): FlashAttention() + (fused_attention): FusedAttention() + (unfused_attention): UnfusedDotProductAttention( + (scale_mask_softmax): FusedScaleMaskSoftmax() + (attention_dropout): Dropout(p=0.0, inplace=False) + ) + ) + (linear_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (linear_qkv): TEColumnParallelLinear(in_features=2048, out_features=5120, bias=True, TP=1) + (q_layernorm): IdentityOp() + (k_layernorm): IdentityOp() + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=global_aux_loss, score_function=softmax, enable_expert_bias=False, fused=True) + (experts): A100GroupedMLP() + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (22): TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (_fused_split_linear): _FusedSplitLinearBoundary() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=global_aux_loss, score_function=softmax, enable_expert_bias=False, fused=True) + (experts): A100GroupedMLP() + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (23): TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): SelfAttention( + (core_attention): TEDotProductAttention( + (flash_attention): FlashAttention() + (fused_attention): FusedAttention() + (unfused_attention): UnfusedDotProductAttention( + (scale_mask_softmax): FusedScaleMaskSoftmax() + (attention_dropout): Dropout(p=0.0, inplace=False) + ) + ) + (linear_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (linear_qkv): TEColumnParallelLinear(in_features=2048, out_features=5120, bias=True, TP=1) + (q_layernorm): IdentityOp() + (k_layernorm): IdentityOp() + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=global_aux_loss, score_function=softmax, enable_expert_bias=False, fused=True) + (experts): A100GroupedMLP() + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (24-45): 22 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (_fused_split_linear): _FusedSplitLinearBoundary() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=global_aux_loss, score_function=softmax, enable_expert_bias=False, fused=True) + (experts): A100GroupedMLP() + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (46): TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): SelfAttention( + (core_attention): TEDotProductAttention( + (flash_attention): FlashAttention() + (fused_attention): FusedAttention() + (unfused_attention): UnfusedDotProductAttention( + (scale_mask_softmax): FusedScaleMaskSoftmax() + (attention_dropout): Dropout(p=0.0, inplace=False) + ) + ) + (linear_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (linear_qkv): TEColumnParallelLinear(in_features=2048, out_features=5120, bias=True, TP=1) + (q_layernorm): IdentityOp() + (k_layernorm): IdentityOp() + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=global_aux_loss, score_function=softmax, enable_expert_bias=False, fused=True) + (experts): A100GroupedMLP() + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (47): TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (_fused_split_linear): _FusedSplitLinearBoundary() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=global_aux_loss, score_function=softmax, enable_expert_bias=False, fused=True) + (experts): A100GroupedMLP() + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (48-49): 2 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): SelfAttention( + (core_attention): TEDotProductAttention( + (flash_attention): FlashAttention() + (fused_attention): FusedAttention() + (unfused_attention): UnfusedDotProductAttention( + (scale_mask_softmax): FusedScaleMaskSoftmax() + (attention_dropout): Dropout(p=0.0, inplace=False) + ) + ) + (linear_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (linear_qkv): TEColumnParallelLinear(in_features=2048, out_features=5120, bias=True, TP=1) + (q_layernorm): IdentityOp() + (k_layernorm): IdentityOp() + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=global_aux_loss, score_function=softmax, enable_expert_bias=False, fused=True) + (experts): A100GroupedMLP() + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + (50-55): 6 x TransformerLayer( + (input_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (self_attention): GatedDeltaNet( + (in_proj_layer_norm): IdentityOp() + (q_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (k_proj): TEColumnParallelLinear(in_features=2048, out_features=512, bias=False, TP=1) + (v_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (z_proj): TEColumnParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + (b_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (a_proj): TEColumnParallelLinear(in_features=2048, out_features=32, bias=False, TP=1) + (_fused_split_linear): _FusedSplitLinearBoundary() + (conv1d): Conv1d(3072, 3072, kernel_size=(4,), stride=(1,), padding=(3,), groups=3072, bias=False) + (out_norm): RMSNorm() + (out_proj): TERowParallelLinear(in_features=2048, out_features=2048, bias=False, TP=1) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + (mlp): MoELayer( + (router): TopKRouter(topk=4, routing_type=global_aux_loss, score_function=softmax, enable_expert_bias=False, fused=True) + (experts): A100GroupedMLP() + (shared_experts): SharedExpertMLP( + (linear_fc1): TEColumnParallelLinear(in_features=2048, out_features=1024, bias=False, TP=1) + (linear_fc2): TERowParallelLinear(in_features=512, out_features=2048, bias=False, TP=1) + ) + ) + ) + ) + (final_layernorm): GatedNorm( + (norm): RMSNorm() + (gate_down): Linear(in_features=2048, out_features=16, bias=False) + (gate_up): Linear(in_features=16, out_features=2048, bias=True) + ) + ) + (output_layer): FusedLinearCrossEntropyColumnParallelLinear(in_features=2048, out_features=99072, bias=False, TP=1) +)] +[gdn-len-ext] applying per-proj LR multipliers: .a_proj.weight=0.5 +INFO:megatron.core.optimizer:Setting up optimizer with config OptimizerConfig(optimizer='adam', lr=0.0005, min_lr=0.0005, decoupled_lr=None, decoupled_min_lr=None, weight_decay=0.1, corrected_weight_decay=True, fp8_recipe='delayed', fp16=False, bf16=True, reuse_grad_buf_for_mxfp8_param_ag=False, params_dtype=torch.bfloat16, use_precision_aware_optimizer=False, store_param_remainders=True, main_grads_dtype=torch.float32, main_params_dtype=torch.float32, exp_avg_dtype=torch.float32, exp_avg_sq_dtype=torch.float32, loss_scale=None, initial_loss_scale=4294967296, min_loss_scale=1.0, loss_scale_window=1000, hysteresis=2, adam_beta1=0.9, adam_beta2=0.95, adam_eps=1e-15, adam_bias_correction=False, decoupled_weight_decay=True, adamw_lr_mup_scaler=False, sgd_momentum=0.9, adamh_scope='lm_head', adamh_radius=None, adamh_eps=1e-08, spectra_scope='matrices', spectra_hyperball=False, spectra_clip=1.0, spectra_grad_clip=None, spectra_ns_steps=10, muon_momentum=0.9, muon_split_qkv=True, muon_qkv_split_mode='component', muon_split_fc1=True, muon_use_nesterov=False, muon_scale_mode='spectral_mup', muon_fp32_matmul_prec='medium', muon_num_ns_steps=5, muon_tp_mode='blockwise', muon_extra_scale_factor=1.0, muon_split_moe_experts=True, muon_vectorize=[], muon_vectorize_attn_dim='hidden_size', muon_scale_vectorized_mode='full', spectral_ball_momentum=0.9, spectral_ball_use_nesterov=True, spectral_ball_split_qkv=True, spectral_ball_qkv_split_mode='component', spectral_ball_split_fc1=True, spectral_ball_split_moe_experts=True, spectral_ball_msign_steps=8, spectral_ball_solver='bisection', spectral_ball_solver_tolerance_f=1e-08, spectral_ball_solver_max_iterations=20, spectral_ball_radius_mode='spectral_mup', spectral_ball_power_iteration_steps=20, spectral_ball_scale_mode='spectral_mup', spectral_ball_retract_mode='hard', spectral_ball_retract_alpha=0.05, muon_ball_momentum=0.9, muon_ball_use_nesterov=True, muon_ball_split_qkv=True, muon_ball_qkv_split_mode='component', muon_ball_split_fc1=True, muon_ball_split_moe_experts=True, muon_ball_msign_steps=5, muon_ball_radius_mode='spectral_mup', muon_ball_power_iteration_steps=10, muon_ball_scale_mode='spectral_mup', muon_ball_retract_mode='hard', muon_ball_retract_alpha=0.05, use_distributed_optimizer=True, overlap_param_gather=True, overlap_param_gather_with_optimizer_step=False, optimizer_cpu_offload=False, optimizer_offload_fraction=1.0, use_torch_optimizer_for_cpu_offload=False, overlap_cpu_optimizer_d2h_h2d=False, pin_cpu_grads=True, pin_cpu_params=True, clip_grad=1.0, log_num_zeros_in_grad=False, log_per_module_update_rms=False, log_per_module_grad_rms=True, barrier_with_L1_time=False, timers=, config_logger_dir='') +setting training iterations to 6855 +INFO:megatron.core.optimizer_param_scheduler:> learning rate decay style: constant +sharded_state_dict metadata loaded from the checkpoint: {'singleton_local_shards': True, 'distrib_optim_sharding_type': 'dp_reshardable', 'chained_optim_avoid_prefix': True} + loading distributed checkpoint from /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage14_120B_8node_gdnlralpha_off_20260925a/checkpoints at iteration 7153 +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/dist_checkpointing/strategies/torch.py:956: FutureWarning: `load_state_dict` is deprecated and will be removed in future versions. Please use `load` instead. + checkpoint.load_state_dict( +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/dist_checkpointing/strategies/torch.py:956: FutureWarning: `load_state_dict` is deprecated and will be removed in future versions. Please use `load` instead. + checkpoint.load_state_dict( +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/dist_checkpointing/strategies/torch.py:956: FutureWarning: `load_state_dict` is deprecated and will be removed in future versions. Please use `load` instead. + checkpoint.load_state_dict( +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/dist_checkpointing/strategies/torch.py:956: FutureWarning: `load_state_dict` is deprecated and will be removed in future versions. Please use `load` instead. + checkpoint.load_state_dict( +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/dist_checkpointing/strategies/torch.py:956: FutureWarning: `load_state_dict` is deprecated and will be removed in future versions. Please use `load` instead. + checkpoint.load_state_dict( +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/dist_checkpointing/strategies/torch.py:956: FutureWarning: `load_state_dict` is deprecated and will be removed in future versions. Please use `load` instead. + checkpoint.load_state_dict( +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/dist_checkpointing/strategies/torch.py:956: FutureWarning: `load_state_dict` is deprecated and will be removed in future versions. Please use `load` instead. + checkpoint.load_state_dict( +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/dist_checkpointing/strategies/torch.py:956: FutureWarning: `load_state_dict` is deprecated and will be removed in future versions. Please use `load` instead. + checkpoint.load_state_dict( +not setting consumed_train_samples, consumed_valid_samples and skipped_train_samples ... (override_opt_param_scheduler is set) + checkpoint version 3.0 +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer_param_scheduler: > overriding learning rate value to 0.0005 +INFO:megatron.core.optimizer_param_scheduler: > overriding minimum learning rate value to 0.0005 +INFO:megatron.core.optimizer_param_scheduler: > overriding warmup iterations value to 0 +INFO:megatron.core.optimizer_param_scheduler: > overriding total number of iterations value to 877440 +INFO:megatron.core.optimizer_param_scheduler: > overriding learning rate decay style value to constant +INFO:megatron.core.optimizer_param_scheduler: > overriding start weight decay value to 0.1 +INFO:megatron.core.optimizer_param_scheduler: > overriding end weight decay value to 0.1 +INFO:megatron.core.optimizer_param_scheduler: > overriding total number of weight decay iterations value to 877440 +INFO:megatron.core.optimizer_param_scheduler: > overriding weight decay incr style value to constant +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable + successfully loaded checkpoint from /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage14_120B_8node_gdnlralpha_off_20260925a/checkpoints [ t 1/1, p 1/1 ] at iteration 0 +NCCL version 2.27.3+cuda12.9 +NCCL version 2.27.3+cuda12.9 +NCCL version 2.27.3+cuda12.9 +NCCL version 2.27.3+cuda12.9 +NCCL version 2.27.3+cuda12.9 +NCCL version 2.27.3+cuda12.9 +NCCL version 2.27.3+cuda12.9 +[after model, optimizer, and learning rate scheduler are built] datetime: 2026-10-03 15:45:27 +> building train, validation, and test datasets ... + train: 877440 + validation: 1280 + test: 1280 +Initializing GlobIndexedDataset +INFO:megatron.core.datasets.blended_megatron_dataset_config:Let split_matrix = [(0, 0.999), (0.999, 1.0), None] +> building train, validation, and test datasets for GPT ... +INFO:megatron.core.datasets.blended_megatron_dataset_builder:Building GPTDataset splits with sizes=(877440, 1280, 1280) and config=GPTDatasetConfig(random_seed=1234, sequence_length=131072, blend=(['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_{0..78}_input_ids_tmp'], None), blend_per_split=None, multiple_validation_sets=False, full_validation=False, split='999,1,0', split_matrix=[(0, 0.999), (0.999, 1.0), None], num_dataset_builder_threads=1, path_to_cache='/mnt/hdd/cpt/binidx_cache_stage15_final_20261001_v2', mmap_bin_files=True, fast_cache_load=False, defer_npy_index_mmap=False, mock=False, tokenizer=, mid_level_dataset_surplus=0.005, allow_ambiguous_pad_tokens=False, reset_position_ids=False, reset_attention_mask=False, eod_mask_loss=False, create_attention_mask=False, drop_last_partial_validation_sequence=True, add_extra_token_to_sequence=True, object_storage_cache_path=None, load_complemental_dataset=['loss_mask'], document_packing_algorithm='obfd', packing_tokens_per_shard=100000000000, enable_debug_logging=False, emit_document_packing_metadata=True, start_samples=0, context_parallel_size=8, data_parallel_size=8, packing_dp_size=None, sequence_parallel_size=0, hybrid_context_parallel=False) +Initializing GlobIndexedDataset +Initializing GlobIndexedDataset +Initializing GlobIndexedDataset +Initializing GlobIndexedDataset +Initializing GlobIndexedDataset +Initializing GlobIndexedDataset +Initializing GlobIndexedDataset +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_0_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_10_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_11_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_12_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_13_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_14_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_15_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_16_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_17_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_18_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_19_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_1_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_20_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_21_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_22_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_23_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_24_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_25_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_26_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_27_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_28_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_29_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_2_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_30_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_31_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_32_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_33_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_34_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_35_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_36_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_37_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_38_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_39_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_3_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_40_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_41_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_42_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_43_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_44_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_45_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_46_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_47_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_48_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_49_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_4_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_50_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_51_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_52_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_53_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_54_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_55_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_56_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_57_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_58_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_59_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_5_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_60_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_61_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_62_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_63_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_64_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_65_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_66_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_67_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_68_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_69_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_6_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_70_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_71_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_72_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_73_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_74_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_75_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_76_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_77_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_78_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_7_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_8_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_9_input_ids_tmp'] +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_0_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_10_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_11_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_12_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_13_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_14_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_15_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_16_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_17_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_18_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_19_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_1_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_20_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_21_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_22_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_23_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_24_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_25_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_26_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_27_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_28_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_29_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_2_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_30_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_31_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_32_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_33_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_34_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_35_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_36_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_37_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_38_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_39_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_3_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_40_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_41_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_42_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_43_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_44_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_45_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_46_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_47_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_48_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_49_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_4_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_50_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_51_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_52_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_53_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_54_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_55_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_56_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_57_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_58_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_59_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_5_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_60_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_61_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_62_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_63_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_64_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_65_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_66_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_67_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_68_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_69_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_6_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_70_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_71_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_72_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_73_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_74_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_75_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_76_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_77_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_78_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_7_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_8_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_9_input_ids_tmp'] +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_0_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_10_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_11_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_12_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_13_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_14_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_15_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_16_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_17_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_18_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_19_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_1_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_20_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_21_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_22_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_23_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_24_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_25_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_26_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_27_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_28_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_29_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_2_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_30_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_31_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_32_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_33_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_34_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_35_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_36_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_37_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_38_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_39_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_3_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_40_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_41_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_42_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_43_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_44_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_45_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_46_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_47_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_48_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_49_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_4_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_50_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_51_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_52_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_53_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_54_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_55_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_56_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_57_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_58_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_59_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_5_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_60_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_61_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_62_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_63_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_64_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_65_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_66_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_67_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_68_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_69_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_6_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_70_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_71_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_72_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_73_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_74_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_75_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_76_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_77_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_78_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_7_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_8_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_9_input_ids_tmp'] +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_0_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_10_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_11_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_12_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_13_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_14_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_15_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_16_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_17_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_18_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_19_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_1_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_20_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_21_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_22_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_23_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_24_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_25_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_26_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_27_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_28_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_29_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_2_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_30_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_31_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_32_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_33_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_34_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_35_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_36_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_37_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_38_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_39_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_3_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_40_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_41_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_42_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_43_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_44_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_45_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_46_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_47_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_48_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_49_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_4_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_50_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_51_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_52_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_53_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_54_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_55_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_56_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_57_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_58_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_59_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_5_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_60_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_61_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_62_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_63_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_64_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_65_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_66_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_67_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_68_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_69_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_6_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_70_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_71_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_72_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_73_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_74_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_75_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_76_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_77_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_78_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_7_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_8_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_9_input_ids_tmp']['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_0_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_10_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_11_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_12_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_13_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_14_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_15_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_16_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_17_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_18_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_19_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_1_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_20_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_21_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_22_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_23_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_24_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_25_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_26_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_27_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_28_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_29_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_2_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_30_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_31_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_32_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_33_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_34_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_35_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_36_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_37_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_38_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_39_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_3_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_40_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_41_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_42_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_43_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_44_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_45_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_46_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_47_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_48_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_49_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_4_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_50_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_51_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_52_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_53_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_54_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_55_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_56_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_57_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_58_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_59_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_5_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_60_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_61_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_62_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_63_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_64_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_65_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_66_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_67_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_68_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_69_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_6_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_70_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_71_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_72_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_73_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_74_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_75_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_76_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_77_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_78_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_7_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_8_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_9_input_ids_tmp']['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_0_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_10_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_11_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_12_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_13_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_14_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_15_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_16_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_17_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_18_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_19_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_1_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_20_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_21_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_22_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_23_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_24_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_25_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_26_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_27_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_28_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_29_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_2_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_30_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_31_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_32_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_33_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_34_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_35_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_36_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_37_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_38_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_39_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_3_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_40_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_41_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_42_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_43_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_44_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_45_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_46_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_47_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_48_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_49_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_4_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_50_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_51_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_52_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_53_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_54_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_55_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_56_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_57_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_58_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_59_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_5_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_60_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_61_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_62_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_63_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_64_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_65_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_66_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_67_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_68_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_69_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_6_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_70_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_71_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_72_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_73_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_74_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_75_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_76_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_77_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_78_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_7_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_8_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_9_input_ids_tmp'] + + +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_0_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_10_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_11_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_12_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_13_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_14_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_15_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_16_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_17_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_18_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_19_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_1_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_20_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_21_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_22_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_23_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_24_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_25_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_26_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_27_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_28_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_29_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_2_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_30_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_31_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_32_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_33_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_34_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_35_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_36_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_37_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_38_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_39_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_3_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_40_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_41_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_42_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_43_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_44_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_45_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_46_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_47_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_48_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_49_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_4_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_50_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_51_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_52_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_53_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_54_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_55_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_56_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_57_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_58_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_59_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_5_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_60_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_61_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_62_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_63_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_64_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_65_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_66_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_67_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_68_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_69_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_6_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_70_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_71_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_72_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_73_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_74_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_75_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_76_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_77_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_78_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_7_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_8_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_9_input_ids_tmp']['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_0_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_10_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_11_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_12_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_13_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_14_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_15_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_16_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_17_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_18_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_19_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_1_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_20_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_21_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_22_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_23_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_24_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_25_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_26_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_27_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_28_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_29_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_2_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_30_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_31_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_32_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_33_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_34_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_35_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_36_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_37_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_38_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_39_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_3_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_40_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_41_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_42_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_43_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_44_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_45_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_46_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_47_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_48_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_49_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_4_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_50_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_51_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_52_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_53_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_54_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_55_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_56_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_57_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_58_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_59_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_5_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_60_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_61_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_62_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_63_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_64_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_65_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_66_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_67_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_68_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_69_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_6_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_70_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_71_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_72_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_73_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_74_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_75_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_76_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_77_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_78_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_7_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_8_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_9_input_ids_tmp'] + +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_0_input_ids_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 6337 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 6337 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_10_input_ids_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 34364 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 34364 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_11_input_ids_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 367336 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 367336 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_12_input_ids_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 16865 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 16865 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_13_input_ids_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 55051 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 55051 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_14_input_ids_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 25677 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 25677 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_15_input_ids_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 420782 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 420782 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_16_input_ids_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 2422 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 2422 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_17_input_ids_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 64627 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 64627 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_18_input_ids_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 190123 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 190123 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_19_input_ids_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 161060 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 161060 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_1_input_ids_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 21716 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 21716 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_20_input_ids_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 90530 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 90530 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_21_input_ids_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 148494 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 148494 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_22_input_ids_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 1022 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 1022 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_23_input_ids_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 1235 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 1235 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_24_input_ids_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 320735 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 320735 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_25_input_ids_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 1477957 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 1477957 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_26_input_ids_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 164732 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 164732 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_27_input_ids_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 1960997 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 1960997 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_28_input_ids_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 105702 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 105702 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_29_input_ids_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 52447 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 52447 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_2_input_ids_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 77593 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 77593 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_30_input_ids_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 37477 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 37477 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_31_input_ids_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 71335 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 71335 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_32_input_ids_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 502031 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 502031 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_33_input_ids_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 7019115 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 7019115 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_34_input_ids_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 5532136 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 5532136 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_35_input_ids_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 450393 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 450393 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_36_input_ids_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 132203 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 132203 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_37_input_ids_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 328579 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 328579 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_38_input_ids_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 446948 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 446948 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_39_input_ids_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 674 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 674 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_3_input_ids_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 283312 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 283312 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_40_input_ids_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 213820 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 213820 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_41_input_ids_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 14548 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 14548 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_42_input_ids_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 267129 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 267129 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_43_input_ids_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 11705 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 11705 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_44_input_ids_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 3098562 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 3098562 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_45_input_ids_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 180011 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 180011 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_46_input_ids_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 211 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 211 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_47_input_ids_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 4144931 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 4144931 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_48_input_ids_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 566913 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 566913 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_49_input_ids_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 939888 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 939888 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_4_input_ids_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 331399 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 331399 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_50_input_ids_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 238697 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 238697 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_51_input_ids_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 144449 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 144449 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_52_input_ids_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 214736 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 214736 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_53_input_ids_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 8629 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 8629 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_54_input_ids_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 49887 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 49887 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_55_input_ids_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 376211 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 376211 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_56_input_ids_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 147841 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 147841 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_57_input_ids_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 22976 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 22976 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_58_input_ids_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 175 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 175 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_59_input_ids_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 177925 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 177925 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_5_input_ids_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 370862 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 370862 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_60_input_ids_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 186 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 186 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_61_input_ids_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 321879 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 321879 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_62_input_ids_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 198643 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 198643 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_63_input_ids_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 994 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 994 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_64_input_ids_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 3891 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 3891 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_65_input_ids_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 20529 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 20529 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_66_input_ids_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 737287 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 737287 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_67_input_ids_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 8984 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 8984 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_68_input_ids_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 39196 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 39196 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_69_input_ids_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 444581 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 444581 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_6_input_ids_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 685240 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 685240 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_70_input_ids_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 268640 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 268640 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_71_input_ids_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 119075 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 119075 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_72_input_ids_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 102314 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 102314 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_73_input_ids_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 18336 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 18336 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_74_input_ids_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 35520 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 35520 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_75_input_ids_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 175716 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 175716 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_76_input_ids_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 60429 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 60429 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_77_input_ids_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 360949 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 360949 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_78_input_ids_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 178581 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 178581 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_7_input_ids_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 619474 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 619474 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_8_input_ids_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 11376 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 11376 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_9_input_ids_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 110763 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 110763 +>>> start_samples: 0 +INFO:megatron.core.datasets.gpt_dataset: > Found complemental loss_mask dataset. Loading from: /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_{0..78}_loss_mask_tmp +Initializing GlobIndexedDataset +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_75_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_76_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_77_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_78_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_9_loss_mask_tmp'] +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_0_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 6337 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 6337 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_10_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 34364 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 34364 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_11_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 367336 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 367336 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_12_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 16865 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 16865 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_13_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 55051 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 55051 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_14_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 25677 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 25677 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_15_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 420782 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 420782 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_16_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 2422 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 2422 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_17_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 64627 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 64627 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_18_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 190123 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 190123 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_19_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 161060 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 161060 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_1_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 21716 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 21716 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_20_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 90530 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 90530 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_21_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 148494 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 148494 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_22_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 1022 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 1022 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_23_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 1235 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 1235 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_24_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 320735 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 320735 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_25_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 1477957 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 1477957 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_26_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 164732 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 164732 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_27_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 1960997 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 1960997 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_28_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 105702 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 105702 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_29_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 52447 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 52447 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_2_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 77593 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 77593 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_30_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 37477 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 37477 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_31_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 71335 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 71335 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_32_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 502031 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 502031 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_33_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 7019115 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 7019115 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_34_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 5532136 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 5532136 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_35_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 450393 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 450393 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_36_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 132203 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 132203 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_37_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 328579 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 328579 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_38_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 446948 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 446948 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_39_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 674 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 674 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_3_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 283312 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 283312 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_40_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 213820 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 213820 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_41_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 14548 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 14548 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_42_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 267129 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 267129 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_43_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 11705 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 11705 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_44_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 3098562 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 3098562 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_45_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 180011 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 180011 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_46_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 211 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 211 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_47_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 4144931 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 4144931 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_48_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 566913 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 566913 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_49_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 939888 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 939888 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_4_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 331399 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 331399 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_50_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 238697 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 238697 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_51_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 144449 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 144449 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_52_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 214736 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 214736 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_53_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 8629 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 8629 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_54_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 49887 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 49887 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_55_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 376211 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 376211 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_56_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 147841 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 147841 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_57_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 22976 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 22976 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_58_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 175 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 175 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_59_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 177925 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 177925 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_5_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 370862 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 370862 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_60_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 186 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 186 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_61_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 321879 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 321879 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_62_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 198643 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 198643 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_63_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 994 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 994 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_64_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 3891 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 3891 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_65_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 20529 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 20529 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_66_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 737287 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 737287 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_67_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 8984 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 8984 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_68_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 39196 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 39196 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_69_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 444581 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 444581 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_6_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 685240 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 685240 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_70_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 268640 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 268640 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_71_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 119075 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 119075 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_72_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 102314 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 102314 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_73_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 18336 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 18336 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_74_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 35520 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 35520 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_75_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 175716 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 175716 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_76_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 60429 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 60429 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_77_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 360949 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 360949 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_78_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 178581 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 178581 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_7_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 619474 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 619474 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_8_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 11376 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 11376 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_9_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 110763 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 110763 +pad_document_marker: 36616095 +/mnt/hdd/cpt/binidx_cache_stage15_final_20261001_v2 d6c36c6d5886334ea5f92e75fcf45d10-GPTDataset-train-obfd cache hit: False +INFO:megatron.core.datasets.gpt_dataset:{ + "class": "GPTDataset", + "dataset_path": "/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_{0..78}_input_ids_tmp", + "num_samples": 877440, + "index_split": "train", + "random_seed": 1234, + "sequence_length": 131072, + "split": "999,1,0", + "split_matrix": [ + [ + 0, + 0.999 + ], + [ + 0.999, + 1.0 + ], + null + ], + "tokenizer": { + "class": "megatron.core.tokenizers.text.models.default_tokenizer.DefaultTokenizerText", + "tokenizer_path": "/mnt/ssd/yulan/pretrain-linear-moe/preprocess/modify_tokenizer/YuLan-Mini-Qwen-Template_new_sp", + "vocab_file": "None", + "merges_file": "None" + } +} +INFO:megatron.core.datasets.gpt_dataset:Build and save the GPTDataset train indices +INFO:megatron.core.datasets.gpt_dataset:pack_documents epoch 0/1, algorithm=obfd, seq_length=131072, pad_marker=36616095 +INFO:megatron.core.datasets.gpt_dataset:Sharded packing: 2 shards, ~100B tokens/shard, 8 workers +INFO:megatron.core.datasets.gpt_dataset:Saved packing cache (epoch 0): /mnt/hdd/cpt/binidx_cache_stage15_final_20261001_v2/09f07acad03ffbf3e26828967bf59cfe-packing.npy +INFO:megatron.core.datasets.gpt_dataset:pack_documents epoch 0/1, algorithm=obfd, seq_length=131072, pad_marker=36616095 +INFO:megatron.core.datasets.gpt_dataset:Sharded packing: 2 shards, ~100B tokens/shard, 8 workers +INFO:megatron.core.datasets.gpt_dataset:Saved packing cache (epoch 0): /mnt/hdd/cpt/binidx_cache_stage15_final_20261001_v2/0d269c8bdb4f1d5d8b925214bd9da477-packing.npy +INFO:megatron.core.datasets.gpt_dataset:Packed document_index token count for sample index: 226724675584 +Saving GPTDataset indexes to /mnt/hdd/cpt/binidx_cache_stage15_final_20261001_v2 +INFO:megatron.core.datasets.gpt_dataset:> total number of samples: 1729771 +INFO:megatron.core.datasets.gpt_dataset:> total number of epochs: 2 +>>> start_samples: 0 +>>> start_samples: 0 +>>> start_samples: 0Initializing GlobIndexedDataset>>> start_samples: 0 + + +>>> start_samples: 0 +Initializing GlobIndexedDataset +>>> start_samples: 0>>> start_samples: 0>>> start_samples: 0 + + +INFO:megatron.core.datasets.gpt_dataset: > Found complemental loss_mask dataset. Loading from: /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_{0..78}_loss_mask_tmp +Initializing GlobIndexedDataset +Initializing GlobIndexedDataset +Initializing GlobIndexedDataset +Initializing GlobIndexedDatasetInitializing GlobIndexedDataset +Initializing GlobIndexedDataset + +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_75_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_76_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_77_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_78_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_9_loss_mask_tmp']['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_75_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_76_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_77_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_78_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_9_loss_mask_tmp']['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_75_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_76_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_77_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_78_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_9_loss_mask_tmp'] + + +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_75_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_76_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_77_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_78_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_9_loss_mask_tmp'] +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_75_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_76_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_77_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_78_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_9_loss_mask_tmp'] +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_75_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_76_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_77_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_78_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_9_loss_mask_tmp']['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_75_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_76_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_77_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_78_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_9_loss_mask_tmp']['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_75_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_76_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_77_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_78_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_9_loss_mask_tmp'] + + +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_0_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 6337 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 6337 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_10_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 34364 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 34364 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_11_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 367336 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 367336 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_12_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 16865 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 16865 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_13_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 55051 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 55051 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_14_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 25677 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 25677 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_15_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 420782 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 420782 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_16_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 2422 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 2422 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_17_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 64627 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 64627 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_18_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 190123 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 190123 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_19_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 161060 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 161060 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_1_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 21716 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 21716 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_20_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 90530 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 90530 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_21_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 148494 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 148494 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_22_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 1022 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 1022 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_23_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 1235 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 1235 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_24_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 320735 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 320735 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_25_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 1477957 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 1477957 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_26_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 164732 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 164732 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_27_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 1960997 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 1960997 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_28_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 105702 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 105702 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_29_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 52447 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 52447 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_2_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 77593 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 77593 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_30_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 37477 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 37477 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_31_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 71335 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 71335 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_32_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 502031 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 502031 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_33_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 7019115 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 7019115 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_34_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 5532136 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 5532136 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_35_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 450393 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 450393 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_36_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 132203 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 132203 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_37_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 328579 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 328579 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_38_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 446948 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 446948 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_39_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 674 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 674 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_3_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 283312 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 283312 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_40_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 213820 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 213820 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_41_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 14548 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 14548 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_42_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 267129 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 267129 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_43_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 11705 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 11705 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_44_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 3098562 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 3098562 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_45_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 180011 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 180011 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_46_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 211 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 211 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_47_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 4144931 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 4144931 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_48_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 566913 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 566913 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_49_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 939888 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 939888 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_4_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 331399 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 331399 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_50_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 238697 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 238697 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_51_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 144449 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 144449 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_52_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 214736 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 214736 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_53_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 8629 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 8629 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_54_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 49887 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 49887 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_55_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 376211 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 376211 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_56_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 147841 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 147841 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_57_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 22976 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 22976 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_58_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 175 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 175 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_59_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 177925 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 177925 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_5_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 370862 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 370862 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_60_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 186 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 186 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_61_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 321879 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 321879 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_62_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 198643 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 198643 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_63_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 994 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 994 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_64_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 3891 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 3891 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_65_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 20529 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 20529 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_66_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 737287 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 737287 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_67_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 8984 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 8984 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_68_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 39196 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 39196 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_69_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 444581 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 444581 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_6_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 685240 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 685240 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_70_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 268640 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 268640 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_71_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 119075 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 119075 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_72_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 102314 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 102314 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_73_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 18336 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 18336 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_74_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 35520 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 35520 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_75_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 175716 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 175716 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_76_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 60429 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 60429 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_77_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 360949 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 360949 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_78_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 178581 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 178581 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_7_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 619474 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 619474 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_8_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 11376 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 11376 +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_9_loss_mask_tmp.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 110763 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 110763 +pad_document_marker: 36616095 +/mnt/hdd/cpt/binidx_cache_stage15_final_20261001_v2 d6c36c6d5886334ea5f92e75fcf45d10-GPTDataset-train-obfd cache hit: True +pad_document_marker: 36616095 +/mnt/hdd/cpt/binidx_cache_stage15_final_20261001_v2 d6c36c6d5886334ea5f92e75fcf45d10-GPTDataset-train-obfd cache hit: True +pad_document_marker: 36616095 +/mnt/hdd/cpt/binidx_cache_stage15_final_20261001_v2 d6c36c6d5886334ea5f92e75fcf45d10-GPTDataset-train-obfd cache hit: True +pad_document_marker: 36616095 +/mnt/hdd/cpt/binidx_cache_stage15_final_20261001_v2 d6c36c6d5886334ea5f92e75fcf45d10-GPTDataset-train-obfd cache hit: True +pad_document_marker: 36616095 +/mnt/hdd/cpt/binidx_cache_stage15_final_20261001_v2 d6c36c6d5886334ea5f92e75fcf45d10-GPTDataset-train-obfd cache hit: True +pad_document_marker: 36616095 +/mnt/hdd/cpt/binidx_cache_stage15_final_20261001_v2 d6c36c6d5886334ea5f92e75fcf45d10-GPTDataset-train-obfd cache hit: True +pad_document_marker: 36616095 +/mnt/hdd/cpt/binidx_cache_stage15_final_20261001_v2 d6c36c6d5886334ea5f92e75fcf45d10-GPTDataset-train-obfd cache hit: True +/mnt/hdd/cpt/binidx_cache_stage15_final_20261001_v2 0642c625420be1073f42e03d3641a6e5-GPTDataset-valid-random cache hit: False +INFO:megatron.core.datasets.gpt_dataset:{ + "class": "GPTDataset", + "dataset_path": "/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_{0..78}_input_ids_tmp", + "num_samples": 1280, + "index_split": "valid", + "random_seed": 1234, + "sequence_length": 131072, + "split": "999,1,0", + "split_matrix": [ + [ + 0, + 0.999 + ], + [ + 0.999, + 1.0 + ], + null + ], + "tokenizer": { + "class": "megatron.core.tokenizers.text.models.default_tokenizer.DefaultTokenizerText", + "tokenizer_path": "/mnt/ssd/yulan/pretrain-linear-moe/preprocess/modify_tokenizer/YuLan-Mini-Qwen-Template_new_sp", + "vocab_file": "None", + "merges_file": "None" + } +} +INFO:megatron.core.datasets.gpt_dataset:Build and save the GPTDataset valid indices +Saving GPTDataset indexes to /mnt/hdd/cpt/binidx_cache_stage15_final_20261001_v2 +INFO:megatron.core.datasets.gpt_dataset:> total number of samples: 17022 +INFO:megatron.core.datasets.gpt_dataset:> total number of epochs: 1 +>>> start_samples: 0>>> start_samples: 0>>> start_samples: 0>>> start_samples: 0>>> start_samples: 0 + +> finished creating GPT datasets ... GPTDataset + + + +> Building pretraining data loader with total train samples: 1729771, consumed train samples: 0 +>>> start_samples: 0Initializing GlobIndexedDatasetInitializing GlobIndexedDataset + + +Initializing GlobIndexedDataset +Initializing GlobIndexedDatasetInitializing GlobIndexedDataset + +>>> start_samples: 0 +Initializing GlobIndexedDataset +Initializing GlobIndexedDataset +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_75_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_76_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_77_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_78_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_9_loss_mask_tmp'] +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_75_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_76_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_77_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_78_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_9_loss_mask_tmp']['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_75_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_76_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_77_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_78_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_9_loss_mask_tmp']['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_75_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_76_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_77_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_78_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_9_loss_mask_tmp']['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_75_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_76_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_77_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_78_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_9_loss_mask_tmp'] +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_75_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_76_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_77_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_78_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_9_loss_mask_tmp']['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_75_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_76_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_77_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_78_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_9_loss_mask_tmp'] + + + + +/mnt/hdd/cpt/binidx_cache_stage15_final_20261001_v2 0642c625420be1073f42e03d3641a6e5-GPTDataset-valid-random cache hit: True +/mnt/hdd/cpt/binidx_cache_stage15_final_20261001_v2 0642c625420be1073f42e03d3641a6e5-GPTDataset-valid-random cache hit: True +/mnt/hdd/cpt/binidx_cache_stage15_final_20261001_v2 0642c625420be1073f42e03d3641a6e5-GPTDataset-valid-random cache hit: True +/mnt/hdd/cpt/binidx_cache_stage15_final_20261001_v2 0642c625420be1073f42e03d3641a6e5-GPTDataset-valid-random cache hit: True +/mnt/hdd/cpt/binidx_cache_stage15_final_20261001_v2 0642c625420be1073f42e03d3641a6e5-GPTDataset-valid-random cache hit: True +/mnt/hdd/cpt/binidx_cache_stage15_final_20261001_v2 0642c625420be1073f42e03d3641a6e5-GPTDataset-valid-random cache hit: True +/mnt/hdd/cpt/binidx_cache_stage15_final_20261001_v2 0642c625420be1073f42e03d3641a6e5-GPTDataset-valid-random cache hit: True +[after dataloaders are built] datetime: 2026-10-03 15:47:07 +done with setup ... +training ... +Overwriting rerun_state_machine.current_iteration from -1 to 0... +[before the start of training step] datetime: 2026-10-03 15:47:07 +Iter 0 increase timing-log-level to 2. +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/optimizer/optimizer.py:65: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /pytorch/build/aten/src/ATen/core/TensorBody.h:489.) + if hasattr(param, grad_attr) and getattr(param, grad_attr) is not None: +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/optimizer/optimizer.py:65: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /pytorch/build/aten/src/ATen/core/TensorBody.h:489.) + if hasattr(param, grad_attr) and getattr(param, grad_attr) is not None: +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/optimizer/optimizer.py:65: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /pytorch/build/aten/src/ATen/core/TensorBody.h:489.) + if hasattr(param, grad_attr) and getattr(param, grad_attr) is not None: +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/optimizer/optimizer.py:65: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /pytorch/build/aten/src/ATen/core/TensorBody.h:489.) + if hasattr(param, grad_attr) and getattr(param, grad_attr) is not None: +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/optimizer/optimizer.py:65: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /pytorch/build/aten/src/ATen/core/TensorBody.h:489.) + if hasattr(param, grad_attr) and getattr(param, grad_attr) is not None: +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/optimizer/optimizer.py:65: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /pytorch/build/aten/src/ATen/core/TensorBody.h:489.) + if hasattr(param, grad_attr) and getattr(param, grad_attr) is not None: +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/optimizer/optimizer.py:65: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /pytorch/build/aten/src/ATen/core/TensorBody.h:489.) + if hasattr(param, grad_attr) and getattr(param, grad_attr) is not None: +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/optimizer/optimizer.py:65: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /pytorch/build/aten/src/ATen/core/TensorBody.h:489.) + if hasattr(param, grad_attr) and getattr(param, grad_attr) is not None: +⚠️ torch 2.8.0+cu128 with triton 3.4.0. inductor patch is not applied.⚠️ torch 2.8.0+cu128 with triton 3.4.0. inductor patch is not applied.⚠️ torch 2.8.0+cu128 with triton 3.4.0. inductor patch is not applied.⚠️ torch 2.8.0+cu128 with triton 3.4.0. inductor patch is not applied.⚠️ torch 2.8.0+cu128 with triton 3.4.0. inductor patch is not applied.⚠️ torch 2.8.0+cu128 with triton 3.4.0. inductor patch is not applied.⚠️ torch 2.8.0+cu128 with triton 3.4.0. inductor patch is not applied. + + + + + + +⚠️ torch 2.8.0+cu128 with triton 3.4.0. inductor patch is not applied. +WARNING:megatron.core.rerun_state_machine:Result validation enabled +Theoretical memory footprints not yet supported for hybrid Mamba-Transformer models. +[Rank 0] (after 1 iterations) memory (MB) | allocated: 47164.259765625 | max allocated: 50495.33154296875 | reserved: 52590.0 | max reserved: 52590.0 | device usage: 61731.875 +[Rank 4] (after 1 iterations) memory (MB) | allocated: 47164.25732421875 | max allocated: 50495.5478515625 | reserved: 52618.0 | max reserved: 52618.0 | device usage: 61137.3125[Rank 6] (after 1 iterations) memory (MB) | allocated: 47164.267578125 | max allocated: 50462.95556640625 | reserved: 52622.0 | max reserved: 52622.0 | device usage: 61141.3125[Rank 3] (after 1 iterations) memory (MB) | allocated: 47164.26318359375 | max allocated: 50444.31396484375 | reserved: 52604.0 | max reserved: 52604.0 | device usage: 61123.3125[Rank 5] (after 1 iterations) memory (MB) | allocated: 47164.25390625 | max allocated: 50532.99755859375 | reserved: 52618.0 | max reserved: 52618.0 | device usage: 61137.3125[Rank 1] (after 1 iterations) memory (MB) | allocated: 47164.263671875 | max allocated: 50551.49609375 | reserved: 52614.0 | max reserved: 52614.0 | device usage: 61133.3125[Rank 7] (after 1 iterations) memory (MB) | allocated: 47164.26953125 | max allocated: 50489.6943359375 | reserved: 52600.0 | max reserved: 52600.0 | device usage: 61119.3125 + + + + + +[Rank 2] (after 1 iterations) memory (MB) | allocated: 47164.26416015625 | max allocated: 50515.3125 | reserved: 52618.0 | max reserved: 52618.0 | device usage: 61137.3125 +Iter 5 reset timing-log-level. checkpoint path: /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints; tensorboard path: /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/tensorboard +saving checkpoint at iteration 40 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints in torch_dist format +Storing distributed optimizer sharded state of type dp_reshardable +W1003 16:47:41.800000 392180 torch/utils/cpp_extension.py:2425] TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +W1003 16:47:41.800000 392180 torch/utils/cpp_extension.py:2425] If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'] to specific architectures. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( + scheduled an async checkpoint save at iteration 40 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints + successfully saved checkpoint from iteration 40 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints [ t 1/1, p 1/1 ] +saving checkpoint at iteration 80 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints in torch_dist format +Storing distributed optimizer sharded state of type dp_reshardable + scheduled an async checkpoint save at iteration 80 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints + successfully saved checkpoint from iteration 80 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints [ t 1/1, p 1/1 ] + successfully deleted checkpoint from iteration 40 at /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints +saving checkpoint at iteration 120 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints in torch_dist format +Storing distributed optimizer sharded state of type dp_reshardable + scheduled an async checkpoint save at iteration 120 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints + successfully saved checkpoint from iteration 120 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints [ t 1/1, p 1/1 ] + successfully deleted checkpoint from iteration 80 at /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints +saving checkpoint at iteration 160 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints in torch_dist format +Storing distributed optimizer sharded state of type dp_reshardable + scheduled an async checkpoint save at iteration 160 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints + successfully saved checkpoint from iteration 160 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints [ t 1/1, p 1/1 ] + successfully deleted checkpoint from iteration 120 at /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints +saving checkpoint at iteration 200 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints in torch_dist format +Storing distributed optimizer sharded state of type dp_reshardable + scheduled an async checkpoint save at iteration 200 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints + successfully saved checkpoint from iteration 200 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints [ t 1/1, p 1/1 ] + successfully deleted checkpoint from iteration 160 at /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints +saving checkpoint at iteration 240 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints in torch_dist format +Storing distributed optimizer sharded state of type dp_reshardable + scheduled an async checkpoint save at iteration 240 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints + successfully saved checkpoint from iteration 240 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints [ t 1/1, p 1/1 ] + successfully deleted checkpoint from iteration 200 at /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints +saving checkpoint at iteration 280 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints in torch_dist format +Storing distributed optimizer sharded state of type dp_reshardable + scheduled an async checkpoint save at iteration 280 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints + successfully saved checkpoint from iteration 280 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints [ t 1/1, p 1/1 ] + successfully deleted checkpoint from iteration 240 at /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints +saving checkpoint at iteration 320 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints in torch_dist format +Storing distributed optimizer sharded state of type dp_reshardable + scheduled an async checkpoint save at iteration 320 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints + successfully saved checkpoint from iteration 320 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints [ t 1/1, p 1/1 ] + successfully deleted checkpoint from iteration 280 at /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints +saving checkpoint at iteration 360 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints in torch_dist format +Storing distributed optimizer sharded state of type dp_reshardable + scheduled an async checkpoint save at iteration 360 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints + successfully saved checkpoint from iteration 360 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints [ t 1/1, p 1/1 ] + successfully deleted checkpoint from iteration 320 at /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints +saving checkpoint at iteration 400 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints in torch_dist format +Storing distributed optimizer sharded state of type dp_reshardable + scheduled an async checkpoint save at iteration 400 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints + successfully saved checkpoint from iteration 400 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints [ t 1/1, p 1/1 ] + successfully deleted checkpoint from iteration 360 at /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints +saving checkpoint at iteration 440 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints in torch_dist format +Storing distributed optimizer sharded state of type dp_reshardable + scheduled an async checkpoint save at iteration 440 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints + successfully saved checkpoint from iteration 440 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints [ t 1/1, p 1/1 ] + successfully deleted checkpoint from iteration 400 at /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints +saving checkpoint at iteration 480 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints in torch_dist format +Storing distributed optimizer sharded state of type dp_reshardable + scheduled an async checkpoint save at iteration 480 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints + successfully saved checkpoint from iteration 480 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints [ t 1/1, p 1/1 ] + successfully deleted checkpoint from iteration 440 at /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints +saving checkpoint at iteration 520 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints in torch_dist format +Storing distributed optimizer sharded state of type dp_reshardable + scheduled an async checkpoint save at iteration 520 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints + successfully saved checkpoint from iteration 520 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints [ t 1/1, p 1/1 ] + successfully deleted checkpoint from iteration 480 at /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints +saving checkpoint at iteration 560 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints in torch_dist format +Storing distributed optimizer sharded state of type dp_reshardable + scheduled an async checkpoint save at iteration 560 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints + successfully saved checkpoint from iteration 560 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints [ t 1/1, p 1/1 ] + successfully deleted checkpoint from iteration 520 at /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints +saving checkpoint at iteration 600 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints in torch_dist format +Storing distributed optimizer sharded state of type dp_reshardable + scheduled an async checkpoint save at iteration 600 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints + successfully saved checkpoint from iteration 600 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints [ t 1/1, p 1/1 ] + successfully deleted checkpoint from iteration 560 at /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints +saving checkpoint at iteration 640 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints in torch_dist format +Storing distributed optimizer sharded state of type dp_reshardable + scheduled an async checkpoint save at iteration 640 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints + successfully saved checkpoint from iteration 640 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints [ t 1/1, p 1/1 ] + successfully deleted checkpoint from iteration 600 at /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints +saving checkpoint at iteration 680 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints in torch_dist format +Storing distributed optimizer sharded state of type dp_reshardable + scheduled an async checkpoint save at iteration 680 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints + successfully saved checkpoint from iteration 680 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints [ t 1/1, p 1/1 ] + successfully deleted checkpoint from iteration 640 at /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints +saving checkpoint at iteration 720 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints in torch_dist format +Storing distributed optimizer sharded state of type dp_reshardable + scheduled an async checkpoint save at iteration 720 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints + successfully saved checkpoint from iteration 720 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints [ t 1/1, p 1/1 ] + successfully deleted checkpoint from iteration 680 at /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints +saving checkpoint at iteration 760 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints in torch_dist format +Storing distributed optimizer sharded state of type dp_reshardable + scheduled an async checkpoint save at iteration 760 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints + successfully saved checkpoint from iteration 760 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints [ t 1/1, p 1/1 ] + successfully deleted checkpoint from iteration 720 at /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints +saving checkpoint at iteration 800 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints in torch_dist format +Storing distributed optimizer sharded state of type dp_reshardable + scheduled an async checkpoint save at iteration 800 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints + successfully saved checkpoint from iteration 800 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints [ t 1/1, p 1/1 ] + successfully deleted checkpoint from iteration 760 at /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints +saving checkpoint at iteration 840 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints in torch_dist format +Storing distributed optimizer sharded state of type dp_reshardable + scheduled an async checkpoint save at iteration 840 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints + successfully saved checkpoint from iteration 840 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints [ t 1/1, p 1/1 ] + successfully deleted checkpoint from iteration 800 at /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints +saving checkpoint at iteration 880 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints in torch_dist format +Storing distributed optimizer sharded state of type dp_reshardable + scheduled an async checkpoint save at iteration 880 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints + successfully saved checkpoint from iteration 880 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints [ t 1/1, p 1/1 ] + successfully deleted checkpoint from iteration 840 at /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints +saving checkpoint at iteration 920 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints in torch_dist format +Storing distributed optimizer sharded state of type dp_reshardable + scheduled an async checkpoint save at iteration 920 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints + successfully saved checkpoint from iteration 920 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints [ t 1/1, p 1/1 ] + successfully deleted checkpoint from iteration 880 at /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints +saving checkpoint at iteration 960 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints in torch_dist format +Storing distributed optimizer sharded state of type dp_reshardable + scheduled an async checkpoint save at iteration 960 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints + successfully saved checkpoint from iteration 960 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints [ t 1/1, p 1/1 ] + successfully deleted checkpoint from iteration 920 at /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints +saving checkpoint at iteration 1000 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints in torch_dist format +Storing distributed optimizer sharded state of type dp_reshardable + scheduled an async checkpoint save at iteration 1000 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints +Iter 1000 increase timing-log-level to 2. + successfully saved checkpoint from iteration 1000 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints [ t 1/1, p 1/1 ] + successfully deleted checkpoint from iteration 960 at /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints +Iter 1005 reset timing-log-level. checkpoint path: /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints; tensorboard path: /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/tensorboard +saving checkpoint at iteration 1040 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints in torch_dist format +Storing distributed optimizer sharded state of type dp_reshardable + scheduled an async checkpoint save at iteration 1040 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints + successfully saved checkpoint from iteration 1040 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints [ t 1/1, p 1/1 ] + successfully deleted checkpoint from iteration 1000 at /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints +saving checkpoint at iteration 1080 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints in torch_dist format +Storing distributed optimizer sharded state of type dp_reshardable + scheduled an async checkpoint save at iteration 1080 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints + successfully saved checkpoint from iteration 1080 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints [ t 1/1, p 1/1 ] + successfully deleted checkpoint from iteration 1040 at /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints +saving checkpoint at iteration 1120 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints in torch_dist format +Storing distributed optimizer sharded state of type dp_reshardable + scheduled an async checkpoint save at iteration 1120 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints + successfully saved checkpoint from iteration 1120 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints [ t 1/1, p 1/1 ] + successfully deleted checkpoint from iteration 1080 at /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints +saving checkpoint at iteration 1160 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints in torch_dist format +Storing distributed optimizer sharded state of type dp_reshardable + scheduled an async checkpoint save at iteration 1160 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints + successfully saved checkpoint from iteration 1160 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints [ t 1/1, p 1/1 ] + successfully deleted checkpoint from iteration 1120 at /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints +saving checkpoint at iteration 1200 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints in torch_dist format +Storing distributed optimizer sharded state of type dp_reshardable + scheduled an async checkpoint save at iteration 1200 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints + successfully saved checkpoint from iteration 1200 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints [ t 1/1, p 1/1 ] + successfully deleted checkpoint from iteration 1160 at /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints +saving checkpoint at iteration 1240 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints in torch_dist format +Storing distributed optimizer sharded state of type dp_reshardable + scheduled an async checkpoint save at iteration 1240 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints + successfully saved checkpoint from iteration 1240 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints [ t 1/1, p 1/1 ] + successfully deleted checkpoint from iteration 1200 at /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints +saving checkpoint at iteration 1280 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints in torch_dist format +Storing distributed optimizer sharded state of type dp_reshardable + scheduled an async checkpoint save at iteration 1280 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints + successfully saved checkpoint from iteration 1280 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints [ t 1/1, p 1/1 ] + successfully deleted checkpoint from iteration 1240 at /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints +saving checkpoint at iteration 1320 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints in torch_dist format +Storing distributed optimizer sharded state of type dp_reshardable + scheduled an async checkpoint save at iteration 1320 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints + successfully saved checkpoint from iteration 1320 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints [ t 1/1, p 1/1 ] + successfully deleted checkpoint from iteration 1280 at /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints +saving checkpoint at iteration 1360 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints in torch_dist format +Storing distributed optimizer sharded state of type dp_reshardable + scheduled an async checkpoint save at iteration 1360 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints + successfully saved checkpoint from iteration 1360 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints [ t 1/1, p 1/1 ] + successfully deleted checkpoint from iteration 1320 at /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints +saving checkpoint at iteration 1400 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints in torch_dist format +Storing distributed optimizer sharded state of type dp_reshardable + scheduled an async checkpoint save at iteration 1400 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints + successfully saved checkpoint from iteration 1400 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints [ t 1/1, p 1/1 ] + successfully deleted checkpoint from iteration 1360 at /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints +saving checkpoint at iteration 1440 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints in torch_dist format +Storing distributed optimizer sharded state of type dp_reshardable + scheduled an async checkpoint save at iteration 1440 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints + successfully saved checkpoint from iteration 1440 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints [ t 1/1, p 1/1 ] + successfully deleted checkpoint from iteration 1400 at /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints +saving checkpoint at iteration 1480 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints in torch_dist format +Storing distributed optimizer sharded state of type dp_reshardable + scheduled an async checkpoint save at iteration 1480 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints + successfully saved checkpoint from iteration 1480 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints [ t 1/1, p 1/1 ] + successfully deleted checkpoint from iteration 1440 at /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints +saving checkpoint at iteration 1520 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints in torch_dist format +Storing distributed optimizer sharded state of type dp_reshardable + scheduled an async checkpoint save at iteration 1520 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints + successfully saved checkpoint from iteration 1520 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints [ t 1/1, p 1/1 ] + successfully deleted checkpoint from iteration 1480 at /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints +saving checkpoint at iteration 1560 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints in torch_dist format +Storing distributed optimizer sharded state of type dp_reshardable + scheduled an async checkpoint save at iteration 1560 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints + successfully saved checkpoint from iteration 1560 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints [ t 1/1, p 1/1 ] + successfully deleted checkpoint from iteration 1520 at /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints +saving checkpoint at iteration 1600 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints in torch_dist format +Storing distributed optimizer sharded state of type dp_reshardable + scheduled an async checkpoint save at iteration 1600 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints + successfully saved checkpoint from iteration 1600 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints [ t 1/1, p 1/1 ] + successfully deleted checkpoint from iteration 1560 at /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints +saving checkpoint at iteration 1640 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints in torch_dist format +Storing distributed optimizer sharded state of type dp_reshardable + scheduled an async checkpoint save at iteration 1640 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints + successfully saved checkpoint from iteration 1640 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints [ t 1/1, p 1/1 ] + successfully deleted checkpoint from iteration 1600 at /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints +saving checkpoint at iteration 1680 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints in torch_dist format +Storing distributed optimizer sharded state of type dp_reshardable + scheduled an async checkpoint save at iteration 1680 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints + successfully saved checkpoint from iteration 1680 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints [ t 1/1, p 1/1 ] + successfully deleted checkpoint from iteration 1640 at /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints +saving checkpoint at iteration 1720 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints in torch_dist format +Storing distributed optimizer sharded state of type dp_reshardable + scheduled an async checkpoint save at iteration 1720 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints + successfully saved checkpoint from iteration 1720 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints [ t 1/1, p 1/1 ] + successfully deleted checkpoint from iteration 1680 at /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints +saving checkpoint at iteration 1760 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints in torch_dist format +Storing distributed optimizer sharded state of type dp_reshardable + scheduled an async checkpoint save at iteration 1760 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints + successfully saved checkpoint from iteration 1760 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints [ t 1/1, p 1/1 ] + successfully deleted checkpoint from iteration 1720 at /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints +saving checkpoint at iteration 1800 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints in torch_dist format +Storing distributed optimizer sharded state of type dp_reshardable + scheduled an async checkpoint save at iteration 1800 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints + successfully saved checkpoint from iteration 1800 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints [ t 1/1, p 1/1 ] + successfully deleted checkpoint from iteration 1760 at /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints +saving checkpoint at iteration 1840 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints in torch_dist format +Storing distributed optimizer sharded state of type dp_reshardable + scheduled an async checkpoint save at iteration 1840 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints + successfully saved checkpoint from iteration 1840 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints [ t 1/1, p 1/1 ] + successfully deleted checkpoint from iteration 1800 at /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints +saving checkpoint at iteration 1880 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints in torch_dist format +Storing distributed optimizer sharded state of type dp_reshardable + scheduled an async checkpoint save at iteration 1880 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints + successfully saved checkpoint from iteration 1880 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints [ t 1/1, p 1/1 ] + successfully deleted checkpoint from iteration 1840 at /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints +saving checkpoint at iteration 1920 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints in torch_dist format +Storing distributed optimizer sharded state of type dp_reshardable + scheduled an async checkpoint save at iteration 1920 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints + successfully saved checkpoint from iteration 1920 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints [ t 1/1, p 1/1 ] + successfully deleted checkpoint from iteration 1880 at /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints +saving checkpoint at iteration 1960 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints in torch_dist format +Storing distributed optimizer sharded state of type dp_reshardable + scheduled an async checkpoint save at iteration 1960 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints + successfully saved checkpoint from iteration 1960 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints [ t 1/1, p 1/1 ] + successfully deleted checkpoint from iteration 1920 at /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints +saving checkpoint at iteration 2000 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints in torch_dist format +Storing distributed optimizer sharded state of type dp_reshardable + scheduled an async checkpoint save at iteration 2000 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints +Iter 2000 increase timing-log-level to 2. + successfully saved checkpoint from iteration 2000 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints [ t 1/1, p 1/1 ] + successfully deleted checkpoint from iteration 1960 at /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints +Iter 2005 reset timing-log-level. checkpoint path: /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints; tensorboard path: /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/tensorboard +saving checkpoint at iteration 2040 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints in torch_dist format +Storing distributed optimizer sharded state of type dp_reshardable + scheduled an async checkpoint save at iteration 2040 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints + successfully saved checkpoint from iteration 2040 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints [ t 1/1, p 1/1 ] + successfully deleted checkpoint from iteration 2000 at /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints +saving checkpoint at iteration 2080 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints in torch_dist format +Storing distributed optimizer sharded state of type dp_reshardable + scheduled an async checkpoint save at iteration 2080 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints + successfully saved checkpoint from iteration 2080 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints [ t 1/1, p 1/1 ] + successfully deleted checkpoint from iteration 2040 at /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints +saving checkpoint at iteration 2120 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints in torch_dist format +Storing distributed optimizer sharded state of type dp_reshardable + scheduled an async checkpoint save at iteration 2120 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints + successfully saved checkpoint from iteration 2120 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints [ t 1/1, p 1/1 ] + successfully deleted checkpoint from iteration 2080 at /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints +saving checkpoint at iteration 2160 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints in torch_dist format +Storing distributed optimizer sharded state of type dp_reshardable + scheduled an async checkpoint save at iteration 2160 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints + successfully saved checkpoint from iteration 2160 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints [ t 1/1, p 1/1 ] + successfully deleted checkpoint from iteration 2120 at /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints +saving checkpoint at iteration 2200 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints in torch_dist format +Storing distributed optimizer sharded state of type dp_reshardable + scheduled an async checkpoint save at iteration 2200 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints + successfully saved checkpoint from iteration 2200 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints [ t 1/1, p 1/1 ] + successfully deleted checkpoint from iteration 2160 at /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints +saving checkpoint at iteration 2240 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints in torch_dist format +Storing distributed optimizer sharded state of type dp_reshardable + scheduled an async checkpoint save at iteration 2240 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints + successfully saved checkpoint from iteration 2240 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints [ t 1/1, p 1/1 ] + successfully deleted checkpoint from iteration 2200 at /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints +saving checkpoint at iteration 2280 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints in torch_dist format +Storing distributed optimizer sharded state of type dp_reshardable + scheduled an async checkpoint save at iteration 2280 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints + successfully saved checkpoint from iteration 2280 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints [ t 1/1, p 1/1 ] + successfully deleted checkpoint from iteration 2240 at /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints +saving checkpoint at iteration 2320 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints in torch_dist format +Storing distributed optimizer sharded state of type dp_reshardable + scheduled an async checkpoint save at iteration 2320 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints + successfully saved checkpoint from iteration 2320 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints [ t 1/1, p 1/1 ] + successfully deleted checkpoint from iteration 2280 at /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints +saving checkpoint at iteration 2360 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints in torch_dist format +Storing distributed optimizer sharded state of type dp_reshardable + scheduled an async checkpoint save at iteration 2360 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints + successfully saved checkpoint from iteration 2360 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints [ t 1/1, p 1/1 ] + successfully deleted checkpoint from iteration 2320 at /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints +saving checkpoint at iteration 2400 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints in torch_dist format +Storing distributed optimizer sharded state of type dp_reshardable + scheduled an async checkpoint save at iteration 2400 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints + successfully saved checkpoint from iteration 2400 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints [ t 1/1, p 1/1 ] + successfully deleted checkpoint from iteration 2360 at /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints +saving checkpoint at iteration 2440 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints in torch_dist format +Storing distributed optimizer sharded state of type dp_reshardable + scheduled an async checkpoint save at iteration 2440 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints + successfully saved checkpoint from iteration 2440 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints [ t 1/1, p 1/1 ] + successfully deleted checkpoint from iteration 2400 at /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints +saving checkpoint at iteration 2480 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints in torch_dist format +Storing distributed optimizer sharded state of type dp_reshardable + scheduled an async checkpoint save at iteration 2480 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints + successfully saved checkpoint from iteration 2480 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints [ t 1/1, p 1/1 ] + successfully deleted checkpoint from iteration 2440 at /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints +saving checkpoint at iteration 2520 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints in torch_dist format +Storing distributed optimizer sharded state of type dp_reshardable + scheduled an async checkpoint save at iteration 2520 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints + successfully saved checkpoint from iteration 2520 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints [ t 1/1, p 1/1 ] + successfully deleted checkpoint from iteration 2480 at /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints +saving checkpoint at iteration 2560 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints in torch_dist format +Storing distributed optimizer sharded state of type dp_reshardable + scheduled an async checkpoint save at iteration 2560 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints + successfully saved checkpoint from iteration 2560 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints [ t 1/1, p 1/1 ] + successfully deleted checkpoint from iteration 2520 at /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints +saving checkpoint at iteration 2600 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints in torch_dist format +Storing distributed optimizer sharded state of type dp_reshardable + scheduled an async checkpoint save at iteration 2600 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints + successfully saved checkpoint from iteration 2600 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints [ t 1/1, p 1/1 ] + successfully deleted checkpoint from iteration 2560 at /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints +saving checkpoint at iteration 2640 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints in torch_dist format +Storing distributed optimizer sharded state of type dp_reshardable + scheduled an async checkpoint save at iteration 2640 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints + successfully saved checkpoint from iteration 2640 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints [ t 1/1, p 1/1 ] + successfully deleted checkpoint from iteration 2600 at /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints +saving checkpoint at iteration 2680 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints in torch_dist format +Storing distributed optimizer sharded state of type dp_reshardable + scheduled an async checkpoint save at iteration 2680 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints + successfully saved checkpoint from iteration 2680 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints [ t 1/1, p 1/1 ] + successfully deleted checkpoint from iteration 2640 at /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints +saving checkpoint at iteration 2720 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints in torch_dist format +Storing distributed optimizer sharded state of type dp_reshardable + scheduled an async checkpoint save at iteration 2720 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints + successfully saved checkpoint from iteration 2720 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints [ t 1/1, p 1/1 ] + successfully deleted checkpoint from iteration 2680 at /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints +saving checkpoint at iteration 2760 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints in torch_dist format +Storing distributed optimizer sharded state of type dp_reshardable + scheduled an async checkpoint save at iteration 2760 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints + successfully saved checkpoint from iteration 2760 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints [ t 1/1, p 1/1 ] + successfully deleted checkpoint from iteration 2720 at /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints +saving checkpoint at iteration 2800 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints in torch_dist format +Storing distributed optimizer sharded state of type dp_reshardable + scheduled an async checkpoint save at iteration 2800 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints + successfully saved checkpoint from iteration 2800 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints [ t 1/1, p 1/1 ] + successfully deleted checkpoint from iteration 2760 at /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints +saving checkpoint at iteration 2840 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints in torch_dist format +Storing distributed optimizer sharded state of type dp_reshardable + scheduled an async checkpoint save at iteration 2840 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints + successfully saved checkpoint from iteration 2840 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints [ t 1/1, p 1/1 ] + successfully deleted checkpoint from iteration 2800 at /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints +saving checkpoint at iteration 2880 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints in torch_dist format +Storing distributed optimizer sharded state of type dp_reshardable + scheduled an async checkpoint save at iteration 2880 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints + successfully saved checkpoint from iteration 2880 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints [ t 1/1, p 1/1 ] + successfully deleted checkpoint from iteration 2840 at /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints +saving checkpoint at iteration 2920 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints in torch_dist format +Storing distributed optimizer sharded state of type dp_reshardable + scheduled an async checkpoint save at iteration 2920 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints + successfully saved checkpoint from iteration 2920 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints [ t 1/1, p 1/1 ] + successfully deleted checkpoint from iteration 2880 at /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints +saving checkpoint at iteration 2960 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints in torch_dist format +Storing distributed optimizer sharded state of type dp_reshardable + scheduled an async checkpoint save at iteration 2960 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints + successfully saved checkpoint from iteration 2960 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints [ t 1/1, p 1/1 ] + successfully deleted checkpoint from iteration 2920 at /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints +saving checkpoint at iteration 3000 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints in torch_dist format +Storing distributed optimizer sharded state of type dp_reshardable + scheduled an async checkpoint save at iteration 3000 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints +Iter 3000 increase timing-log-level to 2. + successfully saved checkpoint from iteration 3000 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints [ t 1/1, p 1/1 ] + successfully deleted checkpoint from iteration 2960 at /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints +Iter 3005 reset timing-log-level. checkpoint path: /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints; tensorboard path: /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/tensorboard +saving checkpoint at iteration 3040 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints in torch_dist format +Storing distributed optimizer sharded state of type dp_reshardable + scheduled an async checkpoint save at iteration 3040 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints + successfully saved checkpoint from iteration 3040 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints [ t 1/1, p 1/1 ] + successfully deleted checkpoint from iteration 3000 at /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints +saving checkpoint at iteration 3080 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints in torch_dist format +Storing distributed optimizer sharded state of type dp_reshardable + scheduled an async checkpoint save at iteration 3080 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints + successfully saved checkpoint from iteration 3080 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints [ t 1/1, p 1/1 ] + successfully deleted checkpoint from iteration 3040 at /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints +saving checkpoint at iteration 3120 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints in torch_dist format +Storing distributed optimizer sharded state of type dp_reshardable + scheduled an async checkpoint save at iteration 3120 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints + successfully saved checkpoint from iteration 3120 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints [ t 1/1, p 1/1 ] + successfully deleted checkpoint from iteration 3080 at /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints +saving checkpoint at iteration 3160 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints in torch_dist format +Storing distributed optimizer sharded state of type dp_reshardable + scheduled an async checkpoint save at iteration 3160 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints + successfully saved checkpoint from iteration 3160 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints [ t 1/1, p 1/1 ] + successfully deleted checkpoint from iteration 3120 at /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints +saving checkpoint at iteration 3200 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints in torch_dist format +Storing distributed optimizer sharded state of type dp_reshardable + scheduled an async checkpoint save at iteration 3200 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints + successfully saved checkpoint from iteration 3200 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints [ t 1/1, p 1/1 ] + successfully deleted checkpoint from iteration 3160 at /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints +saving checkpoint at iteration 3240 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints in torch_dist format +Storing distributed optimizer sharded state of type dp_reshardable + scheduled an async checkpoint save at iteration 3240 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints + successfully saved checkpoint from iteration 3240 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints [ t 1/1, p 1/1 ] + successfully deleted checkpoint from iteration 3200 at /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints +saving checkpoint at iteration 3280 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints in torch_dist format +Storing distributed optimizer sharded state of type dp_reshardable + scheduled an async checkpoint save at iteration 3280 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints + successfully saved checkpoint from iteration 3280 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints [ t 1/1, p 1/1 ] + successfully deleted checkpoint from iteration 3240 at /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints +saving checkpoint at iteration 3320 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints in torch_dist format +Storing distributed optimizer sharded state of type dp_reshardable + scheduled an async checkpoint save at iteration 3320 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints + successfully saved checkpoint from iteration 3320 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints [ t 1/1, p 1/1 ] + successfully deleted checkpoint from iteration 3280 at /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints +saving checkpoint at iteration 3360 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints in torch_dist format +Storing distributed optimizer sharded state of type dp_reshardable + scheduled an async checkpoint save at iteration 3360 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints + successfully saved checkpoint from iteration 3360 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints [ t 1/1, p 1/1 ] + successfully deleted checkpoint from iteration 3320 at /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints +saving checkpoint at iteration 3400 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints in torch_dist format +Storing distributed optimizer sharded state of type dp_reshardable + scheduled an async checkpoint save at iteration 3400 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints + successfully saved checkpoint from iteration 3400 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints [ t 1/1, p 1/1 ] + successfully deleted checkpoint from iteration 3360 at /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints +saving checkpoint at iteration 3440 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints in torch_dist format +Storing distributed optimizer sharded state of type dp_reshardable + scheduled an async checkpoint save at iteration 3440 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints + successfully saved checkpoint from iteration 3440 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints [ t 1/1, p 1/1 ] + successfully deleted checkpoint from iteration 3400 at /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints +saving checkpoint at iteration 3480 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints in torch_dist format +Storing distributed optimizer sharded state of type dp_reshardable + scheduled an async checkpoint save at iteration 3480 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints + successfully saved checkpoint from iteration 3480 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints [ t 1/1, p 1/1 ] + successfully deleted checkpoint from iteration 3440 at /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints +saving checkpoint at iteration 3520 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints in torch_dist format +Storing distributed optimizer sharded state of type dp_reshardable + scheduled an async checkpoint save at iteration 3520 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints + successfully saved checkpoint from iteration 3520 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints [ t 1/1, p 1/1 ] + successfully deleted checkpoint from iteration 3480 at /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints +saving checkpoint at iteration 3560 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints in torch_dist format +Storing distributed optimizer sharded state of type dp_reshardable + scheduled an async checkpoint save at iteration 3560 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints + successfully saved checkpoint from iteration 3560 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints [ t 1/1, p 1/1 ] + successfully deleted checkpoint from iteration 3520 at /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints +saving checkpoint at iteration 3600 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints in torch_dist format +Storing distributed optimizer sharded state of type dp_reshardable + scheduled an async checkpoint save at iteration 3600 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints + successfully saved checkpoint from iteration 3600 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints [ t 1/1, p 1/1 ] + successfully deleted checkpoint from iteration 3560 at /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints +saving checkpoint at iteration 3640 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints in torch_dist format +Storing distributed optimizer sharded state of type dp_reshardable + scheduled an async checkpoint save at iteration 3640 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints + successfully saved checkpoint from iteration 3640 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints [ t 1/1, p 1/1 ] + successfully deleted checkpoint from iteration 3600 at /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints +saving checkpoint at iteration 3680 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints in torch_dist format +Storing distributed optimizer sharded state of type dp_reshardable + scheduled an async checkpoint save at iteration 3680 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints + successfully saved checkpoint from iteration 3680 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints [ t 1/1, p 1/1 ] + successfully deleted checkpoint from iteration 3640 at /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints +saving checkpoint at iteration 3720 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints in torch_dist format +Storing distributed optimizer sharded state of type dp_reshardable + scheduled an async checkpoint save at iteration 3720 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints + successfully saved checkpoint from iteration 3720 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints [ t 1/1, p 1/1 ] + successfully deleted checkpoint from iteration 3680 at /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints +saving checkpoint at iteration 3760 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints in torch_dist format +Storing distributed optimizer sharded state of type dp_reshardable + scheduled an async checkpoint save at iteration 3760 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints + successfully saved checkpoint from iteration 3760 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints [ t 1/1, p 1/1 ] + successfully deleted checkpoint from iteration 3720 at /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints +saving checkpoint at iteration 3800 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints in torch_dist format +Storing distributed optimizer sharded state of type dp_reshardable + scheduled an async checkpoint save at iteration 3800 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints + successfully saved checkpoint from iteration 3800 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints [ t 1/1, p 1/1 ] + successfully deleted checkpoint from iteration 3760 at /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints +saving checkpoint at iteration 3840 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints in torch_dist format +Storing distributed optimizer sharded state of type dp_reshardable + scheduled an async checkpoint save at iteration 3840 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints + successfully saved checkpoint from iteration 3840 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints [ t 1/1, p 1/1 ] + successfully deleted checkpoint from iteration 3800 at /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints +saving checkpoint at iteration 3880 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints in torch_dist format +Storing distributed optimizer sharded state of type dp_reshardable + scheduled an async checkpoint save at iteration 3880 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints + successfully saved checkpoint from iteration 3880 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints [ t 1/1, p 1/1 ] + successfully deleted checkpoint from iteration 3840 at /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints +saving checkpoint at iteration 3920 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints in torch_dist format +Storing distributed optimizer sharded state of type dp_reshardable + scheduled an async checkpoint save at iteration 3920 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints + successfully saved checkpoint from iteration 3920 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints [ t 1/1, p 1/1 ] + successfully deleted checkpoint from iteration 3880 at /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints +saving checkpoint at iteration 3960 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints in torch_dist format +Storing distributed optimizer sharded state of type dp_reshardable + scheduled an async checkpoint save at iteration 3960 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints + successfully saved checkpoint from iteration 3960 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints [ t 1/1, p 1/1 ] + successfully deleted checkpoint from iteration 3920 at /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints +saving checkpoint at iteration 4000 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints in torch_dist format +Storing distributed optimizer sharded state of type dp_reshardable + scheduled an async checkpoint save at iteration 4000 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints +Iter 4000 increase timing-log-level to 2. + successfully saved checkpoint from iteration 4000 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints [ t 1/1, p 1/1 ] + successfully deleted checkpoint from iteration 3960 at /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints +Iter 4005 reset timing-log-level. checkpoint path: /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints; tensorboard path: /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/tensorboard +saving checkpoint at iteration 4040 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints in torch_dist format +Storing distributed optimizer sharded state of type dp_reshardable + scheduled an async checkpoint save at iteration 4040 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints + successfully saved checkpoint from iteration 4040 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints [ t 1/1, p 1/1 ] + successfully deleted checkpoint from iteration 4000 at /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints +saving checkpoint at iteration 4080 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints in torch_dist format +Storing distributed optimizer sharded state of type dp_reshardable + scheduled an async checkpoint save at iteration 4080 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints + successfully saved checkpoint from iteration 4080 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints [ t 1/1, p 1/1 ] + successfully deleted checkpoint from iteration 4040 at /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints +saving checkpoint at iteration 4120 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints in torch_dist format +Storing distributed optimizer sharded state of type dp_reshardable + scheduled an async checkpoint save at iteration 4120 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints + successfully saved checkpoint from iteration 4120 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints [ t 1/1, p 1/1 ] + successfully deleted checkpoint from iteration 4080 at /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints +saving checkpoint at iteration 4160 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints in torch_dist format +Storing distributed optimizer sharded state of type dp_reshardable + scheduled an async checkpoint save at iteration 4160 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints + successfully saved checkpoint from iteration 4160 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints [ t 1/1, p 1/1 ] + successfully deleted checkpoint from iteration 4120 at /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints +saving checkpoint at iteration 4200 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints in torch_dist format +Storing distributed optimizer sharded state of type dp_reshardable + scheduled an async checkpoint save at iteration 4200 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints + successfully saved checkpoint from iteration 4200 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints [ t 1/1, p 1/1 ] + successfully deleted checkpoint from iteration 4160 at /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints +saving checkpoint at iteration 4240 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints in torch_dist format +Storing distributed optimizer sharded state of type dp_reshardable + scheduled an async checkpoint save at iteration 4240 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints + successfully saved checkpoint from iteration 4240 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints [ t 1/1, p 1/1 ] + successfully deleted checkpoint from iteration 4200 at /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints +saving checkpoint at iteration 4280 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints in torch_dist format +Storing distributed optimizer sharded state of type dp_reshardable + scheduled an async checkpoint save at iteration 4280 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints + successfully saved checkpoint from iteration 4280 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints [ t 1/1, p 1/1 ] + successfully deleted checkpoint from iteration 4240 at /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints +saving checkpoint at iteration 4320 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints in torch_dist format +Storing distributed optimizer sharded state of type dp_reshardable + scheduled an async checkpoint save at iteration 4320 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints + successfully saved checkpoint from iteration 4320 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints [ t 1/1, p 1/1 ] + successfully deleted checkpoint from iteration 4280 at /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints +saving checkpoint at iteration 4360 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints in torch_dist format +Storing distributed optimizer sharded state of type dp_reshardable + scheduled an async checkpoint save at iteration 4360 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints + successfully saved checkpoint from iteration 4360 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints [ t 1/1, p 1/1 ] + successfully deleted checkpoint from iteration 4320 at /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints +saving checkpoint at iteration 4400 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints in torch_dist format +Storing distributed optimizer sharded state of type dp_reshardable + scheduled an async checkpoint save at iteration 4400 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints + successfully saved checkpoint from iteration 4400 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints [ t 1/1, p 1/1 ] + successfully deleted checkpoint from iteration 4360 at /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints +saving checkpoint at iteration 4440 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints in torch_dist format +Storing distributed optimizer sharded state of type dp_reshardable + scheduled an async checkpoint save at iteration 4440 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints + successfully saved checkpoint from iteration 4440 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints [ t 1/1, p 1/1 ] + successfully deleted checkpoint from iteration 4400 at /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints +saving checkpoint at iteration 4480 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints in torch_dist format +Storing distributed optimizer sharded state of type dp_reshardable + scheduled an async checkpoint save at iteration 4480 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints + successfully saved checkpoint from iteration 4480 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints [ t 1/1, p 1/1 ] + successfully deleted checkpoint from iteration 4440 at /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints +saving checkpoint at iteration 4520 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints in torch_dist format +Storing distributed optimizer sharded state of type dp_reshardable + scheduled an async checkpoint save at iteration 4520 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints + successfully saved checkpoint from iteration 4520 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints [ t 1/1, p 1/1 ] + successfully deleted checkpoint from iteration 4480 at /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints +saving checkpoint at iteration 4560 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints in torch_dist format +Storing distributed optimizer sharded state of type dp_reshardable + scheduled an async checkpoint save at iteration 4560 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints + successfully saved checkpoint from iteration 4560 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints [ t 1/1, p 1/1 ] + successfully deleted checkpoint from iteration 4520 at /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints +saving checkpoint at iteration 4600 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints in torch_dist format +Storing distributed optimizer sharded state of type dp_reshardable + scheduled an async checkpoint save at iteration 4600 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints + successfully saved checkpoint from iteration 4600 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints [ t 1/1, p 1/1 ] + successfully deleted checkpoint from iteration 4560 at /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints +saving checkpoint at iteration 4640 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints in torch_dist format +Storing distributed optimizer sharded state of type dp_reshardable + scheduled an async checkpoint save at iteration 4640 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints + successfully saved checkpoint from iteration 4640 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints [ t 1/1, p 1/1 ] + successfully deleted checkpoint from iteration 4600 at /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints +saving checkpoint at iteration 4680 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints in torch_dist format +Storing distributed optimizer sharded state of type dp_reshardable + scheduled an async checkpoint save at iteration 4680 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints + successfully saved checkpoint from iteration 4680 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints [ t 1/1, p 1/1 ] + successfully deleted checkpoint from iteration 4640 at /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints +saving checkpoint at iteration 4720 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints in torch_dist format +Storing distributed optimizer sharded state of type dp_reshardable + scheduled an async checkpoint save at iteration 4720 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints + successfully saved checkpoint from iteration 4720 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints [ t 1/1, p 1/1 ] + successfully deleted checkpoint from iteration 4680 at /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints +saving checkpoint at iteration 4760 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints in torch_dist format +Storing distributed optimizer sharded state of type dp_reshardable + scheduled an async checkpoint save at iteration 4760 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints + successfully saved checkpoint from iteration 4760 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints [ t 1/1, p 1/1 ] + successfully deleted checkpoint from iteration 4720 at /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints +saving checkpoint at iteration 4800 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints in torch_dist format +Storing distributed optimizer sharded state of type dp_reshardable + scheduled an async checkpoint save at iteration 4800 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints + successfully saved checkpoint from iteration 4800 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints [ t 1/1, p 1/1 ] + successfully deleted checkpoint from iteration 4760 at /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints +saving checkpoint at iteration 4840 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints in torch_dist format +Storing distributed optimizer sharded state of type dp_reshardable + scheduled an async checkpoint save at iteration 4840 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints + successfully saved checkpoint from iteration 4840 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints [ t 1/1, p 1/1 ] + successfully deleted checkpoint from iteration 4800 at /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints +saving checkpoint at iteration 4880 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints in torch_dist format +Storing distributed optimizer sharded state of type dp_reshardable + scheduled an async checkpoint save at iteration 4880 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints + successfully saved checkpoint from iteration 4880 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints [ t 1/1, p 1/1 ] + successfully deleted checkpoint from iteration 4840 at /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints +saving checkpoint at iteration 4920 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints in torch_dist format +Storing distributed optimizer sharded state of type dp_reshardable + scheduled an async checkpoint save at iteration 4920 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints + successfully saved checkpoint from iteration 4920 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints [ t 1/1, p 1/1 ] + successfully deleted checkpoint from iteration 4880 at /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints +saving checkpoint at iteration 4960 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints in torch_dist format +Storing distributed optimizer sharded state of type dp_reshardable + scheduled an async checkpoint save at iteration 4960 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints + successfully saved checkpoint from iteration 4960 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints [ t 1/1, p 1/1 ] + successfully deleted checkpoint from iteration 4920 at /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints +saving checkpoint at iteration 5000 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints in torch_dist format +Storing distributed optimizer sharded state of type dp_reshardable + scheduled an async checkpoint save at iteration 5000 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints +Iter 5000 increase timing-log-level to 2. + successfully saved checkpoint from iteration 5000 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints [ t 1/1, p 1/1 ] + successfully deleted checkpoint from iteration 4960 at /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints +Iter 5005 reset timing-log-level. checkpoint path: /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints; tensorboard path: /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/tensorboard +saving checkpoint at iteration 5040 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints in torch_dist format +Storing distributed optimizer sharded state of type dp_reshardable + scheduled an async checkpoint save at iteration 5040 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints + successfully saved checkpoint from iteration 5040 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints [ t 1/1, p 1/1 ] + successfully deleted checkpoint from iteration 5000 at /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints +saving checkpoint at iteration 5080 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints in torch_dist format +Storing distributed optimizer sharded state of type dp_reshardable + scheduled an async checkpoint save at iteration 5080 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints + successfully saved checkpoint from iteration 5080 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints [ t 1/1, p 1/1 ] + successfully deleted checkpoint from iteration 5040 at /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints +saving checkpoint at iteration 5120 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints in torch_dist format +Storing distributed optimizer sharded state of type dp_reshardable + scheduled an async checkpoint save at iteration 5120 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints + successfully saved checkpoint from iteration 5120 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints [ t 1/1, p 1/1 ] + successfully deleted checkpoint from iteration 5080 at /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints +saving checkpoint at iteration 5160 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints in torch_dist format +Storing distributed optimizer sharded state of type dp_reshardable + scheduled an async checkpoint save at iteration 5160 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints + successfully saved checkpoint from iteration 5160 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints [ t 1/1, p 1/1 ] + successfully deleted checkpoint from iteration 5120 at /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints +saving checkpoint at iteration 5200 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints in torch_dist format +Storing distributed optimizer sharded state of type dp_reshardable + scheduled an async checkpoint save at iteration 5200 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints + successfully saved checkpoint from iteration 5200 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints [ t 1/1, p 1/1 ] + successfully deleted checkpoint from iteration 5160 at /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints +saving checkpoint at iteration 5240 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints in torch_dist format +Storing distributed optimizer sharded state of type dp_reshardable + scheduled an async checkpoint save at iteration 5240 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints + successfully saved checkpoint from iteration 5240 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints [ t 1/1, p 1/1 ] + successfully deleted checkpoint from iteration 5200 at /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints +saving checkpoint at iteration 5280 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints in torch_dist format +Storing distributed optimizer sharded state of type dp_reshardable + scheduled an async checkpoint save at iteration 5280 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints + successfully saved checkpoint from iteration 5280 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints [ t 1/1, p 1/1 ] + successfully deleted checkpoint from iteration 5240 at /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints +saving checkpoint at iteration 5320 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints in torch_dist format +Storing distributed optimizer sharded state of type dp_reshardable + scheduled an async checkpoint save at iteration 5320 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints + successfully saved checkpoint from iteration 5320 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints [ t 1/1, p 1/1 ] + successfully deleted checkpoint from iteration 5280 at /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints +saving checkpoint at iteration 5360 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints in torch_dist format +Storing distributed optimizer sharded state of type dp_reshardable + scheduled an async checkpoint save at iteration 5360 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints + successfully saved checkpoint from iteration 5360 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints [ t 1/1, p 1/1 ] + successfully deleted checkpoint from iteration 5320 at /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints +saving checkpoint at iteration 5400 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints in torch_dist format +Storing distributed optimizer sharded state of type dp_reshardable + scheduled an async checkpoint save at iteration 5400 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints + successfully saved checkpoint from iteration 5400 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints [ t 1/1, p 1/1 ] + successfully deleted checkpoint from iteration 5360 at /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints +saving checkpoint at iteration 5440 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints in torch_dist format +Storing distributed optimizer sharded state of type dp_reshardable + scheduled an async checkpoint save at iteration 5440 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints + successfully saved checkpoint from iteration 5440 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints [ t 1/1, p 1/1 ] + successfully deleted checkpoint from iteration 5400 at /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints +saving checkpoint at iteration 5480 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints in torch_dist format +Storing distributed optimizer sharded state of type dp_reshardable + scheduled an async checkpoint save at iteration 5480 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints + successfully saved checkpoint from iteration 5480 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints [ t 1/1, p 1/1 ] + successfully deleted checkpoint from iteration 5440 at /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints +saving checkpoint at iteration 5520 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints in torch_dist format +Storing distributed optimizer sharded state of type dp_reshardable + scheduled an async checkpoint save at iteration 5520 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints + successfully saved checkpoint from iteration 5520 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints [ t 1/1, p 1/1 ] + successfully deleted checkpoint from iteration 5480 at /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints +saving checkpoint at iteration 5560 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints in torch_dist format +Storing distributed optimizer sharded state of type dp_reshardable + scheduled an async checkpoint save at iteration 5560 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints + successfully saved checkpoint from iteration 5560 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints [ t 1/1, p 1/1 ] + successfully deleted checkpoint from iteration 5520 at /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints +saving checkpoint at iteration 5600 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints in torch_dist format +Storing distributed optimizer sharded state of type dp_reshardable + scheduled an async checkpoint save at iteration 5600 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints + successfully saved checkpoint from iteration 5600 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints [ t 1/1, p 1/1 ] + successfully deleted checkpoint from iteration 5560 at /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints +saving checkpoint at iteration 5640 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints in torch_dist format +Storing distributed optimizer sharded state of type dp_reshardable + scheduled an async checkpoint save at iteration 5640 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints + successfully saved checkpoint from iteration 5640 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints [ t 1/1, p 1/1 ] + successfully deleted checkpoint from iteration 5600 at /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints +saving checkpoint at iteration 5680 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints in torch_dist format +Storing distributed optimizer sharded state of type dp_reshardable + scheduled an async checkpoint save at iteration 5680 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints + successfully saved checkpoint from iteration 5680 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints [ t 1/1, p 1/1 ] + successfully deleted checkpoint from iteration 5640 at /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints +saving checkpoint at iteration 5720 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints in torch_dist format +Storing distributed optimizer sharded state of type dp_reshardable + scheduled an async checkpoint save at iteration 5720 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints + successfully saved checkpoint from iteration 5720 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints [ t 1/1, p 1/1 ] + successfully deleted checkpoint from iteration 5680 at /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints +saving checkpoint at iteration 5760 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints in torch_dist format +Storing distributed optimizer sharded state of type dp_reshardable + scheduled an async checkpoint save at iteration 5760 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints + successfully saved checkpoint from iteration 5760 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints [ t 1/1, p 1/1 ] + successfully deleted checkpoint from iteration 5720 at /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints +saving checkpoint at iteration 5800 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints in torch_dist format +Storing distributed optimizer sharded state of type dp_reshardable + scheduled an async checkpoint save at iteration 5800 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints + successfully saved checkpoint from iteration 5800 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints [ t 1/1, p 1/1 ] + successfully deleted checkpoint from iteration 5760 at /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints +saving checkpoint at iteration 5840 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints in torch_dist format +Storing distributed optimizer sharded state of type dp_reshardable + scheduled an async checkpoint save at iteration 5840 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints + successfully saved checkpoint from iteration 5840 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints [ t 1/1, p 1/1 ] + successfully deleted checkpoint from iteration 5800 at /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints +saving checkpoint at iteration 5880 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints in torch_dist format +Storing distributed optimizer sharded state of type dp_reshardable + scheduled an async checkpoint save at iteration 5880 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints + successfully saved checkpoint from iteration 5880 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints [ t 1/1, p 1/1 ] + successfully deleted checkpoint from iteration 5840 at /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints +saving checkpoint at iteration 5920 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints in torch_dist format +Storing distributed optimizer sharded state of type dp_reshardable + scheduled an async checkpoint save at iteration 5920 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints + successfully saved checkpoint from iteration 5920 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints [ t 1/1, p 1/1 ] + successfully deleted checkpoint from iteration 5880 at /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints +saving checkpoint at iteration 5960 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints in torch_dist format +Storing distributed optimizer sharded state of type dp_reshardable + scheduled an async checkpoint save at iteration 5960 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints + successfully saved checkpoint from iteration 5960 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints [ t 1/1, p 1/1 ] + successfully deleted checkpoint from iteration 5920 at /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints +saving checkpoint at iteration 6000 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints in torch_dist format +Storing distributed optimizer sharded state of type dp_reshardable + scheduled an async checkpoint save at iteration 6000 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints +Iter 6000 increase timing-log-level to 2. + successfully saved checkpoint from iteration 6000 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints [ t 1/1, p 1/1 ] + successfully deleted checkpoint from iteration 5960 at /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints +Iter 6005 reset timing-log-level. checkpoint path: /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints; tensorboard path: /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/tensorboard +saving checkpoint at iteration 6040 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints in torch_dist format +Storing distributed optimizer sharded state of type dp_reshardable + scheduled an async checkpoint save at iteration 6040 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints + successfully saved checkpoint from iteration 6040 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints [ t 1/1, p 1/1 ] + successfully deleted checkpoint from iteration 6000 at /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints +saving checkpoint at iteration 6080 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints in torch_dist format +Storing distributed optimizer sharded state of type dp_reshardable + scheduled an async checkpoint save at iteration 6080 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints + successfully saved checkpoint from iteration 6080 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints [ t 1/1, p 1/1 ] + successfully deleted checkpoint from iteration 6040 at /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints +saving checkpoint at iteration 6120 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints in torch_dist format +Storing distributed optimizer sharded state of type dp_reshardable + scheduled an async checkpoint save at iteration 6120 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints + successfully saved checkpoint from iteration 6120 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints [ t 1/1, p 1/1 ] + successfully deleted checkpoint from iteration 6080 at /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints +saving checkpoint at iteration 6160 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints in torch_dist format +Storing distributed optimizer sharded state of type dp_reshardable + scheduled an async checkpoint save at iteration 6160 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints + successfully saved checkpoint from iteration 6160 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints [ t 1/1, p 1/1 ] + successfully deleted checkpoint from iteration 6120 at /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints +saving checkpoint at iteration 6200 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints in torch_dist format +Storing distributed optimizer sharded state of type dp_reshardable + scheduled an async checkpoint save at iteration 6200 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints + successfully saved checkpoint from iteration 6200 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints [ t 1/1, p 1/1 ] + successfully deleted checkpoint from iteration 6160 at /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints +saving checkpoint at iteration 6240 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints in torch_dist format +Storing distributed optimizer sharded state of type dp_reshardable + scheduled an async checkpoint save at iteration 6240 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints + successfully saved checkpoint from iteration 6240 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints [ t 1/1, p 1/1 ] + successfully deleted checkpoint from iteration 6200 at /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints +saving checkpoint at iteration 6280 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints in torch_dist format +Storing distributed optimizer sharded state of type dp_reshardable + scheduled an async checkpoint save at iteration 6280 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints + successfully saved checkpoint from iteration 6280 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints [ t 1/1, p 1/1 ] + successfully deleted checkpoint from iteration 6240 at /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints +saving checkpoint at iteration 6320 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints in torch_dist format +Storing distributed optimizer sharded state of type dp_reshardable + scheduled an async checkpoint save at iteration 6320 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints + successfully saved checkpoint from iteration 6320 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints [ t 1/1, p 1/1 ] + successfully deleted checkpoint from iteration 6280 at /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints +saving checkpoint at iteration 6360 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints in torch_dist format +Storing distributed optimizer sharded state of type dp_reshardable + scheduled an async checkpoint save at iteration 6360 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints + successfully saved checkpoint from iteration 6360 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints [ t 1/1, p 1/1 ] + successfully deleted checkpoint from iteration 6320 at /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints +saving checkpoint at iteration 6400 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints in torch_dist format +Storing distributed optimizer sharded state of type dp_reshardable + scheduled an async checkpoint save at iteration 6400 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints + successfully saved checkpoint from iteration 6400 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints [ t 1/1, p 1/1 ] + successfully deleted checkpoint from iteration 6360 at /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints +saving checkpoint at iteration 6440 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints in torch_dist format +Storing distributed optimizer sharded state of type dp_reshardable + scheduled an async checkpoint save at iteration 6440 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints + successfully saved checkpoint from iteration 6440 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints [ t 1/1, p 1/1 ] + successfully deleted checkpoint from iteration 6400 at /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints +saving checkpoint at iteration 6480 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints in torch_dist format +Storing distributed optimizer sharded state of type dp_reshardable + scheduled an async checkpoint save at iteration 6480 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints + successfully saved checkpoint from iteration 6480 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints [ t 1/1, p 1/1 ] + successfully deleted checkpoint from iteration 6440 at /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints +saving checkpoint at iteration 6520 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints in torch_dist format +Storing distributed optimizer sharded state of type dp_reshardable + scheduled an async checkpoint save at iteration 6520 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints + successfully saved checkpoint from iteration 6520 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints [ t 1/1, p 1/1 ] + successfully deleted checkpoint from iteration 6480 at /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints +saving checkpoint at iteration 6560 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints in torch_dist format +Storing distributed optimizer sharded state of type dp_reshardable + scheduled an async checkpoint save at iteration 6560 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints + successfully saved checkpoint from iteration 6560 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints [ t 1/1, p 1/1 ] + successfully deleted checkpoint from iteration 6520 at /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints +saving checkpoint at iteration 6600 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints in torch_dist format +Storing distributed optimizer sharded state of type dp_reshardable + scheduled an async checkpoint save at iteration 6600 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints + successfully saved checkpoint from iteration 6600 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints [ t 1/1, p 1/1 ] + successfully deleted checkpoint from iteration 6560 at /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints +saving checkpoint at iteration 6640 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints in torch_dist format +Storing distributed optimizer sharded state of type dp_reshardable + scheduled an async checkpoint save at iteration 6640 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints + successfully saved checkpoint from iteration 6640 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints [ t 1/1, p 1/1 ] + successfully deleted checkpoint from iteration 6600 at /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints +saving checkpoint at iteration 6680 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints in torch_dist format +Storing distributed optimizer sharded state of type dp_reshardable + scheduled an async checkpoint save at iteration 6680 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints + successfully saved checkpoint from iteration 6680 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints [ t 1/1, p 1/1 ] + successfully deleted checkpoint from iteration 6640 at /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints +saving checkpoint at iteration 6720 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints in torch_dist format +Storing distributed optimizer sharded state of type dp_reshardable + scheduled an async checkpoint save at iteration 6720 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints + successfully saved checkpoint from iteration 6720 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints [ t 1/1, p 1/1 ] + successfully deleted checkpoint from iteration 6680 at /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints +saving checkpoint at iteration 6760 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints in torch_dist format +Storing distributed optimizer sharded state of type dp_reshardable + scheduled an async checkpoint save at iteration 6760 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints + successfully saved checkpoint from iteration 6760 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints [ t 1/1, p 1/1 ] + successfully deleted checkpoint from iteration 6720 at /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints +saving checkpoint at iteration 6800 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints in torch_dist format +Storing distributed optimizer sharded state of type dp_reshardable + scheduled an async checkpoint save at iteration 6800 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints + successfully saved checkpoint from iteration 6800 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints [ t 1/1, p 1/1 ] + successfully deleted checkpoint from iteration 6760 at /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints +saving checkpoint at iteration 6840 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints in torch_dist format +Storing distributed optimizer sharded state of type dp_reshardable + scheduled an async checkpoint save at iteration 6840 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints + successfully saved checkpoint from iteration 6840 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints [ t 1/1, p 1/1 ] + successfully deleted checkpoint from iteration 6800 at /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints +[after training is done] datetime: 2026-10-10 20:21:09 +saving checkpoint at iteration 6855 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints in torch_dist format +Storing distributed optimizer sharded state of type dp_reshardable + scheduled an async checkpoint save at iteration 6855 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints +WARNING:megatron.core.rerun_state_machine:Setting RerunStateMachine mode disabled +Evaluating on 1280 samples +Evaluating iter 1/10 +Evaluating iter 2/10 +Evaluating iter 3/10 +Evaluating iter 4/10 +Evaluating iter 5/10 +Evaluating iter 6/10 +Evaluating iter 7/10 +Evaluating iter 8/10 +Evaluating iter 9/10 +Evaluating iter 10/10 +WARNING:megatron.core.rerun_state_machine:Setting RerunStateMachine mode validate_results +WARNING:megatron.core.rerun_state_machine:Setting RerunStateMachine mode validate_results +Unfinalized async checkpoint saves. Finalizing them synchronously now. + successfully saved checkpoint from iteration 6855 to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints [ t 1/1, p 1/1 ] + successfully deleted checkpoint from iteration 6840 at /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints +[rank5]:[W1010 20:26:00.740164833 ProcessGroupNCCL.cpp:1538] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank6]:[W1010 20:26:01.853699398 ProcessGroupNCCL.cpp:1538] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank2]:[W1010 20:26:01.894177739 ProcessGroupNCCL.cpp:1538] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank3]:[W1010 20:26:01.904558826 ProcessGroupNCCL.cpp:1538] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank7]:[W1010 20:26:01.998760598 ProcessGroupNCCL.cpp:1538] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank1]:[W1010 20:26:01.214401982 ProcessGroupNCCL.cpp:1538] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank4]:[W1010 20:26:01.215443946 ProcessGroupNCCL.cpp:1538] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank0]:[W1010 20:26:01.338442513 ProcessGroupNCCL.cpp:1538] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/node1.log b/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/node1.log new file mode 100644 index 0000000000000000000000000000000000000000..367004b5033ab6ab41ae504064f3dc14e2013835 --- /dev/null +++ b/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/node1.log @@ -0,0 +1,358 @@ +W1003 15:36:37.868000 64347 torch/utils/cpp_extension.py:2425] TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +W1003 15:36:37.868000 64347 torch/utils/cpp_extension.py:2425] If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'] to specific architectures. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +[Gloo] Rank 14 is connected to 63 peer ranks. Expected number of connected peer ranks is : 63 +[rank14]:[W1003 15:37:33.861166768 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank14]:[W1003 15:37:33.861185014 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[Gloo] Rank 10 is connected to 63 peer ranks. Expected number of connected peer ranks is : 63 +[rank10]:[W1003 15:37:33.873324134 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank10]:[W1003 15:37:33.873341775 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[Gloo] Rank 13 is connected to 63 peer ranks. Expected number of connected peer ranks is : 63 +[rank13]:[W1003 15:37:33.874984582 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank13]:[W1003 15:37:33.875000094 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[Gloo] Rank 11 is connected to 63 peer ranks. Expected number of connected peer ranks is : 63 +[rank11]:[W1003 15:37:33.875164483 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank11]:[W1003 15:37:33.875180257 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[Gloo] Rank 9 is connected to 63 peer ranks. Expected number of connected peer ranks is : 63 +[Gloo] Rank [rank9]:[W1003 15:37:33.875518601 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +8[rank9]:[W1003 15:37:33.875554655 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[Gloo] Rank is connected to 1563 is connected to peer ranks. 63Expected number of connected peer ranks is : peer ranks. 63Expected number of connected peer ranks is : +63 +[rank8]:[W1003 15:37:33.875736208 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank8]:[W1003 15:37:33.875752761 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank15]:[W1003 15:37:33.875749107 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank15]:[W1003 15:37:33.875771858 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[Gloo] Rank 12 is connected to 63 peer ranks. Expected number of connected peer ranks is : 63 +[rank12]:[W1003 15:37:33.876077971 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank12]:[W1003 15:37:33.876095184 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 1[Gloo] Rank is connected to 1[Gloo] Rank 7 is connected to 1 peer ranks. 7 is connected to Expected number of connected peer ranks is : peer ranks. 77Expected number of connected peer ranks is : peer ranks. +7Expected number of connected peer ranks is : [Gloo] Rank +71 + is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. + warnings.warn( # warn only once +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. + warnings.warn( # warn only once +[rank13]:[W1003 15:37:33.024895050 ProcessGroupNCCL.cpp:5023] [PG ID 0 PG GUID 0 Rank 13] using GPU 5 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank9]:[W1003 15:37:33.024944742 ProcessGroupNCCL.cpp:5023] [PG ID 0 PG GUID 0 Rank 9] using GPU 1 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. + warnings.warn( # warn only once +[rank15]:[W1003 15:37:33.025160662 ProcessGroupNCCL.cpp:5023] [PG ID 0 PG GUID 0 Rank 15] using GPU 7 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. + warnings.warn( # warn only once +[rank10]:[W1003 15:37:33.026523519 ProcessGroupNCCL.cpp:5023] [PG ID 0 PG GUID 0 Rank 10] using GPU 2 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. + warnings.warn( # warn only once +[rank8]:[W1003 15:37:33.027040814 ProcessGroupNCCL.cpp:5023] [PG ID 0 PG GUID 0 Rank 8] using GPU 0 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. + warnings.warn( # warn only once +[rank12]:[W1003 15:37:33.027758971 ProcessGroupNCCL.cpp:5023] [PG ID 0 PG GUID 0 Rank 12] using GPU 4 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. + warnings.warn( # warn only once +[rank14]:[W1003 15:37:33.027927230 ProcessGroupNCCL.cpp:5023] [PG ID 0 PG GUID 0 Rank 14] using GPU 6 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. + warnings.warn( # warn only once +[rank11]:[W1003 15:37:33.031589527 ProcessGroupNCCL.cpp:5023] [PG ID 0 PG GUID 0 Rank 11] using GPU 3 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank9]:[W1003 15:37:34.022751993 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank10]:[W1003 15:37:34.022778240 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank12]:[W1003 15:37:34.022793467 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank9]:[W1003 15:37:34.022812281 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank15]:[W1003 15:37:34.022822373 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank14]:[W1003 15:37:34.022815044 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank10]:[W1003 15:37:34.022843864 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank11]:[W1003 15:37:34.022849130 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank12]:[W1003 15:37:34.022878347 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank15]:[W1003 15:37:34.022907833 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank14]:[W1003 15:37:34.022939900 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank11]:[W1003 15:37:34.022987935 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank8]:[W1003 15:37:34.023017825 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank8]:[W1003 15:37:34.023056088 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank13]:[W1003 15:37:34.026062685 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank13]:[W1003 15:37:34.026094665 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + + + + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + + + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + + +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/dist_checkpointing/strategies/torch.py:956: FutureWarning: `load_state_dict` is deprecated and will be removed in future versions. Please use `load` instead. + checkpoint.load_state_dict( +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/dist_checkpointing/strategies/torch.py:956: FutureWarning: `load_state_dict` is deprecated and will be removed in future versions. Please use `load` instead. + checkpoint.load_state_dict( +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/dist_checkpointing/strategies/torch.py:956: FutureWarning: `load_state_dict` is deprecated and will be removed in future versions. Please use `load` instead. + checkpoint.load_state_dict( +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/dist_checkpointing/strategies/torch.py:956: FutureWarning: `load_state_dict` is deprecated and will be removed in future versions. Please use `load` instead. + checkpoint.load_state_dict( +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/dist_checkpointing/strategies/torch.py:956: FutureWarning: `load_state_dict` is deprecated and will be removed in future versions. Please use `load` instead. + checkpoint.load_state_dict( +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/dist_checkpointing/strategies/torch.py:956: FutureWarning: `load_state_dict` is deprecated and will be removed in future versions. Please use `load` instead. + checkpoint.load_state_dict( +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/dist_checkpointing/strategies/torch.py:956: FutureWarning: `load_state_dict` is deprecated and will be removed in future versions. Please use `load` instead. + checkpoint.load_state_dict( +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/dist_checkpointing/strategies/torch.py:956: FutureWarning: `load_state_dict` is deprecated and will be removed in future versions. Please use `load` instead. + checkpoint.load_state_dict( +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +NCCL version 2.27.3+cuda12.9 +Initializing GlobIndexedDataset +Initializing GlobIndexedDataset +Initializing GlobIndexedDataset +Initializing GlobIndexedDataset +Initializing GlobIndexedDataset +Initializing GlobIndexedDataset +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_0_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_10_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_11_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_12_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_13_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_14_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_15_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_16_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_17_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_18_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_19_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_1_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_20_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_21_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_22_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_23_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_24_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_25_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_26_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_27_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_28_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_29_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_2_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_30_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_31_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_32_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_33_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_34_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_35_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_36_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_37_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_38_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_39_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_3_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_40_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_41_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_42_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_43_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_44_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_45_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_46_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_47_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_48_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_49_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_4_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_50_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_51_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_52_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_53_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_54_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_55_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_56_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_57_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_58_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_59_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_5_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_60_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_61_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_62_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_63_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_64_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_65_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_66_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_67_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_68_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_69_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_6_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_70_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_71_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_72_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_73_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_74_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_75_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_76_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_77_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_78_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_7_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_8_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_9_input_ids_tmp']['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_0_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_10_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_11_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_12_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_13_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_14_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_15_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_16_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_17_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_18_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_19_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_1_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_20_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_21_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_22_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_23_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_24_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_25_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_26_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_27_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_28_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_29_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_2_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_30_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_31_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_32_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_33_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_34_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_35_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_36_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_37_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_38_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_39_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_3_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_40_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_41_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_42_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_43_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_44_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_45_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_46_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_47_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_48_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_49_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_4_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_50_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_51_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_52_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_53_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_54_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_55_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_56_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_57_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_58_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_59_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_5_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_60_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_61_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_62_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_63_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_64_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_65_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_66_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_67_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_68_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_69_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_6_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_70_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_71_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_72_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_73_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_74_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_75_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_76_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_77_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_78_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_7_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_8_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_9_input_ids_tmp']['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_0_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_10_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_11_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_12_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_13_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_14_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_15_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_16_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_17_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_18_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_19_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_1_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_20_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_21_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_22_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_23_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_24_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_25_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_26_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_27_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_28_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_29_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_2_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_30_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_31_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_32_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_33_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_34_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_35_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_36_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_37_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_38_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_39_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_3_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_40_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_41_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_42_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_43_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_44_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_45_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_46_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_47_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_48_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_49_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_4_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_50_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_51_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_52_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_53_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_54_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_55_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_56_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_57_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_58_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_59_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_5_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_60_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_61_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_62_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_63_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_64_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_65_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_66_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_67_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_68_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_69_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_6_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_70_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_71_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_72_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_73_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_74_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_75_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_76_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_77_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_78_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_7_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_8_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_9_input_ids_tmp']['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_0_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_10_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_11_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_12_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_13_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_14_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_15_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_16_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_17_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_18_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_19_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_1_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_20_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_21_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_22_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_23_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_24_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_25_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_26_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_27_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_28_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_29_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_2_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_30_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_31_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_32_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_33_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_34_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_35_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_36_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_37_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_38_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_39_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_3_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_40_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_41_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_42_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_43_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_44_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_45_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_46_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_47_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_48_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_49_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_4_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_50_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_51_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_52_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_53_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_54_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_55_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_56_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_57_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_58_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_59_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_5_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_60_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_61_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_62_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_63_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_64_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_65_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_66_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_67_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_68_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_69_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_6_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_70_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_71_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_72_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_73_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_74_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_75_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_76_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_77_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_78_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_7_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_8_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_9_input_ids_tmp'] + + + +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_0_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_10_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_11_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_12_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_13_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_14_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_15_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_16_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_17_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_18_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_19_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_1_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_20_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_21_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_22_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_23_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_24_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_25_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_26_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_27_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_28_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_29_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_2_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_30_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_31_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_32_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_33_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_34_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_35_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_36_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_37_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_38_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_39_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_3_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_40_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_41_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_42_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_43_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_44_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_45_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_46_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_47_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_48_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_49_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_4_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_50_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_51_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_52_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_53_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_54_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_55_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_56_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_57_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_58_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_59_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_5_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_60_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_61_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_62_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_63_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_64_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_65_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_66_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_67_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_68_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_69_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_6_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_70_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_71_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_72_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_73_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_74_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_75_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_76_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_77_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_78_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_7_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_8_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_9_input_ids_tmp']['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_0_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_10_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_11_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_12_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_13_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_14_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_15_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_16_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_17_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_18_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_19_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_1_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_20_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_21_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_22_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_23_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_24_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_25_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_26_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_27_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_28_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_29_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_2_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_30_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_31_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_32_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_33_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_34_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_35_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_36_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_37_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_38_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_39_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_3_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_40_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_41_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_42_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_43_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_44_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_45_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_46_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_47_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_48_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_49_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_4_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_50_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_51_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_52_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_53_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_54_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_55_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_56_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_57_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_58_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_59_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_5_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_60_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_61_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_62_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_63_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_64_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_65_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_66_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_67_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_68_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_69_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_6_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_70_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_71_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_72_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_73_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_74_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_75_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_76_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_77_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_78_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_7_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_8_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_9_input_ids_tmp'] + +Initializing GlobIndexedDataset +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_0_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_10_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_11_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_12_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_13_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_14_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_15_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_16_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_17_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_18_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_19_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_1_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_20_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_21_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_22_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_23_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_24_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_25_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_26_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_27_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_28_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_29_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_2_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_30_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_31_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_32_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_33_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_34_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_35_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_36_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_37_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_38_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_39_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_3_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_40_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_41_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_42_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_43_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_44_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_45_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_46_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_47_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_48_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_49_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_4_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_50_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_51_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_52_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_53_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_54_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_55_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_56_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_57_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_58_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_59_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_5_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_60_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_61_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_62_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_63_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_64_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_65_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_66_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_67_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_68_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_69_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_6_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_70_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_71_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_72_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_73_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_74_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_75_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_76_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_77_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_78_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_7_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_8_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_9_input_ids_tmp'] +Initializing GlobIndexedDataset +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_0_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_10_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_11_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_12_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_13_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_14_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_15_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_16_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_17_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_18_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_19_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_1_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_20_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_21_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_22_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_23_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_24_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_25_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_26_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_27_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_28_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_29_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_2_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_30_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_31_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_32_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_33_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_34_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_35_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_36_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_37_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_38_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_39_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_3_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_40_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_41_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_42_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_43_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_44_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_45_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_46_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_47_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_48_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_49_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_4_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_50_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_51_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_52_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_53_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_54_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_55_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_56_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_57_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_58_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_59_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_5_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_60_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_61_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_62_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_63_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_64_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_65_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_66_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_67_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_68_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_69_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_6_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_70_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_71_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_72_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_73_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_74_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_75_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_76_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_77_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_78_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_7_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_8_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_9_input_ids_tmp'] +>>> start_samples: 0>>> start_samples: 0 +>>> start_samples: 0 +>>> start_samples: 0 +>>> start_samples: 0 +>>> start_samples: 0 +>>> start_samples: 0 +>>> start_samples: 0 +Initializing GlobIndexedDataset + +Initializing GlobIndexedDataset +Initializing GlobIndexedDataset +Initializing GlobIndexedDataset +Initializing GlobIndexedDataset +Initializing GlobIndexedDataset +Initializing GlobIndexedDataset +Initializing GlobIndexedDataset +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_75_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_76_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_77_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_78_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_9_loss_mask_tmp'] +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_75_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_76_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_77_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_78_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_9_loss_mask_tmp']['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_75_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_76_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_77_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_78_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_9_loss_mask_tmp'] + +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_75_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_76_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_77_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_78_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_9_loss_mask_tmp']['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_75_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_76_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_77_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_78_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_9_loss_mask_tmp'] + +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_75_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_76_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_77_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_78_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_9_loss_mask_tmp'] +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_75_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_76_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_77_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_78_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_9_loss_mask_tmp']['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_75_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_76_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_77_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_78_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_9_loss_mask_tmp'] + +pad_document_marker: 36616095 +pad_document_marker: 36616095 +pad_document_marker: 36616095 +/mnt/hdd/cpt/binidx_cache_stage15_final_20261001_v2 d6c36c6d5886334ea5f92e75fcf45d10-GPTDataset-train-obfd cache hit: True +/mnt/hdd/cpt/binidx_cache_stage15_final_20261001_v2 d6c36c6d5886334ea5f92e75fcf45d10-GPTDataset-train-obfd cache hit: True/mnt/hdd/cpt/binidx_cache_stage15_final_20261001_v2 d6c36c6d5886334ea5f92e75fcf45d10-GPTDataset-train-obfd cache hit: True + +pad_document_marker: 36616095 +pad_document_marker: 36616095 +/mnt/hdd/cpt/binidx_cache_stage15_final_20261001_v2 d6c36c6d5886334ea5f92e75fcf45d10-GPTDataset-train-obfd cache hit: True +/mnt/hdd/cpt/binidx_cache_stage15_final_20261001_v2 d6c36c6d5886334ea5f92e75fcf45d10-GPTDataset-train-obfd cache hit: True +pad_document_marker: 36616095 +/mnt/hdd/cpt/binidx_cache_stage15_final_20261001_v2 d6c36c6d5886334ea5f92e75fcf45d10-GPTDataset-train-obfd cache hit: True +pad_document_marker: 36616095 +pad_document_marker: 36616095 +/mnt/hdd/cpt/binidx_cache_stage15_final_20261001_v2 d6c36c6d5886334ea5f92e75fcf45d10-GPTDataset-train-obfd cache hit: True +/mnt/hdd/cpt/binidx_cache_stage15_final_20261001_v2 d6c36c6d5886334ea5f92e75fcf45d10-GPTDataset-train-obfd cache hit: True +>>> start_samples: 0>>> start_samples: 0>>> start_samples: 0>>> start_samples: 0>>> start_samples: 0>>> start_samples: 0 + + + + + +>>> start_samples: 0 +>>> start_samples: 0 +Initializing GlobIndexedDatasetInitializing GlobIndexedDatasetInitializing GlobIndexedDatasetInitializing GlobIndexedDataset +Initializing GlobIndexedDataset +Initializing GlobIndexedDataset + + + +Initializing GlobIndexedDataset +Initializing GlobIndexedDataset +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_75_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_76_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_77_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_78_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_9_loss_mask_tmp']['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_75_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_76_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_77_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_78_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_9_loss_mask_tmp']['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_75_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_76_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_77_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_78_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_9_loss_mask_tmp']['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_75_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_76_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_77_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_78_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_9_loss_mask_tmp'] +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_75_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_76_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_77_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_78_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_9_loss_mask_tmp'] +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_75_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_76_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_77_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_78_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_9_loss_mask_tmp']['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_75_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_76_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_77_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_78_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_9_loss_mask_tmp'] +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_75_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_76_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_77_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_78_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_9_loss_mask_tmp'] + + + + +/mnt/hdd/cpt/binidx_cache_stage15_final_20261001_v2 0642c625420be1073f42e03d3641a6e5-GPTDataset-valid-random cache hit: True +/mnt/hdd/cpt/binidx_cache_stage15_final_20261001_v2 0642c625420be1073f42e03d3641a6e5-GPTDataset-valid-random cache hit: True/mnt/hdd/cpt/binidx_cache_stage15_final_20261001_v2 0642c625420be1073f42e03d3641a6e5-GPTDataset-valid-random cache hit: True + +/mnt/hdd/cpt/binidx_cache_stage15_final_20261001_v2 0642c625420be1073f42e03d3641a6e5-GPTDataset-valid-random cache hit: True +/mnt/hdd/cpt/binidx_cache_stage15_final_20261001_v2 0642c625420be1073f42e03d3641a6e5-GPTDataset-valid-random cache hit: True +/mnt/hdd/cpt/binidx_cache_stage15_final_20261001_v2 0642c625420be1073f42e03d3641a6e5-GPTDataset-valid-random cache hit: True +/mnt/hdd/cpt/binidx_cache_stage15_final_20261001_v2 0642c625420be1073f42e03d3641a6e5-GPTDataset-valid-random cache hit: True +/mnt/hdd/cpt/binidx_cache_stage15_final_20261001_v2 0642c625420be1073f42e03d3641a6e5-GPTDataset-valid-random cache hit: True +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/optimizer/optimizer.py:65: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /pytorch/build/aten/src/ATen/core/TensorBody.h:489.) + if hasattr(param, grad_attr) and getattr(param, grad_attr) is not None: +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/optimizer/optimizer.py:65: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /pytorch/build/aten/src/ATen/core/TensorBody.h:489.) + if hasattr(param, grad_attr) and getattr(param, grad_attr) is not None: +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/optimizer/optimizer.py:65: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /pytorch/build/aten/src/ATen/core/TensorBody.h:489.) + if hasattr(param, grad_attr) and getattr(param, grad_attr) is not None: +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/optimizer/optimizer.py:65: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /pytorch/build/aten/src/ATen/core/TensorBody.h:489.) + if hasattr(param, grad_attr) and getattr(param, grad_attr) is not None: +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/optimizer/optimizer.py:65: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /pytorch/build/aten/src/ATen/core/TensorBody.h:489.) + if hasattr(param, grad_attr) and getattr(param, grad_attr) is not None: +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/optimizer/optimizer.py:65: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /pytorch/build/aten/src/ATen/core/TensorBody.h:489.) + if hasattr(param, grad_attr) and getattr(param, grad_attr) is not None: +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/optimizer/optimizer.py:65: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /pytorch/build/aten/src/ATen/core/TensorBody.h:489.) + if hasattr(param, grad_attr) and getattr(param, grad_attr) is not None: +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/optimizer/optimizer.py:65: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /pytorch/build/aten/src/ATen/core/TensorBody.h:489.) + if hasattr(param, grad_attr) and getattr(param, grad_attr) is not None: +⚠️ torch 2.8.0+cu128 with triton 3.4.0. inductor patch is not applied. +⚠️ torch 2.8.0+cu128 with triton 3.4.0. inductor patch is not applied. +⚠️ torch 2.8.0+cu128 with triton 3.4.0. inductor patch is not applied. +⚠️ torch 2.8.0+cu128 with triton 3.4.0. inductor patch is not applied. +⚠️ torch 2.8.0+cu128 with triton 3.4.0. inductor patch is not applied. +⚠️ torch 2.8.0+cu128 with triton 3.4.0. inductor patch is not applied. +⚠️ torch 2.8.0+cu128 with triton 3.4.0. inductor patch is not applied. +⚠️ torch 2.8.0+cu128 with triton 3.4.0. inductor patch is not applied. +W1003 16:47:27.314000 21867 torch/utils/cpp_extension.py:2425] TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +W1003 16:47:27.314000 21867 torch/utils/cpp_extension.py:2425] If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'] to specific architectures. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +[rank8]:[W1010 20:26:00.486010148 ProcessGroupNCCL.cpp:1538] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/node2.log b/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/node2.log new file mode 100644 index 0000000000000000000000000000000000000000..d1b2f6a1b92663c3ada729ecfda037ee6114d0b5 --- /dev/null +++ b/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/node2.log @@ -0,0 +1,358 @@ +W1003 15:36:42.996000 460052 torch/utils/cpp_extension.py:2425] TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +W1003 15:36:42.996000 460052 torch/utils/cpp_extension.py:2425] If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'] to specific architectures. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +[Gloo] Rank 16 is connected to 63 peer ranks. Expected number of connected peer ranks is : 63 +[rank16]:[W1003 15:37:33.429406402 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank16]:[W1003 15:37:33.429427965 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[Gloo] Rank 20 is connected to 63 peer ranks. Expected number of connected peer ranks is : 63 +[rank20]:[W1003 15:37:33.452796000 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank20]:[W1003 15:37:33.452813783 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[Gloo] Rank 17 is connected to 63 peer ranks. Expected number of connected peer ranks is : 63 +[rank17]:[W1003 15:37:33.453204760 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank17]:[W1003 15:37:33.453242442 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[Gloo] Rank [Gloo] Rank 2122 is connected to is connected to 6363 peer ranks. peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : 6363 + +[Gloo] Rank 23[rank21]:[W1003 15:37:33.454043148 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) + is connected to [rank22]:[W1003 15:37:33.454050315 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank21]:[W1003 15:37:33.454066470 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +63[rank22]:[W1003 15:37:33.454083271 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) + peer ranks. Expected number of connected peer ranks is : 63 +[Gloo] Rank [rank23]:[W1003 15:37:33.454206840 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +18[rank23]:[W1003 15:37:33.454251189 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) + is connected to 63 peer ranks. Expected number of connected peer ranks is : 63 +[Gloo] Rank 19 is connected to 63 peer ranks. Expected number of connected peer ranks is : 63 +[rank19]:[W1003 15:37:33.454444337 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank18]:[W1003 15:37:33.454444628 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank19]:[W1003 15:37:33.454472478 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank18]:[W1003 15:37:33.454482136 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank [Gloo] Rank 22 is connected to is connected to 77 peer ranks. peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : 7[Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +7 + +[Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. + warnings.warn( # warn only once +[rank23]:[W1003 15:37:33.617658578 ProcessGroupNCCL.cpp:5023] [PG ID 0 PG GUID 0 Rank 23] using GPU 7 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. + warnings.warn( # warn only once +[rank21]:[W1003 15:37:33.617927770 ProcessGroupNCCL.cpp:5023] [PG ID 0 PG GUID 0 Rank 21] using GPU 5 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. + warnings.warn( # warn only once +[rank17]:[W1003 15:37:33.618273471 ProcessGroupNCCL.cpp:5023] [PG ID 0 PG GUID 0 Rank 17] using GPU 1 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. + warnings.warn( # warn only once +[rank22]:[W1003 15:37:33.619575857 ProcessGroupNCCL.cpp:5023] [PG ID 0 PG GUID 0 Rank 22] using GPU 6 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. + warnings.warn( # warn only once +[rank18]:[W1003 15:37:33.620011528 ProcessGroupNCCL.cpp:5023] [PG ID 0 PG GUID 0 Rank 18] using GPU 2 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. + warnings.warn( # warn only once +[rank19]:[W1003 15:37:33.620481331 ProcessGroupNCCL.cpp:5023] [PG ID 0 PG GUID 0 Rank 19] using GPU 3 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. + warnings.warn( # warn only once +[rank16]:[W1003 15:37:33.623798859 ProcessGroupNCCL.cpp:5023] [PG ID 0 PG GUID 0 Rank 16] using GPU 0 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. + warnings.warn( # warn only once +[rank20]:[W1003 15:37:33.626486843 ProcessGroupNCCL.cpp:5023] [PG ID 0 PG GUID 0 Rank 20] using GPU 4 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank22]:[W1003 15:37:34.608181016 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank22]:[W1003 15:37:34.608234326 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank17]:[W1003 15:37:34.608400638 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank17]:[W1003 15:37:34.608453766 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank21]:[W1003 15:37:34.608611700 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank21]:[W1003 15:37:34.608654367 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank23]:[W1003 15:37:34.608838193 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank23]:[W1003 15:37:34.608892922 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank18]:[W1003 15:37:34.608921150 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank20]:[W1003 15:37:34.608937552 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank18]:[W1003 15:37:34.608956907 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank20]:[W1003 15:37:34.608979420 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank16]:[W1003 15:37:34.609488557 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank19]:[W1003 15:37:34.609527531 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank16]:[W1003 15:37:34.609532975 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank19]:[W1003 15:37:34.609555564 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + + + + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/dist_checkpointing/strategies/torch.py:956: FutureWarning: `load_state_dict` is deprecated and will be removed in future versions. Please use `load` instead. + checkpoint.load_state_dict( +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/dist_checkpointing/strategies/torch.py:956: FutureWarning: `load_state_dict` is deprecated and will be removed in future versions. Please use `load` instead. + checkpoint.load_state_dict( +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/dist_checkpointing/strategies/torch.py:956: FutureWarning: `load_state_dict` is deprecated and will be removed in future versions. Please use `load` instead. + checkpoint.load_state_dict( +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/dist_checkpointing/strategies/torch.py:956: FutureWarning: `load_state_dict` is deprecated and will be removed in future versions. Please use `load` instead. + checkpoint.load_state_dict( +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/dist_checkpointing/strategies/torch.py:956: FutureWarning: `load_state_dict` is deprecated and will be removed in future versions. Please use `load` instead. + checkpoint.load_state_dict( +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/dist_checkpointing/strategies/torch.py:956: FutureWarning: `load_state_dict` is deprecated and will be removed in future versions. Please use `load` instead. + checkpoint.load_state_dict( +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/dist_checkpointing/strategies/torch.py:956: FutureWarning: `load_state_dict` is deprecated and will be removed in future versions. Please use `load` instead. + checkpoint.load_state_dict( +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/dist_checkpointing/strategies/torch.py:956: FutureWarning: `load_state_dict` is deprecated and will be removed in future versions. Please use `load` instead. + checkpoint.load_state_dict( +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +NCCL version 2.27.3+cuda12.9 +Initializing GlobIndexedDataset +Initializing GlobIndexedDataset +Initializing GlobIndexedDataset +Initializing GlobIndexedDataset +Initializing GlobIndexedDataset +Initializing GlobIndexedDataset +Initializing GlobIndexedDataset +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_0_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_10_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_11_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_12_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_13_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_14_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_15_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_16_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_17_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_18_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_19_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_1_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_20_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_21_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_22_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_23_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_24_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_25_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_26_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_27_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_28_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_29_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_2_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_30_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_31_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_32_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_33_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_34_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_35_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_36_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_37_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_38_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_39_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_3_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_40_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_41_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_42_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_43_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_44_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_45_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_46_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_47_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_48_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_49_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_4_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_50_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_51_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_52_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_53_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_54_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_55_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_56_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_57_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_58_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_59_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_5_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_60_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_61_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_62_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_63_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_64_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_65_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_66_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_67_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_68_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_69_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_6_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_70_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_71_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_72_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_73_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_74_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_75_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_76_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_77_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_78_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_7_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_8_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_9_input_ids_tmp'] +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_0_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_10_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_11_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_12_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_13_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_14_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_15_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_16_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_17_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_18_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_19_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_1_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_20_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_21_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_22_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_23_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_24_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_25_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_26_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_27_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_28_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_29_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_2_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_30_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_31_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_32_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_33_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_34_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_35_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_36_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_37_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_38_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_39_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_3_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_40_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_41_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_42_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_43_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_44_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_45_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_46_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_47_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_48_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_49_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_4_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_50_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_51_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_52_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_53_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_54_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_55_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_56_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_57_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_58_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_59_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_5_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_60_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_61_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_62_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_63_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_64_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_65_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_66_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_67_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_68_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_69_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_6_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_70_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_71_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_72_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_73_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_74_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_75_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_76_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_77_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_78_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_7_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_8_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_9_input_ids_tmp']['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_0_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_10_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_11_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_12_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_13_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_14_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_15_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_16_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_17_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_18_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_19_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_1_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_20_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_21_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_22_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_23_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_24_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_25_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_26_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_27_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_28_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_29_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_2_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_30_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_31_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_32_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_33_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_34_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_35_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_36_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_37_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_38_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_39_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_3_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_40_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_41_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_42_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_43_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_44_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_45_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_46_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_47_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_48_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_49_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_4_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_50_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_51_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_52_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_53_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_54_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_55_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_56_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_57_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_58_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_59_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_5_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_60_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_61_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_62_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_63_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_64_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_65_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_66_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_67_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_68_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_69_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_6_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_70_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_71_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_72_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_73_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_74_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_75_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_76_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_77_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_78_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_7_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_8_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_9_input_ids_tmp'] + +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_0_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_10_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_11_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_12_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_13_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_14_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_15_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_16_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_17_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_18_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_19_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_1_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_20_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_21_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_22_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_23_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_24_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_25_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_26_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_27_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_28_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_29_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_2_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_30_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_31_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_32_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_33_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_34_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_35_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_36_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_37_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_38_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_39_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_3_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_40_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_41_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_42_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_43_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_44_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_45_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_46_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_47_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_48_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_49_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_4_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_50_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_51_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_52_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_53_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_54_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_55_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_56_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_57_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_58_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_59_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_5_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_60_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_61_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_62_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_63_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_64_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_65_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_66_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_67_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_68_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_69_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_6_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_70_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_71_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_72_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_73_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_74_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_75_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_76_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_77_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_78_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_7_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_8_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_9_input_ids_tmp'] +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_0_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_10_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_11_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_12_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_13_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_14_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_15_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_16_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_17_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_18_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_19_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_1_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_20_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_21_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_22_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_23_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_24_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_25_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_26_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_27_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_28_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_29_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_2_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_30_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_31_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_32_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_33_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_34_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_35_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_36_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_37_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_38_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_39_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_3_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_40_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_41_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_42_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_43_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_44_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_45_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_46_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_47_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_48_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_49_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_4_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_50_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_51_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_52_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_53_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_54_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_55_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_56_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_57_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_58_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_59_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_5_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_60_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_61_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_62_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_63_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_64_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_65_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_66_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_67_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_68_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_69_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_6_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_70_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_71_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_72_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_73_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_74_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_75_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_76_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_77_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_78_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_7_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_8_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_9_input_ids_tmp']['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_0_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_10_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_11_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_12_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_13_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_14_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_15_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_16_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_17_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_18_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_19_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_1_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_20_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_21_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_22_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_23_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_24_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_25_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_26_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_27_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_28_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_29_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_2_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_30_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_31_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_32_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_33_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_34_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_35_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_36_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_37_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_38_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_39_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_3_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_40_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_41_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_42_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_43_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_44_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_45_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_46_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_47_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_48_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_49_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_4_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_50_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_51_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_52_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_53_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_54_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_55_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_56_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_57_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_58_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_59_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_5_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_60_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_61_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_62_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_63_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_64_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_65_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_66_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_67_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_68_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_69_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_6_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_70_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_71_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_72_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_73_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_74_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_75_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_76_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_77_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_78_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_7_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_8_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_9_input_ids_tmp']['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_0_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_10_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_11_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_12_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_13_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_14_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_15_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_16_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_17_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_18_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_19_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_1_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_20_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_21_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_22_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_23_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_24_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_25_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_26_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_27_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_28_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_29_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_2_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_30_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_31_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_32_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_33_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_34_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_35_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_36_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_37_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_38_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_39_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_3_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_40_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_41_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_42_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_43_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_44_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_45_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_46_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_47_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_48_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_49_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_4_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_50_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_51_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_52_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_53_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_54_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_55_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_56_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_57_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_58_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_59_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_5_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_60_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_61_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_62_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_63_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_64_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_65_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_66_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_67_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_68_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_69_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_6_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_70_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_71_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_72_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_73_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_74_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_75_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_76_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_77_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_78_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_7_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_8_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_9_input_ids_tmp'] + + +Initializing GlobIndexedDataset +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_0_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_10_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_11_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_12_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_13_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_14_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_15_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_16_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_17_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_18_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_19_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_1_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_20_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_21_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_22_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_23_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_24_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_25_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_26_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_27_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_28_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_29_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_2_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_30_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_31_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_32_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_33_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_34_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_35_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_36_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_37_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_38_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_39_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_3_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_40_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_41_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_42_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_43_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_44_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_45_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_46_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_47_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_48_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_49_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_4_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_50_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_51_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_52_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_53_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_54_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_55_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_56_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_57_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_58_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_59_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_5_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_60_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_61_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_62_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_63_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_64_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_65_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_66_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_67_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_68_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_69_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_6_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_70_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_71_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_72_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_73_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_74_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_75_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_76_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_77_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_78_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_7_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_8_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_9_input_ids_tmp'] +>>> start_samples: 0>>> start_samples: 0 +>>> start_samples: 0 +>>> start_samples: 0 +>>> start_samples: 0 +>>> start_samples: 0 +>>> start_samples: 0 +>>> start_samples: 0 + +Initializing GlobIndexedDatasetInitializing GlobIndexedDataset + +Initializing GlobIndexedDatasetInitializing GlobIndexedDataset + +Initializing GlobIndexedDataset +Initializing GlobIndexedDataset +Initializing GlobIndexedDataset +Initializing GlobIndexedDataset +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_75_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_76_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_77_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_78_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_9_loss_mask_tmp']['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_75_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_76_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_77_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_78_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_9_loss_mask_tmp']['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_75_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_76_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_77_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_78_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_9_loss_mask_tmp'] + +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_75_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_76_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_77_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_78_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_9_loss_mask_tmp'] + +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_75_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_76_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_77_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_78_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_9_loss_mask_tmp']['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_75_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_76_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_77_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_78_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_9_loss_mask_tmp'] + +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_75_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_76_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_77_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_78_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_9_loss_mask_tmp']['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_75_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_76_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_77_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_78_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_9_loss_mask_tmp'] + +pad_document_marker: 36616095 +pad_document_marker: 36616095 +/mnt/hdd/cpt/binidx_cache_stage15_final_20261001_v2 d6c36c6d5886334ea5f92e75fcf45d10-GPTDataset-train-obfd cache hit: True +/mnt/hdd/cpt/binidx_cache_stage15_final_20261001_v2 d6c36c6d5886334ea5f92e75fcf45d10-GPTDataset-train-obfd cache hit: True +pad_document_marker: 36616095 +/mnt/hdd/cpt/binidx_cache_stage15_final_20261001_v2 d6c36c6d5886334ea5f92e75fcf45d10-GPTDataset-train-obfd cache hit: True +pad_document_marker: 36616095 +/mnt/hdd/cpt/binidx_cache_stage15_final_20261001_v2 d6c36c6d5886334ea5f92e75fcf45d10-GPTDataset-train-obfd cache hit: True +pad_document_marker: 36616095 +/mnt/hdd/cpt/binidx_cache_stage15_final_20261001_v2 d6c36c6d5886334ea5f92e75fcf45d10-GPTDataset-train-obfd cache hit: True +pad_document_marker: 36616095 +/mnt/hdd/cpt/binidx_cache_stage15_final_20261001_v2 d6c36c6d5886334ea5f92e75fcf45d10-GPTDataset-train-obfd cache hit: True +pad_document_marker: 36616095 +/mnt/hdd/cpt/binidx_cache_stage15_final_20261001_v2 d6c36c6d5886334ea5f92e75fcf45d10-GPTDataset-train-obfd cache hit: True +pad_document_marker: 36616095 +/mnt/hdd/cpt/binidx_cache_stage15_final_20261001_v2 d6c36c6d5886334ea5f92e75fcf45d10-GPTDataset-train-obfd cache hit: True +>>> start_samples: 0>>> start_samples: 0>>> start_samples: 0>>> start_samples: 0>>> start_samples: 0 + +>>> start_samples: 0 + + + +>>> start_samples: 0Initializing GlobIndexedDatasetInitializing GlobIndexedDataset + +Initializing GlobIndexedDataset +Initializing GlobIndexedDatasetInitializing GlobIndexedDataset + + +Initializing GlobIndexedDataset>>> start_samples: 0 + +Initializing GlobIndexedDataset +Initializing GlobIndexedDataset +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_75_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_76_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_77_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_78_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_9_loss_mask_tmp']['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_75_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_76_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_77_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_78_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_9_loss_mask_tmp']['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_75_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_76_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_77_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_78_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_9_loss_mask_tmp']['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_75_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_76_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_77_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_78_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_9_loss_mask_tmp']['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_75_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_76_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_77_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_78_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_9_loss_mask_tmp'] + + + +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_75_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_76_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_77_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_78_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_9_loss_mask_tmp']['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_75_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_76_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_77_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_78_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_9_loss_mask_tmp']['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_75_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_76_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_77_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_78_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_9_loss_mask_tmp'] + + + +/mnt/hdd/cpt/binidx_cache_stage15_final_20261001_v2 0642c625420be1073f42e03d3641a6e5-GPTDataset-valid-random cache hit: True +/mnt/hdd/cpt/binidx_cache_stage15_final_20261001_v2 0642c625420be1073f42e03d3641a6e5-GPTDataset-valid-random cache hit: True +/mnt/hdd/cpt/binidx_cache_stage15_final_20261001_v2 0642c625420be1073f42e03d3641a6e5-GPTDataset-valid-random cache hit: True +/mnt/hdd/cpt/binidx_cache_stage15_final_20261001_v2 0642c625420be1073f42e03d3641a6e5-GPTDataset-valid-random cache hit: True +/mnt/hdd/cpt/binidx_cache_stage15_final_20261001_v2 0642c625420be1073f42e03d3641a6e5-GPTDataset-valid-random cache hit: True +/mnt/hdd/cpt/binidx_cache_stage15_final_20261001_v2 0642c625420be1073f42e03d3641a6e5-GPTDataset-valid-random cache hit: True +/mnt/hdd/cpt/binidx_cache_stage15_final_20261001_v2 0642c625420be1073f42e03d3641a6e5-GPTDataset-valid-random cache hit: True +/mnt/hdd/cpt/binidx_cache_stage15_final_20261001_v2 0642c625420be1073f42e03d3641a6e5-GPTDataset-valid-random cache hit: True +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/optimizer/optimizer.py:65: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /pytorch/build/aten/src/ATen/core/TensorBody.h:489.) + if hasattr(param, grad_attr) and getattr(param, grad_attr) is not None: +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/optimizer/optimizer.py:65: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /pytorch/build/aten/src/ATen/core/TensorBody.h:489.) + if hasattr(param, grad_attr) and getattr(param, grad_attr) is not None: +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/optimizer/optimizer.py:65: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /pytorch/build/aten/src/ATen/core/TensorBody.h:489.) + if hasattr(param, grad_attr) and getattr(param, grad_attr) is not None: +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/optimizer/optimizer.py:65: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /pytorch/build/aten/src/ATen/core/TensorBody.h:489.) + if hasattr(param, grad_attr) and getattr(param, grad_attr) is not None: +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/optimizer/optimizer.py:65: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /pytorch/build/aten/src/ATen/core/TensorBody.h:489.) + if hasattr(param, grad_attr) and getattr(param, grad_attr) is not None: +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/optimizer/optimizer.py:65: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /pytorch/build/aten/src/ATen/core/TensorBody.h:489.) + if hasattr(param, grad_attr) and getattr(param, grad_attr) is not None: +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/optimizer/optimizer.py:65: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /pytorch/build/aten/src/ATen/core/TensorBody.h:489.) + if hasattr(param, grad_attr) and getattr(param, grad_attr) is not None: +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/optimizer/optimizer.py:65: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /pytorch/build/aten/src/ATen/core/TensorBody.h:489.) + if hasattr(param, grad_attr) and getattr(param, grad_attr) is not None: +⚠️ torch 2.8.0+cu128 with triton 3.4.0. inductor patch is not applied. +⚠️ torch 2.8.0+cu128 with triton 3.4.0. inductor patch is not applied. +⚠️ torch 2.8.0+cu128 with triton 3.4.0. inductor patch is not applied. +⚠️ torch 2.8.0+cu128 with triton 3.4.0. inductor patch is not applied. +⚠️ torch 2.8.0+cu128 with triton 3.4.0. inductor patch is not applied. +⚠️ torch 2.8.0+cu128 with triton 3.4.0. inductor patch is not applied. +⚠️ torch 2.8.0+cu128 with triton 3.4.0. inductor patch is not applied. +⚠️ torch 2.8.0+cu128 with triton 3.4.0. inductor patch is not applied. +W1003 16:47:32.294000 538226 torch/utils/cpp_extension.py:2425] TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +W1003 16:47:32.294000 538226 torch/utils/cpp_extension.py:2425] If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'] to specific architectures. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +[rank16]:[W1010 20:26:01.340100649 ProcessGroupNCCL.cpp:1538] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/node3.log b/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/node3.log new file mode 100644 index 0000000000000000000000000000000000000000..cf8ff85128680a0e2dd0d1348401a63609458471 --- /dev/null +++ b/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/node3.log @@ -0,0 +1,358 @@ +W1003 15:36:53.983000 63001 torch/utils/cpp_extension.py:2425] TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +W1003 15:36:53.983000 63001 torch/utils/cpp_extension.py:2425] If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'] to specific architectures. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +[Gloo] Rank 24 is connected to 63[Gloo] Rank peer ranks. 25Expected number of connected peer ranks is : is connected to [Gloo] Rank 636326 + peer ranks. is connected to [Gloo] Rank Expected number of connected peer ranks is : 632763 peer ranks. is connected to +Expected number of connected peer ranks is : [Gloo] Rank 636328 peer ranks. + is connected to Expected number of connected peer ranks is : [Gloo] Rank [Gloo] Rank 63633029 peer ranks. + is connected to is connected to Expected number of connected peer ranks is : [Gloo] Rank [rank24]:[W1003 15:37:33.285222647 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank26]:[W1003 15:37:33.285235568 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +63[rank25]:[W1003 15:37:33.285244997 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +636331[rank24]:[W1003 15:37:33.285402773 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank26]:[W1003 15:37:33.285432875 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) + peer ranks. [rank25]:[W1003 15:37:33.285504046 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) + peer ranks. +[rank27]:[W1003 15:37:33.285552307 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) + is connected to Expected number of connected peer ranks is : Expected number of connected peer ranks is : [rank27]:[W1003 15:37:33.285820855 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +636363[rank28]:[W1003 15:37:33.285978561 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) + peer ranks. + +[rank28]:[W1003 15:37:33.286116432 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +Expected number of connected peer ranks is : 63 +[rank30]:[W1003 15:37:33.286414663 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank30]:[W1003 15:37:33.286457896 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank29]:[W1003 15:37:33.286636555 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank29]:[W1003 15:37:33.286713593 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank31]:[W1003 15:37:33.286787207 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank31]:[W1003 15:37:33.286861730 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[Gloo] Rank 3 is connected to [Gloo] Rank 73 peer ranks. [Gloo] Rank is connected to Expected number of connected peer ranks is : 377 is connected to peer ranks. +7Expected number of connected peer ranks is : peer ranks. 7Expected number of connected peer ranks is : +7 +[Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 3 is connected to 7[Gloo] Rank peer ranks. 3Expected number of connected peer ranks is : is connected to 77 +[Gloo] Rank peer ranks. [Gloo] Rank 3Expected number of connected peer ranks is : 3 is connected to 7 is connected to 7 +7 peer ranks. peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : 77 + +[Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 3 is connected to [Gloo] Rank 73 peer ranks. is connected to Expected number of connected peer ranks is : 77 peer ranks. +Expected number of connected peer ranks is : 7 +[Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : [Gloo] Rank 73 + is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. + warnings.warn( # warn only once +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. + warnings.warn( # warn only once +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. + warnings.warn( # warn only once +[rank31]:[W1003 15:37:33.376424429 ProcessGroupNCCL.cpp:5023] [PG ID 0 PG GUID 0 Rank 31] using GPU 7 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank28]:[W1003 15:37:33.376618594 ProcessGroupNCCL.cpp:5023] [PG ID 0 PG GUID 0 Rank 28] using GPU 4 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank30]:[W1003 15:37:33.376618594 ProcessGroupNCCL.cpp:5023] [PG ID 0 PG GUID 0 Rank 30] using GPU 6 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. + warnings.warn( # warn only once +[rank29]:[W1003 15:37:33.377198084 ProcessGroupNCCL.cpp:5023] [PG ID 0 PG GUID 0 Rank 29] using GPU 5 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. + warnings.warn( # warn only once +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. + warnings.warn( # warn only once +[rank26]:[W1003 15:37:33.378540822 ProcessGroupNCCL.cpp:5023] [PG ID 0 PG GUID 0 Rank 26] using GPU 2 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank24]:[W1003 15:37:33.378569388 ProcessGroupNCCL.cpp:5023] [PG ID 0 PG GUID 0 Rank 24] using GPU 0 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. + warnings.warn( # warn only once +[rank27]:[W1003 15:37:33.379580999 ProcessGroupNCCL.cpp:5023] [PG ID 0 PG GUID 0 Rank 27] using GPU 3 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. + warnings.warn( # warn only once +[rank25]:[W1003 15:37:33.379965836 ProcessGroupNCCL.cpp:5023] [PG ID 0 PG GUID 0 Rank 25] using GPU 1 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank30]:[W1003 15:37:34.401847860 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank29]:[W1003 15:37:34.401910369 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank30]:[W1003 15:37:34.401939773 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank31]:[W1003 15:37:34.402630594 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank27]:[W1003 15:37:34.402929314 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank25]:[W1003 15:37:34.403196814 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank24]:[W1003 15:37:34.403469133 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank26]:[W1003 15:37:34.403634662 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank28]:[W1003 15:37:34.403682295 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank29]:[W1003 15:37:34.406798565 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank31]:[W1003 15:37:34.406905635 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank27]:[W1003 15:37:34.406975478 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank25]:[W1003 15:37:34.407062293 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank24]:[W1003 15:37:34.407145826 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank26]:[W1003 15:37:34.407257017 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank28]:[W1003 15:37:34.407332308 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + + + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + + + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + + + + + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/dist_checkpointing/strategies/torch.py:956: FutureWarning: `load_state_dict` is deprecated and will be removed in future versions. Please use `load` instead. + checkpoint.load_state_dict( +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/dist_checkpointing/strategies/torch.py:956: FutureWarning: `load_state_dict` is deprecated and will be removed in future versions. Please use `load` instead. + checkpoint.load_state_dict( +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/dist_checkpointing/strategies/torch.py:956: FutureWarning: `load_state_dict` is deprecated and will be removed in future versions. Please use `load` instead. + checkpoint.load_state_dict( +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/dist_checkpointing/strategies/torch.py:956: FutureWarning: `load_state_dict` is deprecated and will be removed in future versions. Please use `load` instead. + checkpoint.load_state_dict( +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/dist_checkpointing/strategies/torch.py:956: FutureWarning: `load_state_dict` is deprecated and will be removed in future versions. Please use `load` instead. + checkpoint.load_state_dict( +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/dist_checkpointing/strategies/torch.py:956: FutureWarning: `load_state_dict` is deprecated and will be removed in future versions. Please use `load` instead. + checkpoint.load_state_dict( +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/dist_checkpointing/strategies/torch.py:956: FutureWarning: `load_state_dict` is deprecated and will be removed in future versions. Please use `load` instead. + checkpoint.load_state_dict( +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/dist_checkpointing/strategies/torch.py:956: FutureWarning: `load_state_dict` is deprecated and will be removed in future versions. Please use `load` instead. + checkpoint.load_state_dict( +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +NCCL version 2.27.3+cuda12.9 +Initializing GlobIndexedDataset +Initializing GlobIndexedDatasetInitializing GlobIndexedDataset + +Initializing GlobIndexedDataset +Initializing GlobIndexedDataset +Initializing GlobIndexedDataset +Initializing GlobIndexedDataset +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_0_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_10_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_11_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_12_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_13_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_14_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_15_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_16_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_17_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_18_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_19_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_1_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_20_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_21_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_22_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_23_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_24_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_25_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_26_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_27_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_28_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_29_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_2_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_30_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_31_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_32_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_33_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_34_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_35_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_36_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_37_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_38_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_39_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_3_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_40_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_41_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_42_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_43_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_44_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_45_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_46_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_47_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_48_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_49_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_4_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_50_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_51_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_52_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_53_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_54_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_55_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_56_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_57_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_58_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_59_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_5_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_60_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_61_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_62_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_63_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_64_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_65_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_66_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_67_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_68_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_69_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_6_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_70_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_71_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_72_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_73_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_74_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_75_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_76_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_77_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_78_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_7_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_8_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_9_input_ids_tmp']['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_0_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_10_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_11_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_12_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_13_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_14_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_15_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_16_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_17_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_18_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_19_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_1_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_20_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_21_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_22_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_23_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_24_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_25_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_26_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_27_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_28_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_29_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_2_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_30_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_31_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_32_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_33_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_34_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_35_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_36_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_37_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_38_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_39_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_3_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_40_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_41_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_42_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_43_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_44_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_45_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_46_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_47_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_48_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_49_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_4_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_50_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_51_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_52_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_53_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_54_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_55_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_56_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_57_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_58_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_59_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_5_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_60_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_61_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_62_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_63_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_64_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_65_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_66_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_67_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_68_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_69_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_6_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_70_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_71_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_72_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_73_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_74_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_75_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_76_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_77_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_78_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_7_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_8_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_9_input_ids_tmp']['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_0_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_10_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_11_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_12_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_13_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_14_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_15_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_16_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_17_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_18_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_19_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_1_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_20_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_21_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_22_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_23_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_24_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_25_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_26_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_27_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_28_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_29_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_2_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_30_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_31_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_32_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_33_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_34_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_35_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_36_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_37_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_38_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_39_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_3_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_40_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_41_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_42_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_43_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_44_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_45_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_46_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_47_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_48_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_49_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_4_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_50_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_51_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_52_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_53_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_54_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_55_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_56_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_57_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_58_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_59_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_5_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_60_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_61_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_62_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_63_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_64_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_65_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_66_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_67_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_68_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_69_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_6_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_70_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_71_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_72_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_73_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_74_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_75_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_76_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_77_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_78_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_7_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_8_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_9_input_ids_tmp'] + +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_0_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_10_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_11_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_12_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_13_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_14_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_15_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_16_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_17_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_18_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_19_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_1_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_20_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_21_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_22_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_23_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_24_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_25_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_26_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_27_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_28_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_29_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_2_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_30_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_31_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_32_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_33_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_34_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_35_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_36_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_37_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_38_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_39_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_3_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_40_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_41_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_42_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_43_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_44_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_45_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_46_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_47_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_48_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_49_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_4_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_50_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_51_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_52_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_53_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_54_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_55_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_56_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_57_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_58_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_59_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_5_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_60_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_61_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_62_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_63_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_64_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_65_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_66_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_67_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_68_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_69_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_6_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_70_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_71_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_72_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_73_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_74_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_75_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_76_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_77_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_78_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_7_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_8_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_9_input_ids_tmp'] + +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_0_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_10_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_11_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_12_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_13_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_14_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_15_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_16_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_17_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_18_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_19_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_1_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_20_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_21_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_22_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_23_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_24_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_25_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_26_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_27_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_28_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_29_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_2_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_30_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_31_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_32_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_33_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_34_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_35_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_36_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_37_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_38_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_39_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_3_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_40_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_41_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_42_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_43_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_44_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_45_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_46_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_47_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_48_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_49_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_4_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_50_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_51_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_52_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_53_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_54_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_55_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_56_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_57_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_58_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_59_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_5_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_60_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_61_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_62_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_63_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_64_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_65_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_66_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_67_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_68_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_69_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_6_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_70_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_71_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_72_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_73_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_74_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_75_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_76_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_77_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_78_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_7_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_8_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_9_input_ids_tmp']['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_0_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_10_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_11_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_12_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_13_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_14_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_15_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_16_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_17_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_18_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_19_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_1_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_20_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_21_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_22_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_23_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_24_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_25_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_26_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_27_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_28_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_29_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_2_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_30_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_31_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_32_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_33_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_34_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_35_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_36_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_37_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_38_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_39_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_3_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_40_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_41_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_42_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_43_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_44_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_45_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_46_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_47_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_48_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_49_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_4_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_50_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_51_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_52_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_53_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_54_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_55_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_56_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_57_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_58_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_59_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_5_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_60_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_61_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_62_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_63_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_64_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_65_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_66_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_67_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_68_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_69_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_6_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_70_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_71_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_72_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_73_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_74_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_75_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_76_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_77_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_78_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_7_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_8_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_9_input_ids_tmp'] + +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_0_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_10_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_11_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_12_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_13_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_14_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_15_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_16_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_17_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_18_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_19_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_1_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_20_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_21_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_22_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_23_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_24_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_25_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_26_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_27_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_28_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_29_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_2_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_30_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_31_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_32_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_33_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_34_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_35_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_36_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_37_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_38_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_39_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_3_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_40_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_41_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_42_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_43_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_44_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_45_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_46_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_47_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_48_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_49_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_4_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_50_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_51_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_52_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_53_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_54_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_55_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_56_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_57_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_58_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_59_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_5_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_60_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_61_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_62_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_63_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_64_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_65_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_66_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_67_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_68_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_69_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_6_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_70_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_71_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_72_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_73_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_74_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_75_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_76_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_77_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_78_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_7_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_8_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_9_input_ids_tmp'] +Initializing GlobIndexedDataset +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_0_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_10_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_11_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_12_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_13_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_14_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_15_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_16_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_17_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_18_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_19_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_1_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_20_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_21_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_22_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_23_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_24_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_25_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_26_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_27_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_28_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_29_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_2_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_30_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_31_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_32_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_33_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_34_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_35_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_36_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_37_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_38_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_39_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_3_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_40_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_41_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_42_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_43_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_44_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_45_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_46_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_47_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_48_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_49_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_4_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_50_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_51_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_52_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_53_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_54_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_55_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_56_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_57_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_58_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_59_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_5_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_60_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_61_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_62_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_63_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_64_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_65_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_66_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_67_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_68_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_69_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_6_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_70_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_71_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_72_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_73_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_74_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_75_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_76_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_77_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_78_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_7_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_8_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_9_input_ids_tmp'] +>>> start_samples: 0>>> start_samples: 0 +>>> start_samples: 0 +>>> start_samples: 0 +>>> start_samples: 0 +>>> start_samples: 0Initializing GlobIndexedDataset>>> start_samples: 0 +>>> start_samples: 0Initializing GlobIndexedDataset + +Initializing GlobIndexedDataset +Initializing GlobIndexedDataset + + + +Initializing GlobIndexedDataset +Initializing GlobIndexedDataset +Initializing GlobIndexedDataset +Initializing GlobIndexedDataset +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_75_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_76_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_77_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_78_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_9_loss_mask_tmp'] +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_75_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_76_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_77_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_78_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_9_loss_mask_tmp']['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_75_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_76_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_77_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_78_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_9_loss_mask_tmp'] + +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_75_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_76_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_77_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_78_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_9_loss_mask_tmp']['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_75_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_76_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_77_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_78_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_9_loss_mask_tmp'] +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_75_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_76_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_77_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_78_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_9_loss_mask_tmp']['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_75_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_76_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_77_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_78_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_9_loss_mask_tmp'] + +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_75_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_76_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_77_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_78_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_9_loss_mask_tmp'] + +pad_document_marker: 36616095pad_document_marker: 36616095pad_document_marker: 36616095 + + +pad_document_marker: 36616095 +pad_document_marker: 36616095 +pad_document_marker: 36616095 +pad_document_marker: 36616095pad_document_marker: 36616095 +/mnt/hdd/cpt/binidx_cache_stage15_final_20261001_v2 d6c36c6d5886334ea5f92e75fcf45d10-GPTDataset-train-obfd cache hit: True/mnt/hdd/cpt/binidx_cache_stage15_final_20261001_v2 d6c36c6d5886334ea5f92e75fcf45d10-GPTDataset-train-obfd cache hit: True + + +/mnt/hdd/cpt/binidx_cache_stage15_final_20261001_v2 d6c36c6d5886334ea5f92e75fcf45d10-GPTDataset-train-obfd cache hit: True/mnt/hdd/cpt/binidx_cache_stage15_final_20261001_v2 d6c36c6d5886334ea5f92e75fcf45d10-GPTDataset-train-obfd cache hit: True/mnt/hdd/cpt/binidx_cache_stage15_final_20261001_v2 d6c36c6d5886334ea5f92e75fcf45d10-GPTDataset-train-obfd cache hit: True + +/mnt/hdd/cpt/binidx_cache_stage15_final_20261001_v2 d6c36c6d5886334ea5f92e75fcf45d10-GPTDataset-train-obfd cache hit: True +/mnt/hdd/cpt/binidx_cache_stage15_final_20261001_v2 d6c36c6d5886334ea5f92e75fcf45d10-GPTDataset-train-obfd cache hit: True +/mnt/hdd/cpt/binidx_cache_stage15_final_20261001_v2 d6c36c6d5886334ea5f92e75fcf45d10-GPTDataset-train-obfd cache hit: True + +>>> start_samples: 0>>> start_samples: 0>>> start_samples: 0>>> start_samples: 0 + + +>>> start_samples: 0>>> start_samples: 0>>> start_samples: 0 +>>> start_samples: 0 + +Initializing GlobIndexedDatasetInitializing GlobIndexedDataset +Initializing GlobIndexedDataset + + +Initializing GlobIndexedDataset + +Initializing GlobIndexedDatasetInitializing GlobIndexedDataset + +Initializing GlobIndexedDataset +Initializing GlobIndexedDataset +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_75_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_76_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_77_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_78_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_9_loss_mask_tmp']['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_75_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_76_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_77_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_78_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_9_loss_mask_tmp']['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_75_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_76_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_77_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_78_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_9_loss_mask_tmp'] +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_75_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_76_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_77_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_78_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_9_loss_mask_tmp'] + + +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_75_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_76_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_77_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_78_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_9_loss_mask_tmp']['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_75_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_76_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_77_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_78_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_9_loss_mask_tmp']['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_75_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_76_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_77_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_78_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_9_loss_mask_tmp'] + + +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_75_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_76_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_77_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_78_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_9_loss_mask_tmp'] +/mnt/hdd/cpt/binidx_cache_stage15_final_20261001_v2 0642c625420be1073f42e03d3641a6e5-GPTDataset-valid-random cache hit: True +/mnt/hdd/cpt/binidx_cache_stage15_final_20261001_v2 0642c625420be1073f42e03d3641a6e5-GPTDataset-valid-random cache hit: True +/mnt/hdd/cpt/binidx_cache_stage15_final_20261001_v2 0642c625420be1073f42e03d3641a6e5-GPTDataset-valid-random cache hit: True +/mnt/hdd/cpt/binidx_cache_stage15_final_20261001_v2 0642c625420be1073f42e03d3641a6e5-GPTDataset-valid-random cache hit: True +/mnt/hdd/cpt/binidx_cache_stage15_final_20261001_v2 0642c625420be1073f42e03d3641a6e5-GPTDataset-valid-random cache hit: True +/mnt/hdd/cpt/binidx_cache_stage15_final_20261001_v2 0642c625420be1073f42e03d3641a6e5-GPTDataset-valid-random cache hit: True +/mnt/hdd/cpt/binidx_cache_stage15_final_20261001_v2 0642c625420be1073f42e03d3641a6e5-GPTDataset-valid-random cache hit: True +/mnt/hdd/cpt/binidx_cache_stage15_final_20261001_v2 0642c625420be1073f42e03d3641a6e5-GPTDataset-valid-random cache hit: True +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/optimizer/optimizer.py:65: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /pytorch/build/aten/src/ATen/core/TensorBody.h:489.) + if hasattr(param, grad_attr) and getattr(param, grad_attr) is not None: +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/optimizer/optimizer.py:65: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /pytorch/build/aten/src/ATen/core/TensorBody.h:489.) + if hasattr(param, grad_attr) and getattr(param, grad_attr) is not None: +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/optimizer/optimizer.py:65: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /pytorch/build/aten/src/ATen/core/TensorBody.h:489.) + if hasattr(param, grad_attr) and getattr(param, grad_attr) is not None: +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/optimizer/optimizer.py:65: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /pytorch/build/aten/src/ATen/core/TensorBody.h:489.) + if hasattr(param, grad_attr) and getattr(param, grad_attr) is not None: +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/optimizer/optimizer.py:65: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /pytorch/build/aten/src/ATen/core/TensorBody.h:489.) + if hasattr(param, grad_attr) and getattr(param, grad_attr) is not None: +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/optimizer/optimizer.py:65: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /pytorch/build/aten/src/ATen/core/TensorBody.h:489.) + if hasattr(param, grad_attr) and getattr(param, grad_attr) is not None: +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/optimizer/optimizer.py:65: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /pytorch/build/aten/src/ATen/core/TensorBody.h:489.) + if hasattr(param, grad_attr) and getattr(param, grad_attr) is not None: +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/optimizer/optimizer.py:65: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /pytorch/build/aten/src/ATen/core/TensorBody.h:489.) + if hasattr(param, grad_attr) and getattr(param, grad_attr) is not None: +⚠️ torch 2.8.0+cu128 with triton 3.4.0. inductor patch is not applied.⚠️ torch 2.8.0+cu128 with triton 3.4.0. inductor patch is not applied.⚠️ torch 2.8.0+cu128 with triton 3.4.0. inductor patch is not applied. +⚠️ torch 2.8.0+cu128 with triton 3.4.0. inductor patch is not applied. +⚠️ torch 2.8.0+cu128 with triton 3.4.0. inductor patch is not applied. +⚠️ torch 2.8.0+cu128 with triton 3.4.0. inductor patch is not applied. +⚠️ torch 2.8.0+cu128 with triton 3.4.0. inductor patch is not applied. + + +⚠️ torch 2.8.0+cu128 with triton 3.4.0. inductor patch is not applied. +W1003 16:47:56.034000 31305 torch/utils/cpp_extension.py:2425] TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +W1003 16:47:56.034000 31305 torch/utils/cpp_extension.py:2425] If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'] to specific architectures. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +[rank24]:[W1010 20:26:02.984383757 ProcessGroupNCCL.cpp:1538] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/node4.log b/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/node4.log new file mode 100644 index 0000000000000000000000000000000000000000..112c46659997697840a1e5fc462567d18b5af709 --- /dev/null +++ b/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/node4.log @@ -0,0 +1,358 @@ +W1003 15:36:45.469000 21691 torch/utils/cpp_extension.py:2425] TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +W1003 15:36:45.469000 21691 torch/utils/cpp_extension.py:2425] If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'] to specific architectures. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +[Gloo] Rank 32 is connected to 63 peer ranks. Expected number of connected peer ranks is : 63 +[Gloo] Rank 33 is connected to 63 peer ranks. Expected number of connected peer ranks is : 63 +[Gloo] Rank 34 is connected to 63 peer ranks. Expected number of connected peer ranks is : 63 +[rank33]:[W1003 15:37:33.116868127 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank32]:[W1003 15:37:33.116868147 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank33]:[W1003 15:37:33.116894628 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank32]:[W1003 15:37:33.116911289 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank34]:[W1003 15:37:33.117038842 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank34]:[W1003 15:37:33.117058349 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[Gloo] Rank 35 is connected to 63 peer ranks. Expected number of connected peer ranks is : 63 +[rank35]:[W1003 15:37:33.117486746 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank35]:[W1003 15:37:33.117507175 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[Gloo] Rank [Gloo] Rank [Gloo] Rank 393736 is connected to is connected to is connected to 636363 peer ranks. peer ranks. peer ranks. [Gloo] Rank Expected number of connected peer ranks is : Expected number of connected peer ranks is : Expected number of connected peer ranks is : 38636363 is connected to + + +63 peer ranks. Expected number of connected peer ranks is : 63 +[rank37]:[W1003 15:37:33.139415499 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank37]:[W1003 15:37:33.139434004 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank39]:[W1003 15:37:33.139456076 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank39]:[W1003 15:37:33.139480202 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank36]:[W1003 15:37:33.139513716 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank38]:[W1003 15:37:33.139537511 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank36]:[W1003 15:37:33.139561357 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank38]:[W1003 15:37:33.139577487 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 4 is connected to 7 peer ranks. [Gloo] Rank Expected number of connected peer ranks is : 47 is connected to +7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 4 is connected to [Gloo] Rank 74 peer ranks. is connected to Expected number of connected peer ranks is : 77 peer ranks. +Expected number of connected peer ranks is : 7 +[Gloo] Rank 4 is connected to 7[Gloo] Rank peer ranks. 4Expected number of connected peer ranks is : is connected to 77 + peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. + warnings.warn( # warn only once +[rank33]:[W1003 15:37:33.224656496 ProcessGroupNCCL.cpp:5023] [PG ID 0 PG GUID 0 Rank 33] using GPU 1 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. + warnings.warn( # warn only once +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. + warnings.warn( # warn only once +[rank36]:[W1003 15:37:33.225677010 ProcessGroupNCCL.cpp:5023] [PG ID 0 PG GUID 0 Rank 36] using GPU 4 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank32]:[W1003 15:37:33.225701838 ProcessGroupNCCL.cpp:5023] [PG ID 0 PG GUID 0 Rank 32] using GPU 0 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. + warnings.warn( # warn only once +[rank34]:[W1003 15:37:33.226008171 ProcessGroupNCCL.cpp:5023] [PG ID 0 PG GUID 0 Rank 34] using GPU 2 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. + warnings.warn( # warn only once +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. + warnings.warn( # warn only once +[rank35]:[W1003 15:37:33.227458695 ProcessGroupNCCL.cpp:5023] [PG ID 0 PG GUID 0 Rank 35] using GPU 3 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. + warnings.warn( # warn only once +[rank38]:[W1003 15:37:33.227583473 ProcessGroupNCCL.cpp:5023] [PG ID 0 PG GUID 0 Rank 38] using GPU 6 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank39]:[W1003 15:37:33.227623249 ProcessGroupNCCL.cpp:5023] [PG ID 0 PG GUID 0 Rank 39] using GPU 7 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. + warnings.warn( # warn only once +[rank37]:[W1003 15:37:33.227850812 ProcessGroupNCCL.cpp:5023] [PG ID 0 PG GUID 0 Rank 37] using GPU 5 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank37]:[W1003 15:37:34.245640421 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank36]:[W1003 15:37:34.245699895 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank37]:[W1003 15:37:34.245736214 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank36]:[W1003 15:37:34.245815395 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank34]:[W1003 15:37:34.246474671 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank34]:[W1003 15:37:34.246576054 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank33]:[W1003 15:37:34.247268023 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank33]:[W1003 15:37:34.247363004 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank35]:[W1003 15:37:34.248780134 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank35]:[W1003 15:37:34.248882689 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank32]:[W1003 15:37:34.249293031 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank39]:[W1003 15:37:34.249334440 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank32]:[W1003 15:37:34.249349068 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank39]:[W1003 15:37:34.249375338 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank38]:[W1003 15:37:34.250054352 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank38]:[W1003 15:37:34.250147399 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + + + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + + + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + + + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + + + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/dist_checkpointing/strategies/torch.py:956: FutureWarning: `load_state_dict` is deprecated and will be removed in future versions. Please use `load` instead. + checkpoint.load_state_dict( +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/dist_checkpointing/strategies/torch.py:956: FutureWarning: `load_state_dict` is deprecated and will be removed in future versions. Please use `load` instead. + checkpoint.load_state_dict( +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/dist_checkpointing/strategies/torch.py:956: FutureWarning: `load_state_dict` is deprecated and will be removed in future versions. Please use `load` instead. + checkpoint.load_state_dict( +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/dist_checkpointing/strategies/torch.py:956: FutureWarning: `load_state_dict` is deprecated and will be removed in future versions. Please use `load` instead. + checkpoint.load_state_dict( +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/dist_checkpointing/strategies/torch.py:956: FutureWarning: `load_state_dict` is deprecated and will be removed in future versions. Please use `load` instead. + checkpoint.load_state_dict( +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/dist_checkpointing/strategies/torch.py:956: FutureWarning: `load_state_dict` is deprecated and will be removed in future versions. Please use `load` instead. + checkpoint.load_state_dict( +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/dist_checkpointing/strategies/torch.py:956: FutureWarning: `load_state_dict` is deprecated and will be removed in future versions. Please use `load` instead. + checkpoint.load_state_dict( +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/dist_checkpointing/strategies/torch.py:956: FutureWarning: `load_state_dict` is deprecated and will be removed in future versions. Please use `load` instead. + checkpoint.load_state_dict( +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +NCCL version 2.27.3+cuda12.9 +Initializing GlobIndexedDataset +Initializing GlobIndexedDataset +Initializing GlobIndexedDataset +Initializing GlobIndexedDataset +Initializing GlobIndexedDataset +Initializing GlobIndexedDataset +Initializing GlobIndexedDataset +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_0_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_10_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_11_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_12_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_13_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_14_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_15_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_16_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_17_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_18_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_19_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_1_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_20_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_21_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_22_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_23_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_24_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_25_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_26_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_27_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_28_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_29_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_2_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_30_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_31_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_32_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_33_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_34_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_35_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_36_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_37_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_38_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_39_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_3_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_40_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_41_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_42_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_43_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_44_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_45_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_46_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_47_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_48_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_49_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_4_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_50_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_51_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_52_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_53_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_54_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_55_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_56_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_57_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_58_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_59_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_5_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_60_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_61_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_62_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_63_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_64_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_65_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_66_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_67_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_68_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_69_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_6_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_70_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_71_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_72_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_73_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_74_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_75_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_76_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_77_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_78_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_7_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_8_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_9_input_ids_tmp'] +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_0_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_10_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_11_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_12_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_13_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_14_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_15_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_16_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_17_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_18_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_19_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_1_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_20_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_21_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_22_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_23_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_24_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_25_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_26_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_27_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_28_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_29_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_2_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_30_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_31_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_32_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_33_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_34_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_35_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_36_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_37_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_38_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_39_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_3_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_40_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_41_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_42_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_43_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_44_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_45_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_46_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_47_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_48_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_49_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_4_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_50_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_51_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_52_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_53_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_54_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_55_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_56_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_57_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_58_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_59_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_5_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_60_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_61_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_62_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_63_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_64_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_65_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_66_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_67_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_68_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_69_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_6_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_70_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_71_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_72_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_73_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_74_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_75_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_76_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_77_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_78_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_7_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_8_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_9_input_ids_tmp']['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_0_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_10_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_11_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_12_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_13_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_14_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_15_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_16_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_17_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_18_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_19_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_1_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_20_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_21_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_22_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_23_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_24_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_25_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_26_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_27_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_28_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_29_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_2_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_30_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_31_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_32_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_33_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_34_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_35_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_36_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_37_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_38_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_39_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_3_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_40_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_41_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_42_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_43_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_44_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_45_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_46_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_47_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_48_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_49_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_4_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_50_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_51_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_52_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_53_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_54_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_55_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_56_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_57_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_58_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_59_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_5_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_60_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_61_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_62_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_63_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_64_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_65_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_66_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_67_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_68_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_69_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_6_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_70_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_71_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_72_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_73_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_74_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_75_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_76_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_77_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_78_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_7_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_8_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_9_input_ids_tmp']['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_0_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_10_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_11_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_12_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_13_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_14_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_15_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_16_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_17_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_18_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_19_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_1_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_20_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_21_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_22_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_23_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_24_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_25_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_26_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_27_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_28_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_29_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_2_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_30_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_31_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_32_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_33_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_34_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_35_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_36_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_37_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_38_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_39_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_3_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_40_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_41_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_42_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_43_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_44_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_45_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_46_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_47_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_48_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_49_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_4_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_50_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_51_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_52_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_53_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_54_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_55_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_56_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_57_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_58_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_59_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_5_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_60_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_61_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_62_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_63_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_64_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_65_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_66_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_67_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_68_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_69_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_6_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_70_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_71_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_72_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_73_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_74_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_75_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_76_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_77_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_78_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_7_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_8_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_9_input_ids_tmp'] + + +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_0_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_10_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_11_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_12_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_13_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_14_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_15_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_16_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_17_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_18_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_19_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_1_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_20_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_21_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_22_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_23_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_24_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_25_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_26_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_27_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_28_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_29_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_2_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_30_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_31_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_32_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_33_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_34_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_35_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_36_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_37_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_38_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_39_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_3_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_40_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_41_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_42_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_43_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_44_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_45_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_46_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_47_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_48_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_49_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_4_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_50_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_51_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_52_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_53_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_54_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_55_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_56_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_57_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_58_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_59_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_5_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_60_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_61_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_62_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_63_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_64_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_65_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_66_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_67_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_68_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_69_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_6_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_70_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_71_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_72_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_73_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_74_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_75_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_76_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_77_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_78_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_7_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_8_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_9_input_ids_tmp'] +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_0_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_10_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_11_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_12_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_13_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_14_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_15_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_16_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_17_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_18_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_19_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_1_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_20_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_21_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_22_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_23_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_24_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_25_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_26_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_27_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_28_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_29_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_2_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_30_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_31_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_32_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_33_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_34_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_35_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_36_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_37_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_38_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_39_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_3_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_40_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_41_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_42_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_43_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_44_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_45_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_46_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_47_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_48_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_49_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_4_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_50_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_51_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_52_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_53_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_54_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_55_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_56_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_57_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_58_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_59_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_5_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_60_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_61_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_62_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_63_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_64_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_65_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_66_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_67_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_68_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_69_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_6_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_70_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_71_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_72_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_73_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_74_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_75_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_76_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_77_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_78_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_7_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_8_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_9_input_ids_tmp']['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_0_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_10_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_11_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_12_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_13_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_14_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_15_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_16_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_17_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_18_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_19_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_1_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_20_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_21_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_22_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_23_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_24_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_25_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_26_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_27_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_28_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_29_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_2_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_30_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_31_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_32_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_33_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_34_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_35_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_36_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_37_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_38_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_39_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_3_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_40_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_41_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_42_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_43_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_44_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_45_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_46_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_47_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_48_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_49_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_4_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_50_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_51_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_52_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_53_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_54_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_55_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_56_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_57_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_58_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_59_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_5_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_60_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_61_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_62_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_63_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_64_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_65_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_66_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_67_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_68_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_69_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_6_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_70_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_71_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_72_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_73_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_74_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_75_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_76_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_77_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_78_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_7_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_8_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_9_input_ids_tmp'] + +Initializing GlobIndexedDataset +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_0_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_10_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_11_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_12_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_13_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_14_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_15_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_16_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_17_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_18_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_19_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_1_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_20_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_21_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_22_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_23_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_24_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_25_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_26_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_27_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_28_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_29_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_2_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_30_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_31_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_32_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_33_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_34_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_35_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_36_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_37_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_38_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_39_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_3_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_40_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_41_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_42_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_43_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_44_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_45_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_46_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_47_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_48_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_49_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_4_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_50_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_51_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_52_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_53_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_54_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_55_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_56_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_57_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_58_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_59_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_5_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_60_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_61_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_62_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_63_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_64_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_65_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_66_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_67_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_68_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_69_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_6_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_70_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_71_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_72_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_73_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_74_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_75_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_76_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_77_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_78_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_7_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_8_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_9_input_ids_tmp'] +>>> start_samples: 0>>> start_samples: 0 +>>> start_samples: 0 +>>> start_samples: 0 +>>> start_samples: 0 +>>> start_samples: 0 +>>> start_samples: 0 +>>> start_samples: 0Initializing GlobIndexedDataset +Initializing GlobIndexedDataset + + +Initializing GlobIndexedDataset +Initializing GlobIndexedDataset +Initializing GlobIndexedDataset +Initializing GlobIndexedDataset +Initializing GlobIndexedDataset +Initializing GlobIndexedDataset +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_75_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_76_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_77_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_78_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_9_loss_mask_tmp'] +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_75_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_76_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_77_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_78_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_9_loss_mask_tmp'] +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_75_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_76_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_77_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_78_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_9_loss_mask_tmp'] +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_75_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_76_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_77_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_78_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_9_loss_mask_tmp']['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_75_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_76_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_77_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_78_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_9_loss_mask_tmp'] +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_75_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_76_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_77_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_78_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_9_loss_mask_tmp'] + +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_75_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_76_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_77_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_78_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_9_loss_mask_tmp']['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_75_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_76_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_77_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_78_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_9_loss_mask_tmp'] + +pad_document_marker: 36616095 +pad_document_marker: 36616095 +/mnt/hdd/cpt/binidx_cache_stage15_final_20261001_v2 d6c36c6d5886334ea5f92e75fcf45d10-GPTDataset-train-obfd cache hit: True +pad_document_marker: 36616095 +pad_document_marker: 36616095 +/mnt/hdd/cpt/binidx_cache_stage15_final_20261001_v2 d6c36c6d5886334ea5f92e75fcf45d10-GPTDataset-train-obfd cache hit: True +/mnt/hdd/cpt/binidx_cache_stage15_final_20261001_v2 d6c36c6d5886334ea5f92e75fcf45d10-GPTDataset-train-obfd cache hit: True/mnt/hdd/cpt/binidx_cache_stage15_final_20261001_v2 d6c36c6d5886334ea5f92e75fcf45d10-GPTDataset-train-obfd cache hit: True + +pad_document_marker: 36616095 +/mnt/hdd/cpt/binidx_cache_stage15_final_20261001_v2 d6c36c6d5886334ea5f92e75fcf45d10-GPTDataset-train-obfd cache hit: True +pad_document_marker: 36616095 +pad_document_marker: 36616095 +pad_document_marker: 36616095 +/mnt/hdd/cpt/binidx_cache_stage15_final_20261001_v2 d6c36c6d5886334ea5f92e75fcf45d10-GPTDataset-train-obfd cache hit: True/mnt/hdd/cpt/binidx_cache_stage15_final_20261001_v2 d6c36c6d5886334ea5f92e75fcf45d10-GPTDataset-train-obfd cache hit: True + +/mnt/hdd/cpt/binidx_cache_stage15_final_20261001_v2 d6c36c6d5886334ea5f92e75fcf45d10-GPTDataset-train-obfd cache hit: True +>>> start_samples: 0>>> start_samples: 0>>> start_samples: 0>>> start_samples: 0 +>>> start_samples: 0 + + +>>> start_samples: 0>>> start_samples: 0 + + +Initializing GlobIndexedDataset +Initializing GlobIndexedDataset +Initializing GlobIndexedDatasetInitializing GlobIndexedDataset + +Initializing GlobIndexedDataset +Initializing GlobIndexedDatasetInitializing GlobIndexedDataset + +>>> start_samples: 0 +Initializing GlobIndexedDataset +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_75_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_76_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_77_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_78_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_9_loss_mask_tmp']['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_75_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_76_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_77_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_78_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_9_loss_mask_tmp'] +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_75_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_76_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_77_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_78_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_9_loss_mask_tmp']['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_75_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_76_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_77_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_78_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_9_loss_mask_tmp'] + + +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_75_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_76_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_77_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_78_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_9_loss_mask_tmp']['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_75_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_76_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_77_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_78_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_9_loss_mask_tmp'] + +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_75_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_76_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_77_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_78_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_9_loss_mask_tmp'] +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_75_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_76_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_77_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_78_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_9_loss_mask_tmp'] +/mnt/hdd/cpt/binidx_cache_stage15_final_20261001_v2 0642c625420be1073f42e03d3641a6e5-GPTDataset-valid-random cache hit: True +/mnt/hdd/cpt/binidx_cache_stage15_final_20261001_v2 0642c625420be1073f42e03d3641a6e5-GPTDataset-valid-random cache hit: True +/mnt/hdd/cpt/binidx_cache_stage15_final_20261001_v2 0642c625420be1073f42e03d3641a6e5-GPTDataset-valid-random cache hit: True +/mnt/hdd/cpt/binidx_cache_stage15_final_20261001_v2 0642c625420be1073f42e03d3641a6e5-GPTDataset-valid-random cache hit: True +/mnt/hdd/cpt/binidx_cache_stage15_final_20261001_v2 0642c625420be1073f42e03d3641a6e5-GPTDataset-valid-random cache hit: True +/mnt/hdd/cpt/binidx_cache_stage15_final_20261001_v2 0642c625420be1073f42e03d3641a6e5-GPTDataset-valid-random cache hit: True +/mnt/hdd/cpt/binidx_cache_stage15_final_20261001_v2 0642c625420be1073f42e03d3641a6e5-GPTDataset-valid-random cache hit: True +/mnt/hdd/cpt/binidx_cache_stage15_final_20261001_v2 0642c625420be1073f42e03d3641a6e5-GPTDataset-valid-random cache hit: True +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/optimizer/optimizer.py:65: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /pytorch/build/aten/src/ATen/core/TensorBody.h:489.) + if hasattr(param, grad_attr) and getattr(param, grad_attr) is not None: +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/optimizer/optimizer.py:65: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /pytorch/build/aten/src/ATen/core/TensorBody.h:489.) + if hasattr(param, grad_attr) and getattr(param, grad_attr) is not None: +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/optimizer/optimizer.py:65: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /pytorch/build/aten/src/ATen/core/TensorBody.h:489.) + if hasattr(param, grad_attr) and getattr(param, grad_attr) is not None: +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/optimizer/optimizer.py:65: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /pytorch/build/aten/src/ATen/core/TensorBody.h:489.) + if hasattr(param, grad_attr) and getattr(param, grad_attr) is not None: +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/optimizer/optimizer.py:65: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /pytorch/build/aten/src/ATen/core/TensorBody.h:489.) + if hasattr(param, grad_attr) and getattr(param, grad_attr) is not None: +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/optimizer/optimizer.py:65: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /pytorch/build/aten/src/ATen/core/TensorBody.h:489.) + if hasattr(param, grad_attr) and getattr(param, grad_attr) is not None: +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/optimizer/optimizer.py:65: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /pytorch/build/aten/src/ATen/core/TensorBody.h:489.) + if hasattr(param, grad_attr) and getattr(param, grad_attr) is not None: +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/optimizer/optimizer.py:65: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /pytorch/build/aten/src/ATen/core/TensorBody.h:489.) + if hasattr(param, grad_attr) and getattr(param, grad_attr) is not None: +⚠️ torch 2.8.0+cu128 with triton 3.4.0. inductor patch is not applied. +⚠️ torch 2.8.0+cu128 with triton 3.4.0. inductor patch is not applied. +⚠️ torch 2.8.0+cu128 with triton 3.4.0. inductor patch is not applied. +⚠️ torch 2.8.0+cu128 with triton 3.4.0. inductor patch is not applied. +⚠️ torch 2.8.0+cu128 with triton 3.4.0. inductor patch is not applied. +⚠️ torch 2.8.0+cu128 with triton 3.4.0. inductor patch is not applied. +⚠️ torch 2.8.0+cu128 with triton 3.4.0. inductor patch is not applied. +⚠️ torch 2.8.0+cu128 with triton 3.4.0. inductor patch is not applied. +W1003 16:47:34.962000 41446 torch/utils/cpp_extension.py:2425] TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +W1003 16:47:34.962000 41446 torch/utils/cpp_extension.py:2425] If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'] to specific architectures. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +[rank32]:[W1010 20:26:01.566999951 ProcessGroupNCCL.cpp:1538] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/node5.log b/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/node5.log new file mode 100644 index 0000000000000000000000000000000000000000..9d072dc7449a693e290015f333c8da3be7cd86cc --- /dev/null +++ b/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/node5.log @@ -0,0 +1,358 @@ +W1003 15:36:42.844000 28123 torch/utils/cpp_extension.py:2425] TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +W1003 15:36:42.844000 28123 torch/utils/cpp_extension.py:2425] If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'] to specific architectures. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +[Gloo] Rank 41 is connected to 63 peer ranks. Expected number of connected peer ranks is : 63 +[Gloo] Rank 40 is connected to 63 peer ranks. Expected number of connected peer ranks is : 63 +[Gloo] Rank [Gloo] Rank [Gloo] Rank [Gloo] Rank 43444245 is connected to is connected to [Gloo] Rank is connected to [Gloo] Rank is connected to 63634763[rank41]:[W1003 15:37:33.848657172 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank40]:[W1003 15:37:33.848657266 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +4663 peer ranks. peer ranks. is connected to peer ranks. [rank41]:[W1003 15:37:33.848710712 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank40]:[W1003 15:37:33.848720919 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) + is connected to peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : 63Expected number of connected peer ranks is : 63Expected number of connected peer ranks is : 6363 peer ranks. 63 peer ranks. 63 + +Expected number of connected peer ranks is : +Expected number of connected peer ranks is : +6363 + +[rank44]:[W1003 15:37:33.848968772 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank43]:[W1003 15:37:33.848973463 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank44]:[W1003 15:37:33.848987617 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank45]:[W1003 15:37:33.848985631 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank43]:[W1003 15:37:33.848995364 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank45]:[W1003 15:37:33.849009732 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank47]:[W1003 15:37:33.849008891 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank46]:[W1003 15:37:33.849008568 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank42]:[W1003 15:37:33.849016229 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank47]:[W1003 15:37:33.849029037 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank46]:[W1003 15:37:33.849039243 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank42]:[W1003 15:37:33.849064894 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[Gloo] Rank 5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank [Gloo] Rank 5[Gloo] Rank 5 is connected to 5 is connected to 7 is connected to 7 peer ranks. 7 peer ranks. Expected number of connected peer ranks is : peer ranks. Expected number of connected peer ranks is : 7Expected number of connected peer ranks is : 7 +7[Gloo] Rank + +5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. + warnings.warn( # warn only once +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. + warnings.warn( # warn only once +[rank41]:[W1003 15:37:33.966142619 ProcessGroupNCCL.cpp:5023] [PG ID 0 PG GUID 0 Rank 41] using GPU 1 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank45]:[W1003 15:37:33.966189509 ProcessGroupNCCL.cpp:5023] [PG ID 0 PG GUID 0 Rank 45] using GPU 5 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. + warnings.warn( # warn only once +[rank47]:[W1003 15:37:33.966422776 ProcessGroupNCCL.cpp:5023] [PG ID 0 PG GUID 0 Rank 47] using GPU 7 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. + warnings.warn( # warn only once +[rank44]:[W1003 15:37:33.966656775 ProcessGroupNCCL.cpp:5023] [PG ID 0 PG GUID 0 Rank 44] using GPU 4 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. + warnings.warn( # warn only once +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. + warnings.warn( # warn only once +[rank43]:[W1003 15:37:33.967642655 ProcessGroupNCCL.cpp:5023] [PG ID 0 PG GUID 0 Rank 43] using GPU 3 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank46]:[W1003 15:37:33.967688668 ProcessGroupNCCL.cpp:5023] [PG ID 0 PG GUID 0 Rank 46] using GPU 6 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. + warnings.warn( # warn only once +[rank40]:[W1003 15:37:33.968320900 ProcessGroupNCCL.cpp:5023] [PG ID 0 PG GUID 0 Rank 40] using GPU 0 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. + warnings.warn( # warn only once +[rank42]:[W1003 15:37:33.969371825 ProcessGroupNCCL.cpp:5023] [PG ID 0 PG GUID 0 Rank 42] using GPU 2 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank43]:[W1003 15:37:34.962793772 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank43]:[W1003 15:37:34.962845909 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank46]:[W1003 15:37:34.962982332 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank47]:[W1003 15:37:34.962994481 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank44]:[W1003 15:37:34.963011310 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank46]:[W1003 15:37:34.963029488 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank47]:[W1003 15:37:34.963056746 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank44]:[W1003 15:37:34.963080975 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank42]:[W1003 15:37:34.963084640 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank42]:[W1003 15:37:34.963151099 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank41]:[W1003 15:37:34.963179894 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank41]:[W1003 15:37:34.963215842 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank40]:[W1003 15:37:34.963213743 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank40]:[W1003 15:37:34.963252209 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank45]:[W1003 15:37:34.963289543 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank45]:[W1003 15:37:34.963324081 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + + + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/dist_checkpointing/strategies/torch.py:956: FutureWarning: `load_state_dict` is deprecated and will be removed in future versions. Please use `load` instead. + checkpoint.load_state_dict( +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/dist_checkpointing/strategies/torch.py:956: FutureWarning: `load_state_dict` is deprecated and will be removed in future versions. Please use `load` instead. + checkpoint.load_state_dict( +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/dist_checkpointing/strategies/torch.py:956: FutureWarning: `load_state_dict` is deprecated and will be removed in future versions. Please use `load` instead. + checkpoint.load_state_dict( +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/dist_checkpointing/strategies/torch.py:956: FutureWarning: `load_state_dict` is deprecated and will be removed in future versions. Please use `load` instead. + checkpoint.load_state_dict( +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/dist_checkpointing/strategies/torch.py:956: FutureWarning: `load_state_dict` is deprecated and will be removed in future versions. Please use `load` instead. + checkpoint.load_state_dict( +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/dist_checkpointing/strategies/torch.py:956: FutureWarning: `load_state_dict` is deprecated and will be removed in future versions. Please use `load` instead. + checkpoint.load_state_dict( +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/dist_checkpointing/strategies/torch.py:956: FutureWarning: `load_state_dict` is deprecated and will be removed in future versions. Please use `load` instead. + checkpoint.load_state_dict( +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/dist_checkpointing/strategies/torch.py:956: FutureWarning: `load_state_dict` is deprecated and will be removed in future versions. Please use `load` instead. + checkpoint.load_state_dict( +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +NCCL version 2.27.3+cuda12.9 +Initializing GlobIndexedDatasetInitializing GlobIndexedDataset + +Initializing GlobIndexedDataset +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_0_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_10_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_11_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_12_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_13_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_14_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_15_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_16_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_17_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_18_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_19_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_1_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_20_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_21_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_22_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_23_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_24_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_25_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_26_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_27_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_28_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_29_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_2_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_30_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_31_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_32_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_33_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_34_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_35_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_36_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_37_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_38_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_39_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_3_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_40_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_41_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_42_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_43_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_44_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_45_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_46_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_47_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_48_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_49_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_4_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_50_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_51_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_52_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_53_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_54_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_55_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_56_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_57_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_58_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_59_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_5_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_60_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_61_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_62_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_63_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_64_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_65_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_66_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_67_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_68_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_69_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_6_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_70_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_71_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_72_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_73_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_74_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_75_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_76_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_77_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_78_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_7_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_8_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_9_input_ids_tmp']['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_0_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_10_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_11_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_12_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_13_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_14_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_15_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_16_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_17_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_18_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_19_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_1_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_20_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_21_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_22_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_23_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_24_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_25_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_26_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_27_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_28_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_29_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_2_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_30_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_31_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_32_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_33_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_34_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_35_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_36_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_37_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_38_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_39_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_3_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_40_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_41_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_42_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_43_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_44_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_45_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_46_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_47_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_48_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_49_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_4_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_50_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_51_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_52_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_53_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_54_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_55_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_56_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_57_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_58_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_59_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_5_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_60_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_61_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_62_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_63_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_64_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_65_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_66_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_67_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_68_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_69_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_6_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_70_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_71_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_72_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_73_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_74_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_75_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_76_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_77_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_78_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_7_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_8_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_9_input_ids_tmp'] + +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_0_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_10_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_11_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_12_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_13_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_14_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_15_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_16_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_17_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_18_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_19_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_1_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_20_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_21_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_22_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_23_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_24_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_25_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_26_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_27_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_28_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_29_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_2_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_30_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_31_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_32_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_33_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_34_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_35_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_36_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_37_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_38_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_39_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_3_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_40_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_41_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_42_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_43_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_44_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_45_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_46_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_47_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_48_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_49_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_4_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_50_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_51_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_52_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_53_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_54_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_55_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_56_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_57_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_58_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_59_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_5_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_60_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_61_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_62_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_63_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_64_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_65_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_66_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_67_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_68_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_69_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_6_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_70_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_71_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_72_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_73_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_74_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_75_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_76_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_77_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_78_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_7_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_8_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_9_input_ids_tmp'] +Initializing GlobIndexedDataset +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_0_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_10_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_11_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_12_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_13_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_14_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_15_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_16_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_17_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_18_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_19_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_1_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_20_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_21_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_22_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_23_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_24_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_25_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_26_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_27_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_28_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_29_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_2_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_30_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_31_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_32_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_33_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_34_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_35_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_36_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_37_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_38_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_39_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_3_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_40_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_41_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_42_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_43_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_44_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_45_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_46_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_47_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_48_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_49_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_4_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_50_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_51_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_52_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_53_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_54_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_55_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_56_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_57_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_58_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_59_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_5_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_60_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_61_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_62_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_63_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_64_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_65_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_66_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_67_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_68_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_69_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_6_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_70_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_71_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_72_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_73_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_74_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_75_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_76_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_77_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_78_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_7_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_8_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_9_input_ids_tmp'] +Initializing GlobIndexedDataset +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_0_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_10_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_11_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_12_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_13_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_14_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_15_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_16_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_17_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_18_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_19_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_1_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_20_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_21_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_22_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_23_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_24_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_25_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_26_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_27_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_28_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_29_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_2_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_30_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_31_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_32_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_33_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_34_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_35_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_36_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_37_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_38_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_39_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_3_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_40_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_41_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_42_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_43_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_44_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_45_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_46_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_47_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_48_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_49_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_4_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_50_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_51_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_52_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_53_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_54_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_55_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_56_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_57_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_58_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_59_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_5_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_60_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_61_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_62_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_63_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_64_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_65_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_66_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_67_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_68_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_69_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_6_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_70_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_71_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_72_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_73_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_74_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_75_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_76_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_77_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_78_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_7_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_8_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_9_input_ids_tmp'] +Initializing GlobIndexedDataset +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_0_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_10_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_11_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_12_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_13_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_14_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_15_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_16_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_17_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_18_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_19_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_1_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_20_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_21_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_22_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_23_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_24_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_25_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_26_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_27_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_28_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_29_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_2_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_30_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_31_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_32_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_33_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_34_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_35_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_36_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_37_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_38_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_39_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_3_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_40_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_41_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_42_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_43_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_44_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_45_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_46_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_47_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_48_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_49_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_4_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_50_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_51_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_52_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_53_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_54_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_55_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_56_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_57_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_58_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_59_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_5_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_60_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_61_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_62_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_63_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_64_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_65_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_66_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_67_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_68_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_69_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_6_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_70_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_71_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_72_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_73_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_74_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_75_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_76_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_77_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_78_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_7_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_8_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_9_input_ids_tmp'] +Initializing GlobIndexedDataset +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_0_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_10_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_11_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_12_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_13_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_14_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_15_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_16_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_17_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_18_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_19_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_1_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_20_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_21_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_22_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_23_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_24_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_25_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_26_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_27_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_28_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_29_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_2_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_30_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_31_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_32_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_33_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_34_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_35_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_36_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_37_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_38_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_39_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_3_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_40_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_41_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_42_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_43_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_44_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_45_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_46_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_47_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_48_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_49_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_4_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_50_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_51_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_52_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_53_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_54_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_55_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_56_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_57_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_58_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_59_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_5_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_60_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_61_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_62_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_63_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_64_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_65_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_66_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_67_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_68_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_69_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_6_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_70_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_71_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_72_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_73_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_74_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_75_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_76_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_77_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_78_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_7_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_8_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_9_input_ids_tmp'] +Initializing GlobIndexedDataset +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_0_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_10_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_11_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_12_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_13_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_14_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_15_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_16_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_17_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_18_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_19_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_1_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_20_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_21_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_22_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_23_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_24_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_25_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_26_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_27_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_28_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_29_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_2_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_30_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_31_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_32_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_33_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_34_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_35_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_36_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_37_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_38_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_39_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_3_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_40_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_41_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_42_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_43_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_44_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_45_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_46_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_47_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_48_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_49_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_4_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_50_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_51_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_52_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_53_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_54_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_55_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_56_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_57_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_58_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_59_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_5_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_60_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_61_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_62_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_63_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_64_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_65_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_66_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_67_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_68_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_69_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_6_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_70_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_71_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_72_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_73_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_74_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_75_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_76_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_77_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_78_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_7_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_8_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_9_input_ids_tmp'] +>>> start_samples: 0>>> start_samples: 0 +>>> start_samples: 0 +>>> start_samples: 0 +>>> start_samples: 0 +>>> start_samples: 0 +>>> start_samples: 0 +>>> start_samples: 0 + +Initializing GlobIndexedDataset +Initializing GlobIndexedDataset +Initializing GlobIndexedDataset +Initializing GlobIndexedDataset +Initializing GlobIndexedDataset +Initializing GlobIndexedDataset +Initializing GlobIndexedDataset +Initializing GlobIndexedDataset +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_75_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_76_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_77_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_78_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_9_loss_mask_tmp']['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_75_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_76_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_77_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_78_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_9_loss_mask_tmp'] + +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_75_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_76_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_77_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_78_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_9_loss_mask_tmp'] +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_75_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_76_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_77_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_78_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_9_loss_mask_tmp'] +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_75_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_76_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_77_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_78_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_9_loss_mask_tmp']['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_75_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_76_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_77_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_78_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_9_loss_mask_tmp'] + +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_75_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_76_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_77_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_78_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_9_loss_mask_tmp']['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_75_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_76_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_77_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_78_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_9_loss_mask_tmp'] + +pad_document_marker: 36616095 +pad_document_marker: 36616095 +/mnt/hdd/cpt/binidx_cache_stage15_final_20261001_v2 d6c36c6d5886334ea5f92e75fcf45d10-GPTDataset-train-obfd cache hit: True/mnt/hdd/cpt/binidx_cache_stage15_final_20261001_v2 d6c36c6d5886334ea5f92e75fcf45d10-GPTDataset-train-obfd cache hit: True + +pad_document_marker: 36616095 +/mnt/hdd/cpt/binidx_cache_stage15_final_20261001_v2 d6c36c6d5886334ea5f92e75fcf45d10-GPTDataset-train-obfd cache hit: True +pad_document_marker: 36616095 +/mnt/hdd/cpt/binidx_cache_stage15_final_20261001_v2 d6c36c6d5886334ea5f92e75fcf45d10-GPTDataset-train-obfd cache hit: True +pad_document_marker: 36616095 +/mnt/hdd/cpt/binidx_cache_stage15_final_20261001_v2 d6c36c6d5886334ea5f92e75fcf45d10-GPTDataset-train-obfd cache hit: True +pad_document_marker: 36616095 +/mnt/hdd/cpt/binidx_cache_stage15_final_20261001_v2 d6c36c6d5886334ea5f92e75fcf45d10-GPTDataset-train-obfd cache hit: True +pad_document_marker: 36616095 +/mnt/hdd/cpt/binidx_cache_stage15_final_20261001_v2 d6c36c6d5886334ea5f92e75fcf45d10-GPTDataset-train-obfd cache hit: True +pad_document_marker: 36616095 +/mnt/hdd/cpt/binidx_cache_stage15_final_20261001_v2 d6c36c6d5886334ea5f92e75fcf45d10-GPTDataset-train-obfd cache hit: True +>>> start_samples: 0>>> start_samples: 0>>> start_samples: 0>>> start_samples: 0>>> start_samples: 0 + +>>> start_samples: 0>>> start_samples: 0 + + + + +>>> start_samples: 0 +Initializing GlobIndexedDatasetInitializing GlobIndexedDataset + +Initializing GlobIndexedDataset +Initializing GlobIndexedDatasetInitializing GlobIndexedDataset + +Initializing GlobIndexedDatasetInitializing GlobIndexedDataset + +Initializing GlobIndexedDataset +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_75_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_76_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_77_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_78_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_9_loss_mask_tmp']['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_75_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_76_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_77_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_78_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_9_loss_mask_tmp']['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_75_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_76_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_77_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_78_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_9_loss_mask_tmp']['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_75_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_76_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_77_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_78_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_9_loss_mask_tmp']['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_75_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_76_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_77_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_78_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_9_loss_mask_tmp'] + +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_75_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_76_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_77_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_78_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_9_loss_mask_tmp']['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_75_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_76_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_77_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_78_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_9_loss_mask_tmp'] +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_75_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_76_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_77_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_78_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_9_loss_mask_tmp'] + + + + +/mnt/hdd/cpt/binidx_cache_stage15_final_20261001_v2 0642c625420be1073f42e03d3641a6e5-GPTDataset-valid-random cache hit: True +/mnt/hdd/cpt/binidx_cache_stage15_final_20261001_v2 0642c625420be1073f42e03d3641a6e5-GPTDataset-valid-random cache hit: True +/mnt/hdd/cpt/binidx_cache_stage15_final_20261001_v2 0642c625420be1073f42e03d3641a6e5-GPTDataset-valid-random cache hit: True +/mnt/hdd/cpt/binidx_cache_stage15_final_20261001_v2 0642c625420be1073f42e03d3641a6e5-GPTDataset-valid-random cache hit: True +/mnt/hdd/cpt/binidx_cache_stage15_final_20261001_v2 0642c625420be1073f42e03d3641a6e5-GPTDataset-valid-random cache hit: True +/mnt/hdd/cpt/binidx_cache_stage15_final_20261001_v2 0642c625420be1073f42e03d3641a6e5-GPTDataset-valid-random cache hit: True +/mnt/hdd/cpt/binidx_cache_stage15_final_20261001_v2 0642c625420be1073f42e03d3641a6e5-GPTDataset-valid-random cache hit: True +/mnt/hdd/cpt/binidx_cache_stage15_final_20261001_v2 0642c625420be1073f42e03d3641a6e5-GPTDataset-valid-random cache hit: True +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/optimizer/optimizer.py:65: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /pytorch/build/aten/src/ATen/core/TensorBody.h:489.) + if hasattr(param, grad_attr) and getattr(param, grad_attr) is not None: +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/optimizer/optimizer.py:65: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /pytorch/build/aten/src/ATen/core/TensorBody.h:489.) + if hasattr(param, grad_attr) and getattr(param, grad_attr) is not None: +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/optimizer/optimizer.py:65: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /pytorch/build/aten/src/ATen/core/TensorBody.h:489.) + if hasattr(param, grad_attr) and getattr(param, grad_attr) is not None: +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/optimizer/optimizer.py:65: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /pytorch/build/aten/src/ATen/core/TensorBody.h:489.) + if hasattr(param, grad_attr) and getattr(param, grad_attr) is not None: +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/optimizer/optimizer.py:65: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /pytorch/build/aten/src/ATen/core/TensorBody.h:489.) + if hasattr(param, grad_attr) and getattr(param, grad_attr) is not None: +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/optimizer/optimizer.py:65: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /pytorch/build/aten/src/ATen/core/TensorBody.h:489.) + if hasattr(param, grad_attr) and getattr(param, grad_attr) is not None: +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/optimizer/optimizer.py:65: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /pytorch/build/aten/src/ATen/core/TensorBody.h:489.) + if hasattr(param, grad_attr) and getattr(param, grad_attr) is not None: +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/optimizer/optimizer.py:65: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /pytorch/build/aten/src/ATen/core/TensorBody.h:489.) + if hasattr(param, grad_attr) and getattr(param, grad_attr) is not None: +⚠️ torch 2.8.0+cu128 with triton 3.4.0. inductor patch is not applied. +⚠️ torch 2.8.0+cu128 with triton 3.4.0. inductor patch is not applied. +⚠️ torch 2.8.0+cu128 with triton 3.4.0. inductor patch is not applied. +⚠️ torch 2.8.0+cu128 with triton 3.4.0. inductor patch is not applied. +⚠️ torch 2.8.0+cu128 with triton 3.4.0. inductor patch is not applied. +⚠️ torch 2.8.0+cu128 with triton 3.4.0. inductor patch is not applied. +⚠️ torch 2.8.0+cu128 with triton 3.4.0. inductor patch is not applied. +⚠️ torch 2.8.0+cu128 with triton 3.4.0. inductor patch is not applied. +W1003 16:47:33.272000 49972 torch/utils/cpp_extension.py:2425] TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +W1003 16:47:33.272000 49972 torch/utils/cpp_extension.py:2425] If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'] to specific architectures. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +[rank40]:[W1010 20:26:01.507444267 ProcessGroupNCCL.cpp:1538] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/node6.log b/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/node6.log new file mode 100644 index 0000000000000000000000000000000000000000..983ca3ad12f962f36f0f0b80211d011fdf96c87c --- /dev/null +++ b/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/node6.log @@ -0,0 +1,358 @@ +W1003 15:36:46.436000 42982 torch/utils/cpp_extension.py:2425] TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +W1003 15:36:46.436000 42982 torch/utils/cpp_extension.py:2425] If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'] to specific architectures. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +[Gloo] Rank 50 is connected to 63 peer ranks. Expected number of connected peer ranks is : 63 +[rank50]:[W1003 15:37:33.437585283 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank50]:[W1003 15:37:33.437608978 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[Gloo] Rank [Gloo] Rank 4849 is connected to is connected to 63[Gloo] Rank 63 peer ranks. 51 peer ranks. [Gloo] Rank Expected number of connected peer ranks is : [Gloo] Rank [Gloo] Rank is connected to Expected number of connected peer ranks is : [Gloo] Rank 53635452636355 is connected to + is connected to is connected to peer ranks. + is connected to 636363Expected number of connected peer ranks is : 63 peer ranks. peer ranks. peer ranks. 63 peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : Expected number of connected peer ranks is : +Expected number of connected peer ranks is : 63636363 + + + +[rank48]:[W1003 15:37:33.438794951 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank48]:[W1003 15:37:33.438820149 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank49]:[W1003 15:37:33.438823746 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank49]:[W1003 15:37:33.438853342 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank51]:[W1003 15:37:33.438890613 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank51]:[W1003 15:37:33.438916393 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank52]:[W1003 15:37:33.438956118 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank55]:[W1003 15:37:33.438957461 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank54]:[W1003 15:37:33.438956579 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank53]:[W1003 15:37:33.438970737 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank52]:[W1003 15:37:33.438976588 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank55]:[W1003 15:37:33.438989312 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank54]:[W1003 15:37:33.439005904 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank53]:[W1003 15:37:33.439016283 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[Gloo] Rank 6 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 6 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 6 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 6 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 6 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank [Gloo] Rank 66 is connected to is connected to 77 peer ranks. peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : 77 + +[Gloo] Rank 6 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 6 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 6 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 6 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank [Gloo] Rank 66 is connected to is connected to 77 peer ranks. peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : 77 + +[Gloo] Rank 6 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 6 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 6 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. + warnings.warn( # warn only once +[rank48]:[W1003 15:37:33.549126037 ProcessGroupNCCL.cpp:5023] [PG ID 0 PG GUID 0 Rank 48] using GPU 0 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. + warnings.warn( # warn only once +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. + warnings.warn( # warn only once +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. + warnings.warn( # warn only once +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. + warnings.warn( # warn only once +[rank50]:[W1003 15:37:33.551298102 ProcessGroupNCCL.cpp:5023] [PG ID 0 PG GUID 0 Rank 50] using GPU 2 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank52]:[W1003 15:37:33.551290818 ProcessGroupNCCL.cpp:5023] [PG ID 0 PG GUID 0 Rank 52] using GPU 4 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank53]:[W1003 15:37:33.551346304 ProcessGroupNCCL.cpp:5023] [PG ID 0 PG GUID 0 Rank 53] using GPU 5 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank54]:[W1003 15:37:33.551381641 ProcessGroupNCCL.cpp:5023] [PG ID 0 PG GUID 0 Rank 54] using GPU 6 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. + warnings.warn( # warn only once +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. + warnings.warn( # warn only once +[rank55]:[W1003 15:37:33.551886224 ProcessGroupNCCL.cpp:5023] [PG ID 0 PG GUID 0 Rank 55] using GPU 7 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank51]:[W1003 15:37:33.551907615 ProcessGroupNCCL.cpp:5023] [PG ID 0 PG GUID 0 Rank 51] using GPU 3 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. + warnings.warn( # warn only once +[rank49]:[W1003 15:37:33.552427397 ProcessGroupNCCL.cpp:5023] [PG ID 0 PG GUID 0 Rank 49] using GPU 1 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank51]:[W1003 15:37:34.564675293 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank51]:[W1003 15:37:34.564766227 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank52]:[W1003 15:37:34.566301867 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank52]:[W1003 15:37:34.566340240 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank48]:[W1003 15:37:34.566458446 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank49]:[W1003 15:37:34.566507970 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank48]:[W1003 15:37:34.566544640 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank49]:[W1003 15:37:34.566607650 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank53]:[W1003 15:37:34.566923222 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank50]:[W1003 15:37:34.566959071 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank54]:[W1003 15:37:34.567006040 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank53]:[W1003 15:37:34.567028914 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank54]:[W1003 15:37:34.567095210 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank50]:[W1003 15:37:34.567066756 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[rank55]:[W1003 15:37:34.568859126 Utils.hpp:137] Warning: Environment variable TORCH_NCCL_TRACE_BUFFER_SIZE is deprecated; use TORCH_FR_BUFFER_SIZE instead (function operator()) +[rank55]:[W1003 15:37:34.568900265 Utils.hpp:166] Warning: Environment variable TORCH_NCCL_TRACE_CPP_STACK is deprecated; use TORCH_FR_CPP_STACK instead (function operator()) +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + + + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + + + + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True + +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/dist_checkpointing/strategies/torch.py:956: FutureWarning: `load_state_dict` is deprecated and will be removed in future versions. Please use `load` instead. + checkpoint.load_state_dict( +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/dist_checkpointing/strategies/torch.py:956: FutureWarning: `load_state_dict` is deprecated and will be removed in future versions. Please use `load` instead. + checkpoint.load_state_dict( +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/dist_checkpointing/strategies/torch.py:956: FutureWarning: `load_state_dict` is deprecated and will be removed in future versions. Please use `load` instead. + checkpoint.load_state_dict( +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/dist_checkpointing/strategies/torch.py:956: FutureWarning: `load_state_dict` is deprecated and will be removed in future versions. Please use `load` instead. + checkpoint.load_state_dict( +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/dist_checkpointing/strategies/torch.py:956: FutureWarning: `load_state_dict` is deprecated and will be removed in future versions. Please use `load` instead. + checkpoint.load_state_dict( +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/dist_checkpointing/strategies/torch.py:956: FutureWarning: `load_state_dict` is deprecated and will be removed in future versions. Please use `load` instead. + checkpoint.load_state_dict( +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/dist_checkpointing/strategies/torch.py:956: FutureWarning: `load_state_dict` is deprecated and will be removed in future versions. Please use `load` instead. + checkpoint.load_state_dict( +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/dist_checkpointing/strategies/torch.py:956: FutureWarning: `load_state_dict` is deprecated and will be removed in future versions. Please use `load` instead. + checkpoint.load_state_dict( +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +NCCL version 2.27.3+cuda12.9 +Initializing GlobIndexedDataset +Initializing GlobIndexedDataset +Initializing GlobIndexedDataset +Initializing GlobIndexedDataset +Initializing GlobIndexedDataset +Initializing GlobIndexedDataset +Initializing GlobIndexedDataset +Initializing GlobIndexedDataset +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_0_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_10_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_11_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_12_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_13_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_14_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_15_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_16_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_17_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_18_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_19_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_1_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_20_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_21_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_22_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_23_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_24_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_25_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_26_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_27_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_28_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_29_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_2_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_30_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_31_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_32_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_33_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_34_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_35_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_36_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_37_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_38_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_39_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_3_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_40_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_41_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_42_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_43_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_44_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_45_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_46_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_47_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_48_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_49_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_4_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_50_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_51_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_52_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_53_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_54_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_55_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_56_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_57_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_58_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_59_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_5_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_60_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_61_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_62_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_63_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_64_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_65_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_66_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_67_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_68_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_69_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_6_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_70_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_71_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_72_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_73_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_74_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_75_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_76_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_77_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_78_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_7_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_8_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_9_input_ids_tmp'] +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_0_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_10_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_11_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_12_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_13_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_14_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_15_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_16_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_17_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_18_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_19_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_1_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_20_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_21_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_22_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_23_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_24_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_25_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_26_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_27_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_28_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_29_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_2_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_30_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_31_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_32_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_33_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_34_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_35_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_36_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_37_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_38_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_39_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_3_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_40_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_41_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_42_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_43_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_44_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_45_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_46_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_47_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_48_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_49_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_4_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_50_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_51_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_52_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_53_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_54_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_55_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_56_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_57_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_58_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_59_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_5_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_60_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_61_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_62_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_63_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_64_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_65_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_66_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_67_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_68_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_69_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_6_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_70_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_71_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_72_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_73_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_74_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_75_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_76_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_77_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_78_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_7_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_8_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_9_input_ids_tmp']['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_0_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_10_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_11_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_12_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_13_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_14_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_15_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_16_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_17_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_18_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_19_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_1_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_20_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_21_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_22_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_23_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_24_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_25_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_26_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_27_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_28_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_29_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_2_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_30_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_31_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_32_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_33_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_34_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_35_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_36_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_37_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_38_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_39_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_3_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_40_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_41_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_42_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_43_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_44_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_45_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_46_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_47_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_48_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_49_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_4_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_50_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_51_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_52_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_53_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_54_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_55_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_56_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_57_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_58_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_59_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_5_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_60_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_61_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_62_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_63_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_64_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_65_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_66_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_67_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_68_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_69_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_6_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_70_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_71_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_72_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_73_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_74_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_75_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_76_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_77_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_78_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_7_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_8_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_9_input_ids_tmp']['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_0_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_10_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_11_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_12_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_13_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_14_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_15_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_16_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_17_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_18_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_19_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_1_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_20_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_21_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_22_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_23_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_24_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_25_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_26_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_27_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_28_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_29_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_2_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_30_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_31_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_32_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_33_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_34_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_35_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_36_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_37_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_38_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_39_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_3_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_40_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_41_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_42_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_43_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_44_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_45_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_46_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_47_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_48_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_49_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_4_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_50_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_51_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_52_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_53_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_54_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_55_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_56_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_57_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_58_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_59_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_5_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_60_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_61_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_62_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_63_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_64_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_65_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_66_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_67_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_68_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_69_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_6_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_70_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_71_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_72_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_73_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_74_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_75_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_76_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_77_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_78_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_7_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_8_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_9_input_ids_tmp']['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_0_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_10_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_11_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_12_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_13_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_14_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_15_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_16_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_17_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_18_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_19_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_1_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_20_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_21_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_22_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_23_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_24_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_25_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_26_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_27_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_28_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_29_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_2_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_30_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_31_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_32_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_33_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_34_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_35_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_36_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_37_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_38_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_39_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_3_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_40_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_41_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_42_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_43_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_44_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_45_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_46_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_47_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_48_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_49_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_4_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_50_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_51_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_52_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_53_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_54_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_55_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_56_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_57_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_58_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_59_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_5_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_60_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_61_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_62_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_63_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_64_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_65_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_66_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_67_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_68_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_69_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_6_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_70_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_71_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_72_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_73_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_74_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_75_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_76_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_77_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_78_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_7_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_8_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_9_input_ids_tmp']['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_0_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_10_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_11_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_12_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_13_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_14_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_15_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_16_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_17_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_18_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_19_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_1_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_20_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_21_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_22_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_23_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_24_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_25_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_26_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_27_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_28_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_29_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_2_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_30_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_31_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_32_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_33_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_34_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_35_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_36_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_37_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_38_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_39_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_3_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_40_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_41_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_42_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_43_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_44_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_45_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_46_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_47_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_48_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_49_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_4_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_50_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_51_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_52_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_53_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_54_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_55_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_56_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_57_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_58_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_59_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_5_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_60_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_61_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_62_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_63_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_64_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_65_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_66_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_67_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_68_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_69_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_6_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_70_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_71_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_72_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_73_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_74_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_75_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_76_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_77_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_78_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_7_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_8_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_9_input_ids_tmp']['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_0_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_10_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_11_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_12_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_13_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_14_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_15_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_16_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_17_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_18_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_19_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_1_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_20_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_21_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_22_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_23_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_24_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_25_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_26_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_27_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_28_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_29_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_2_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_30_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_31_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_32_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_33_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_34_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_35_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_36_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_37_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_38_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_39_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_3_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_40_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_41_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_42_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_43_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_44_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_45_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_46_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_47_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_48_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_49_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_4_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_50_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_51_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_52_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_53_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_54_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_55_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_56_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_57_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_58_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_59_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_5_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_60_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_61_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_62_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_63_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_64_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_65_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_66_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_67_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_68_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_69_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_6_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_70_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_71_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_72_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_73_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_74_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_75_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_76_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_77_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_78_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_7_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_8_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_9_input_ids_tmp'] + + + + +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_0_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_10_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_11_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_12_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_13_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_14_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_15_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_16_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_17_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_18_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_19_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_1_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_20_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_21_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_22_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_23_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_24_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_25_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_26_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_27_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_28_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_29_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_2_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_30_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_31_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_32_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_33_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_34_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_35_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_36_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_37_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_38_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_39_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_3_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_40_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_41_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_42_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_43_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_44_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_45_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_46_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_47_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_48_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_49_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_4_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_50_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_51_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_52_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_53_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_54_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_55_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_56_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_57_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_58_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_59_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_5_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_60_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_61_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_62_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_63_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_64_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_65_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_66_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_67_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_68_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_69_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_6_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_70_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_71_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_72_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_73_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_74_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_75_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_76_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_77_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_78_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_7_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_8_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_9_input_ids_tmp'] + +>>> start_samples: 0>>> start_samples: 0 +>>> start_samples: 0 +>>> start_samples: 0 +>>> start_samples: 0 +>>> start_samples: 0 +>>> start_samples: 0 +>>> start_samples: 0Initializing GlobIndexedDataset +Initializing GlobIndexedDataset + +Initializing GlobIndexedDataset + +Initializing GlobIndexedDataset +Initializing GlobIndexedDataset +Initializing GlobIndexedDataset +Initializing GlobIndexedDataset +Initializing GlobIndexedDataset +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_75_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_76_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_77_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_78_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_9_loss_mask_tmp']['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_75_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_76_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_77_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_78_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_9_loss_mask_tmp']['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_75_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_76_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_77_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_78_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_9_loss_mask_tmp'] + + +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_75_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_76_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_77_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_78_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_9_loss_mask_tmp']['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_75_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_76_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_77_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_78_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_9_loss_mask_tmp'] +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_75_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_76_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_77_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_78_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_9_loss_mask_tmp']['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_75_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_76_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_77_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_78_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_9_loss_mask_tmp'] +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_75_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_76_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_77_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_78_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_9_loss_mask_tmp'] + + +pad_document_marker: 36616095 +pad_document_marker: 36616095 +pad_document_marker: 36616095 +/mnt/hdd/cpt/binidx_cache_stage15_final_20261001_v2 d6c36c6d5886334ea5f92e75fcf45d10-GPTDataset-train-obfd cache hit: True/mnt/hdd/cpt/binidx_cache_stage15_final_20261001_v2 d6c36c6d5886334ea5f92e75fcf45d10-GPTDataset-train-obfd cache hit: True/mnt/hdd/cpt/binidx_cache_stage15_final_20261001_v2 d6c36c6d5886334ea5f92e75fcf45d10-GPTDataset-train-obfd cache hit: True + + +pad_document_marker: 36616095 +/mnt/hdd/cpt/binidx_cache_stage15_final_20261001_v2 d6c36c6d5886334ea5f92e75fcf45d10-GPTDataset-train-obfd cache hit: True +pad_document_marker: 36616095 +/mnt/hdd/cpt/binidx_cache_stage15_final_20261001_v2 d6c36c6d5886334ea5f92e75fcf45d10-GPTDataset-train-obfd cache hit: True +pad_document_marker: 36616095 +pad_document_marker: 36616095pad_document_marker: 36616095 +/mnt/hdd/cpt/binidx_cache_stage15_final_20261001_v2 d6c36c6d5886334ea5f92e75fcf45d10-GPTDataset-train-obfd cache hit: True + +/mnt/hdd/cpt/binidx_cache_stage15_final_20261001_v2 d6c36c6d5886334ea5f92e75fcf45d10-GPTDataset-train-obfd cache hit: True +/mnt/hdd/cpt/binidx_cache_stage15_final_20261001_v2 d6c36c6d5886334ea5f92e75fcf45d10-GPTDataset-train-obfd cache hit: True +>>> start_samples: 0 +>>> start_samples: 0>>> start_samples: 0 +>>> start_samples: 0>>> start_samples: 0>>> start_samples: 0 +>>> start_samples: 0>>> start_samples: 0Initializing GlobIndexedDataset + + + + + +Initializing GlobIndexedDataset +Initializing GlobIndexedDataset +Initializing GlobIndexedDatasetInitializing GlobIndexedDataset +Initializing GlobIndexedDataset +Initializing GlobIndexedDatasetInitializing GlobIndexedDataset + + +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_75_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_76_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_77_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_78_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_9_loss_mask_tmp'] +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_75_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_76_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_77_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_78_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_9_loss_mask_tmp']['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_75_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_76_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_77_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_78_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_9_loss_mask_tmp'] + +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_75_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_76_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_77_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_78_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_9_loss_mask_tmp']['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_75_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_76_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_77_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_78_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_9_loss_mask_tmp']['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_75_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_76_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_77_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_78_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_9_loss_mask_tmp']['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_75_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_76_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_77_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_78_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_9_loss_mask_tmp'] +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_47_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_48_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_49_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_4_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_50_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_51_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_52_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_53_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_54_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_55_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_56_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_57_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_58_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_59_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_5_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_60_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_61_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_62_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_63_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_64_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_65_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_66_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_67_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_68_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_69_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_6_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_70_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_71_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_72_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_73_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_74_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_75_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_76_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_77_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_78_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_7_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_8_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_9_loss_mask_tmp'] + + + +/mnt/hdd/cpt/binidx_cache_stage15_final_20261001_v2 0642c625420be1073f42e03d3641a6e5-GPTDataset-valid-random cache hit: True/mnt/hdd/cpt/binidx_cache_stage15_final_20261001_v2 0642c625420be1073f42e03d3641a6e5-GPTDataset-valid-random cache hit: True + +/mnt/hdd/cpt/binidx_cache_stage15_final_20261001_v2 0642c625420be1073f42e03d3641a6e5-GPTDataset-valid-random cache hit: True +/mnt/hdd/cpt/binidx_cache_stage15_final_20261001_v2 0642c625420be1073f42e03d3641a6e5-GPTDataset-valid-random cache hit: True +/mnt/hdd/cpt/binidx_cache_stage15_final_20261001_v2 0642c625420be1073f42e03d3641a6e5-GPTDataset-valid-random cache hit: True +/mnt/hdd/cpt/binidx_cache_stage15_final_20261001_v2 0642c625420be1073f42e03d3641a6e5-GPTDataset-valid-random cache hit: True +/mnt/hdd/cpt/binidx_cache_stage15_final_20261001_v2 0642c625420be1073f42e03d3641a6e5-GPTDataset-valid-random cache hit: True +/mnt/hdd/cpt/binidx_cache_stage15_final_20261001_v2 0642c625420be1073f42e03d3641a6e5-GPTDataset-valid-random cache hit: True +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/optimizer/optimizer.py:65: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /pytorch/build/aten/src/ATen/core/TensorBody.h:489.) + if hasattr(param, grad_attr) and getattr(param, grad_attr) is not None: +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/optimizer/optimizer.py:65: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /pytorch/build/aten/src/ATen/core/TensorBody.h:489.) + if hasattr(param, grad_attr) and getattr(param, grad_attr) is not None: +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/optimizer/optimizer.py:65: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /pytorch/build/aten/src/ATen/core/TensorBody.h:489.) + if hasattr(param, grad_attr) and getattr(param, grad_attr) is not None: +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/optimizer/optimizer.py:65: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /pytorch/build/aten/src/ATen/core/TensorBody.h:489.) + if hasattr(param, grad_attr) and getattr(param, grad_attr) is not None: +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/optimizer/optimizer.py:65: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /pytorch/build/aten/src/ATen/core/TensorBody.h:489.) + if hasattr(param, grad_attr) and getattr(param, grad_attr) is not None: +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/optimizer/optimizer.py:65: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /pytorch/build/aten/src/ATen/core/TensorBody.h:489.) + if hasattr(param, grad_attr) and getattr(param, grad_attr) is not None: +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/optimizer/optimizer.py:65: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /pytorch/build/aten/src/ATen/core/TensorBody.h:489.) + if hasattr(param, grad_attr) and getattr(param, grad_attr) is not None: +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/optimizer/optimizer.py:65: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /pytorch/build/aten/src/ATen/core/TensorBody.h:489.) + if hasattr(param, grad_attr) and getattr(param, grad_attr) is not None: +⚠️ torch 2.8.0+cu128 with triton 3.4.0. inductor patch is not applied. +⚠️ torch 2.8.0+cu128 with triton 3.4.0. inductor patch is not applied. +⚠️ torch 2.8.0+cu128 with triton 3.4.0. inductor patch is not applied. +⚠️ torch 2.8.0+cu128 with triton 3.4.0. inductor patch is not applied. +⚠️ torch 2.8.0+cu128 with triton 3.4.0. inductor patch is not applied. +⚠️ torch 2.8.0+cu128 with triton 3.4.0. inductor patch is not applied. +⚠️ torch 2.8.0+cu128 with triton 3.4.0. inductor patch is not applied. +⚠️ torch 2.8.0+cu128 with triton 3.4.0. inductor patch is not applied. +W1003 16:47:34.163000 3638 torch/utils/cpp_extension.py:2425] TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation. +W1003 16:47:34.163000 3638 torch/utils/cpp_extension.py:2425] If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'] to specific architectures. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience. +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/modelopt/torch/__init__.py:36: UserWarning: transformers version 5.13.0 is not tested with nvidia-modelopt and may cause issues. Please install recommended version with `pip install nvidia-modelopt[hf]` if working with HF models. + _warnings.warn( +[rank48]:[W1010 20:26:01.580847940 ProcessGroupNCCL.cpp:1538] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/wandb/wandb/run-20261003_153655-neb76hvj/files/config.yaml b/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/wandb/wandb/run-20261003_153655-neb76hvj/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8aa54cc71295d8c2d23d434127e9461edc6f6538 --- /dev/null +++ b/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/wandb/wandb/run-20261003_153655-neb76hvj/files/config.yaml @@ -0,0 +1,2177 @@ +_wandb: + value: + cli_version: 0.23.0 + e: + i4bo5dz9n6n9rgehz11zfcuyno7peo6e: + args: + - --use-mcore-models + - --hybrid-attention-ratio + - "0.0625" + - --hybrid-mlp-ratio + - "0.5" + - --is-hybrid-model + - --linear-attention-type + - gated_delta_net + - --linear-conv-kernel-dim + - "4" + - --linear-key-head-dim + - "64" + - --linear-value-head-dim + - "64" + - --linear-num-key-heads + - "8" + - --linear-num-value-heads + - "32" + - --disable-bias-linear + - --seq-length + - "131072" + - --max-position-embeddings + - "131072" + - --num-layers + - "56" + - --hidden-size + - "2048" + - --ffn-hidden-size + - "4800" + - --num-attention-heads + - "32" + - --init-method-std + - "0.02" + - --attention-dropout + - "0.0" + - --hidden-dropout + - "0.0" + - --normalization + - RMSNorm + - --swiglu + - --group-query-attention + - --num-query-groups + - "8" + - --no-masked-softmax-fusion + - --use-flash-attn + - --add-qkv-bias + - --mtp-num-layers + - "0" + - --mtp-loss-scaling-factor + - "0.1" + - --norm-epsilon + - "1e-06" + - --mtp-linear-attention-type + - gated_delta_net + - --position-embedding-type + - none + - --linear-attention-freq + - '[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1]' + - --optimizer + - adam + - --adam-beta1 + - "0.9" + - --adam-beta2 + - "0.95" + - --adam-eps + - "1e-15" + - --weight-decay + - "0.1" + - --clip-grad + - "1.0" + - --data-path + - /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_{0..78}_input_ids_tmp + - --split + - 999,1,0 + - --data-cache-path + - /mnt/hdd/cpt/binidx_cache_stage15_final_20261001_v2 + - --no-create-attention-mask-in-dataloader + - --num-experts + - "192" + - --moe-grouped-gemm + - --moe-router-topk + - "4" + - --moe-router-score-function + - softmax + - --overlap-param-gather + - --overlap-grad-reduce + - --moe-expert-capacity-factor + - "0" + - --moe-shared-expert-intermediate-size + - "512" + - --moe-router-dtype + - fp32 + - --moe-permute-fusion + - --moe-ffn-hidden-size + - "512" + - --moe-shared-expert-gate + - --moe-shared-expert-overlap + - --moe-layer-freq + - ([0]*2+[1]*54) + - --moe-aux-loss-coeff + - "0.05" + - --moe-router-load-balancing-type + - global_aux_loss + - --micro-batch-size + - "1" + - --global-batch-size + - "128" + - --lr + - "5e-4" + - --train-samples + - "877440" + - --lr-warmup-samples + - "0" + - --lr-decay-samples + - "877440" + - --lr-decay-style + - constant + - --min-lr + - "0.0005" + - --bf16 + - --async-save + - --ckpt-fully-parallel-load + - --auto-detect-ckpt-format + - --load-complemental-dataset + - loss_mask + - --tensor-model-parallel-size + - "1" + - --pipeline-model-parallel-size + - "1" + - --context-parallel-size + - "8" + - --expert-model-parallel-size + - "8" + - --sequence-parallel + - --use-distributed-optimizer + - --log-interval + - "1" + - --log-throughput + - --save-interval + - "40" + - --eval-interval + - "99999999" + - --eval-iters + - "10" + - --increase-log-level-iters + - "5" + - --timing-log-level + - "0" + - --wandb-project + - qwen2.5_3b_mom_cpt + - --tokenizer-type + - HuggingFaceTokenizer + - --tokenizer-model + - /mnt/ssd/yulan/pretrain-linear-moe/preprocess/modify_tokenizer/YuLan-Mini-Qwen-Template_new_sp + - --no-adam-bias-correction + - --corrected-weight-decay + - --auto-generate-cu-seqlens + - --cp-comm-type + - a2a + - --log-moe-activations + - --final-logits-clamp-value + - "30" + - --attention-output-gate + - --gated-norm + - --gated-norm-lora-rank + - "16" + - --dataloader-type + - cyclic + - --lm-loss-normalization + - global-token + - --prefetch-factor + - "8" + - --activation-func-clamp-value + - "7" + - --activation-func-clamp-mode + - soft + - --gdn-split-in-proj + - --gdn-fused-split-forward + - --gdn-alpha-weight-scale + - "1.0" + - --gdn-alpha-weight-scale-mode + - matrix + - --gdn-lr-mult-alpha + - "0.5" + - --start-samples + - "0" + - --ckpt-format + - torch_dist + - --rerun-mode + - validate_results + - --distributed-timeout-minutes + - "120" + - --dist-ckpt-strictness + - assume_ok_unexpected + - --gdn-backend + - fla + - --gdn-gated-norm-backend + - linghe + - --moe-router-gemm-backend + - te + - --log-params-norm + - --timing-log-option + - all + - --no-barrier-with-level-1-timing + - --log-metrics-interval + - "10" + - --cross-entropy-loss-fusion + - --cross-entropy-fusion-impl + - linear_triton + - --linear-cross-entropy-num-chunks + - "8" + - --num-workers + - "4" + - --moe-router-sqrt-gate + - --moe-router-fusion + - --moe-token-dispatcher-type + - alltoall + - --log-hidden-states + - embeddings + - residual_attn + - input_layernorm + - attention::q_layernorm + - attention::k_layernorm + - attention::core_attention + - attention::o_proj + - residual_mlp + - pre_mlp_layernorm + - mlp + - moe::router + - moe::shared_experts + - moe::routed_experts + - gated_delta::in_proj + - gated_delta::conv1d + - gated_delta::query + - gated_delta::key + - gated_delta::value + - gated_delta::gate + - gated_delta::beta + - gated_delta::alpha + - gated_delta::g + - gated_delta::core_attention + - gated_delta::out_norm + - gated_delta::out_proj + - final_layernorm + - --log-params + - attention::linear_qkv + - attention::o_proj + - mlp::linear_fc1 + - mlp::linear_fc2 + - input_layernorm + - pre_mlp_layernorm + - embedding + - lm_head + - gated_delta::in_proj + - gated_delta::in_proj_bias + - gated_delta::conv1d + - gated_delta::conv1d_bias + - gated_delta::dt_bias + - gated_delta::A_log + - gated_delta::out_norm + - gated_delta::out_norm_bias + - gated_delta::out_proj + - gated_delta::out_proj_bias + - --log-update-rms + - --log-per-module-grad-rms + - --log-per-layer-grad-rms + - --document-packing-algorithm + - obfd + - --exit-on-missing-checkpoint + - --no-initialization + - --untie-embeddings-and-output-weights + - --recompute-granularity + - full + - --recompute-method + - block + - --recompute-num-layers + - "56" + - --no-save-step-one + - --save-retain-interval + - "1000000000" + - --exit-signal-handler + - --fast-thd-a2a + - --hybrid-adamh + - --reuse-training-communicators + - --ddp-bucket-size + - "40000000" + - --load + - /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage14_120B_8node_gdnlralpha_off_20260925a/checkpoints + - --continual-pretrain + - --save + - /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints + - --tensorboard-dir + - /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/tensorboard + - --wandb-save-dir + - /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/wandb + - --wandb-exp-name + - stage14_120B_8node_gdnlralpha_off_20260925a + - --runtime-audit-dir + - /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/runtime_audit + cpu_count: 64 + cpu_count_logical: 128 + cudaVersion: "13.3" + disk: + /: + total: "31526391808" + used: "1319878656" + email: huyiwen0301@hotmail.com + executable: /mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python + gpu: NVIDIA A800-SXM4-80GB + gpu_count: 8 + gpu_nvidia: + - architecture: Ampere + cudaCores: 6912 + memoryTotal: "85899345920" + name: NVIDIA A800-SXM4-80GB + uuid: GPU-40999d1b-2b80-14f4-48d7-334e7dc3e650 + - architecture: Ampere + cudaCores: 6912 + memoryTotal: "85899345920" + name: NVIDIA A800-SXM4-80GB + uuid: GPU-c854c8e4-7352-b317-f9e8-7ab1d7a12c08 + - architecture: Ampere + cudaCores: 6912 + memoryTotal: "85899345920" + name: NVIDIA A800-SXM4-80GB + uuid: GPU-f7d2842f-48e9-649c-55a4-4efe7a6e20f7 + - architecture: Ampere + cudaCores: 6912 + memoryTotal: "85899345920" + name: NVIDIA A800-SXM4-80GB + uuid: GPU-03925119-14d4-a11c-d740-0ba854285052 + - architecture: Ampere + cudaCores: 6912 + memoryTotal: "85899345920" + name: NVIDIA A800-SXM4-80GB + uuid: GPU-7cbabdfb-9388-3fbd-babc-d0b03ac9f1bb + - architecture: Ampere + cudaCores: 6912 + memoryTotal: "85899345920" + name: NVIDIA A800-SXM4-80GB + uuid: GPU-4999b9f4-9851-ce6a-4296-c230e2813b5f + - architecture: Ampere + cudaCores: 6912 + memoryTotal: "85899345920" + name: NVIDIA A800-SXM4-80GB + uuid: GPU-9e23b382-2f8e-e424-f025-c2b311d96c54 + - architecture: Ampere + cudaCores: 6912 + memoryTotal: "85899345920" + name: NVIDIA A800-SXM4-80GB + uuid: GPU-4241d9ca-7546-c1c3-dba8-bdc5786a3777 + host: cs-3554a-53569-server + memory: + total: "1081804955648" + os: Linux-5.15.0-94-generic-x86_64-with-glibc2.39 + program: /mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/pretrain_gpt.py + python: CPython 3.10.19 + root: /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/wandb + startedAt: "2026-10-03T07:36:55.078128Z" + writerId: i4bo5dz9n6n9rgehz11zfcuyno7peo6e + m: [] + python_version: 3.10.19 + t: + "1": + - 1 + - 11 + - 41 + - 49 + - 51 + - 71 + "2": + - 1 + - 11 + - 41 + - 49 + - 51 + - 71 + "3": + - 2 + - 13 + - 16 + - 61 + "4": 3.10.19 + "5": 0.23.0 + "6": 5.13.0 + "12": 0.23.0 + "13": linux-x86_64 +account_for_embedding_in_pipeline_split: + value: false +account_for_loss_in_pipeline_split: + value: false +accumulate_allreduce_grads_in_fp32: + value: true +activation_func_clamp_mode: + value: soft +activation_func_clamp_value: + value: 7 +adam_beta1: + value: 0.9 +adam_beta2: + value: 0.95 +adam_bias_correction: + value: false +adam_eps: + value: 1e-15 +adamh_eps: + value: 1e-08 +adamh_radius: + value: null +adamh_scope: + value: lm_head +adamw_lr_mup_scaler: + value: false +add_bias_linear: + value: false +add_position_embedding: + value: true +add_qkv_bias: + value: true +adlr_autoresume: + value: false +adlr_autoresume_interval: + value: 1000 +align_grad_reduce: + value: true +align_param_gather: + value: false +allow_ambiguous_pad_tokens: + value: false +app_tag_run_name: + value: null +app_tag_run_version: + value: 0.0.0 +apply_layernorm_1p: + value: false +apply_query_key_layer_scaling: + value: false +apply_residual_connection_post_layernorm: + value: false +apply_rope_fusion: + value: false +async_save: + value: true +async_tensor_model_parallel_allreduce: + value: true +attention_backend: + value: auto +attention_dropout: + value: 0 +attention_output_gate: + value: true +attention_output_gate_upcycling: + value: false +attention_softmax_in_fp32: + value: false +attn_k_token_shift: + value: null +attn_output_gate: + value: null +attn_q_token_shift: + value: null +attn_token_shift: + value: null +attn_v_token_shift: + value: null +auto_detect_ckpt_format: + value: true +auto_generate_cu_seqlens: + value: true +barrier_with_L1_time: + value: false +benchmark_eval: + value: false +benchmark_global_batch: + value: null +benchmark_interval: + value: null +benchmark_micro_batch: + value: null +benchmark_sequence_length: + value: null +benchmark_tasks: + value: null +bert_binary_head: + value: true +bert_embedder_type: + value: megatron +bert_load: + value: null +bf16: + value: true +bias_dropout_fusion: + value: true +bias_gelu_fusion: + value: false +bias_swiglu_fusion: + value: true +biencoder_projection_dim: + value: 0 +biencoder_shared_query_context_model: + value: false +block_data_path: + value: null +cache_mla_latents: + value: false +calc_ft_timeouts: + value: false +calculate_per_token_loss: + value: false +check_for_large_grads: + value: false +check_for_nan_in_loss_and_grad: + value: true +check_for_spiky_loss: + value: false +check_weight_hash_across_dp_replicas_interval: + value: null +ckpt_assume_constant_structure: + value: false +ckpt_convert_format: + value: null +ckpt_convert_save: + value: null +ckpt_convert_update_legacy_dist_opt_format: + value: false +ckpt_format: + value: torch_dist +ckpt_fully_parallel_load: + value: true +ckpt_fully_parallel_save: + value: true +ckpt_fully_parallel_save_deprecated: + value: false +ckpt_step: + value: null +classes_fraction: + value: 1 +clip_grad: + value: 1 +clone_scatter_output_in_embedding: + value: true +config_logger_dir: + value: "" +consumed_train_samples: + value: 0 +consumed_valid_samples: + value: 0 +context_parallel_size: + value: 8 +continual_pretrain: + value: true +continual_pretrain_preserve_opt_param_scheduler: + value: false +corrected_weight_decay: + value: true +cp_comm_type: + value: + - a2a +create_attention_mask_in_dataloader: + value: false +cross_entropy_fusion_impl: + value: linear_triton +cross_entropy_loss_fusion: + value: true +cuda_graph_impl: + value: none +cuda_graph_scope: + value: [] +cuda_graph_warmup_steps: + value: 3 +data_args_path: + value: null +data_cache_path: + value: /mnt/hdd/cpt/binidx_cache_stage15_final_20261001_v2 +data_parallel_random_init: + value: false +data_parallel_sharding_strategy: + value: no_shard +data_parallel_size: + value: 8 +data_path: + value: + - /mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_{0..78}_input_ids_tmp +data_per_class_fraction: + value: 1 +data_sharding: + value: true +dataloader_defer_npy_index_mmap: + value: false +dataloader_fast_cache_load: + value: false +dataloader_type: + value: cyclic +dc_shortconv: + value: false +dc_shortconv_optimizer_state: + value: conservative +dc_shortconv_upcycling: + value: false +ddp_average_in_collective: + value: false +ddp_bucket_size: + value: 40000000 +ddp_num_buckets: + value: null +ddp_pad_buckets_for_high_nccl_busbw: + value: false +decode_only_cuda_graphs: + value: false +decoder_first_pipeline_num_layers: + value: null +decoder_last_pipeline_num_layers: + value: null +decoder_num_layers: + value: null +decoder_seq_length: + value: null +decoupled_lr: + value: null +decoupled_min_lr: + value: null +decrease_batch_size_if_needed: + value: false +defer_embedding_wgrad_compute: + value: false +defer_grad_checks: + value: false +delay_wgrad_compute: + value: false +deprecated_use_mcore_models: + value: true +deterministic_mode: + value: false +dino_bottleneck_size: + value: 256 +dino_freeze_last_layer: + value: 1 +dino_head_hidden_size: + value: 2048 +dino_local_crops_number: + value: 10 +dino_local_img_size: + value: 96 +dino_norm_last_layer: + value: false +dino_teacher_temp: + value: 0.07 +dino_warmup_teacher_temp: + value: 0.04 +dino_warmup_teacher_temp_epochs: + value: 30 +disable_bf16_reduced_precision_matmul: + value: false +disable_chunked_prefill: + value: false +disable_explicit_attention_mask: + value: false +disable_mamba_mem_eff_path: + value: false +disable_straggler_on_startup: + value: false +disable_symmetric_registration: + value: false +dist_ckpt_format_deprecated: + value: null +dist_ckpt_optim_fully_reshardable: + value: false +dist_ckpt_save_pre_mcore_014: + value: false +dist_ckpt_strictness: + value: assume_ok_unexpected +distrib_optim_fully_reshardable_mem_efficient: + value: false +distribute_saved_activations: + value: false +distributed_backend: + value: nccl +distributed_timeout_minutes: + value: 120 +distributed_timeout_seconds_after_init: + value: null +document_packing_algorithm: + value: obfd +document_packing_dp_size: + value: null +dump_param_to_param_group_map: + value: null +dynamic_context_parallel: + value: false +dynamic_context_parallel_sizes: + value: null +dynamic_cp_candidate_sizes: + value: null +dynamic_cp_gdn_policy: + value: null +dynamic_cp_moe_policy: + value: null +dynamic_cp_on_error: + value: error +dynamic_cp_scheduler: + value: default_dynamic_cp +dynamic_cp_wave_coexecution: + value: false +dynamic_cp_workspace_max_tokens: + value: null +emb_deviation_loss_coeff: + value: 0 +emb_deviation_type: + value: null +embedding_init_method_std: + value: null +embedding_path: + value: null +empty_unused_memory_level: + value: 0 +enable_cuda_graph: + value: false +enable_debug_logging: + value: false +enable_experimental: + value: false +enable_ft_package: + value: false +enable_full_sharding_in_hsdp: + value: false +enable_gloo_process_groups: + value: true +enable_msc: + value: true +enable_one_logger: + value: true +encoder_num_layers: + value: 56 +encoder_seq_length: + value: 131072 +end_weight_decay: + value: 0.1 +eod_mask_loss: + value: false +error_injection_rate: + value: 0 +error_injection_type: + value: transient_error +eval_interval: + value: 99999999 +eval_iters: + value: 10 +evidence_data_path: + value: null +exit_duration_in_mins: + value: null +exit_interval: + value: null +exit_on_missing_checkpoint: + value: true +exit_signal_handler: + value: true +exp_avg_dtype: + value: torch.float32 +exp_avg_sq_dtype: + value: torch.float32 +expert_model_parallel_size: + value: 8 +expert_tensor_parallel_size: + value: 1 +export_force_local_attention: + value: false +export_kd_cfg: + value: null +export_kd_teacher_ckpt_format: + value: null +export_kd_teacher_load: + value: null +export_kv_cache_quant: + value: false +export_legacy_megatron: + value: false +export_model_type: + value: GPTModel +export_moe_apply_probs_on_input: + value: false +export_offline_model: + value: false +export_qk_l2_norm: + value: false +export_quant_cfg: + value: null +export_real_quant_cfg: + value: None +export_te_mcore_model: + value: false +external_cuda_graph: + value: false +fast_thd_a2a: + value: true +ffn_hidden_size: + value: 4800 +ffn_intermediate_token_shift: + value: null +ffn_token_shift: + value: null +final_logits_clamp_value: + value: 30 +fine_grained_activation_offloading: + value: false +finetune: + value: false +finetune_data_split: + value: train +finetune_hf_dataset: + value: null +first_last_layers_bf16: + value: false +flash_decode: + value: false +fp4: + value: null +fp4_param: + value: false +fp4_recipe: + value: nvfp4 +fp8: + value: null +fp8_amax_compute_algo: + value: most_recent +fp8_amax_history_len: + value: 1 +fp8_interval: + value: 1 +fp8_margin: + value: 0 +fp8_param_gather: + value: false +fp8_recipe: + value: delayed +fp8_wgrad: + value: true +fp16: + value: false +fp16_lm_cross_entropy: + value: false +fp32_residual_connection: + value: false +freeze_layernorm_weight: + value: false +freeze_non_mamba: + value: false +fsdp_double_buffer: + value: false +full_validation: + value: false +gated_norm: + value: true +gated_norm_lora_rank: + value: 16 +gated_norm_upcycling: + value: false +gdn_alpha_weight_scale: + value: 1 +gdn_alpha_weight_scale_mode: + value: matrix +gdn_backend: + value: fla +gdn_fused_split_forward: + value: true +gdn_gated_norm_backend: + value: linghe +gdn_lr_mult_alpha: + value: 0.5 +gdn_lr_mult_beta: + value: 1 +gdn_lr_mult_k: + value: 1 +gdn_lr_mult_q: + value: 1 +gdn_lr_mult_v: + value: 1 +gdn_postinit_alog_shift: + value: 0 +gdn_postinit_alpha_scale: + value: 1 +gdn_postinit_key_scale: + value: 1 +gdn_pre_gdr_fusion: + value: false +gdn_pre_gdr_fusion_deterministic_weight_gradient: + value: false +gdn_pre_gdr_fusion_high_precision_backward: + value: false +gdn_shortconv_separate_gate: + value: false +gdn_shortconv_separate_gate_upcycling: + value: false +gdn_split_in_proj: + value: true +geglu: + value: false +global_batch_size: + value: 128 +glu_linear_offset: + value: 0 +grad_reduce_in_bf16: + value: false +gradient_accumulation_fusion: + value: true +gradient_reduce_div_fusion: + value: true +group_query_attention: + value: true +grpo_clamp_eps_lower: + value: 0.01 +grpo_clamp_eps_upper: + value: 0.01 +grpo_default_temperature: + value: 1 +grpo_default_top_p: + value: 0 +grpo_entropy_term_weight: + value: 0 +grpo_filter_groups_with_same_reward: + value: false +grpo_group_size: + value: 2 +grpo_iterations: + value: 2 +grpo_kl_beta: + value: 0.001 +grpo_prompts_per_step: + value: 32 +head_lr_mult: + value: 1 +heterogeneous_layers_config_encoded_json: + value: null +heterogeneous_layers_config_path: + value: null +hidden_dropout: + value: 0 +hidden_size: + value: 2048 +hidden_states_shrink: + value: 1 +hierarchical_context_parallel_sizes: + value: null +high_priority_stream_groups: + value: [] +hybrid_adamh: + value: true +hybrid_attention_ratio: + value: 0.0625 +hybrid_context_parallel: + value: false +hybrid_mlp_ratio: + value: 0.5 +hybrid_override_pattern: + value: null +hysteresis: + value: 2 +ict_head_size: + value: null +ict_load: + value: null +img_h: + value: 224 +img_w: + value: 224 +increase_log_level_interval: + value: 1000 +increase_log_level_iters: + value: 5 +indexer_batch_size: + value: 128 +indexer_log_interval: + value: 1000 +inference_batch_times_seqlen_threshold: + value: -1 +inference_dynamic_batching: + value: false +inference_dynamic_batching_block_size: + value: 256 +inference_dynamic_batching_buffer_guaranteed_fraction: + value: 0.2 +inference_dynamic_batching_buffer_overflow_factor: + value: null +inference_dynamic_batching_buffer_size_gb: + value: 40 +inference_dynamic_batching_max_requests_override: + value: null +inference_dynamic_batching_max_tokens_override: + value: null +inference_dynamic_batching_num_cuda_graphs: + value: 16 +inference_dynamic_batching_track_paused_request_events: + value: false +inference_dynamic_batching_unified_memory_level: + value: 0 +inference_max_batch_size: + value: 8 +inference_max_seq_length: + value: 2560 +inference_rng_tracker: + value: false +init_method_std: + value: 0.02 +init_method_xavier_uniform: + value: false +init_model_with_meta_device: + value: false +initial_loss_scale: + value: 4294967296 +inprocess_active_world_size: + value: 64 +inprocess_barrier_timeout: + value: 120 +inprocess_completion_timeout: + value: 120 +inprocess_empty_cuda_cache: + value: false +inprocess_granularity: + value: node +inprocess_hard_timeout: + value: 90 +inprocess_heartbeat_interval: + value: 30 +inprocess_heartbeat_timeout: + value: 60 +inprocess_last_call_wait: + value: 1 +inprocess_max_iterations: + value: null +inprocess_monitor_process_interval: + value: 1 +inprocess_monitor_thread_interval: + value: 1 +inprocess_progress_watchdog_interval: + value: 1 +inprocess_restart: + value: false +inprocess_soft_timeout: + value: 60 +inprocess_termination_grace_time: + value: 1 +is_hybrid_model: + value: true +iter_per_epoch: + value: 1250 +iterations_to_skip: + value: [] +keep_fp8_transpose_cache: + value: false +kitchen_config_file: + value: null +kitchen_recipe_number: + value: null +kv_channels: + value: 64 +kv_lora_rank: + value: 32 +langrl_env_config: + value: null +langrl_external_server: + value: false +langrl_inference_server_conversation_template: + value: null +langrl_inference_server_type: + value: inplace_megatron +lazy_mpu_init: + value: null +legacy_tokenizer: + value: false +linear_attention_freq: + value: + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 0 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 0 + - 0 + - 1 + - 0 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 0 + - 1 + - 0 + - 0 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 +linear_attention_type: + value: gated_delta_net +linear_conv_kernel_dim: + value: 4 +linear_cross_entropy_accumulate_grad_in_fp32: + value: true +linear_cross_entropy_num_chunks: + value: 8 +linear_key_head_dim: + value: 64 +linear_num_key_heads: + value: 8 +linear_num_value_heads: + value: 32 +linear_rope_mode: + value: none +linear_value_head_dim: + value: 64 +lm_head_alpha: + value: 1 +lm_loss_normalization: + value: global-token +load: + value: /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage14_120B_8node_gdnlralpha_off_20260925a/checkpoints +load_complemental_dataset: + value: + - loss_mask +load_main_params_from_ckpt: + value: null +local_rank: + value: 7 +log_attn_logits: + value: false +log_energy: + value: false +log_hidden_states: + value: + - embeddings + - residual_attn + - input_layernorm + - attention::q_layernorm + - attention::k_layernorm + - attention::core_attention + - attention::o_proj + - residual_mlp + - pre_mlp_layernorm + - mlp + - moe::router + - moe::shared_experts + - moe::routed_experts + - gated_delta::in_proj + - gated_delta::conv1d + - gated_delta::query + - gated_delta::key + - gated_delta::value + - gated_delta::gate + - gated_delta::beta + - gated_delta::alpha + - gated_delta::g + - gated_delta::core_attention + - gated_delta::out_norm + - gated_delta::out_proj + - final_layernorm +log_interval: + value: 1 +log_logits_z_loss: + value: false +log_loss_scale_to_tensorboard: + value: true +log_memory_to_tensorboard: + value: false +log_metrics_interval: + value: 10 +log_moe_activations: + value: true +log_moe_params: + value: false +log_moe_router_metrics: + value: false +log_num_zeros_in_grad: + value: false +log_padding_stats: + value: false +log_params: + value: + - attention::linear_qkv + - attention::o_proj + - mlp::linear_fc1 + - mlp::linear_fc2 + - input_layernorm + - pre_mlp_layernorm + - embedding + - lm_head + - gated_delta::in_proj + - gated_delta::in_proj_bias + - gated_delta::conv1d + - gated_delta::conv1d_bias + - gated_delta::dt_bias + - gated_delta::A_log + - gated_delta::out_norm + - gated_delta::out_norm_bias + - gated_delta::out_proj + - gated_delta::out_proj_bias +log_params_norm: + value: true +log_per_layer_grad_rms: + value: true +log_per_module_grad_rms: + value: true +log_per_module_update_rms: + value: false +log_progress: + value: false +log_spike_diagnostics: + value: false +log_straggler: + value: false +log_throughput: + value: true +log_timers_to_tensorboard: + value: false +log_update_rms: + value: true +log_validation_ppl_to_tensorboard: + value: false +log_world_size_to_tensorboard: + value: false +logging_level: + value: null +loss_scale: + value: null +loss_scale_window: + value: 1000 +lr: + value: 0.0005 +lr_decay_iters: + value: null +lr_decay_samples: + value: 877440 +lr_decay_style: + value: constant +lr_warmup_fraction: + value: null +lr_warmup_init: + value: 0 +lr_warmup_iters: + value: 0 +lr_warmup_samples: + value: 0 +lr_wsd_decay_iters: + value: null +lr_wsd_decay_samples: + value: null +lr_wsd_decay_style: + value: exponential +main_grads_dtype: + value: torch.float32 +main_params_dtype: + value: torch.float32 +make_vocab_size_divisible_by: + value: 128 +mamba_disable_cp: + value: false +mamba_expand: + value: 2 +mamba_head_dim: + value: 64 +mamba_num_groups: + value: 8 +mamba_num_heads: + value: null +mamba_state_dim: + value: 128 +manual_gc: + value: false +manual_gc_eval: + value: true +manual_gc_interval: + value: 0 +mask_factor: + value: 1 +mask_prob: + value: 0.15 +mask_type: + value: random +masked_softmax_fusion: + value: false +max_position_embeddings: + value: 131072 +max_seqlen_per_cp_rank: + value: null +max_tokens_to_oom: + value: 12000 +memory_snapshot_path: + value: null +merge_file: + value: null +micro_batch_size: + value: 1 +microbatch_group_size_per_vp_stage: + value: null +mid_level_dataset_surplus: + value: 0.005 +min_loss_scale: + value: 1 +min_lr: + value: 0.0005 +min_offloaded_tensor_size: + value: 1048576 +mlp_chunks_for_prefill: + value: 1 +mmap_bin_files: + value: true +mock_data: + value: false +modelopt_enabled: + value: false +moe_apply_probs_on_input: + value: false +moe_aux_loss_coeff: + value: 0.05 +moe_deepep_num_sms: + value: 20 +moe_enable_deepep: + value: false +moe_expert_capacity_factor: + value: null +moe_extended_tp: + value: false +moe_ffn_hidden_size: + value: 512 +moe_flex_dispatcher_backend: + value: deepep +moe_gaux_overlap: + value: false +moe_grouped_gemm: + value: true +moe_hybridep_num_sms: + value: 16 +moe_input_jitter_eps: + value: null +moe_layer_freq: + value: + - 0 + - 0 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 +moe_layer_recompute: + value: false +moe_monitor_layers: + value: "" +moe_monitor_log_interval: + value: null +moe_pad_expert_input_to_capacity: + value: false +moe_pad_experts_for_cuda_graph_inference: + value: false +moe_per_layer_logging: + value: false +moe_permute_backend: + value: auto +moe_permute_fusion: + value: true +moe_router_aux_loss_fusion: + value: auto +moe_router_bias_update_method: + value: sign +moe_router_bias_update_rate: + value: 0.001 +moe_router_dtype: + value: fp32 +moe_router_enable_expert_bias: + value: false +moe_router_force_load_balancing: + value: false +moe_router_fusion: + value: true +moe_router_gemm_backend: + value: te +moe_router_group_topk: + value: null +moe_router_load_balancing_type: + value: global_aux_loss +moe_router_num_groups: + value: null +moe_router_padding_for_fp8: + value: false +moe_router_padding_for_quantization: + value: false +moe_router_pre_softmax: + value: false +moe_router_score_function: + value: softmax +moe_router_sqrt_gate: + value: true +moe_router_topk: + value: 4 +moe_router_topk_scaling_factor: + value: null +moe_shared_expert_gate: + value: true +moe_shared_expert_intermediate_size: + value: 512 +moe_shared_expert_overlap: + value: true +moe_token_dispatcher_type: + value: alltoall +moe_token_drop_policy: + value: probs +moe_triton_dispatcher_exclude_padding_tokens: + value: false +moe_triton_mega_activation_sms: + value: 4 +moe_triton_mega_combine_sms: + value: 32 +moe_triton_mega_compute_sms: + value: 24 +moe_triton_mega_dispatch_sms: + value: 48 +moe_triton_mega_num_warps: + value: 16 +moe_triton_mega_reduce_sms: + value: 32 +moe_triton_mega_tail_sms: + value: 16 +moe_triton_shared_expert_overlap: + value: false +moe_triton_workspace_expand: + value: 4 +moe_upcycling_granularity: + value: 1 +moe_use_legacy_grouped_gemm: + value: false +moe_use_upcycling: + value: false +moe_z_loss_coeff: + value: null +mrope_section: + value: null +mscale: + value: 1 +mscale_all_dim: + value: 0 +mtp_linear_attention_type: + value: gated_delta_net +mtp_loss_scaling_factor: + value: 0.1 +mtp_num_layers: + value: 0 +multi_latent_attention: + value: false +multiple_validation_sets: + value: false +muon_ball_momentum: + value: 0.9 +muon_ball_msign_steps: + value: 5 +muon_ball_power_iteration_steps: + value: 10 +muon_ball_qkv_split_mode: + value: component +muon_ball_radius_mode: + value: spectral_mup +muon_ball_retract_alpha: + value: 0.05 +muon_ball_retract_mode: + value: hard +muon_ball_scale_mode: + value: spectral_mup +muon_ball_split_fc1: + value: true +muon_ball_split_moe_experts: + value: true +muon_ball_split_qkv: + value: true +muon_ball_use_nesterov: + value: true +muon_extra_scale_factor: + value: 1 +muon_fp32_matmul_prec: + value: medium +muon_momentum: + value: 0.9 +muon_num_ns_steps: + value: 5 +muon_qkv_split_mode: + value: component +muon_scale_mode: + value: spectral_mup +muon_scale_vectorized_mode: + value: full +muon_split_fc1: + value: true +muon_split_moe_experts: + value: true +muon_split_qkv: + value: true +muon_tp_mode: + value: blockwise +muon_use_nesterov: + value: false +muon_vectorize: + value: [] +muon_vectorize_attn_dim: + value: hidden_size +mup_lr_mult: + value: 1 +nccl_all_reduce_for_prefill: + value: false +nccl_communicator_config_path: + value: null +nccl_ub: + value: false +no_load_optim: + value: null +no_load_rng: + value: true +no_load_scheduler: + value: null +no_persist_layer_norm: + value: false +no_rope_freq: + value: null +no_save_optim: + value: null +no_save_rng: + value: null +no_save_step_one: + value: true +no_weight_decay_cond_type: + value: null +non_persistent_ckpt_type: + value: null +non_persistent_global_ckpt_dir: + value: null +non_persistent_keep_last: + value: 1 +non_persistent_local_ckpt_algo: + value: fully_parallel +non_persistent_local_ckpt_dir: + value: null +non_persistent_save_interval: + value: null +norm_epsilon: + value: 1e-06 +normalization: + value: RMSNorm +num_attention_heads: + value: 32 +num_channels: + value: 3 +num_classes: + value: 1000 +num_dataset_builder_threads: + value: 1 +num_distributed_optimizer_instances: + value: 1 +num_experts: + value: 192 +num_layers: + value: 56 +num_layers_at_end_in_bf16: + value: 1 +num_layers_at_start_in_bf16: + value: 1 +num_layers_per_virtual_pipeline_stage: + value: null +num_query_groups: + value: 8 +num_virtual_stages_per_pipeline_rank: + value: null +num_workers: + value: 4 +object_storage_cache_path: + value: null +offload_modules: + value: [] +one_logger_async: + value: false +one_logger_project: + value: megatron-lm +one_logger_run_name: + value: null +onnx_safe: + value: null +openai_gelu: + value: false +optimizer: + value: adam +optimizer_cpu_offload: + value: false +optimizer_offload_fraction: + value: 1 +output_bert_embeddings: + value: false +output_logits_z_loss_coeff: + value: null +overlap_cpu_optimizer_d2h_h2d: + value: false +overlap_grad_reduce: + value: true +overlap_moe_expert_parallel_comm: + value: false +overlap_p2p_comm: + value: false +overlap_p2p_comm_warmup_flush: + value: false +overlap_param_gather: + value: true +overlap_param_gather_with_optimizer_step: + value: false +override_hf_eod_token_id: + value: null +override_opt_param_scheduler: + value: true +packing_tokens_per_shard: + value: 100000000000 +padded_vocab_size: + value: 99072 +params_dtype: + value: torch.bfloat16 +patch_dim: + value: 16 +per_split_data_args_path: + value: null +perform_initialization: + value: false +perform_rl_step: + value: false +pin_cpu_grads: + value: true +pin_cpu_params: + value: true +pipeline_model_parallel_comm_backend: + value: null +pipeline_model_parallel_layout: + value: null +pipeline_model_parallel_size: + value: 1 +position_embedding_type: + value: none +pp_aware_finegrained_recompute: + value: false +pp_aware_finegrained_recompute_global_layers: + value: null +pp_aware_finegrained_recompute_pp_ranks: + value: null +prefetch_factor: + value: 8 +pretrained_checkpoint: + value: null +previous_stage_seq_length: + value: null +profile: + value: false +profile_microbatch_count: + value: 2 +profile_microbatch_dir: + value: null +profile_microbatch_iteration: + value: null +profile_microbatch_start: + value: 0 +profile_ranks: + value: + - 0 +profile_step_end: + value: null +profile_step_start: + value: null +q_lora_rank: + value: null +qk_head_dim: + value: 128 +qk_l2_norm: + value: false +qk_layernorm: + value: false +qk_pos_emb_head_dim: + value: 64 +query_in_block_prob: + value: 0.1 +quick_geglu: + value: false +rampup_batch_size: + value: null +rank: + value: 63 +recompute_granularity: + value: full +recompute_method: + value: block +recompute_modules: + value: null +recompute_num_layers: + value: 56 +record_memory_history: + value: false +relative_attention_max_distance: + value: 128 +relative_attention_num_buckets: + value: 32 +reparam_keys: + value: null +replication: + value: false +replication_factor: + value: 2 +replication_jump: + value: null +rerun_mode: + value: validate_results +reset_attention_mask: + value: false +reset_iteration_one_to_zero: + value: false +reset_position_ids: + value: false +result_rejected_tracker_filename: + value: null +retriever_report_topk_accuracies: + value: [] +retriever_score_scaling: + value: false +retriever_seq_length: + value: 256 +retro_add_retriever: + value: false +retro_attention_gate: + value: 1 +retro_cyclic_train_iters: + value: null +retro_encoder_attention_dropout: + value: 0.1 +retro_encoder_hidden_dropout: + value: 0.1 +retro_encoder_layers: + value: 2 +retro_num_neighbors: + value: 2 +retro_num_retrieved_chunks: + value: 2 +retro_project_dir: + value: null +retro_verify_neighbor_count: + value: true +reuse_grad_buf_for_mxfp8_param_ag: + value: false +reuse_training_communicators: + value: true +rl_calculate_intra_group_similarity: + value: false +rl_importance_sampling_truncation_coef: + value: null +rl_inference_logprobs_is_correction: + value: false +rl_offload_kv_cache_during_training: + value: false +rl_offload_optimizer_during_inference: + value: false +rl_partial_rollouts: + value: false +rl_prompts_per_eval: + value: 32 +rl_remove_kv_cache_during_training: + value: false +rl_reset_cuda_graphs: + value: false +rl_sequence_packing_algo: + value: fifo +rl_sequence_packing_bin_size: + value: 8192 +rl_use_sequence_packing: + value: false +rope_scaling_factor: + value: 8 +rope_type: + value: null +rotary_base: + value: 10000 +rotary_interleaved: + value: false +rotary_percent: + value: 1 +rotary_scaling_factor: + value: 1 +rotary_seq_len_interpolation_factor: + value: null +run_workload_inspector_server: + value: false +runtime_audit_dir: + value: /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/runtime_audit +sample_rate: + value: 1 +save: + value: /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints +save_after_load: + value: false +save_interval: + value: 40 +save_retain_interval: + value: 1000000000 +scale_emb: + value: 1 +scatter_gather_tensors_in_pipeline: + value: true +seed: + value: 1234 +seq_length: + value: 131072 +sequence_packing_scheduler: + value: default_dynamic_cp +sequence_parallel: + value: false +sft: + value: false +sft_tokenizer_prompt_format: + value: nemotron-h-aligned +sgd_momentum: + value: 0.9 +sharp_enabled_group: + value: null +short_seq_prob: + value: 0.1 +skip_train: + value: false +skipped_train_samples: + value: 0 +slurm_job_name: + value: N/A +softmax_type: + value: vanilla +spec: + value: null +spectra_clip: + value: 1 +spectra_grad_clip: + value: null +spectra_hyperball: + value: false +spectra_ns_steps: + value: 10 +spectra_scope: + value: matrices +spectral_ball_momentum: + value: 0.9 +spectral_ball_msign_steps: + value: 8 +spectral_ball_power_iteration_steps: + value: 20 +spectral_ball_qkv_split_mode: + value: component +spectral_ball_radius_mode: + value: spectral_mup +spectral_ball_retract_alpha: + value: 0.05 +spectral_ball_retract_mode: + value: hard +spectral_ball_scale_mode: + value: spectral_mup +spectral_ball_solver: + value: bisection +spectral_ball_solver_max_iterations: + value: 20 +spectral_ball_solver_tolerance_f: + value: 1e-08 +spectral_ball_split_fc1: + value: true +spectral_ball_split_moe_experts: + value: true +spectral_ball_split_qkv: + value: true +spectral_ball_use_nesterov: + value: true +spectral_mup_init: + value: false +spike_diagnostics_dir: + value: null +spike_diagnostics_end: + value: 21 +spike_diagnostics_start: + value: 2 +split: + value: 999,1,0 +split_expert_init: + value: true +split_fc1_init: + value: true +split_qkv_init: + value: true +split_qkv_init_mode: + value: group +sqreglu: + value: false +squared_relu: + value: false +start_samples: + value: 0 +start_weight_decay: + value: 0.1 +straggler_ctrlr_port: + value: 65535 +straggler_minmax_count: + value: 1 +strict_fsdp_dtensor_load: + value: true +suggested_communication_unit_size: + value: null +swanlab_exp_name: + value: "" +swanlab_project: + value: "" +swanlab_save_dir: + value: "" +swanlab_workspace: + value: "" +swiglu: + value: true +swin_backbone_type: + value: tiny +symmetric_ar_type: + value: null +te_rng_tracker: + value: false +teacher_model_config: + value: null +tensor_model_parallel_size: + value: 1 +tensorboard_dir: + value: /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/tensorboard +tensorboard_log_interval: + value: 1 +tensorboard_queue_size: + value: 1000 +test_data_path: + value: null +test_mode: + value: false +tiktoken_num_special_tokens: + value: 1000 +tiktoken_pattern: + value: null +tiktoken_special_tokens: + value: null +timing_log_level: + value: 0 +timing_log_option: + value: all +titles_data_path: + value: null +token_shift_conv_init: + value: default +token_shift_conv_size: + value: 4 +tokenizer_metadata: + value: null +tokenizer_model: + value: /mnt/ssd/yulan/pretrain-linear-moe/preprocess/modify_tokenizer/YuLan-Mini-Qwen-Template_new_sp +tokenizer_type: + value: HuggingFaceTokenizer +torch_fsdp2_reshard_after_forward: + value: true +tp_comm_bootstrap_backend: + value: nccl +tp_comm_bulk_dgrad: + value: true +tp_comm_bulk_wgrad: + value: true +tp_comm_overlap: + value: false +tp_comm_overlap_ag: + value: true +tp_comm_overlap_cfg: + value: null +tp_comm_overlap_rs: + value: true +tp_comm_overlap_rs_dgrad: + value: false +tp_comm_split_ag: + value: true +tp_comm_split_rs: + value: true +train_data_path: + value: null +train_iters: + value: null +train_samples: + value: 877440 +train_sync_interval: + value: null +train_until_stopped: + value: false +transformer_impl: + value: transformer_engine +transformer_pipeline_model_parallel_size: + value: 1 +trust_remote_code: + value: false +untie_embeddings_and_output_weights: + value: true +use_checkpoint_args: + value: false +use_checkpoint_opt_param_scheduler: + value: false +use_cpu_initialization: + value: null +use_dist_ckpt: + value: true +use_dist_ckpt_deprecated: + value: false +use_distributed_optimizer: + value: true +use_flash_attn: + value: true +use_fused_weighted_squared_relu: + value: false +use_legacy_models: + value: false +use_megatron_fsdp: + value: false +use_mp_args_from_checkpoint_args: + value: false +use_one_sent_docs: + value: false +use_persistent_ckpt_worker: + value: false +use_precision_aware_optimizer: + value: false +use_pytorch_profiler: + value: false +use_ring_exchange_p2p: + value: false +use_rope_scaling: + value: false +use_rotary_position_embeddings: + value: false +use_sharp: + value: false +use_te_activation_func: + value: false +use_tokenizer_model_from_checkpoint_args: + value: true +use_torch_fsdp2: + value: false +use_torch_optimizer_for_cpu_offload: + value: false +use_tp_pp_dp_mapping: + value: false +v_head_dim: + value: 128 +valid_data_path: + value: null +variable_seq_lengths: + value: false +virtual_pipeline_model_parallel_size: + value: null +vision_backbone_type: + value: vit +vision_pretraining: + value: false +vision_pretraining_type: + value: classify +vocab_extra_ids: + value: 0 +vocab_file: + value: null +vocab_size: + value: null +wandb_entity: + value: "" +wandb_exp_name: + value: stage14_120B_8node_gdnlralpha_off_20260925a +wandb_project: + value: qwen2.5_3b_mom_cpt +wandb_save_dir: + value: /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/wandb +weight_decay: + value: 0.1 +weight_decay_incr_style: + value: constant +wgrad_deferral_limit: + value: 0 +window_attn_skip_freq: + value: null +window_size: + value: null +word_embedding_dropout_prob: + value: 0 +world_size: + value: 64 +yaml_cfg: + value: null diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/wandb/wandb/run-20261003_153655-neb76hvj/files/output.log b/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/wandb/wandb/run-20261003_153655-neb76hvj/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..7f50051847e997d40d8013a7a4949c8597dbf624 --- /dev/null +++ b/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/wandb/wandb/run-20261003_153655-neb76hvj/files/output.log @@ -0,0 +1,45772 @@ +WARNING: one_logger package is required to enable e2e metrics tracking. please go to https://confluence.nvidia.com/display/MLWFO/Package+Repositories for details to install it +/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. + warnings.warn( # warn only once +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +[GPT_LAYER_SPEC] log_hidden_states=['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'] reparam_keys=[] disable_in_ln_fusion=True disable_pre_mlp_ln_fusion=True +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/dist_checkpointing/strategies/torch.py:956: FutureWarning: `load_state_dict` is deprecated and will be removed in future versions. Please use `load` instead. + checkpoint.load_state_dict( +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable +INFO:megatron.core.optimizer.distrib_optimizer:Loading distributed optimizer sharded state of type dp_reshardable + failed to find checkpoint /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage14_120B_8node_gdnlralpha_off_20260925a/checkpoints/iter_0007153 in wandb +times across ranks (ms): + load-checkpoint: + rank 0: 129567.63 + rank 1: 129567.76 + rank 2: 129567.71 + rank 3: 129567.71 + rank 4: 129567.80 + rank 5: 129567.70 + rank 6: 129567.84 + rank 7: 129567.65 + rank 8: 129564.70 + rank 9: 129564.69 + rank 10: 129564.62 + rank 11: 129564.60 + rank 12: 129564.62 + rank 13: 129564.61 + rank 14: 129564.67 + rank 15: 129564.78 + rank 16: 129564.88 + rank 17: 129564.12 + rank 18: 129564.52 + rank 19: 129564.43 + rank 20: 129564.28 + rank 21: 129564.38 + rank 22: 129564.51 + rank 23: 129564.46 + rank 24: 129556.57 + rank 25: 129556.77 + rank 26: 129557.00 + rank 27: 129556.62 + rank 28: 129556.62 + rank 29: 129556.63 + rank 30: 129556.59 + rank 31: 129556.54 + rank 32: 129563.61 + rank 33: 129563.44 + rank 34: 129563.44 + rank 35: 129563.61 + rank 36: 129563.47 + rank 37: 129563.45 + rank 38: 129563.97 + rank 39: 129563.55 + rank 40: 129562.17 + rank 41: 129562.30 + rank 42: 129562.33 + rank 43: 129562.31 + rank 44: 129562.16 + rank 45: 129562.17 + rank 46: 129562.43 + rank 47: 129562.20 + rank 48: 129560.97 + rank 49: 129560.86 + rank 50: 129561.10 + rank 51: 129561.26 + rank 52: 129561.10 + rank 53: 129561.10 + rank 54: 129560.62 + rank 55: 129560.89 + rank 56: 129563.13 + rank 57: 129563.15 + rank 58: 129563.10 + rank 59: 129563.12 + rank 60: 129563.13 + rank 61: 129563.12 + rank 62: 129563.20 + rank 63: 129563.05 +Initializing GlobIndexedDataset +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_0_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_10_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_11_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_12_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_13_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_14_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_15_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_16_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_17_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_18_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_19_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_1_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_20_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_21_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_22_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_23_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_24_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_25_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_26_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_27_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_28_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_29_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_2_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_30_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_31_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_32_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_33_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_34_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_35_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_36_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_37_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_38_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_39_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_3_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_40_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_41_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_42_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_43_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_44_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_45_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_46_input_ids_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15 +>>> start_samples: 0 +Initializing GlobIndexedDataset +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15 +pad_document_marker: 36616095 +/mnt/hdd/cpt/binidx_cache_stage15_final_20261001_v2 d6c36c6d5886334ea5f92e75fcf45d10-GPTDataset-train-obfd cache hit: True +>>> start_samples: 0 +Initializing GlobIndexedDataset +['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_0_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_10_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_11_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_12_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_13_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_14_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_15_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_16_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_17_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_18_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_19_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_1_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_20_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_21_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_22_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_23_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_24_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_25_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_26_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_27_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_28_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_29_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_2_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_30_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_31_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_32_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_33_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_34_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_35_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_36_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_37_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_38_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_39_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_3_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_40_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_41_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_42_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_43_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_44_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_45_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_46_loss_mask_tmp', '/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15 +/mnt/hdd/cpt/binidx_cache_stage15_final_20261001_v2 0642c625420be1073f42e03d3641a6e5-GPTDataset-valid-random cache hit: True +times across ranks (ms): + model-and-optimizer-setup: + rank 0: 155969.56 + rank 1: 155958.91 + rank 2: 155959.02 + rank 3: 155962.28 + rank 4: 155966.44 + rank 5: 155966.77 + rank 6: 155967.91 + rank 7: 155969.27 + rank 8: 155961.72 + rank 9: 155955.70 + rank 10: 155958.75 + rank 11: 155962.28 + rank 12: 155964.25 + rank 13: 155963.12 + rank 14: 155963.69 + rank 15: 155962.72 + rank 16: 155961.69 + rank 17: 155961.98 + rank 18: 155967.17 + rank 19: 155966.72 + rank 20: 155968.08 + rank 21: 155965.19 + rank 22: 155964.72 + rank 23: 155964.06 + rank 24: 155955.98 + rank 25: 155954.59 + rank 26: 155962.94 + rank 27: 155961.55 + rank 28: 155959.81 + rank 29: 155960.80 + rank 30: 155957.89 + rank 31: 155959.12 + rank 32: 155968.09 + rank 33: 155963.91 + rank 34: 155966.33 + rank 35: 155964.45 + rank 36: 155966.88 + rank 37: 155965.33 + rank 38: 155968.09 + rank 39: 155962.98 + rank 40: 155964.81 + rank 41: 155965.45 + rank 42: 155962.08 + rank 43: 155962.94 + rank 44: 155964.08 + rank 45: 155960.16 + rank 46: 155957.61 + rank 47: 155961.03 + rank 48: 155963.31 + rank 49: 155962.23 + rank 50: 155961.55 + rank 51: 155964.03 + rank 52: 155959.42 + rank 53: 155953.25 + rank 54: 155960.44 + rank 55: 155956.97 + rank 56: 155962.05 + rank 57: 155961.06 + rank 58: 155953.45 + rank 59: 155952.27 + rank 60: 155956.50 + rank 61: 155961.09 + rank 62: 155960.38 + rank 63: 155962.28 + train/valid/test-data-iterators-setup: + rank 0: 85449.81 + rank 1: 95673.00 + rank 2: 95673.02 + rank 3: 95673.03 + rank 4: 95673.03 + rank 5: 95673.02 + rank 6: 95673.03 + rank 7: 95590.84 + rank 8: 95671.82 + rank 9: 96030.42 + rank 10: 95671.74 + rank 11: 95671.70 + rank 12: 95671.60 + rank 13: 95671.75 + rank 14: 95671.77 + rank 15: 95671.62 + rank 16: 96030.83 + rank 17: 96030.10 + rank 18: 96030.09 + rank 19: 96030.04 + rank 20: 96030.04 + rank 21: 96030.03 + rank 22: 96030.10 + rank 23: 96030.04 + rank 24: 99323.93 + rank 25: 99719.16 + rank 26: 99720.02 + rank 27: 99719.14 + rank 28: 99719.19 + rank 29: 99719.22 + rank 30: 99719.29 + rank 31: 99719.07 + rank 32: 99723.68 + rank 33: 99724.70 + rank 34: 99723.69 + rank 35: 99723.80 + rank 36: 99723.66 + rank 37: 99723.73 + rank 38: 99723.71 + rank 39: 99723.66 + rank 40: 99722.92 + rank 41: 99723.12 + rank 42: 99722.98 + rank 43: 99722.80 + rank 44: 99722.91 + rank 45: 99722.90 + rank 46: 99723.05 + rank 47: 99722.89 + rank 48: 99722.22 + rank 49: 99722.22 + rank 50: 99722.19 + rank 51: 99722.38 + rank 52: 99722.23 + rank 53: 99722.18 + rank 54: 99722.22 + rank 55: 99722.52 + rank 56: 99723.74 + rank 57: 99723.73 + rank 58: 99723.70 + rank 59: 99723.72 + rank 60: 99723.70 + rank 61: 99723.72 + rank 62: 99723.67 + rank 63: 99723.67 +/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/megatron/core/optimizer/optimizer.py:65: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at /pytorch/build/aten/src/ATen/core/TensorBody.h:489.) + if hasattr(param, grad_attr) and getattr(param, grad_attr) is not None: +⚠️ torch 2.8.0+cu128 with triton 3.4.0. inductor patch is not applied. + [2026-10-03 15:49:28] iteration 1/ 6855 | consumed samples: 128 | elapsed time per iteration (ms): 140440.6 | throughput per GPU (TFLOP/s/GPU): 26.3 | MFU 8.43% | FLOPs estimator: workload-aware | logical tokens/s: 119461.2 | valid tokens/s: 119461.2 | physical tokens/s: 119710.1 | loss tokens/s: 90532.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.604183E-01 | lm-global-supervised-tokens: 1.271444E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.943439E+01 | lm-token-buffer-mib: 5.809886E+01 | global_load_balancing_loss: 9.995751E-01 | loss scale: 1.0 | grad norm: 0.141 | params norm: 11681.705 | params rms: 0.063253 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 11 days, 3:22:59.897986 | finish at 2026-10-14 19:12:28 +times across ranks (ms): + forward-backward: + rank 0: 119862.99 + rank 1: 119863.15 + rank 2: 119871.55 + rank 3: 119890.97 + rank 4: 119861.76 + rank 5: 119873.04 + rank 6: 119858.74 + rank 7: 119869.45 + rank 8: 119853.78 + rank 9: 119853.61 + rank 10: 119862.03 + rank 11: 119881.88 + rank 12: 119852.76 + rank 13: 119863.74 + rank 14: 119849.31 + rank 15: 119860.01 + rank 16: 119857.07 + rank 17: 119856.48 + rank 18: 119864.70 + rank 19: 119884.38 + rank 20: 119854.88 + rank 21: 119866.92 + rank 22: 119852.02 + rank 23: 119862.67 + rank 24: 119874.85 + rank 25: 119873.40 + rank 26: 119882.23 + rank 27: 119901.09 + rank 28: 119871.84 + rank 29: 119883.60 + rank 30: 119868.47 + rank 31: 119880.12 + rank 32: 119867.22 + rank 33: 119867.74 + rank 34: 119875.31 + rank 35: 119895.12 + rank 36: 119865.44 + rank 37: 119876.95 + rank 38: 119863.78 + rank 39: 119873.07 + rank 40: 119852.99 + rank 41: 119852.91 + rank 42: 119861.14 + rank 43: 119880.68 + rank 44: 119851.71 + rank 45: 119863.01 + rank 46: 119848.58 + rank 47: 119859.24 + rank 48: 119872.62 + rank 49: 119873.15 + rank 50: 119880.73 + rank 51: 119900.99 + rank 52: 119871.20 + rank 53: 119882.52 + rank 54: 119868.52 + rank 55: 119879.11 + rank 56: 119853.81 + rank 57: 119855.48 + rank 58: 119863.98 + rank 59: 119883.59 + rank 60: 119854.20 + rank 61: 119865.81 + rank 62: 119851.11 + rank 63: 119861.95 + forward-compute: + rank 0: 57790.12 + rank 1: 57791.22 + rank 2: 57799.77 + rank 3: 57818.45 + rank 4: 57790.31 + rank 5: 57801.23 + rank 6: 57785.70 + rank 7: 57798.85 + rank 8: 55138.51 + rank 9: 55137.30 + rank 10: 55147.21 + rank 11: 55171.55 + rank 12: 55143.15 + rank 13: 55151.22 + rank 14: 55133.17 + rank 15: 55146.14 + rank 16: 57886.33 + rank 17: 57890.13 + rank 18: 57897.19 + rank 19: 57919.18 + rank 20: 57889.40 + rank 21: 57902.66 + rank 22: 57891.02 + rank 23: 57899.97 + rank 24: 49650.75 + rank 25: 49656.44 + rank 26: 49671.53 + rank 27: 49691.18 + rank 28: 49680.87 + rank 29: 49690.07 + rank 30: 49665.61 + rank 31: 49679.34 + rank 32: 56044.65 + rank 33: 56047.64 + rank 34: 56055.79 + rank 35: 56076.41 + rank 36: 56049.81 + rank 37: 56060.61 + rank 38: 56047.73 + rank 39: 56056.70 + rank 40: 58735.14 + rank 41: 58736.60 + rank 42: 58745.49 + rank 43: 58759.90 + rank 44: 58733.62 + rank 45: 58741.90 + rank 46: 58732.28 + rank 47: 58744.95 + rank 48: 57849.70 + rank 49: 57853.43 + rank 50: 57864.34 + rank 51: 57885.04 + rank 52: 57856.83 + rank 53: 57867.82 + rank 54: 57852.12 + rank 55: 57864.19 + rank 56: 57322.67 + rank 57: 57327.56 + rank 58: 57333.96 + rank 59: 57353.84 + rank 60: 57323.91 + rank 61: 57338.63 + rank 62: 57318.72 + rank 63: 57335.67 + backward-compute: + rank 0: 62006.93 + rank 1: 62004.55 + rank 2: 62003.82 + rank 3: 61999.11 + rank 4: 62001.99 + rank 5: 62004.01 + rank 6: 62004.07 + rank 7: 61997.21 + rank 8: 64654.62 + rank 9: 64656.73 + rank 10: 64655.29 + rank 11: 64651.06 + rank 12: 64650.04 + rank 13: 64655.01 + rank 14: 64654.32 + rank 15: 64653.06 + rank 16: 61895.26 + rank 17: 61891.65 + rank 18: 61896.25 + rank 19: 61895.81 + rank 20: 61893.79 + rank 21: 61884.81 + rank 22: 61889.12 + rank 23: 61887.62 + rank 24: 70118.20 + rank 25: 70125.23 + rank 26: 70103.18 + rank 27: 70114.55 + rank 28: 70096.34 + rank 29: 70102.45 + rank 30: 70107.29 + rank 31: 70102.06 + rank 32: 63732.38 + rank 33: 63726.96 + rank 34: 63730.15 + rank 35: 63730.85 + rank 36: 63729.17 + rank 37: 63729.36 + rank 38: 63725.10 + rank 39: 63723.33 + rank 40: 61051.64 + rank 41: 61050.20 + rank 42: 61048.30 + rank 43: 61050.43 + rank 44: 61049.62 + rank 45: 61052.38 + rank 46: 61050.58 + rank 47: 61049.73 + rank 48: 61941.12 + rank 49: 61934.65 + rank 50: 61933.39 + rank 51: 61928.46 + rank 52: 61928.76 + rank 53: 61927.16 + rank 54: 61930.38 + rank 55: 61926.11 + rank 56: 62472.75 + rank 57: 62467.90 + rank 58: 62469.88 + rank 59: 62465.66 + rank 60: 62464.64 + rank 61: 62465.36 + rank 62: 62470.35 + rank 63: 62466.79 + batch-generator: + rank 0: 59.71 + rank 1: 58.09 + rank 2: 66.27 + rank 3: 89.00 + rank 4: 58.99 + rank 5: 68.00 + rank 6: 53.52 + rank 7: 67.41 + rank 8: 51.36 + rank 9: 50.91 + rank 10: 58.79 + rank 11: 79.60 + rank 12: 50.49 + rank 13: 61.60 + rank 14: 49.10 + rank 15: 59.60 + rank 16: 52.02 + rank 17: 47.45 + rank 18: 57.61 + rank 19: 77.89 + rank 20: 45.47 + rank 21: 62.36 + rank 22: 45.03 + rank 23: 54.03 + rank 24: 96.38 + rank 25: 84.69 + rank 26: 121.05 + rank 27: 119.64 + rank 28: 87.14 + rank 29: 95.07 + rank 30: 82.06 + rank 31: 94.33 + rank 32: 56.68 + rank 33: 57.05 + rank 34: 64.55 + rank 35: 84.55 + rank 36: 55.00 + rank 37: 67.56 + rank 38: 55.76 + rank 39: 62.32 + rank 40: 44.05 + rank 41: 43.58 + rank 42: 51.40 + rank 43: 67.91 + rank 44: 44.33 + rank 45: 53.68 + rank 46: 42.98 + rank 47: 49.33 + rank 48: 59.93 + rank 49: 68.52 + rank 50: 73.77 + rank 51: 97.38 + rank 52: 66.00 + rank 53: 80.10 + rank 54: 65.17 + rank 55: 80.28 + rank 56: 48.12 + rank 57: 48.98 + rank 58: 51.27 + rank 59: 69.70 + rank 60: 48.02 + rank 61: 53.10 + rank 62: 47.72 + rank 63: 51.15 + embedding-grads-all-reduce: + rank 0: 0.03 + rank 1: 0.03 + rank 2: 0.03 + rank 3: 0.03 + rank 4: 0.03 + rank 5: 0.03 + rank 6: 0.03 + rank 7: 0.03 + rank 8: 0.02 + rank 9: 0.02 + rank 10: 0.02 + rank 11: 0.03 + rank 12: 0.03 + rank 13: 0.03 + rank 14: 0.03 + rank 15: 0.03 + rank 16: 0.03 + rank 17: 0.03 + rank 18: 0.03 + rank 19: 0.03 + rank 20: 0.02 + rank 21: 0.03 + rank 22: 0.03 + rank 23: 0.03 + rank 24: 0.05 + rank 25: 0.06 + rank 26: 0.05 + rank 27: 0.04 + rank 28: 0.04 + rank 29: 0.04 + rank 30: 0.04 + rank 31: 0.04 + rank 32: 0.03 + rank 33: 0.03 + rank 34: 0.03 + rank 35: 0.03 + rank 36: 0.03 + rank 37: 0.03 + rank 38: 0.03 + rank 39: 0.03 + rank 40: 0.03 + rank 41: 0.03 + rank 42: 0.03 + rank 43: 0.02 + rank 44: 0.03 + rank 45: 0.03 + rank 46: 0.03 + rank 47: 0.03 + rank 48: 0.03 + rank 49: 0.03 + rank 50: 0.03 + rank 51: 0.04 + rank 52: 0.03 + rank 53: 0.04 + rank 54: 0.04 + rank 55: 0.03 + rank 56: 0.03 + rank 57: 0.03 + rank 58: 0.03 + rank 59: 0.03 + rank 60: 0.03 + rank 61: 0.03 + rank 62: 0.03 + rank 63: 0.03 + all-grads-sync: + rank 0: 0.81 + rank 1: 0.75 + rank 2: 0.79 + rank 3: 0.80 + rank 4: 0.79 + rank 5: 0.73 + rank 6: 1.05 + rank 7: 0.85 + rank 8: 0.72 + rank 9: 0.69 + rank 10: 0.74 + rank 11: 0.77 + rank 12: 0.72 + rank 13: 0.75 + rank 14: 0.76 + rank 15: 1.14 + rank 16: 0.98 + rank 17: 0.73 + rank 18: 0.78 + rank 19: 0.75 + rank 20: 0.75 + rank 21: 0.99 + rank 22: 0.74 + rank 23: 0.73 + rank 24: 1.65 + rank 25: 1.17 + rank 26: 3.09 + rank 27: 1.19 + rank 28: 1.30 + rank 29: 1.17 + rank 30: 1.19 + rank 31: 1.16 + rank 32: 1.10 + rank 33: 1.30 + rank 34: 1.17 + rank 35: 1.11 + rank 36: 1.08 + rank 37: 1.49 + rank 38: 1.17 + rank 39: 1.18 + rank 40: 0.73 + rank 41: 0.73 + rank 42: 0.72 + rank 43: 0.73 + rank 44: 1.07 + rank 45: 0.75 + rank 46: 0.74 + rank 47: 0.78 + rank 48: 1.10 + rank 49: 1.16 + rank 50: 0.91 + rank 51: 1.14 + rank 52: 1.01 + rank 53: 1.29 + rank 54: 1.05 + rank 55: 1.30 + rank 56: 0.79 + rank 57: 0.74 + rank 58: 0.76 + rank 59: 1.01 + rank 60: 0.75 + rank 61: 0.73 + rank 62: 0.81 + rank 63: 0.78 + params-all-gather: + rank 0: 0.03 + rank 1: 0.02 + rank 2: 0.02 + rank 3: 0.02 + rank 4: 0.02 + rank 5: 0.02 + rank 6: 0.03 + rank 7: 0.02 + rank 8: 0.02 + rank 9: 0.03 + rank 10: 0.02 + rank 11: 0.02 + rank 12: 0.02 + rank 13: 0.02 + rank 14: 0.02 + rank 15: 0.02 + rank 16: 0.02 + rank 17: 0.02 + rank 18: 0.02 + rank 19: 0.02 + rank 20: 0.02 + rank 21: 0.03 + rank 22: 0.02 + rank 23: 0.02 + rank 24: 0.04 + rank 25: 0.03 + rank 26: 0.04 + rank 27: 0.03 + rank 28: 0.04 + rank 29: 0.03 + rank 30: 0.03 + rank 31: 0.04 + rank 32: 0.03 + rank 33: 0.03 + rank 34: 0.03 + rank 35: 0.02 + rank 36: 0.02 + rank 37: 0.03 + rank 38: 0.03 + rank 39: 0.03 + rank 40: 0.02 + rank 41: 0.02 + rank 42: 0.02 + rank 43: 0.02 + rank 44: 0.02 + rank 45: 0.02 + rank 46: 0.02 + rank 47: 0.02 + rank 48: 0.03 + rank 49: 0.03 + rank 50: 0.03 + rank 51: 0.03 + rank 52: 0.03 + rank 53: 0.03 + rank 54: 0.03 + rank 55: 0.03 + rank 56: 0.02 + rank 57: 0.03 + rank 58: 0.02 + rank 59: 0.02 + rank 60: 0.02 + rank 61: 0.02 + rank 62: 0.02 + rank 63: 0.02 + optimizer-copy-to-main-grad: + rank 0: 2.58 + rank 1: 2.17 + rank 2: 2.00 + rank 3: 2.49 + rank 4: 2.08 + rank 5: 2.19 + rank 6: 2.26 + rank 7: 2.26 + rank 8: 1.78 + rank 9: 2.45 + rank 10: 1.82 + rank 11: 2.12 + rank 12: 2.25 + rank 13: 2.10 + rank 14: 2.28 + rank 15: 2.38 + rank 16: 2.74 + rank 17: 2.54 + rank 18: 2.18 + rank 19: 2.59 + rank 20: 2.21 + rank 21: 2.93 + rank 22: 2.94 + rank 23: 2.35 + rank 24: 2.76 + rank 25: 2.60 + rank 26: 2.26 + rank 27: 2.13 + rank 28: 2.66 + rank 29: 2.66 + rank 30: 2.73 + rank 31: 2.15 + rank 32: 2.41 + rank 33: 2.24 + rank 34: 2.43 + rank 35: 2.79 + rank 36: 2.47 + rank 37: 2.41 + rank 38: 2.74 + rank 39: 2.03 + rank 40: 2.06 + rank 41: 2.24 + rank 42: 2.24 + rank 43: 2.11 + rank 44: 2.21 + rank 45: 2.36 + rank 46: 2.12 + rank 47: 2.07 + rank 48: 2.42 + rank 49: 2.39 + rank 50: 2.84 + rank 51: 2.77 + rank 52: 2.38 + rank 53: 2.81 + rank 54: 2.62 + rank 55: 2.93 + rank 56: 2.18 + rank 57: 2.80 + rank 58: 2.25 + rank 59: 2.08 + rank 60: 1.87 + rank 61: 2.15 + rank 62: 2.25 + rank 63: 2.11 + optimizer-inner-step: + rank 0: 109.46 + rank 1: 104.77 + rank 2: 104.87 + rank 3: 104.64 + rank 4: 104.84 + rank 5: 104.72 + rank 6: 104.68 + rank 7: 104.77 + rank 8: 96.57 + rank 9: 96.24 + rank 10: 96.56 + rank 11: 96.60 + rank 12: 96.39 + rank 13: 96.61 + rank 14: 96.48 + rank 15: 96.37 + rank 16: 108.88 + rank 17: 109.02 + rank 18: 108.83 + rank 19: 109.00 + rank 20: 108.88 + rank 21: 108.81 + rank 22: 108.60 + rank 23: 108.77 + rank 24: 111.04 + rank 25: 111.58 + rank 26: 112.08 + rank 27: 111.70 + rank 28: 111.73 + rank 29: 111.93 + rank 30: 111.62 + rank 31: 111.93 + rank 32: 104.76 + rank 33: 104.85 + rank 34: 104.81 + rank 35: 104.48 + rank 36: 104.66 + rank 37: 104.61 + rank 38: 104.72 + rank 39: 104.73 + rank 40: 97.89 + rank 41: 97.74 + rank 42: 97.65 + rank 43: 97.83 + rank 44: 97.85 + rank 45: 97.75 + rank 46: 97.86 + rank 47: 97.88 + rank 48: 108.98 + rank 49: 109.13 + rank 50: 108.70 + rank 51: 109.24 + rank 52: 109.08 + rank 53: 109.18 + rank 54: 109.30 + rank 55: 109.34 + rank 56: 98.37 + rank 57: 98.09 + rank 58: 98.40 + rank 59: 98.53 + rank 60: 98.53 + rank 61: 98.50 + rank 62: 98.37 + rank 63: 98.52 + optimizer-copy-main-to-model-params: + rank 0: 4.67 + rank 1: 4.11 + rank 2: 3.87 + rank 3: 4.67 + rank 4: 3.97 + rank 5: 4.21 + rank 6: 4.11 + rank 7: 4.30 + rank 8: 4.04 + rank 9: 4.76 + rank 10: 4.04 + rank 11: 4.16 + rank 12: 4.28 + rank 13: 4.11 + rank 14: 4.30 + rank 15: 4.56 + rank 16: 4.79 + rank 17: 4.32 + rank 18: 4.18 + rank 19: 4.29 + rank 20: 4.21 + rank 21: 4.61 + rank 22: 4.78 + rank 23: 4.44 + rank 24: 4.89 + rank 25: 4.90 + rank 26: 4.80 + rank 27: 4.66 + rank 28: 5.12 + rank 29: 5.28 + rank 30: 4.99 + rank 31: 4.82 + rank 32: 4.65 + rank 33: 4.67 + rank 34: 4.85 + rank 35: 5.08 + rank 36: 4.83 + rank 37: 4.72 + rank 38: 5.23 + rank 39: 4.45 + rank 40: 3.93 + rank 41: 4.31 + rank 42: 4.04 + rank 43: 4.25 + rank 44: 4.26 + rank 45: 4.52 + rank 46: 3.86 + rank 47: 4.02 + rank 48: 4.58 + rank 49: 4.41 + rank 50: 5.32 + rank 51: 4.61 + rank 52: 4.72 + rank 53: 4.49 + rank 54: 4.38 + rank 55: 4.58 + rank 56: 4.08 + rank 57: 5.05 + rank 58: 4.15 + rank 59: 4.06 + rank 60: 4.02 + rank 61: 4.04 + rank 62: 4.18 + rank 63: 3.98 + optimizer: + rank 0: 132.53 + rank 1: 127.02 + rank 2: 126.86 + rank 3: 127.70 + rank 4: 126.91 + rank 5: 127.14 + rank 6: 126.99 + rank 7: 127.31 + rank 8: 118.74 + rank 9: 119.38 + rank 10: 118.71 + rank 11: 118.87 + rank 12: 118.86 + rank 13: 118.88 + rank 14: 118.99 + rank 15: 119.24 + rank 16: 132.23 + rank 17: 131.73 + rank 18: 131.44 + rank 19: 131.70 + rank 20: 131.48 + rank 21: 131.92 + rank 22: 132.03 + rank 23: 131.68 + rank 24: 135.44 + rank 25: 134.85 + rank 26: 135.13 + rank 27: 134.64 + rank 28: 135.26 + rank 29: 135.59 + rank 30: 134.92 + rank 31: 134.99 + rank 32: 127.57 + rank 33: 127.62 + rank 34: 127.92 + rank 35: 127.87 + rank 36: 127.72 + rank 37: 127.48 + rank 38: 128.27 + rank 39: 127.27 + rank 40: 119.90 + rank 41: 120.22 + rank 42: 119.82 + rank 43: 120.23 + rank 44: 120.27 + rank 45: 120.54 + rank 46: 119.76 + rank 47: 119.97 + rank 48: 131.79 + rank 49: 131.68 + rank 50: 132.52 + rank 51: 131.98 + rank 52: 132.04 + rank 53: 131.79 + rank 54: 131.80 + rank 55: 132.05 + rank 56: 120.59 + rank 57: 121.66 + rank 58: 120.72 + rank 59: 120.69 + rank 60: 120.67 + rank 61: 120.64 + rank 62: 120.71 + rank 63: 120.59 + interval-time: + rank 0: 9.36 + rank 1: 9.53 + rank 2: 9.61 + rank 3: 9.51 + rank 4: 9.49 + rank 5: 9.52 + rank 6: 9.49 + rank 7: 9.54 + rank 8: 1.72 + rank 9: 1.88 + rank 10: 1.68 + rank 11: 1.77 + rank 12: 1.79 + rank 13: 1.71 + rank 14: 1.60 + rank 15: 1.69 + rank 16: 1.86 + rank 17: 1.78 + rank 18: 1.68 + rank 19: 1.85 + rank 20: 1.72 + rank 21: 1.87 + rank 22: 1.67 + rank 23: 1.66 + rank 24: 2.78 + rank 25: 2.37 + rank 26: 2.65 + rank 27: 2.15 + rank 28: 2.22 + rank 29: 3.00 + rank 30: 2.26 + rank 31: 2.18 + rank 32: 2.11 + rank 33: 2.20 + rank 34: 2.00 + rank 35: 2.09 + rank 36: 2.11 + rank 37: 2.28 + rank 38: 2.79 + rank 39: 1.85 + rank 40: 1.68 + rank 41: 1.74 + rank 42: 1.70 + rank 43: 1.72 + rank 44: 1.66 + rank 45: 1.68 + rank 46: 1.82 + rank 47: 1.74 + rank 48: 2.00 + rank 49: 2.01 + rank 50: 1.87 + rank 51: 2.34 + rank 52: 2.01 + rank 53: 2.20 + rank 54: 2.19 + rank 55: 2.26 + rank 56: 1.81 + rank 57: 1.85 + rank 58: 1.75 + rank 59: 1.75 + rank 60: 1.75 + rank 61: 1.99 + rank 62: 1.79 + rank 63: 3.30 + [2026-10-03 15:50:59] iteration 2/ 6855 | consumed samples: 256 | elapsed time per iteration (ms): 91236.2 | throughput per GPU (TFLOP/s/GPU): 41.6 | MFU 13.34% | FLOPs estimator: workload-aware | logical tokens/s: 183887.2 | valid tokens/s: 183887.2 | physical tokens/s: 184281.0 | loss tokens/s: 136776.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.356030E-01 | lm-global-supervised-tokens: 1.247893E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.437729E-02 | lm-token-buffer-mib: 5.815647E+01 | global_load_balancing_loss: 9.997712E-01 | loss scale: 1.0 | grad norm: 0.075 | params norm: 11681.610 | params rms: 0.063252 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 7 days, 5:40:41.983148 | finish at 2026-10-10 21:31:41 +times across ranks (ms): + forward-backward: + rank 0: 91021.51 + rank 1: 91020.66 + rank 2: 91021.66 + rank 3: 91022.20 + rank 4: 91022.99 + rank 5: 91022.06 + rank 6: 91019.45 + rank 7: 91020.20 + rank 8: 91016.16 + rank 9: 91016.58 + rank 10: 91017.22 + rank 11: 91018.05 + rank 12: 91018.16 + rank 13: 91019.01 + rank 14: 91015.40 + rank 15: 91015.96 + rank 16: 91013.59 + rank 17: 91013.51 + rank 18: 91013.26 + rank 19: 91014.48 + rank 20: 91013.77 + rank 21: 91015.88 + rank 22: 91011.82 + rank 23: 91011.95 + rank 24: 91010.34 + rank 25: 91009.06 + rank 26: 91012.62 + rank 27: 91009.53 + rank 28: 91009.55 + rank 29: 91010.30 + rank 30: 91006.89 + rank 31: 91006.46 + rank 32: 91005.97 + rank 33: 91006.56 + rank 34: 91006.23 + rank 35: 91006.39 + rank 36: 91008.16 + rank 37: 91007.49 + rank 38: 91008.13 + rank 39: 91007.40 + rank 40: 91009.59 + rank 41: 91009.95 + rank 42: 91008.95 + rank 43: 91008.59 + rank 44: 91008.76 + rank 45: 91007.88 + rank 46: 91011.17 + rank 47: 91010.48 + rank 48: 91015.92 + rank 49: 91015.62 + rank 50: 91015.23 + rank 51: 91016.80 + rank 52: 91013.84 + rank 53: 91015.15 + rank 54: 91017.08 + rank 55: 91018.68 + rank 56: 91018.84 + rank 57: 91018.40 + rank 58: 91018.27 + rank 59: 91018.27 + rank 60: 91017.35 + rank 61: 91017.41 + rank 62: 91020.07 + rank 63: 91020.06 + forward-compute: + rank 0: 37375.11 + rank 1: 37377.95 + rank 2: 37379.12 + rank 3: 37379.91 + rank 4: 37378.63 + rank 5: 37378.76 + rank 6: 37377.92 + rank 7: 37376.55 + rank 8: 33667.63 + rank 9: 33665.87 + rank 10: 33671.12 + rank 11: 33674.74 + rank 12: 33681.41 + rank 13: 33683.68 + rank 14: 33675.65 + rank 15: 33677.79 + rank 16: 35143.30 + rank 17: 35138.37 + rank 18: 35141.16 + rank 19: 35143.19 + rank 20: 35141.26 + rank 21: 35144.79 + rank 22: 35140.65 + rank 23: 35138.20 + rank 24: 33995.75 + rank 25: 34001.89 + rank 26: 34006.71 + rank 27: 34002.00 + rank 28: 34013.86 + rank 29: 34010.52 + rank 30: 34009.27 + rank 31: 34009.02 + rank 32: 37228.71 + rank 33: 37232.86 + rank 34: 37230.29 + rank 35: 37232.41 + rank 36: 37234.98 + rank 37: 37238.38 + rank 38: 37238.42 + rank 39: 37238.30 + rank 40: 34346.35 + rank 41: 34344.89 + rank 42: 34348.55 + rank 43: 34348.62 + rank 44: 34347.77 + rank 45: 34350.22 + rank 46: 34350.19 + rank 47: 34350.63 + rank 48: 36966.46 + rank 49: 36966.22 + rank 50: 36967.73 + rank 51: 36964.59 + rank 52: 36970.23 + rank 53: 36968.71 + rank 54: 36963.99 + rank 55: 36967.68 + rank 56: 38563.72 + rank 57: 38562.38 + rank 58: 38566.96 + rank 59: 38566.54 + rank 60: 38566.35 + rank 61: 38576.55 + rank 62: 38572.09 + rank 63: 38570.27 + backward-compute: + rank 0: 53611.24 + rank 1: 53610.81 + rank 2: 53610.20 + rank 3: 53608.29 + rank 4: 53611.26 + rank 5: 53612.33 + rank 6: 53609.53 + rank 7: 53610.77 + rank 8: 57318.62 + rank 9: 57321.04 + rank 10: 57316.78 + rank 11: 57313.08 + rank 12: 57306.34 + rank 13: 57304.61 + rank 14: 57310.18 + rank 15: 57308.09 + rank 16: 55834.57 + rank 17: 55841.66 + rank 18: 55840.04 + rank 19: 55840.37 + rank 20: 55838.99 + rank 21: 55826.83 + rank 22: 55839.86 + rank 23: 55841.88 + rank 24: 56971.05 + rank 25: 56971.09 + rank 26: 56940.62 + rank 27: 56967.79 + rank 28: 56954.63 + rank 29: 56962.78 + rank 30: 56958.59 + rank 31: 56961.47 + rank 32: 53742.12 + rank 33: 53733.97 + rank 34: 53740.71 + rank 35: 53738.82 + rank 36: 53737.94 + rank 37: 53733.04 + rank 38: 53730.75 + rank 39: 53733.12 + rank 40: 56633.83 + rank 41: 56634.91 + rank 42: 56631.13 + rank 43: 56630.42 + rank 44: 56629.56 + rank 45: 56627.47 + rank 46: 56631.02 + rank 47: 56629.41 + rank 48: 54015.88 + rank 49: 54014.05 + rank 50: 53988.66 + rank 51: 54011.36 + rank 52: 54010.03 + rank 53: 54011.34 + rank 54: 54019.55 + rank 55: 54006.34 + rank 56: 52424.79 + rank 57: 52424.89 + rank 58: 52420.75 + rank 59: 52418.01 + rank 60: 52418.81 + rank 61: 52409.93 + rank 62: 52415.45 + rank 63: 52418.00 + batch-generator: + rank 0: 21.15 + rank 1: 18.69 + rank 2: 18.74 + rank 3: 21.76 + rank 4: 21.87 + rank 5: 18.58 + rank 6: 18.91 + rank 7: 22.08 + rank 8: 18.58 + rank 9: 18.44 + rank 10: 18.44 + rank 11: 18.10 + rank 12: 18.89 + rank 13: 18.44 + rank 14: 19.10 + rank 15: 19.28 + rank 16: 24.56 + rank 17: 18.77 + rank 18: 20.03 + rank 19: 19.29 + rank 20: 19.18 + rank 21: 24.35 + rank 22: 19.18 + rank 23: 19.91 + rank 24: 38.29 + rank 25: 28.33 + rank 26: 44.85 + rank 27: 30.23 + rank 28: 34.29 + rank 29: 28.13 + rank 30: 29.38 + rank 31: 26.14 + rank 32: 29.87 + rank 33: 31.59 + rank 34: 27.49 + rank 35: 28.07 + rank 36: 26.68 + rank 37: 30.50 + rank 38: 31.37 + rank 39: 29.79 + rank 40: 18.81 + rank 41: 18.90 + rank 42: 18.93 + rank 43: 18.67 + rank 44: 19.46 + rank 45: 18.80 + rank 46: 19.74 + rank 47: 18.77 + rank 48: 19.59 + rank 49: 23.49 + rank 50: 32.48 + rank 51: 31.65 + rank 52: 24.13 + rank 53: 24.55 + rank 54: 28.29 + rank 55: 30.72 + rank 56: 18.77 + rank 57: 18.58 + rank 58: 19.02 + rank 59: 19.12 + rank 60: 18.98 + rank 61: 19.67 + rank 62: 17.97 + rank 63: 19.52 + embedding-grads-all-reduce: + rank 0: 0.02 + rank 1: 0.02 + rank 2: 0.02 + rank 3: 0.02 + rank 4: 0.13 + rank 5: 0.02 + rank 6: 0.02 + rank 7: 0.02 + rank 8: 0.02 + rank 9: 0.02 + rank 10: 0.03 + rank 11: 0.02 + rank 12: 0.02 + rank 13: 0.02 + rank 14: 0.02 + rank 15: 0.02 + rank 16: 0.03 + rank 17: 0.02 + rank 18: 0.02 + rank 19: 0.02 + rank 20: 0.02 + rank 21: 0.02 + rank 22: 0.02 + rank 23: 0.02 + rank 24: 0.06 + rank 25: 0.03 + rank 26: 0.04 + rank 27: 0.03 + rank 28: 0.03 + rank 29: 0.03 + rank 30: 0.04 + rank 31: 0.04 + rank 32: 0.04 + rank 33: 0.03 + rank 34: 0.02 + rank 35: 0.02 + rank 36: 0.03 + rank 37: 0.03 + rank 38: 0.03 + rank 39: 0.02 + rank 40: 0.02 + rank 41: 0.02 + rank 42: 0.02 + rank 43: 0.02 + rank 44: 0.02 + rank 45: 0.02 + rank 46: 0.02 + rank 47: 0.03 + rank 48: 0.02 + rank 49: 0.03 + rank 50: 0.05 + rank 51: 0.03 + rank 52: 0.03 + rank 53: 0.03 + rank 54: 0.03 + rank 55: 0.03 + rank 56: 0.02 + rank 57: 0.02 + rank 58: 0.02 + rank 59: 0.02 + rank 60: 0.02 + rank 61: 0.03 + rank 62: 0.02 + rank 63: 0.02 + all-grads-sync: + rank 0: 0.77 + rank 1: 0.75 + rank 2: 0.76 + rank 3: 0.78 + rank 4: 0.90 + rank 5: 0.74 + rank 6: 0.71 + rank 7: 0.77 + rank 8: 0.80 + rank 9: 0.75 + rank 10: 1.13 + rank 11: 0.79 + rank 12: 0.85 + rank 13: 0.78 + rank 14: 0.75 + rank 15: 0.79 + rank 16: 1.17 + rank 17: 0.76 + rank 18: 0.82 + rank 19: 1.18 + rank 20: 0.76 + rank 21: 0.93 + rank 22: 0.88 + rank 23: 0.79 + rank 24: 1.84 + rank 25: 1.19 + rank 26: 2.16 + rank 27: 1.29 + rank 28: 0.98 + rank 29: 1.14 + rank 30: 1.13 + rank 31: 0.92 + rank 32: 1.11 + rank 33: 1.28 + rank 34: 1.03 + rank 35: 1.18 + rank 36: 1.05 + rank 37: 0.87 + rank 38: 1.46 + rank 39: 1.14 + rank 40: 0.79 + rank 41: 0.75 + rank 42: 0.80 + rank 43: 0.79 + rank 44: 0.77 + rank 45: 0.77 + rank 46: 0.77 + rank 47: 0.80 + rank 48: 1.05 + rank 49: 1.12 + rank 50: 1.06 + rank 51: 1.37 + rank 52: 0.93 + rank 53: 1.10 + rank 54: 0.98 + rank 55: 1.65 + rank 56: 0.79 + rank 57: 0.74 + rank 58: 0.76 + rank 59: 0.75 + rank 60: 0.77 + rank 61: 1.05 + rank 62: 0.81 + rank 63: 1.08 + params-all-gather: + rank 0: 0.03 + rank 1: 0.02 + rank 2: 0.02 + rank 3: 0.02 + rank 4: 0.02 + rank 5: 0.02 + rank 6: 0.02 + rank 7: 0.02 + rank 8: 0.02 + rank 9: 0.02 + rank 10: 0.02 + rank 11: 0.02 + rank 12: 0.02 + rank 13: 0.02 + rank 14: 0.02 + rank 15: 0.02 + rank 16: 0.02 + rank 17: 0.02 + rank 18: 0.02 + rank 19: 0.02 + rank 20: 0.02 + rank 21: 0.02 + rank 22: 0.02 + rank 23: 0.02 + rank 24: 0.03 + rank 25: 0.03 + rank 26: 0.03 + rank 27: 0.03 + rank 28: 0.04 + rank 29: 0.03 + rank 30: 0.03 + rank 31: 0.03 + rank 32: 0.03 + rank 33: 0.02 + rank 34: 0.02 + rank 35: 0.03 + rank 36: 0.02 + rank 37: 0.03 + rank 38: 0.03 + rank 39: 0.02 + rank 40: 0.02 + rank 41: 0.02 + rank 42: 0.02 + rank 43: 0.02 + rank 44: 0.02 + rank 45: 0.03 + rank 46: 0.02 + rank 47: 0.02 + rank 48: 0.03 + rank 49: 0.03 + rank 50: 0.02 + rank 51: 0.03 + rank 52: 0.03 + rank 53: 0.03 + rank 54: 0.03 + rank 55: 0.03 + rank 56: 0.02 + rank 57: 0.02 + rank 58: 0.02 + rank 59: 0.02 + rank 60: 0.02 + rank 61: 0.02 + rank 62: 0.02 + rank 63: 0.02 + optimizer-copy-to-main-grad: + rank 0: 2.07 + rank 1: 1.74 + rank 2: 1.65 + rank 3: 2.05 + rank 4: 1.71 + rank 5: 1.77 + rank 6: 1.82 + rank 7: 1.89 + rank 8: 1.78 + rank 9: 2.08 + rank 10: 1.71 + rank 11: 1.73 + rank 12: 1.87 + rank 13: 1.73 + rank 14: 1.90 + rank 15: 2.01 + rank 16: 2.57 + rank 17: 2.12 + rank 18: 2.11 + rank 19: 2.14 + rank 20: 2.09 + rank 21: 2.41 + rank 22: 2.50 + rank 23: 2.27 + rank 24: 2.23 + rank 25: 2.29 + rank 26: 2.35 + rank 27: 2.16 + rank 28: 2.10 + rank 29: 2.20 + rank 30: 2.23 + rank 31: 2.01 + rank 32: 2.19 + rank 33: 2.22 + rank 34: 2.23 + rank 35: 2.42 + rank 36: 2.19 + rank 37: 2.21 + rank 38: 2.50 + rank 39: 1.87 + rank 40: 1.67 + rank 41: 1.86 + rank 42: 1.78 + rank 43: 1.78 + rank 44: 1.84 + rank 45: 2.02 + rank 46: 1.75 + rank 47: 1.69 + rank 48: 1.94 + rank 49: 1.80 + rank 50: 2.60 + rank 51: 2.02 + rank 52: 1.96 + rank 53: 1.93 + rank 54: 1.79 + rank 55: 2.01 + rank 56: 1.71 + rank 57: 2.24 + rank 58: 1.76 + rank 59: 1.72 + rank 60: 1.72 + rank 61: 1.70 + rank 62: 1.82 + rank 63: 1.66 + optimizer-inner-step: + rank 0: 47.77 + rank 1: 41.49 + rank 2: 41.24 + rank 3: 41.44 + rank 4: 41.21 + rank 5: 41.33 + rank 6: 41.69 + rank 7: 41.65 + rank 8: 41.42 + rank 9: 41.38 + rank 10: 41.30 + rank 11: 41.44 + rank 12: 41.34 + rank 13: 41.58 + rank 14: 41.40 + rank 15: 41.25 + rank 16: 41.73 + rank 17: 41.50 + rank 18: 41.51 + rank 19: 41.51 + rank 20: 41.47 + rank 21: 41.36 + rank 22: 41.89 + rank 23: 41.58 + rank 24: 41.10 + rank 25: 43.58 + rank 26: 41.34 + rank 27: 42.47 + rank 28: 47.71 + rank 29: 41.39 + rank 30: 41.29 + rank 31: 41.55 + rank 32: 41.45 + rank 33: 41.24 + rank 34: 41.38 + rank 35: 47.84 + rank 36: 41.47 + rank 37: 41.48 + rank 38: 41.81 + rank 39: 41.26 + rank 40: 41.41 + rank 41: 41.48 + rank 42: 41.55 + rank 43: 41.64 + rank 44: 41.16 + rank 45: 41.74 + rank 46: 41.41 + rank 47: 41.06 + rank 48: 42.13 + rank 49: 41.25 + rank 50: 52.97 + rank 51: 41.08 + rank 52: 41.43 + rank 53: 41.26 + rank 54: 41.50 + rank 55: 41.34 + rank 56: 41.03 + rank 57: 46.81 + rank 58: 41.36 + rank 59: 41.15 + rank 60: 41.56 + rank 61: 41.16 + rank 62: 41.31 + rank 63: 41.17 + optimizer-copy-main-to-model-params: + rank 0: 5.65 + rank 1: 3.94 + rank 2: 3.78 + rank 3: 4.52 + rank 4: 3.92 + rank 5: 3.94 + rank 6: 3.96 + rank 7: 4.15 + rank 8: 3.95 + rank 9: 4.52 + rank 10: 3.84 + rank 11: 3.87 + rank 12: 3.99 + rank 13: 3.89 + rank 14: 4.07 + rank 15: 4.34 + rank 16: 4.58 + rank 17: 4.13 + rank 18: 4.01 + rank 19: 4.15 + rank 20: 4.06 + rank 21: 4.31 + rank 22: 4.68 + rank 23: 4.22 + rank 24: 4.19 + rank 25: 4.38 + rank 26: 4.41 + rank 27: 4.29 + rank 28: 4.50 + rank 29: 4.61 + rank 30: 4.41 + rank 31: 4.24 + rank 32: 4.25 + rank 33: 4.10 + rank 34: 4.18 + rank 35: 4.79 + rank 36: 4.39 + rank 37: 4.39 + rank 38: 4.71 + rank 39: 3.99 + rank 40: 3.75 + rank 41: 4.05 + rank 42: 3.96 + rank 43: 3.98 + rank 44: 4.05 + rank 45: 4.29 + rank 46: 3.83 + rank 47: 3.79 + rank 48: 4.24 + rank 49: 4.04 + rank 50: 5.42 + rank 51: 4.13 + rank 52: 4.37 + rank 53: 4.06 + rank 54: 4.00 + rank 55: 4.08 + rank 56: 3.87 + rank 57: 4.74 + rank 58: 3.90 + rank 59: 3.81 + rank 60: 3.83 + rank 61: 3.73 + rank 62: 4.00 + rank 63: 3.76 + optimizer: + rank 0: 61.09 + rank 1: 52.32 + rank 2: 51.87 + rank 3: 53.04 + rank 4: 51.97 + rank 5: 52.17 + rank 6: 52.55 + rank 7: 52.74 + rank 8: 52.24 + rank 9: 53.07 + rank 10: 52.01 + rank 11: 52.14 + rank 12: 52.26 + rank 13: 52.34 + rank 14: 52.40 + rank 15: 52.61 + rank 16: 53.63 + rank 17: 52.73 + rank 18: 52.69 + rank 19: 52.81 + rank 20: 52.63 + rank 21: 52.92 + rank 22: 53.91 + rank 23: 53.01 + rank 24: 52.16 + rank 25: 54.94 + rank 26: 52.61 + rank 27: 53.66 + rank 28: 59.23 + rank 29: 53.00 + rank 30: 52.70 + rank 31: 52.69 + rank 32: 52.58 + rank 33: 52.16 + rank 34: 52.39 + rank 35: 59.68 + rank 36: 52.80 + rank 37: 52.79 + rank 38: 53.58 + rank 39: 52.07 + rank 40: 51.98 + rank 41: 52.46 + rank 42: 52.42 + rank 43: 52.53 + rank 44: 52.12 + rank 45: 53.12 + rank 46: 52.08 + rank 47: 51.68 + rank 48: 53.30 + rank 49: 52.16 + rank 50: 65.62 + rank 51: 52.10 + rank 52: 52.76 + rank 53: 52.13 + rank 54: 52.33 + rank 55: 52.25 + rank 56: 51.73 + rank 57: 58.79 + rank 58: 52.14 + rank 59: 51.78 + rank 60: 52.25 + rank 61: 51.71 + rank 62: 52.21 + rank 63: 51.74 + interval-time: + rank 0: 0.81 + rank 1: 0.76 + rank 2: 0.67 + rank 3: 0.71 + rank 4: 0.80 + rank 5: 0.67 + rank 6: 0.67 + rank 7: 0.81 + rank 8: 0.77 + rank 9: 0.70 + rank 10: 0.69 + rank 11: 0.76 + rank 12: 0.77 + rank 13: 0.76 + rank 14: 0.69 + rank 15: 0.69 + rank 16: 0.86 + rank 17: 0.78 + rank 18: 0.76 + rank 19: 0.77 + rank 20: 0.75 + rank 21: 0.81 + rank 22: 0.67 + rank 23: 0.77 + rank 24: 0.98 + rank 25: 0.87 + rank 26: 0.98 + rank 27: 0.89 + rank 28: 0.88 + rank 29: 0.80 + rank 30: 0.76 + rank 31: 0.91 + rank 32: 0.75 + rank 33: 0.82 + rank 34: 0.82 + rank 35: 0.75 + rank 36: 0.86 + rank 37: 0.74 + rank 38: 0.78 + rank 39: 0.85 + rank 40: 1.02 + rank 41: 1.00 + rank 42: 0.93 + rank 43: 0.78 + rank 44: 1.03 + rank 45: 0.77 + rank 46: 1.00 + rank 47: 0.96 + rank 48: 0.86 + rank 49: 0.89 + rank 50: 0.78 + rank 51: 0.82 + rank 52: 0.78 + rank 53: 0.89 + rank 54: 0.91 + rank 55: 0.91 + rank 56: 0.67 + rank 57: 0.67 + rank 58: 0.76 + rank 59: 0.68 + rank 60: 0.76 + rank 61: 0.74 + rank 62: 0.68 + rank 63: 2.11 + [2026-10-03 15:52:28] iteration 3/ 6855 | consumed samples: 384 | elapsed time per iteration (ms): 88822.4 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.44% | FLOPs estimator: workload-aware | logical tokens/s: 188884.7 | valid tokens/s: 188884.7 | physical tokens/s: 189326.5 | loss tokens/s: 137075.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.093949E-01 | lm-global-supervised-tokens: 1.217536E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.171072E-02 | lm-token-buffer-mib: 5.811095E+01 | global_load_balancing_loss: 1.000152E+00 | loss scale: 1.0 | grad norm: 0.195 | params norm: 11681.515 | params rms: 0.063252 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 7 days, 1:03:30.987419 | finish at 2026-10-10 16:55:59 +times across ranks (ms): + forward-backward: + rank 0: 88613.30 + rank 1: 88613.62 + rank 2: 88614.15 + rank 3: 88615.48 + rank 4: 88615.66 + rank 5: 88615.82 + rank 6: 88612.82 + rank 7: 88613.75 + rank 8: 88609.21 + rank 9: 88609.95 + rank 10: 88609.72 + rank 11: 88611.19 + rank 12: 88611.56 + rank 13: 88612.30 + rank 14: 88608.13 + rank 15: 88608.89 + rank 16: 88606.53 + rank 17: 88606.84 + rank 18: 88605.78 + rank 19: 88607.00 + rank 20: 88607.45 + rank 21: 88609.16 + rank 22: 88605.29 + rank 23: 88605.44 + rank 24: 88602.24 + rank 25: 88603.09 + rank 26: 88604.53 + rank 27: 88602.80 + rank 28: 88603.41 + rank 29: 88603.27 + rank 30: 88599.85 + rank 31: 88600.12 + rank 32: 88599.91 + rank 33: 88601.56 + rank 34: 88599.64 + rank 35: 88600.48 + rank 36: 88601.67 + rank 37: 88601.93 + rank 38: 88602.02 + rank 39: 88602.44 + rank 40: 88605.30 + rank 41: 88602.80 + rank 42: 88601.81 + rank 43: 88601.91 + rank 44: 88602.26 + rank 45: 88600.93 + rank 46: 88605.27 + rank 47: 88603.77 + rank 48: 88609.30 + rank 49: 88609.06 + rank 50: 88607.58 + rank 51: 88609.38 + rank 52: 88606.59 + rank 53: 88607.73 + rank 54: 88610.20 + rank 55: 88612.02 + rank 56: 88613.02 + rank 57: 88612.02 + rank 58: 88611.09 + rank 59: 88611.31 + rank 60: 88610.98 + rank 61: 88610.70 + rank 62: 88613.32 + rank 63: 88613.69 + forward-compute: + rank 0: 34439.96 + rank 1: 34445.30 + rank 2: 34447.32 + rank 3: 34444.49 + rank 4: 34448.41 + rank 5: 34445.70 + rank 6: 34448.71 + rank 7: 34449.87 + rank 8: 35103.03 + rank 9: 35104.34 + rank 10: 35111.02 + rank 11: 35114.41 + rank 12: 35113.62 + rank 13: 35115.18 + rank 14: 35110.65 + rank 15: 35110.99 + rank 16: 31505.59 + rank 17: 31500.05 + rank 18: 31499.01 + rank 19: 31497.53 + rank 20: 31497.89 + rank 21: 31502.62 + rank 22: 31499.74 + rank 23: 31497.56 + rank 24: 37792.65 + rank 25: 37793.13 + rank 26: 37791.78 + rank 27: 37785.88 + rank 28: 37786.35 + rank 29: 37786.34 + rank 30: 37785.79 + rank 31: 37784.79 + rank 32: 33408.27 + rank 33: 33415.48 + rank 34: 33419.61 + rank 35: 33422.25 + rank 36: 33438.73 + rank 37: 33443.15 + rank 38: 33440.43 + rank 39: 33423.50 + rank 40: 37251.61 + rank 41: 37254.92 + rank 42: 37255.27 + rank 43: 37254.88 + rank 44: 37256.55 + rank 45: 37256.65 + rank 46: 37256.37 + rank 47: 37257.60 + rank 48: 33277.71 + rank 49: 33276.50 + rank 50: 33274.57 + rank 51: 33282.14 + rank 52: 33285.44 + rank 53: 33288.22 + rank 54: 33281.88 + rank 55: 33289.19 + rank 56: 32609.87 + rank 57: 32618.15 + rank 58: 32617.80 + rank 59: 32618.29 + rank 60: 32622.75 + rank 61: 32617.18 + rank 62: 32615.98 + rank 63: 32619.43 + backward-compute: + rank 0: 54141.02 + rank 1: 54138.77 + rank 2: 54136.07 + rank 3: 54137.82 + rank 4: 54134.11 + rank 5: 54138.58 + rank 6: 54133.23 + rank 7: 54130.19 + rank 8: 53475.20 + rank 9: 53476.21 + rank 10: 53468.50 + rank 11: 53464.82 + rank 12: 53465.30 + rank 13: 53465.12 + rank 14: 53465.86 + rank 15: 53468.00 + rank 16: 57066.47 + rank 17: 57075.31 + rank 18: 57075.96 + rank 19: 57079.46 + rank 20: 57080.07 + rank 21: 57064.02 + rank 22: 57075.64 + rank 23: 57076.67 + rank 24: 50764.86 + rank 25: 50766.16 + rank 26: 50756.95 + rank 27: 50774.77 + rank 28: 50779.50 + rank 29: 50781.56 + rank 30: 50780.82 + rank 31: 50777.38 + rank 32: 55158.58 + rank 33: 55147.82 + rank 34: 55144.42 + rank 35: 55144.44 + rank 36: 55126.85 + rank 37: 55122.66 + rank 38: 55125.71 + rank 39: 55143.22 + rank 40: 51303.93 + rank 41: 51317.54 + rank 42: 51315.70 + rank 43: 51316.23 + rank 44: 51313.00 + rank 45: 51312.43 + rank 46: 51314.61 + rank 47: 51313.83 + rank 48: 55295.59 + rank 49: 55299.39 + rank 50: 55290.25 + rank 51: 55293.26 + rank 52: 55289.65 + rank 53: 55286.36 + rank 54: 55294.45 + rank 55: 55280.36 + rank 56: 55972.20 + rank 57: 55964.21 + rank 58: 55963.26 + rank 59: 55961.72 + rank 60: 55957.51 + rank 61: 55963.06 + rank 62: 55966.66 + rank 63: 55963.21 + batch-generator: + rank 0: 22.49 + rank 1: 18.76 + rank 2: 19.80 + rank 3: 21.36 + rank 4: 21.82 + rank 5: 18.25 + rank 6: 18.73 + rank 7: 22.53 + rank 8: 18.79 + rank 9: 18.70 + rank 10: 19.24 + rank 11: 18.71 + rank 12: 19.58 + rank 13: 19.27 + rank 14: 19.40 + rank 15: 18.96 + rank 16: 25.45 + rank 17: 19.15 + rank 18: 19.76 + rank 19: 19.06 + rank 20: 18.92 + rank 21: 24.26 + rank 22: 19.06 + rank 23: 19.08 + rank 24: 39.48 + rank 25: 37.70 + rank 26: 40.91 + rank 27: 31.22 + rank 28: 21.26 + rank 29: 26.39 + rank 30: 26.05 + rank 31: 30.31 + rank 32: 26.90 + rank 33: 31.19 + rank 34: 30.18 + rank 35: 29.19 + rank 36: 26.31 + rank 37: 27.23 + rank 38: 24.51 + rank 39: 29.52 + rank 40: 22.87 + rank 41: 19.42 + rank 42: 18.97 + rank 43: 19.35 + rank 44: 19.63 + rank 45: 19.12 + rank 46: 19.73 + rank 47: 18.91 + rank 48: 22.80 + rank 49: 22.15 + rank 50: 28.82 + rank 51: 31.01 + rank 52: 25.07 + rank 53: 23.81 + rank 54: 27.73 + rank 55: 31.45 + rank 56: 22.79 + rank 57: 18.65 + rank 58: 19.57 + rank 59: 20.10 + rank 60: 19.18 + rank 61: 19.08 + rank 62: 18.44 + rank 63: 19.00 + embedding-grads-all-reduce: + rank 0: 0.02 + rank 1: 0.02 + rank 2: 0.02 + rank 3: 0.02 + rank 4: 0.02 + rank 5: 0.02 + rank 6: 0.02 + rank 7: 0.02 + rank 8: 0.02 + rank 9: 0.02 + rank 10: 0.02 + rank 11: 0.02 + rank 12: 0.02 + rank 13: 0.02 + rank 14: 0.02 + rank 15: 0.02 + rank 16: 0.04 + rank 17: 0.02 + rank 18: 0.02 + rank 19: 0.02 + rank 20: 0.03 + rank 21: 0.02 + rank 22: 0.02 + rank 23: 0.03 + rank 24: 0.06 + rank 25: 0.03 + rank 26: 0.04 + rank 27: 0.03 + rank 28: 0.03 + rank 29: 0.03 + rank 30: 0.03 + rank 31: 0.04 + rank 32: 0.04 + rank 33: 0.04 + rank 34: 0.04 + rank 35: 0.03 + rank 36: 0.03 + rank 37: 0.03 + rank 38: 0.03 + rank 39: 0.03 + rank 40: 0.05 + rank 41: 0.02 + rank 42: 0.02 + rank 43: 0.02 + rank 44: 0.02 + rank 45: 0.02 + rank 46: 0.02 + rank 47: 0.02 + rank 48: 0.03 + rank 49: 0.02 + rank 50: 0.03 + rank 51: 0.03 + rank 52: 0.02 + rank 53: 0.03 + rank 54: 0.03 + rank 55: 0.03 + rank 56: 0.02 + rank 57: 0.02 + rank 58: 0.02 + rank 59: 0.02 + rank 60: 0.02 + rank 61: 0.02 + rank 62: 0.02 + rank 63: 0.02 + all-grads-sync: + rank 0: 0.78 + rank 1: 0.76 + rank 2: 0.83 + rank 3: 0.82 + rank 4: 0.76 + rank 5: 0.79 + rank 6: 0.82 + rank 7: 0.80 + rank 8: 0.74 + rank 9: 0.74 + rank 10: 0.80 + rank 11: 0.71 + rank 12: 0.75 + rank 13: 0.77 + rank 14: 0.73 + rank 15: 0.78 + rank 16: 1.10 + rank 17: 1.03 + rank 18: 0.77 + rank 19: 0.76 + rank 20: 0.77 + rank 21: 0.98 + rank 22: 0.94 + rank 23: 0.75 + rank 24: 1.31 + rank 25: 1.08 + rank 26: 2.28 + rank 27: 1.11 + rank 28: 0.89 + rank 29: 0.97 + rank 30: 1.03 + rank 31: 1.04 + rank 32: 1.09 + rank 33: 1.28 + rank 34: 1.13 + rank 35: 1.15 + rank 36: 1.13 + rank 37: 1.12 + rank 38: 1.25 + rank 39: 1.14 + rank 40: 1.13 + rank 41: 0.77 + rank 42: 0.78 + rank 43: 0.74 + rank 44: 0.75 + rank 45: 0.79 + rank 46: 1.18 + rank 47: 0.74 + rank 48: 1.01 + rank 49: 0.97 + rank 50: 1.00 + rank 51: 1.15 + rank 52: 0.97 + rank 53: 1.05 + rank 54: 1.06 + rank 55: 1.36 + rank 56: 0.85 + rank 57: 0.75 + rank 58: 0.79 + rank 59: 0.83 + rank 60: 0.78 + rank 61: 0.72 + rank 62: 0.82 + rank 63: 0.75 + params-all-gather: + rank 0: 0.02 + rank 1: 0.02 + rank 2: 0.02 + rank 3: 0.02 + rank 4: 0.02 + rank 5: 0.02 + rank 6: 0.02 + rank 7: 0.02 + rank 8: 0.02 + rank 9: 0.02 + rank 10: 0.02 + rank 11: 0.02 + rank 12: 0.02 + rank 13: 0.02 + rank 14: 0.02 + rank 15: 0.02 + rank 16: 0.02 + rank 17: 0.02 + rank 18: 0.02 + rank 19: 0.02 + rank 20: 0.02 + rank 21: 0.02 + rank 22: 0.02 + rank 23: 0.02 + rank 24: 0.04 + rank 25: 0.03 + rank 26: 0.03 + rank 27: 0.03 + rank 28: 0.04 + rank 29: 0.03 + rank 30: 0.03 + rank 31: 0.04 + rank 32: 0.02 + rank 33: 0.02 + rank 34: 0.02 + rank 35: 0.02 + rank 36: 0.02 + rank 37: 0.02 + rank 38: 0.02 + rank 39: 0.02 + rank 40: 0.02 + rank 41: 0.02 + rank 42: 0.02 + rank 43: 0.02 + rank 44: 0.02 + rank 45: 0.02 + rank 46: 0.03 + rank 47: 0.02 + rank 48: 0.03 + rank 49: 0.03 + rank 50: 0.02 + rank 51: 0.02 + rank 52: 0.02 + rank 53: 0.02 + rank 54: 0.03 + rank 55: 0.03 + rank 56: 0.02 + rank 57: 0.02 + rank 58: 0.02 + rank 59: 0.02 + rank 60: 0.02 + rank 61: 0.02 + rank 62: 0.02 + rank 63: 0.02 + optimizer-copy-to-main-grad: + rank 0: 2.12 + rank 1: 1.80 + rank 2: 1.68 + rank 3: 2.07 + rank 4: 1.68 + rank 5: 1.83 + rank 6: 1.83 + rank 7: 1.86 + rank 8: 1.78 + rank 9: 2.07 + rank 10: 1.75 + rank 11: 1.68 + rank 12: 1.77 + rank 13: 1.75 + rank 14: 1.81 + rank 15: 1.94 + rank 16: 2.60 + rank 17: 2.07 + rank 18: 2.13 + rank 19: 2.11 + rank 20: 2.13 + rank 21: 2.42 + rank 22: 2.47 + rank 23: 2.29 + rank 24: 2.08 + rank 25: 2.15 + rank 26: 2.08 + rank 27: 2.14 + rank 28: 2.17 + rank 29: 2.05 + rank 30: 2.06 + rank 31: 1.92 + rank 32: 2.20 + rank 33: 2.16 + rank 34: 2.12 + rank 35: 2.28 + rank 36: 2.16 + rank 37: 2.01 + rank 38: 2.43 + rank 39: 1.93 + rank 40: 2.67 + rank 41: 1.82 + rank 42: 1.82 + rank 43: 1.81 + rank 44: 1.84 + rank 45: 1.98 + rank 46: 1.82 + rank 47: 1.68 + rank 48: 1.94 + rank 49: 1.83 + rank 50: 3.20 + rank 51: 1.91 + rank 52: 1.98 + rank 53: 1.79 + rank 54: 1.80 + rank 55: 1.93 + rank 56: 1.79 + rank 57: 2.32 + rank 58: 1.76 + rank 59: 1.71 + rank 60: 1.68 + rank 61: 1.71 + rank 62: 1.84 + rank 63: 1.73 + optimizer-inner-step: + rank 0: 43.08 + rank 1: 42.96 + rank 2: 42.74 + rank 3: 42.84 + rank 4: 42.68 + rank 5: 42.84 + rank 6: 43.19 + rank 7: 43.13 + rank 8: 42.88 + rank 9: 42.93 + rank 10: 42.75 + rank 11: 42.91 + rank 12: 42.83 + rank 13: 43.02 + rank 14: 42.81 + rank 15: 42.70 + rank 16: 43.21 + rank 17: 43.01 + rank 18: 43.01 + rank 19: 43.01 + rank 20: 42.98 + rank 21: 42.85 + rank 22: 43.33 + rank 23: 42.99 + rank 24: 42.61 + rank 25: 42.86 + rank 26: 42.81 + rank 27: 43.01 + rank 28: 45.62 + rank 29: 42.72 + rank 30: 42.73 + rank 31: 42.99 + rank 32: 42.91 + rank 33: 42.69 + rank 34: 42.86 + rank 35: 47.28 + rank 36: 42.94 + rank 37: 42.96 + rank 38: 42.76 + rank 39: 42.74 + rank 40: 42.87 + rank 41: 42.95 + rank 42: 43.03 + rank 43: 43.10 + rank 44: 42.65 + rank 45: 43.23 + rank 46: 42.91 + rank 47: 42.54 + rank 48: 42.92 + rank 49: 42.74 + rank 50: 54.13 + rank 51: 42.59 + rank 52: 42.85 + rank 53: 42.79 + rank 54: 42.94 + rank 55: 42.83 + rank 56: 42.51 + rank 57: 47.04 + rank 58: 42.83 + rank 59: 42.68 + rank 60: 43.02 + rank 61: 42.65 + rank 62: 42.74 + rank 63: 42.63 + optimizer-copy-main-to-model-params: + rank 0: 4.49 + rank 1: 3.91 + rank 2: 3.84 + rank 3: 4.39 + rank 4: 3.77 + rank 5: 4.03 + rank 6: 4.05 + rank 7: 4.03 + rank 8: 3.92 + rank 9: 4.46 + rank 10: 3.91 + rank 11: 3.85 + rank 12: 4.01 + rank 13: 3.86 + rank 14: 4.12 + rank 15: 4.29 + rank 16: 4.53 + rank 17: 4.07 + rank 18: 4.06 + rank 19: 4.06 + rank 20: 4.08 + rank 21: 4.30 + rank 22: 4.61 + rank 23: 4.24 + rank 24: 4.16 + rank 25: 4.37 + rank 26: 4.25 + rank 27: 4.56 + rank 28: 4.41 + rank 29: 4.50 + rank 30: 4.45 + rank 31: 4.18 + rank 32: 4.31 + rank 33: 4.12 + rank 34: 4.18 + rank 35: 4.65 + rank 36: 4.50 + rank 37: 4.22 + rank 38: 4.72 + rank 39: 4.03 + rank 40: 3.89 + rank 41: 4.02 + rank 42: 3.95 + rank 43: 4.06 + rank 44: 4.02 + rank 45: 4.24 + rank 46: 3.86 + rank 47: 3.75 + rank 48: 4.26 + rank 49: 4.04 + rank 50: 5.17 + rank 51: 4.16 + rank 52: 4.34 + rank 53: 4.03 + rank 54: 4.03 + rank 55: 4.09 + rank 56: 3.89 + rank 57: 4.89 + rank 58: 3.90 + rank 59: 3.85 + rank 60: 3.85 + rank 61: 3.85 + rank 62: 4.02 + rank 63: 3.91 + optimizer: + rank 0: 55.63 + rank 1: 54.77 + rank 2: 54.41 + rank 3: 55.27 + rank 4: 54.24 + rank 5: 54.77 + rank 6: 55.17 + rank 7: 55.07 + rank 8: 54.66 + rank 9: 55.50 + rank 10: 54.56 + rank 11: 54.58 + rank 12: 54.80 + rank 13: 54.77 + rank 14: 54.86 + rank 15: 55.03 + rank 16: 56.07 + rank 17: 55.18 + rank 18: 55.23 + rank 19: 55.19 + rank 20: 55.17 + rank 21: 55.40 + rank 22: 56.29 + rank 23: 55.44 + rank 24: 54.67 + rank 25: 55.20 + rank 26: 54.95 + rank 27: 55.50 + rank 28: 58.05 + rank 29: 55.26 + rank 30: 55.17 + rank 31: 55.09 + rank 32: 55.10 + rank 33: 54.62 + rank 34: 54.87 + rank 35: 59.96 + rank 36: 55.36 + rank 37: 55.08 + rank 38: 55.52 + rank 39: 54.59 + rank 40: 54.65 + rank 41: 54.91 + rank 42: 54.90 + rank 43: 55.07 + rank 44: 54.61 + rank 45: 55.51 + rank 46: 54.64 + rank 47: 54.13 + rank 48: 55.13 + rank 49: 54.67 + rank 50: 67.62 + rank 51: 54.61 + rank 52: 55.14 + rank 53: 54.64 + rank 54: 54.80 + rank 55: 54.80 + rank 56: 54.24 + rank 57: 60.19 + rank 58: 54.61 + rank 59: 54.34 + rank 60: 54.71 + rank 61: 54.33 + rank 62: 54.64 + rank 63: 54.37 + interval-time: + rank 0: 0.78 + rank 1: 0.80 + rank 2: 0.68 + rank 3: 0.77 + rank 4: 0.83 + rank 5: 0.76 + rank 6: 0.76 + rank 7: 0.78 + rank 8: 0.68 + rank 9: 0.69 + rank 10: 0.79 + rank 11: 0.75 + rank 12: 0.70 + rank 13: 0.77 + rank 14: 0.69 + rank 15: 0.70 + rank 16: 0.82 + rank 17: 0.69 + rank 18: 0.78 + rank 19: 0.76 + rank 20: 0.76 + rank 21: 0.81 + rank 22: 0.68 + rank 23: 0.77 + rank 24: 0.96 + rank 25: 0.90 + rank 26: 0.99 + rank 27: 0.98 + rank 28: 0.85 + rank 29: 0.91 + rank 30: 0.78 + rank 31: 0.83 + rank 32: 0.76 + rank 33: 0.78 + rank 34: 0.75 + rank 35: 0.83 + rank 36: 0.83 + rank 37: 0.75 + rank 38: 0.78 + rank 39: 0.85 + rank 40: 0.83 + rank 41: 0.70 + rank 42: 0.75 + rank 43: 0.77 + rank 44: 0.91 + rank 45: 0.76 + rank 46: 0.74 + rank 47: 0.68 + rank 48: 0.86 + rank 49: 0.88 + rank 50: 0.82 + rank 51: 0.77 + rank 52: 0.79 + rank 53: 0.78 + rank 54: 0.84 + rank 55: 0.94 + rank 56: 0.69 + rank 57: 0.69 + rank 58: 0.80 + rank 59: 0.77 + rank 60: 0.76 + rank 61: 0.76 + rank 62: 0.68 + rank 63: 2.23 + [2026-10-03 15:53:54] iteration 4/ 6855 | consumed samples: 512 | elapsed time per iteration (ms): 86067.3 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.74% | FLOPs estimator: workload-aware | logical tokens/s: 194931.4 | valid tokens/s: 194931.4 | physical tokens/s: 195223.9 | loss tokens/s: 137961.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.617708E-01 | lm-global-supervised-tokens: 1.187400E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.146064E-02 | lm-token-buffer-mib: 5.804692E+01 | global_load_balancing_loss: 1.000061E+00 | loss scale: 1.0 | grad norm: 0.058 | params norm: 11681.418 | params rms: 0.063251 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 19:47:26.747809 | finish at 2026-10-10 11:41:20 +times across ranks (ms): + forward-backward: + rank 0: 85859.88 + rank 1: 85860.27 + rank 2: 85860.65 + rank 3: 85860.83 + rank 4: 85861.87 + rank 5: 85861.63 + rank 6: 85859.05 + rank 7: 85860.09 + rank 8: 85855.73 + rank 9: 85856.30 + rank 10: 85857.14 + rank 11: 85857.17 + rank 12: 85857.72 + rank 13: 85857.97 + rank 14: 85854.74 + rank 15: 85855.53 + rank 16: 85853.40 + rank 17: 85852.89 + rank 18: 85852.92 + rank 19: 85853.38 + rank 20: 85854.00 + rank 21: 85855.73 + rank 22: 85851.48 + rank 23: 85852.38 + rank 24: 85850.58 + rank 25: 85849.79 + rank 26: 85852.14 + rank 27: 85849.96 + rank 28: 85851.79 + rank 29: 85850.43 + rank 30: 85848.02 + rank 31: 85847.68 + rank 32: 85848.59 + rank 33: 85848.82 + rank 34: 85847.70 + rank 35: 85847.85 + rank 36: 85849.20 + rank 37: 85848.26 + rank 38: 85847.66 + rank 39: 85849.60 + rank 40: 85849.89 + rank 41: 85849.58 + rank 42: 85848.63 + rank 43: 85848.09 + rank 44: 85848.38 + rank 45: 85847.77 + rank 46: 85850.92 + rank 47: 85850.24 + rank 48: 85855.52 + rank 49: 85858.45 + rank 50: 85854.11 + rank 51: 85855.32 + rank 52: 85853.48 + rank 53: 85854.62 + rank 54: 85857.31 + rank 55: 85857.88 + rank 56: 85858.67 + rank 57: 85858.87 + rank 58: 85858.30 + rank 59: 85857.30 + rank 60: 85857.63 + rank 61: 85857.33 + rank 62: 85859.66 + rank 63: 85859.73 + forward-compute: + rank 0: 33461.39 + rank 1: 33465.89 + rank 2: 33464.45 + rank 3: 33462.43 + rank 4: 33464.22 + rank 5: 33463.11 + rank 6: 33462.40 + rank 7: 33465.55 + rank 8: 32244.13 + rank 9: 32243.98 + rank 10: 32235.98 + rank 11: 32239.49 + rank 12: 32240.77 + rank 13: 32242.51 + rank 14: 32239.73 + rank 15: 32239.68 + rank 16: 31125.29 + rank 17: 31127.63 + rank 18: 31128.18 + rank 19: 31127.03 + rank 20: 31127.59 + rank 21: 31129.72 + rank 22: 31123.92 + rank 23: 31128.24 + rank 24: 33156.13 + rank 25: 33159.94 + rank 26: 33156.73 + rank 27: 33155.44 + rank 28: 33165.45 + rank 29: 33162.30 + rank 30: 33161.11 + rank 31: 33161.75 + rank 32: 32142.10 + rank 33: 32142.85 + rank 34: 32143.87 + rank 35: 32145.91 + rank 36: 32149.01 + rank 37: 32150.66 + rank 38: 32150.26 + rank 39: 32147.35 + rank 40: 34481.28 + rank 41: 34481.59 + rank 42: 34484.60 + rank 43: 34485.32 + rank 44: 34485.57 + rank 45: 34486.31 + rank 46: 34485.04 + rank 47: 34486.21 + rank 48: 31656.90 + rank 49: 31654.04 + rank 50: 31653.47 + rank 51: 31651.46 + rank 52: 31656.44 + rank 53: 31655.35 + rank 54: 31652.21 + rank 55: 31655.46 + rank 56: 27838.95 + rank 57: 27832.77 + rank 58: 27836.63 + rank 59: 27827.65 + rank 60: 27837.63 + rank 61: 27849.18 + rank 62: 27839.93 + rank 63: 27847.49 + backward-compute: + rank 0: 52364.96 + rank 1: 52364.06 + rank 2: 52364.74 + rank 3: 52364.66 + rank 4: 52364.80 + rank 5: 52368.53 + rank 6: 52366.13 + rank 7: 52361.52 + rank 8: 53581.52 + rank 9: 53583.06 + rank 10: 53590.85 + rank 11: 53586.93 + rank 12: 53586.71 + rank 13: 53585.89 + rank 14: 53584.64 + rank 15: 53585.47 + rank 16: 54688.96 + rank 17: 54694.37 + rank 18: 54694.25 + rank 19: 54694.90 + rank 20: 54696.86 + rank 21: 54682.58 + rank 22: 54697.32 + rank 23: 54692.30 + rank 24: 52644.44 + rank 25: 52647.45 + rank 26: 52639.43 + rank 27: 52652.79 + rank 28: 52645.22 + rank 29: 52650.33 + rank 30: 52647.64 + rank 31: 52646.69 + rank 32: 53669.52 + rank 33: 53666.95 + rank 34: 53667.13 + rank 35: 53665.71 + rank 36: 53662.96 + rank 37: 53662.00 + rank 38: 53659.35 + rank 39: 53665.36 + rank 40: 51333.01 + rank 41: 51336.54 + rank 42: 51331.59 + rank 43: 51324.38 + rank 44: 51331.19 + rank 45: 51330.71 + rank 46: 51333.17 + rank 47: 51334.33 + rank 48: 54166.15 + rank 49: 54153.97 + rank 50: 54167.76 + rank 51: 54168.71 + rank 52: 54166.99 + rank 53: 54165.14 + rank 54: 54172.06 + rank 55: 54161.98 + rank 56: 57990.83 + rank 57: 57997.37 + rank 58: 57991.74 + rank 59: 58000.88 + rank 60: 57990.32 + rank 61: 57979.01 + rank 62: 57991.41 + rank 63: 57983.36 + batch-generator: + rank 0: 21.85 + rank 1: 18.80 + rank 2: 18.81 + rank 3: 21.80 + rank 4: 21.85 + rank 5: 18.96 + rank 6: 18.65 + rank 7: 22.15 + rank 8: 19.10 + rank 9: 19.78 + rank 10: 21.69 + rank 11: 18.44 + rank 12: 19.02 + rank 13: 19.51 + rank 14: 19.32 + rank 15: 19.19 + rank 16: 24.94 + rank 17: 18.74 + rank 18: 19.46 + rank 19: 18.60 + rank 20: 19.10 + rank 21: 25.60 + rank 22: 18.75 + rank 23: 19.27 + rank 24: 39.18 + rank 25: 33.39 + rank 26: 41.59 + rank 27: 34.32 + rank 28: 27.36 + rank 29: 29.04 + rank 30: 29.64 + rank 31: 27.40 + rank 32: 28.25 + rank 33: 28.49 + rank 34: 29.52 + rank 35: 28.76 + rank 36: 26.00 + rank 37: 28.33 + rank 38: 31.35 + rank 39: 26.85 + rank 40: 22.20 + rank 41: 19.10 + rank 42: 18.83 + rank 43: 20.53 + rank 44: 18.76 + rank 45: 18.58 + rank 46: 19.81 + rank 47: 19.06 + rank 48: 20.39 + rank 49: 39.51 + rank 50: 21.56 + rank 51: 28.85 + rank 52: 24.06 + rank 53: 26.93 + rank 54: 27.66 + rank 55: 29.87 + rank 56: 20.35 + rank 57: 20.74 + rank 58: 18.64 + rank 59: 19.78 + rank 60: 19.11 + rank 61: 20.94 + rank 62: 18.34 + rank 63: 19.26 + embedding-grads-all-reduce: + rank 0: 0.02 + rank 1: 0.02 + rank 2: 0.02 + rank 3: 0.02 + rank 4: 0.02 + rank 5: 0.02 + rank 6: 0.02 + rank 7: 0.03 + rank 8: 0.02 + rank 9: 0.02 + rank 10: 0.02 + rank 11: 0.02 + rank 12: 0.02 + rank 13: 0.02 + rank 14: 0.02 + rank 15: 0.02 + rank 16: 0.03 + rank 17: 0.02 + rank 18: 0.02 + rank 19: 0.02 + rank 20: 0.03 + rank 21: 0.02 + rank 22: 0.02 + rank 23: 0.02 + rank 24: 0.04 + rank 25: 0.03 + rank 26: 0.03 + rank 27: 0.03 + rank 28: 0.03 + rank 29: 0.03 + rank 30: 0.03 + rank 31: 0.04 + rank 32: 0.03 + rank 33: 0.03 + rank 34: 0.03 + rank 35: 0.03 + rank 36: 0.03 + rank 37: 0.03 + rank 38: 0.04 + rank 39: 0.03 + rank 40: 0.02 + rank 41: 0.02 + rank 42: 0.02 + rank 43: 0.02 + rank 44: 0.02 + rank 45: 0.02 + rank 46: 0.02 + rank 47: 0.02 + rank 48: 0.03 + rank 49: 0.04 + rank 50: 0.03 + rank 51: 0.03 + rank 52: 0.03 + rank 53: 0.03 + rank 54: 0.03 + rank 55: 0.03 + rank 56: 0.02 + rank 57: 0.02 + rank 58: 0.02 + rank 59: 0.02 + rank 60: 0.02 + rank 61: 0.02 + rank 62: 0.02 + rank 63: 0.02 + all-grads-sync: + rank 0: 0.88 + rank 1: 0.80 + rank 2: 0.75 + rank 3: 0.79 + rank 4: 0.83 + rank 5: 0.77 + rank 6: 0.83 + rank 7: 0.96 + rank 8: 0.77 + rank 9: 0.75 + rank 10: 0.78 + rank 11: 0.72 + rank 12: 0.73 + rank 13: 0.80 + rank 14: 0.80 + rank 15: 0.82 + rank 16: 1.01 + rank 17: 0.88 + rank 18: 0.78 + rank 19: 0.79 + rank 20: 0.80 + rank 21: 0.97 + rank 22: 0.74 + rank 23: 1.22 + rank 24: 1.80 + rank 25: 1.33 + rank 26: 1.87 + rank 27: 1.22 + rank 28: 1.17 + rank 29: 0.96 + rank 30: 1.15 + rank 31: 1.21 + rank 32: 1.08 + rank 33: 1.25 + rank 34: 1.09 + rank 35: 1.17 + rank 36: 1.12 + rank 37: 1.16 + rank 38: 1.22 + rank 39: 1.11 + rank 40: 0.81 + rank 41: 0.78 + rank 42: 0.89 + rank 43: 0.72 + rank 44: 0.87 + rank 45: 0.73 + rank 46: 0.78 + rank 47: 0.74 + rank 48: 0.85 + rank 49: 1.56 + rank 50: 0.92 + rank 51: 1.09 + rank 52: 0.86 + rank 53: 1.09 + rank 54: 1.43 + rank 55: 1.11 + rank 56: 0.80 + rank 57: 0.77 + rank 58: 0.78 + rank 59: 0.76 + rank 60: 0.74 + rank 61: 0.92 + rank 62: 0.79 + rank 63: 0.77 + params-all-gather: + rank 0: 0.02 + rank 1: 0.02 + rank 2: 0.02 + rank 3: 0.02 + rank 4: 0.03 + rank 5: 0.02 + rank 6: 0.02 + rank 7: 0.02 + rank 8: 0.02 + rank 9: 0.02 + rank 10: 0.02 + rank 11: 0.02 + rank 12: 0.02 + rank 13: 0.02 + rank 14: 0.02 + rank 15: 0.02 + rank 16: 0.02 + rank 17: 0.02 + rank 18: 0.02 + rank 19: 0.02 + rank 20: 0.02 + rank 21: 0.02 + rank 22: 0.02 + rank 23: 0.02 + rank 24: 0.04 + rank 25: 0.03 + rank 26: 0.03 + rank 27: 0.04 + rank 28: 0.03 + rank 29: 0.03 + rank 30: 0.04 + rank 31: 0.06 + rank 32: 0.02 + rank 33: 0.03 + rank 34: 0.02 + rank 35: 0.02 + rank 36: 0.03 + rank 37: 0.03 + rank 38: 0.02 + rank 39: 0.03 + rank 40: 0.02 + rank 41: 0.02 + rank 42: 0.02 + rank 43: 0.02 + rank 44: 0.02 + rank 45: 0.02 + rank 46: 0.02 + rank 47: 0.02 + rank 48: 0.03 + rank 49: 0.03 + rank 50: 0.03 + rank 51: 0.02 + rank 52: 0.03 + rank 53: 0.02 + rank 54: 0.02 + rank 55: 0.03 + rank 56: 0.02 + rank 57: 0.02 + rank 58: 0.02 + rank 59: 0.02 + rank 60: 0.02 + rank 61: 0.02 + rank 62: 0.02 + rank 63: 0.02 + optimizer-copy-to-main-grad: + rank 0: 2.12 + rank 1: 1.79 + rank 2: 1.65 + rank 3: 2.05 + rank 4: 1.70 + rank 5: 1.80 + rank 6: 1.82 + rank 7: 1.88 + rank 8: 1.74 + rank 9: 2.08 + rank 10: 1.76 + rank 11: 1.72 + rank 12: 1.77 + rank 13: 1.70 + rank 14: 1.86 + rank 15: 1.97 + rank 16: 2.55 + rank 17: 2.12 + rank 18: 2.15 + rank 19: 2.20 + rank 20: 2.14 + rank 21: 2.44 + rank 22: 2.49 + rank 23: 2.33 + rank 24: 3.18 + rank 25: 2.30 + rank 26: 2.92 + rank 27: 2.82 + rank 28: 2.34 + rank 29: 2.19 + rank 30: 2.37 + rank 31: 2.11 + rank 32: 1.92 + rank 33: 2.10 + rank 34: 1.85 + rank 35: 2.13 + rank 36: 2.17 + rank 37: 1.88 + rank 38: 2.12 + rank 39: 1.80 + rank 40: 1.77 + rank 41: 1.79 + rank 42: 1.78 + rank 43: 1.79 + rank 44: 1.78 + rank 45: 1.95 + rank 46: 1.75 + rank 47: 1.62 + rank 48: 1.92 + rank 49: 2.75 + rank 50: 2.36 + rank 51: 1.86 + rank 52: 1.88 + rank 53: 1.80 + rank 54: 1.83 + rank 55: 2.04 + rank 56: 1.67 + rank 57: 2.33 + rank 58: 1.75 + rank 59: 1.65 + rank 60: 1.68 + rank 61: 1.68 + rank 62: 1.80 + rank 63: 1.70 + optimizer-inner-step: + rank 0: 41.58 + rank 1: 41.37 + rank 2: 41.13 + rank 3: 41.33 + rank 4: 41.15 + rank 5: 41.28 + rank 6: 41.60 + rank 7: 41.59 + rank 8: 41.38 + rank 9: 41.32 + rank 10: 41.19 + rank 11: 41.33 + rank 12: 41.32 + rank 13: 41.50 + rank 14: 41.26 + rank 15: 41.20 + rank 16: 41.65 + rank 17: 41.40 + rank 18: 41.45 + rank 19: 41.41 + rank 20: 41.41 + rank 21: 41.29 + rank 22: 41.71 + rank 23: 41.48 + rank 24: 40.79 + rank 25: 41.36 + rank 26: 40.96 + rank 27: 41.65 + rank 28: 45.87 + rank 29: 41.21 + rank 30: 41.16 + rank 31: 41.45 + rank 32: 41.36 + rank 33: 41.21 + rank 34: 41.37 + rank 35: 47.37 + rank 36: 41.85 + rank 37: 41.54 + rank 38: 41.85 + rank 39: 41.26 + rank 40: 41.33 + rank 41: 41.41 + rank 42: 41.52 + rank 43: 41.53 + rank 44: 41.14 + rank 45: 41.71 + rank 46: 41.33 + rank 47: 41.04 + rank 48: 42.23 + rank 49: 41.23 + rank 50: 52.63 + rank 51: 41.06 + rank 52: 41.32 + rank 53: 41.26 + rank 54: 41.50 + rank 55: 41.32 + rank 56: 40.95 + rank 57: 47.00 + rank 58: 41.27 + rank 59: 41.17 + rank 60: 41.45 + rank 61: 41.06 + rank 62: 41.20 + rank 63: 41.08 + optimizer-copy-main-to-model-params: + rank 0: 4.58 + rank 1: 3.93 + rank 2: 3.75 + rank 3: 4.48 + rank 4: 3.84 + rank 5: 3.94 + rank 6: 3.94 + rank 7: 4.12 + rank 8: 3.86 + rank 9: 4.51 + rank 10: 3.87 + rank 11: 3.85 + rank 12: 3.98 + rank 13: 3.84 + rank 14: 4.07 + rank 15: 4.30 + rank 16: 4.58 + rank 17: 4.14 + rank 18: 4.09 + rank 19: 4.15 + rank 20: 4.08 + rank 21: 4.31 + rank 22: 4.67 + rank 23: 4.23 + rank 24: 4.24 + rank 25: 4.33 + rank 26: 4.33 + rank 27: 4.18 + rank 28: 4.67 + rank 29: 4.66 + rank 30: 4.52 + rank 31: 4.39 + rank 32: 4.30 + rank 33: 4.48 + rank 34: 4.22 + rank 35: 4.70 + rank 36: 4.54 + rank 37: 4.35 + rank 38: 4.81 + rank 39: 4.12 + rank 40: 3.74 + rank 41: 4.03 + rank 42: 3.95 + rank 43: 4.00 + rank 44: 3.97 + rank 45: 4.24 + rank 46: 3.85 + rank 47: 3.80 + rank 48: 4.48 + rank 49: 4.18 + rank 50: 5.09 + rank 51: 4.08 + rank 52: 4.33 + rank 53: 4.07 + rank 54: 4.02 + rank 55: 4.11 + rank 56: 3.89 + rank 57: 4.80 + rank 58: 3.95 + rank 59: 3.79 + rank 60: 3.86 + rank 61: 3.73 + rank 62: 4.03 + rank 63: 3.76 + optimizer: + rank 0: 54.25 + rank 1: 53.18 + rank 2: 52.73 + rank 3: 53.89 + rank 4: 52.81 + rank 5: 53.13 + rank 6: 53.48 + rank 7: 53.67 + rank 8: 53.13 + rank 9: 53.99 + rank 10: 52.94 + rank 11: 53.01 + rank 12: 53.22 + rank 13: 53.23 + rank 14: 53.29 + rank 15: 53.54 + rank 16: 54.56 + rank 17: 53.69 + rank 18: 53.72 + rank 19: 53.71 + rank 20: 53.61 + rank 21: 53.87 + rank 22: 54.78 + rank 23: 53.96 + rank 24: 53.11 + rank 25: 53.67 + rank 26: 53.43 + rank 27: 53.99 + rank 28: 58.61 + rank 29: 53.94 + rank 30: 53.72 + rank 31: 53.81 + rank 32: 53.56 + rank 33: 53.72 + rank 34: 53.46 + rank 35: 60.12 + rank 36: 54.35 + rank 37: 53.79 + rank 38: 54.71 + rank 39: 53.23 + rank 40: 52.94 + rank 41: 53.39 + rank 42: 53.39 + rank 43: 53.45 + rank 44: 53.02 + rank 45: 54.01 + rank 46: 53.07 + rank 47: 52.68 + rank 48: 54.70 + rank 49: 53.33 + rank 50: 65.96 + rank 51: 53.01 + rank 52: 53.60 + rank 53: 53.18 + rank 54: 53.37 + rank 55: 53.29 + rank 56: 52.71 + rank 57: 60.07 + rank 58: 53.11 + rank 59: 52.80 + rank 60: 53.18 + rank 61: 52.65 + rank 62: 53.13 + rank 63: 52.68 + interval-time: + rank 0: 0.72 + rank 1: 0.75 + rank 2: 0.68 + rank 3: 0.73 + rank 4: 0.78 + rank 5: 0.74 + rank 6: 0.75 + rank 7: 0.80 + rank 8: 0.76 + rank 9: 0.78 + rank 10: 0.79 + rank 11: 0.73 + rank 12: 0.67 + rank 13: 0.75 + rank 14: 0.76 + rank 15: 0.69 + rank 16: 0.85 + rank 17: 0.70 + rank 18: 0.79 + rank 19: 0.79 + rank 20: 0.77 + rank 21: 0.85 + rank 22: 0.69 + rank 23: 0.78 + rank 24: 0.97 + rank 25: 0.79 + rank 26: 1.06 + rank 27: 0.91 + rank 28: 0.79 + rank 29: 0.92 + rank 30: 0.78 + rank 31: 0.80 + rank 32: 0.85 + rank 33: 0.84 + rank 34: 0.76 + rank 35: 0.80 + rank 36: 0.84 + rank 37: 0.74 + rank 38: 0.78 + rank 39: 0.86 + rank 40: 0.77 + rank 41: 0.77 + rank 42: 0.73 + rank 43: 0.75 + rank 44: 0.75 + rank 45: 0.74 + rank 46: 0.69 + rank 47: 0.71 + rank 48: 0.87 + rank 49: 1.05 + rank 50: 0.77 + rank 51: 0.85 + rank 52: 0.87 + rank 53: 0.85 + rank 54: 0.85 + rank 55: 0.89 + rank 56: 0.68 + rank 57: 0.69 + rank 58: 0.76 + rank 59: 0.77 + rank 60: 0.76 + rank 61: 0.95 + rank 62: 0.68 + rank 63: 2.28 + [2026-10-03 15:55:20] iteration 5/ 6855 | consumed samples: 640 | elapsed time per iteration (ms): 85947.4 | throughput per GPU (TFLOP/s/GPU): 42.8 | MFU 13.72% | FLOPs estimator: workload-aware | logical tokens/s: 195203.2 | valid tokens/s: 195203.2 | physical tokens/s: 195647.5 | loss tokens/s: 148824.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.923957E-01 | lm-global-supervised-tokens: 1.279109E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.189252E-02 | lm-token-buffer-mib: 5.808097E+01 | global_load_balancing_loss: 9.998234E-01 | loss scale: 1.0 | grad norm: 0.052 | params norm: 11681.319 | params rms: 0.063251 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 19:32:19.744532 | finish at 2026-10-10 11:27:39 +times across ranks (ms): + forward-backward: + rank 0: 85611.67 + rank 1: 85611.74 + rank 2: 85610.93 + rank 3: 85614.19 + rank 4: 85614.01 + rank 5: 85613.41 + rank 6: 85611.09 + rank 7: 85610.97 + rank 8: 85607.83 + rank 9: 85607.95 + rank 10: 85607.30 + rank 11: 85609.90 + rank 12: 85610.44 + rank 13: 85610.34 + rank 14: 85607.15 + rank 15: 85607.95 + rank 16: 85607.73 + rank 17: 85604.27 + rank 18: 85603.57 + rank 19: 85605.70 + rank 20: 85606.59 + rank 21: 85608.04 + rank 22: 85604.19 + rank 23: 85603.94 + rank 24: 85603.80 + rank 25: 85601.66 + rank 26: 85603.02 + rank 27: 85602.91 + rank 28: 85603.47 + rank 29: 85603.59 + rank 30: 85600.79 + rank 31: 85599.96 + rank 32: 85598.27 + rank 33: 85598.91 + rank 34: 85597.53 + rank 35: 85600.10 + rank 36: 85601.13 + rank 37: 85599.96 + rank 38: 85599.24 + rank 39: 85600.14 + rank 40: 85602.34 + rank 41: 85601.27 + rank 42: 85599.55 + rank 43: 85604.02 + rank 44: 85600.68 + rank 45: 85599.49 + rank 46: 85603.90 + rank 47: 85602.40 + rank 48: 85607.45 + rank 49: 85612.74 + rank 50: 85605.09 + rank 51: 85608.96 + rank 52: 85606.41 + rank 53: 85607.45 + rank 54: 85609.91 + rank 55: 85610.16 + rank 56: 85611.56 + rank 57: 85611.76 + rank 58: 85652.20 + rank 59: 85611.51 + rank 60: 85610.16 + rank 61: 85610.25 + rank 62: 85612.88 + rank 63: 85612.31 + forward-compute: + rank 0: 32919.53 + rank 1: 32922.78 + rank 2: 32926.48 + rank 3: 32925.46 + rank 4: 32928.39 + rank 5: 32925.96 + rank 6: 32928.03 + rank 7: 32935.96 + rank 8: 33411.75 + rank 9: 33415.06 + rank 10: 33413.32 + rank 11: 33417.36 + rank 12: 33416.69 + rank 13: 33417.74 + rank 14: 33418.10 + rank 15: 33419.05 + rank 16: 33612.55 + rank 17: 33609.91 + rank 18: 33608.71 + rank 19: 33614.82 + rank 20: 33610.64 + rank 21: 33614.89 + rank 22: 33614.22 + rank 23: 33617.55 + rank 24: 31148.66 + rank 25: 31153.74 + rank 26: 31155.45 + rank 27: 31153.83 + rank 28: 31165.92 + rank 29: 31167.54 + rank 30: 31162.35 + rank 31: 31159.18 + rank 32: 31367.64 + rank 33: 31380.16 + rank 34: 31380.12 + rank 35: 31386.11 + rank 36: 31390.17 + rank 37: 31395.81 + rank 38: 31394.83 + rank 39: 31387.71 + rank 40: 32696.41 + rank 41: 32695.58 + rank 42: 32695.80 + rank 43: 32697.57 + rank 44: 32696.88 + rank 45: 32698.06 + rank 46: 32700.26 + rank 47: 32699.46 + rank 48: 34026.99 + rank 49: 34032.01 + rank 50: 34033.28 + rank 51: 34035.66 + rank 52: 34041.38 + rank 53: 34036.67 + rank 54: 34035.90 + rank 55: 34035.88 + rank 56: 31616.27 + rank 57: 31621.50 + rank 58: 31619.64 + rank 59: 31619.90 + rank 60: 31627.73 + rank 61: 31627.96 + rank 62: 31623.36 + rank 63: 31630.19 + backward-compute: + rank 0: 52660.65 + rank 1: 52658.09 + rank 2: 52653.33 + rank 3: 52654.89 + rank 4: 52652.65 + rank 5: 52657.49 + rank 6: 52652.83 + rank 7: 52643.20 + rank 8: 52165.45 + rank 9: 52163.12 + rank 10: 52164.02 + rank 11: 52160.27 + rank 12: 52161.38 + rank 13: 52162.86 + rank 14: 52158.02 + rank 15: 52158.77 + rank 16: 51953.71 + rank 17: 51963.87 + rank 18: 51963.64 + rank 19: 51960.48 + rank 20: 51965.05 + rank 21: 51950.66 + rank 22: 51957.80 + rank 23: 51954.45 + rank 24: 54407.65 + rank 25: 54408.81 + rank 26: 54391.46 + rank 27: 54408.20 + rank 28: 54399.41 + rank 29: 54399.75 + rank 30: 54398.90 + rank 31: 54404.32 + rank 32: 54197.07 + rank 33: 54184.23 + rank 34: 54183.64 + rank 35: 54178.48 + rank 36: 54176.64 + rank 37: 54170.10 + rank 38: 54168.67 + rank 39: 54176.55 + rank 40: 52873.51 + rank 41: 52876.28 + rank 42: 52872.79 + rank 43: 52857.43 + rank 44: 52873.13 + rank 45: 52871.44 + rank 46: 52871.65 + rank 47: 52872.88 + rank 48: 51548.39 + rank 49: 51487.45 + rank 50: 51537.90 + rank 51: 51536.65 + rank 52: 51531.44 + rank 53: 51537.20 + rank 54: 51540.36 + rank 55: 51533.13 + rank 56: 53964.40 + rank 57: 53958.64 + rank 58: 53959.31 + rank 59: 53956.73 + rank 60: 53952.03 + rank 61: 53951.89 + rank 62: 53958.73 + rank 63: 53950.95 + batch-generator: + rank 0: 22.00 + rank 1: 19.37 + rank 2: 19.30 + rank 3: 21.99 + rank 4: 22.08 + rank 5: 19.06 + rank 6: 18.87 + rank 7: 21.88 + rank 8: 18.91 + rank 9: 19.28 + rank 10: 20.61 + rank 11: 18.94 + rank 12: 19.04 + rank 13: 18.86 + rank 14: 19.38 + rank 15: 18.84 + rank 16: 30.01 + rank 17: 18.84 + rank 18: 19.41 + rank 19: 19.28 + rank 20: 18.77 + rank 21: 25.40 + rank 22: 18.67 + rank 23: 19.16 + rank 24: 45.74 + rank 25: 31.27 + rank 26: 44.03 + rank 27: 32.76 + rank 28: 29.21 + rank 29: 29.81 + rank 30: 30.40 + rank 31: 30.14 + rank 32: 29.32 + rank 33: 28.40 + rank 34: 30.30 + rank 35: 30.86 + rank 36: 29.13 + rank 37: 29.22 + rank 38: 31.79 + rank 39: 28.78 + rank 40: 19.96 + rank 41: 18.92 + rank 42: 19.33 + rank 43: 27.94 + rank 44: 18.83 + rank 45: 19.07 + rank 46: 19.91 + rank 47: 19.18 + rank 48: 19.69 + rank 49: 49.85 + rank 50: 20.79 + rank 51: 28.86 + rank 52: 25.29 + rank 53: 26.76 + rank 54: 26.33 + rank 55: 32.99 + rank 56: 19.15 + rank 57: 18.82 + rank 58: 18.69 + rank 59: 22.65 + rank 60: 19.22 + rank 61: 18.89 + rank 62: 17.88 + rank 63: 19.20 + embedding-grads-all-reduce: + rank 0: 0.02 + rank 1: 0.02 + rank 2: 0.02 + rank 3: 0.02 + rank 4: 0.02 + rank 5: 0.02 + rank 6: 0.02 + rank 7: 0.02 + rank 8: 0.02 + rank 9: 0.02 + rank 10: 0.02 + rank 11: 0.02 + rank 12: 0.02 + rank 13: 0.02 + rank 14: 0.02 + rank 15: 0.02 + rank 16: 0.03 + rank 17: 0.02 + rank 18: 0.02 + rank 19: 0.02 + rank 20: 0.03 + rank 21: 0.02 + rank 22: 0.02 + rank 23: 0.02 + rank 24: 0.04 + rank 25: 0.03 + rank 26: 0.03 + rank 27: 0.03 + rank 28: 0.03 + rank 29: 0.03 + rank 30: 0.03 + rank 31: 0.03 + rank 32: 0.03 + rank 33: 0.03 + rank 34: 0.03 + rank 35: 0.03 + rank 36: 0.02 + rank 37: 0.03 + rank 38: 0.03 + rank 39: 0.02 + rank 40: 0.03 + rank 41: 0.02 + rank 42: 0.02 + rank 43: 0.03 + rank 44: 0.02 + rank 45: 0.02 + rank 46: 0.02 + rank 47: 0.02 + rank 48: 0.02 + rank 49: 0.03 + rank 50: 0.03 + rank 51: 0.03 + rank 52: 0.03 + rank 53: 0.03 + rank 54: 0.03 + rank 55: 0.03 + rank 56: 0.02 + rank 57: 0.02 + rank 58: 0.10 + rank 59: 0.02 + rank 60: 0.02 + rank 61: 0.03 + rank 62: 0.02 + rank 63: 0.02 + all-grads-sync: + rank 0: 0.98 + rank 1: 0.75 + rank 2: 0.74 + rank 3: 1.02 + rank 4: 0.74 + rank 5: 0.73 + rank 6: 0.71 + rank 7: 0.72 + rank 8: 0.73 + rank 9: 0.74 + rank 10: 0.79 + rank 11: 0.74 + rank 12: 0.93 + rank 13: 0.77 + rank 14: 0.76 + rank 15: 0.76 + rank 16: 2.30 + rank 17: 0.73 + rank 18: 0.74 + rank 19: 0.77 + rank 20: 0.78 + rank 21: 1.11 + rank 22: 0.74 + rank 23: 0.75 + rank 24: 2.58 + rank 25: 1.19 + rank 26: 1.76 + rank 27: 1.20 + rank 28: 1.13 + rank 29: 1.22 + rank 30: 1.29 + rank 31: 1.24 + rank 32: 1.14 + rank 33: 1.19 + rank 34: 1.14 + rank 35: 1.18 + rank 36: 1.25 + rank 37: 1.12 + rank 38: 1.19 + rank 39: 1.11 + rank 40: 0.76 + rank 41: 0.89 + rank 42: 0.75 + rank 43: 0.99 + rank 44: 0.73 + rank 45: 0.75 + rank 46: 0.80 + rank 47: 0.81 + rank 48: 0.82 + rank 49: 2.84 + rank 50: 0.92 + rank 51: 1.14 + rank 52: 1.05 + rank 53: 1.32 + rank 54: 1.03 + rank 55: 1.26 + rank 56: 0.81 + rank 57: 1.41 + rank 58: 3.42 + rank 59: 1.08 + rank 60: 0.76 + rank 61: 0.76 + rank 62: 0.79 + rank 63: 0.76 + params-all-gather: + rank 0: 0.02 + rank 1: 0.02 + rank 2: 0.02 + rank 3: 0.02 + rank 4: 0.02 + rank 5: 0.02 + rank 6: 0.02 + rank 7: 0.02 + rank 8: 0.02 + rank 9: 0.02 + rank 10: 0.02 + rank 11: 0.02 + rank 12: 0.02 + rank 13: 0.02 + rank 14: 0.02 + rank 15: 0.02 + rank 16: 0.02 + rank 17: 0.02 + rank 18: 0.02 + rank 19: 0.02 + rank 20: 0.02 + rank 21: 0.02 + rank 22: 0.02 + rank 23: 0.02 + rank 24: 0.03 + rank 25: 0.03 + rank 26: 0.03 + rank 27: 0.03 + rank 28: 0.05 + rank 29: 0.04 + rank 30: 0.03 + rank 31: 0.04 + rank 32: 0.02 + rank 33: 0.02 + rank 34: 0.02 + rank 35: 0.02 + rank 36: 0.02 + rank 37: 0.02 + rank 38: 0.02 + rank 39: 0.02 + rank 40: 0.02 + rank 41: 0.02 + rank 42: 0.02 + rank 43: 0.02 + rank 44: 0.02 + rank 45: 0.02 + rank 46: 0.02 + rank 47: 0.02 + rank 48: 0.02 + rank 49: 0.03 + rank 50: 0.02 + rank 51: 0.02 + rank 52: 0.03 + rank 53: 0.03 + rank 54: 0.03 + rank 55: 0.02 + rank 56: 0.02 + rank 57: 0.02 + rank 58: 0.02 + rank 59: 0.02 + rank 60: 0.02 + rank 61: 0.02 + rank 62: 0.02 + rank 63: 0.02 + optimizer-copy-to-main-grad: + rank 0: 2.06 + rank 1: 1.77 + rank 2: 1.64 + rank 3: 2.03 + rank 4: 1.67 + rank 5: 1.74 + rank 6: 1.81 + rank 7: 1.83 + rank 8: 1.75 + rank 9: 2.08 + rank 10: 1.76 + rank 11: 1.71 + rank 12: 1.81 + rank 13: 1.74 + rank 14: 1.87 + rank 15: 1.99 + rank 16: 2.68 + rank 17: 2.08 + rank 18: 2.17 + rank 19: 2.12 + rank 20: 2.14 + rank 21: 2.37 + rank 22: 2.52 + rank 23: 2.30 + rank 24: 2.28 + rank 25: 2.38 + rank 26: 2.30 + rank 27: 2.23 + rank 28: 2.37 + rank 29: 2.21 + rank 30: 2.34 + rank 31: 2.15 + rank 32: 2.16 + rank 33: 2.09 + rank 34: 2.05 + rank 35: 2.33 + rank 36: 2.11 + rank 37: 2.05 + rank 38: 2.33 + rank 39: 1.92 + rank 40: 1.75 + rank 41: 1.85 + rank 42: 1.82 + rank 43: 3.05 + rank 44: 1.84 + rank 45: 2.03 + rank 46: 1.78 + rank 47: 1.70 + rank 48: 1.92 + rank 49: 2.16 + rank 50: 2.49 + rank 51: 1.89 + rank 52: 2.03 + rank 53: 1.98 + rank 54: 1.84 + rank 55: 1.96 + rank 56: 1.85 + rank 57: 2.49 + rank 58: 5.89 + rank 59: 1.85 + rank 60: 1.82 + rank 61: 1.83 + rank 62: 1.96 + rank 63: 1.81 + optimizer-inner-step: + rank 0: 65.25 + rank 1: 65.13 + rank 2: 64.84 + rank 3: 65.00 + rank 4: 64.78 + rank 5: 64.94 + rank 6: 65.35 + rank 7: 65.28 + rank 8: 64.83 + rank 9: 65.02 + rank 10: 64.90 + rank 11: 64.99 + rank 12: 65.01 + rank 13: 65.20 + rank 14: 64.97 + rank 15: 64.92 + rank 16: 65.36 + rank 17: 65.04 + rank 18: 65.19 + rank 19: 65.18 + rank 20: 65.18 + rank 21: 65.05 + rank 22: 65.53 + rank 23: 127.07 + rank 24: 64.75 + rank 25: 65.05 + rank 26: 64.92 + rank 27: 65.15 + rank 28: 65.44 + rank 29: 64.91 + rank 30: 64.85 + rank 31: 65.13 + rank 32: 65.06 + rank 33: 64.82 + rank 34: 65.02 + rank 35: 65.39 + rank 36: 65.14 + rank 37: 65.13 + rank 38: 64.95 + rank 39: 64.93 + rank 40: 65.14 + rank 41: 65.15 + rank 42: 65.24 + rank 43: 65.25 + rank 44: 64.85 + rank 45: 65.40 + rank 46: 65.07 + rank 47: 64.69 + rank 48: 65.08 + rank 49: 64.93 + rank 50: 65.58 + rank 51: 64.70 + rank 52: 65.05 + rank 53: 64.99 + rank 54: 65.12 + rank 55: 64.95 + rank 56: 64.58 + rank 57: 65.94 + rank 58: 67.52 + rank 59: 64.79 + rank 60: 65.06 + rank 61: 64.70 + rank 62: 64.88 + rank 63: 64.70 + optimizer-copy-main-to-model-params: + rank 0: 4.43 + rank 1: 3.93 + rank 2: 3.87 + rank 3: 4.33 + rank 4: 3.78 + rank 5: 3.95 + rank 6: 4.05 + rank 7: 4.05 + rank 8: 4.06 + rank 9: 4.50 + rank 10: 3.84 + rank 11: 3.93 + rank 12: 4.05 + rank 13: 3.81 + rank 14: 4.13 + rank 15: 4.32 + rank 16: 4.64 + rank 17: 4.12 + rank 18: 4.13 + rank 19: 4.10 + rank 20: 4.11 + rank 21: 4.40 + rank 22: 4.73 + rank 23: 4.40 + rank 24: 4.99 + rank 25: 4.69 + rank 26: 4.25 + rank 27: 4.43 + rank 28: 4.47 + rank 29: 4.48 + rank 30: 4.41 + rank 31: 4.13 + rank 32: 4.27 + rank 33: 4.05 + rank 34: 4.21 + rank 35: 4.71 + rank 36: 4.44 + rank 37: 4.25 + rank 38: 4.74 + rank 39: 4.04 + rank 40: 3.91 + rank 41: 4.06 + rank 42: 4.08 + rank 43: 4.49 + rank 44: 4.03 + rank 45: 4.31 + rank 46: 3.92 + rank 47: 3.82 + rank 48: 4.29 + rank 49: 4.46 + rank 50: 5.33 + rank 51: 4.14 + rank 52: 4.38 + rank 53: 4.36 + rank 54: 4.06 + rank 55: 4.10 + rank 56: 4.15 + rank 57: 5.10 + rank 58: 4.09 + rank 59: 4.08 + rank 60: 4.09 + rank 61: 4.07 + rank 62: 4.23 + rank 63: 4.03 + optimizer: + rank 0: 82.59 + rank 1: 81.76 + rank 2: 81.39 + rank 3: 82.22 + rank 4: 81.20 + rank 5: 81.60 + rank 6: 82.14 + rank 7: 82.08 + rank 8: 81.81 + rank 9: 82.48 + rank 10: 81.43 + rank 11: 81.57 + rank 12: 81.79 + rank 13: 81.69 + rank 14: 81.84 + rank 15: 82.08 + rank 16: 83.12 + rank 17: 82.17 + rank 18: 82.30 + rank 19: 82.22 + rank 20: 82.26 + rank 21: 82.50 + rank 22: 83.43 + rank 23: 144.60 + rank 24: 82.42 + rank 25: 82.46 + rank 26: 81.86 + rank 27: 82.31 + rank 28: 82.78 + rank 29: 82.23 + rank 30: 82.08 + rank 31: 81.98 + rank 32: 82.03 + rank 33: 81.49 + rank 34: 81.89 + rank 35: 82.93 + rank 36: 82.33 + rank 37: 82.08 + rank 38: 82.55 + rank 39: 81.60 + rank 40: 81.69 + rank 41: 81.95 + rank 42: 82.04 + rank 43: 82.48 + rank 44: 81.61 + rank 45: 82.56 + rank 46: 81.66 + rank 47: 81.17 + rank 48: 82.12 + rank 49: 82.10 + rank 50: 83.95 + rank 51: 81.49 + rank 52: 82.19 + rank 53: 81.93 + rank 54: 81.82 + rank 55: 81.70 + rank 56: 81.54 + rank 57: 84.18 + rank 58: 86.54 + rank 59: 81.60 + rank 60: 81.95 + rank 61: 81.54 + rank 62: 81.93 + rank 63: 81.47 + interval-time: + rank 0: 0.67 + rank 1: 0.75 + rank 2: 0.76 + rank 3: 0.71 + rank 4: 0.76 + rank 5: 0.74 + rank 6: 0.74 + rank 7: 0.75 + rank 8: 0.78 + rank 9: 0.77 + rank 10: 0.77 + rank 11: 0.77 + rank 12: 0.76 + rank 13: 0.75 + rank 14: 0.70 + rank 15: 0.69 + rank 16: 0.83 + rank 17: 0.70 + rank 18: 0.81 + rank 19: 0.77 + rank 20: 0.79 + rank 21: 0.85 + rank 22: 0.68 + rank 23: 0.69 + rank 24: 0.99 + rank 25: 0.84 + rank 26: 0.99 + rank 27: 0.90 + rank 28: 0.79 + rank 29: 0.87 + rank 30: 0.78 + rank 31: 0.81 + rank 32: 0.85 + rank 33: 0.78 + rank 34: 0.76 + rank 35: 0.83 + rank 36: 0.82 + rank 37: 0.73 + rank 38: 0.78 + rank 39: 0.83 + rank 40: 0.80 + rank 41: 0.79 + rank 42: 0.77 + rank 43: 0.90 + rank 44: 0.79 + rank 45: 0.77 + rank 46: 0.80 + rank 47: 0.69 + rank 48: 0.86 + rank 49: 0.98 + rank 50: 0.84 + rank 51: 0.89 + rank 52: 0.87 + rank 53: 0.93 + rank 54: 0.77 + rank 55: 0.89 + rank 56: 0.84 + rank 57: 0.84 + rank 58: 0.93 + rank 59: 0.90 + rank 60: 0.92 + rank 61: 0.91 + rank 62: 0.83 + rank 63: 2.71 + [2026-10-03 15:56:48] iteration 6/ 6855 | consumed samples: 768 | elapsed time per iteration (ms): 88168.7 | throughput per GPU (TFLOP/s/GPU): 41.6 | MFU 13.34% | FLOPs estimator: workload-aware | logical tokens/s: 190284.9 | valid tokens/s: 190284.9 | physical tokens/s: 190680.5 | loss tokens/s: 148389.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.520971E-01 | lm-global-supervised-tokens: 1.308330E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.191971E-02 | lm-token-buffer-mib: 5.821401E+01 | global_load_balancing_loss: 1.000294E+00 | loss scale: 1.0 | grad norm: 0.097 | params norm: 11681.218 | params rms: 0.063250 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 23:44:27.326553 | finish at 2026-10-10 15:41:15 + [2026-10-03 15:58:17] iteration 7/ 6855 | consumed samples: 896 | elapsed time per iteration (ms): 88699.4 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.67% | FLOPs estimator: workload-aware | logical tokens/s: 189146.2 | valid tokens/s: 189146.2 | physical tokens/s: 189714.8 | loss tokens/s: 143860.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.320268E-01 | lm-global-supervised-tokens: 1.276035E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.205771E-02 | lm-token-buffer-mib: 5.818698E+01 | global_load_balancing_loss: 1.000775E+00 | loss scale: 1.0 | grad norm: 0.079 | params norm: 11681.116 | params rms: 0.063249 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 7 days, 0:43:33.187164 | finish at 2026-10-10 16:41:50 + [2026-10-03 15:59:43] iteration 8/ 6855 | consumed samples: 1024 | elapsed time per iteration (ms): 86676.2 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.76% | FLOPs estimator: workload-aware | logical tokens/s: 193561.4 | valid tokens/s: 193561.4 | physical tokens/s: 193923.9 | loss tokens/s: 141264.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.950675E-01 | lm-global-supervised-tokens: 1.224424E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.032033E-02 | lm-token-buffer-mib: 5.805603E+01 | global_load_balancing_loss: 1.000667E+00 | loss scale: 1.0 | grad norm: 0.043 | params norm: 11681.013 | params rms: 0.063249 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 20:51:11.684510 | finish at 2026-10-10 12:50:55 + [2026-10-03 16:01:11] iteration 9/ 6855 | consumed samples: 1152 | elapsed time per iteration (ms): 87971.5 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.79% | FLOPs estimator: workload-aware | logical tokens/s: 190712.0 | valid tokens/s: 190712.0 | physical tokens/s: 191082.9 | loss tokens/s: 137057.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.994282E-01 | lm-global-supervised-tokens: 1.205717E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.232534E-02 | lm-token-buffer-mib: 5.810181E+01 | global_load_balancing_loss: 1.001399E+00 | loss scale: 1.0 | grad norm: 0.094 | params norm: 11680.908 | params rms: 0.063248 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 23:17:32.689322 | finish at 2026-10-10 15:18:44 + [2026-10-03 16:02:45] iteration 10/ 6855 | consumed samples: 1280 | elapsed time per iteration (ms): 93342.2 | throughput per GPU (TFLOP/s/GPU): 39.2 | MFU 12.56% | FLOPs estimator: workload-aware | logical tokens/s: 179737.7 | valid tokens/s: 179737.6 | physical tokens/s: 180309.7 | loss tokens/s: 133479.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.775400E-01 | lm-global-supervised-tokens: 1.245924E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.151955E-02 | lm-token-buffer-mib: 5.831451E+01 | global_load_balancing_loss: 1.000223E+00 | loss scale: 1.0 | grad norm: 0.064 | params norm: 11680.803 | params rms: 0.063248 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 7 days, 9:28:47.021461 | finish at 2026-10-11 01:31:32 + [2026-10-03 16:04:10] iteration 11/ 6855 | consumed samples: 1408 | elapsed time per iteration (ms): 85630.5 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.79% | FLOPs estimator: workload-aware | logical tokens/s: 195925.5 | valid tokens/s: 195925.5 | physical tokens/s: 196376.9 | loss tokens/s: 144615.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.635357E-01 | lm-global-supervised-tokens: 1.238349E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.148968E-02 | lm-token-buffer-mib: 5.815474E+01 | global_load_balancing_loss: 9.995468E-01 | loss scale: 1.0 | grad norm: 0.088 | params norm: 11680.697 | params rms: 0.063247 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 18:47:35.131113 | finish at 2026-10-10 10:51:45 + [2026-10-03 16:05:38] iteration 12/ 6855 | consumed samples: 1536 | elapsed time per iteration (ms): 87743.2 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.64% | FLOPs estimator: workload-aware | logical tokens/s: 191206.9 | valid tokens/s: 191206.9 | physical tokens/s: 191577.1 | loss tokens/s: 143275.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.560428E-01 | lm-global-supervised-tokens: 1.257146E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.519432E-02 | lm-token-buffer-mib: 5.808842E+01 | global_load_balancing_loss: 1.000064E+00 | loss scale: 1.0 | grad norm: 0.036 | params norm: 11680.589 | params rms: 0.063247 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 22:47:06.693563 | finish at 2026-10-10 14:52:45 + [2026-10-03 16:07:06] iteration 13/ 6855 | consumed samples: 1664 | elapsed time per iteration (ms): 87594.1 | throughput per GPU (TFLOP/s/GPU): 42.8 | MFU 13.71% | FLOPs estimator: workload-aware | logical tokens/s: 191533.6 | valid tokens/s: 191533.6 | physical tokens/s: 191903.5 | loss tokens/s: 141217.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.347817E-01 | lm-global-supervised-tokens: 1.236984E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.137506E-02 | lm-token-buffer-mib: 5.816574E+01 | global_load_balancing_loss: 9.990418E-01 | loss scale: 1.0 | grad norm: 0.073 | params norm: 11680.481 | params rms: 0.063246 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 22:28:39.024678 | finish at 2026-10-10 14:35:45 + [2026-10-03 16:08:35] iteration 14/ 6855 | consumed samples: 1792 | elapsed time per iteration (ms): 89728.3 | throughput per GPU (TFLOP/s/GPU): 41.7 | MFU 13.37% | FLOPs estimator: workload-aware | logical tokens/s: 186977.7 | valid tokens/s: 186977.7 | physical tokens/s: 187451.3 | loss tokens/s: 136757.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.772115E-01 | lm-global-supervised-tokens: 1.227105E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.134089E-02 | lm-token-buffer-mib: 5.810478E+01 | global_load_balancing_loss: 9.996521E-01 | loss scale: 1.0 | grad norm: 0.036 | params norm: 11680.371 | params rms: 0.063245 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 7 days, 2:30:31.576590 | finish at 2026-10-10 18:39:07 + [2026-10-03 16:10:04] iteration 15/ 6855 | consumed samples: 1920 | elapsed time per iteration (ms): 88346.5 | throughput per GPU (TFLOP/s/GPU): 41.5 | MFU 13.29% | FLOPs estimator: workload-aware | logical tokens/s: 189901.9 | valid tokens/s: 189901.8 | physical tokens/s: 190394.0 | loss tokens/s: 139390.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.475797E-01 | lm-global-supervised-tokens: 1.231466E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.160435E-02 | lm-token-buffer-mib: 5.820992E+01 | global_load_balancing_loss: 1.000752E+00 | loss scale: 1.0 | grad norm: 0.053 | params norm: 11680.260 | params rms: 0.063245 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 23:51:30.054560 | finish at 2026-10-10 16:01:34 + [2026-10-03 16:11:40] iteration 16/ 6855 | consumed samples: 2048 | elapsed time per iteration (ms): 96516.2 | throughput per GPU (TFLOP/s/GPU): 39.1 | MFU 12.54% | FLOPs estimator: workload-aware | logical tokens/s: 173827.2 | valid tokens/s: 173827.2 | physical tokens/s: 174323.5 | loss tokens/s: 127452.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.821101E-01 | lm-global-supervised-tokens: 1.230123E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.149699E-02 | lm-token-buffer-mib: 5.810744E+01 | global_load_balancing_loss: 1.000891E+00 | loss scale: 1.0 | grad norm: 0.070 | params norm: 11680.148 | params rms: 0.063244 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 7 days, 15:21:14.390081 | finish at 2026-10-11 07:32:55 + [2026-10-03 16:13:08] iteration 17/ 6855 | consumed samples: 2176 | elapsed time per iteration (ms): 87799.5 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.40% | FLOPs estimator: workload-aware | logical tokens/s: 191085.4 | valid tokens/s: 191085.4 | physical tokens/s: 191613.1 | loss tokens/s: 140089.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.927003E-01 | lm-global-supervised-tokens: 1.229978E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.002125E-02 | lm-token-buffer-mib: 5.829067E+01 | global_load_balancing_loss: 1.001508E+00 | loss scale: 1.0 | grad norm: 0.073 | params norm: 11680.036 | params rms: 0.063244 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 22:46:12.912449 | finish at 2026-10-10 14:59:21 + [2026-10-03 16:14:35] iteration 18/ 6855 | consumed samples: 2304 | elapsed time per iteration (ms): 86849.2 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.64% | FLOPs estimator: workload-aware | logical tokens/s: 193176.0 | valid tokens/s: 193176.0 | physical tokens/s: 193576.3 | loss tokens/s: 136517.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.453120E-01 | lm-global-supervised-tokens: 1.185641E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.691006E-02 | lm-token-buffer-mib: 5.810365E+01 | global_load_balancing_loss: 1.000219E+00 | loss scale: 1.0 | grad norm: 0.095 | params norm: 11679.923 | params rms: 0.063243 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 20:56:28.022852 | finish at 2026-10-10 13:11:03 + [2026-10-03 16:16:02] iteration 19/ 6855 | consumed samples: 2432 | elapsed time per iteration (ms): 87229.9 | throughput per GPU (TFLOP/s/GPU): 43.1 | MFU 13.81% | FLOPs estimator: workload-aware | logical tokens/s: 192332.7 | valid tokens/s: 192332.7 | physical tokens/s: 192728.1 | loss tokens/s: 145661.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.834517E-01 | lm-global-supervised-tokens: 1.270601E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.429407E-02 | lm-token-buffer-mib: 5.812680E+01 | global_load_balancing_loss: 1.000488E+00 | loss scale: 1.0 | grad norm: 0.051 | params norm: 11679.810 | params rms: 0.063242 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 21:38:23.424470 | finish at 2026-10-10 13:54:25 + [2026-10-03 16:17:32] iteration 20/ 6855 | consumed samples: 2560 | elapsed time per iteration (ms): 90444.3 | throughput per GPU (TFLOP/s/GPU): 40.5 | MFU 12.97% | FLOPs estimator: workload-aware | logical tokens/s: 185497.7 | valid tokens/s: 185497.7 | physical tokens/s: 185815.4 | loss tokens/s: 136682.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.346424E-01 | lm-global-supervised-tokens: 1.236211E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.153970E-02 | lm-token-buffer-mib: 5.808388E+01 | global_load_balancing_loss: 9.994228E-01 | loss scale: 1.0 | grad norm: 0.039 | params norm: 11679.695 | params rms: 0.063242 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 7 days, 3:43:06.599402 | finish at 2026-10-10 20:00:39 + [2026-10-03 16:18:58] iteration 21/ 6855 | consumed samples: 2688 | elapsed time per iteration (ms): 86007.3 | throughput per GPU (TFLOP/s/GPU): 41.6 | MFU 13.32% | FLOPs estimator: workload-aware | logical tokens/s: 195066.2 | valid tokens/s: 195066.2 | physical tokens/s: 195749.3 | loss tokens/s: 149550.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.251236E-01 | lm-global-supervised-tokens: 1.286248E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.121731E-02 | lm-token-buffer-mib: 5.825092E+01 | global_load_balancing_loss: 9.989905E-01 | loss scale: 1.0 | grad norm: 0.089 | params norm: 11679.579 | params rms: 0.063241 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 19:16:14.042305 | finish at 2026-10-10 11:35:12 + [2026-10-03 16:20:26] iteration 22/ 6855 | consumed samples: 2816 | elapsed time per iteration (ms): 87665.0 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.39% | FLOPs estimator: workload-aware | logical tokens/s: 191378.1 | valid tokens/s: 191378.1 | physical tokens/s: 192045.1 | loss tokens/s: 138452.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.935586E-01 | lm-global-supervised-tokens: 1.213742E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.252584E-02 | lm-token-buffer-mib: 5.825320E+01 | global_load_balancing_loss: 1.000177E+00 | loss scale: 1.0 | grad norm: 0.035 | params norm: 11679.463 | params rms: 0.063240 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 22:23:34.755762 | finish at 2026-10-10 14:44:01 + [2026-10-03 16:21:52] iteration 23/ 6855 | consumed samples: 2944 | elapsed time per iteration (ms): 86015.3 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.55% | FLOPs estimator: workload-aware | logical tokens/s: 195048.8 | valid tokens/s: 195048.8 | physical tokens/s: 195577.3 | loss tokens/s: 147155.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.530855E-01 | lm-global-supervised-tokens: 1.265765E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.062782E-02 | lm-token-buffer-mib: 5.817621E+01 | global_load_balancing_loss: 1.000255E+00 | loss scale: 1.0 | grad norm: 0.057 | params norm: 11679.345 | params rms: 0.063240 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 19:14:16.520069 | finish at 2026-10-10 11:36:09 + [2026-10-03 16:23:18] iteration 24/ 6855 | consumed samples: 3072 | elapsed time per iteration (ms): 86207.3 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.43% | FLOPs estimator: workload-aware | logical tokens/s: 194614.0 | valid tokens/s: 194614.0 | physical tokens/s: 195130.1 | loss tokens/s: 148484.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.859960E-01 | lm-global-supervised-tokens: 1.280040E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.269812E-02 | lm-token-buffer-mib: 5.804120E+01 | global_load_balancing_loss: 1.001618E+00 | loss scale: 1.0 | grad norm: 0.039 | params norm: 11679.226 | params rms: 0.063239 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 19:34:41.891679 | finish at 2026-10-10 11:58:00 + [2026-10-03 16:24:45] iteration 25/ 6855 | consumed samples: 3200 | elapsed time per iteration (ms): 86298.5 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.47% | FLOPs estimator: workload-aware | logical tokens/s: 194409.0 | valid tokens/s: 194409.0 | physical tokens/s: 194900.0 | loss tokens/s: 143830.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.392472E-01 | lm-global-supervised-tokens: 1.241234E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.160657E-02 | lm-token-buffer-mib: 5.809883E+01 | global_load_balancing_loss: 1.001197E+00 | loss scale: 1.0 | grad norm: 0.108 | params norm: 11679.107 | params rms: 0.063239 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 19:43:38.449278 | finish at 2026-10-10 12:08:23 + [2026-10-03 16:26:11] iteration 26/ 6855 | consumed samples: 3328 | elapsed time per iteration (ms): 86679.4 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.49% | FLOPs estimator: workload-aware | logical tokens/s: 193554.7 | valid tokens/s: 193554.7 | physical tokens/s: 193945.0 | loss tokens/s: 141142.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.917882E-01 | lm-global-supervised-tokens: 1.223416E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.198927E-02 | lm-token-buffer-mib: 5.815668E+01 | global_load_balancing_loss: 1.000084E+00 | loss scale: 1.0 | grad norm: 0.072 | params norm: 11678.989 | params rms: 0.063238 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 20:25:33.723322 | finish at 2026-10-10 12:51:45 + [2026-10-03 16:27:38] iteration 27/ 6855 | consumed samples: 3456 | elapsed time per iteration (ms): 86975.6 | throughput per GPU (TFLOP/s/GPU): 43.1 | MFU 13.82% | FLOPs estimator: workload-aware | logical tokens/s: 192895.5 | valid tokens/s: 192895.5 | physical tokens/s: 193258.6 | loss tokens/s: 136847.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.319033E-01 | lm-global-supervised-tokens: 1.190242E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.145155E-02 | lm-token-buffer-mib: 5.807953E+01 | global_load_balancing_loss: 9.993385E-01 | loss scale: 1.0 | grad norm: 0.091 | params norm: 11678.870 | params rms: 0.063237 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 20:57:49.439155 | finish at 2026-10-10 13:25:28 + [2026-10-03 16:29:09] iteration 28/ 6855 | consumed samples: 3584 | elapsed time per iteration (ms): 90647.6 | throughput per GPU (TFLOP/s/GPU): 41.0 | MFU 13.13% | FLOPs estimator: workload-aware | logical tokens/s: 185081.3 | valid tokens/s: 185081.3 | physical tokens/s: 185467.4 | loss tokens/s: 135464.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.389781E-01 | lm-global-supervised-tokens: 1.227957E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.150691E-02 | lm-token-buffer-mib: 5.805988E+01 | global_load_balancing_loss: 9.989486E-01 | loss scale: 1.0 | grad norm: 0.103 | params norm: 11678.751 | params rms: 0.063237 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 7 days, 3:54:11.138717 | finish at 2026-10-10 20:23:20 + [2026-10-03 16:30:41] iteration 29/ 6855 | consumed samples: 3712 | elapsed time per iteration (ms): 91701.8 | throughput per GPU (TFLOP/s/GPU): 40.3 | MFU 12.92% | FLOPs estimator: workload-aware | logical tokens/s: 182953.2 | valid tokens/s: 182953.2 | physical tokens/s: 183326.8 | loss tokens/s: 136869.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.271786E-01 | lm-global-supervised-tokens: 1.255116E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.307421E-02 | lm-token-buffer-mib: 5.817998E+01 | global_load_balancing_loss: 9.994800E-01 | loss scale: 1.0 | grad norm: 0.037 | params norm: 11678.631 | params rms: 0.063236 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 7 days, 5:52:36.290616 | finish at 2026-10-10 22:23:17 + [2026-10-03 16:32:16] iteration 30/ 6855 | consumed samples: 3840 | elapsed time per iteration (ms): 95618.3 | throughput per GPU (TFLOP/s/GPU): 38.4 | MFU 12.32% | FLOPs estimator: workload-aware | logical tokens/s: 175459.5 | valid tokens/s: 175459.5 | physical tokens/s: 175868.1 | loss tokens/s: 128187.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.829391E-01 | lm-global-supervised-tokens: 1.225706E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.209945E-02 | lm-token-buffer-mib: 5.810712E+01 | global_load_balancing_loss: 1.001461E+00 | loss scale: 1.0 | grad norm: 0.088 | params norm: 11678.512 | params rms: 0.063235 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 7 days, 13:16:34.880962 | finish at 2026-10-11 05:48:51 + [2026-10-03 16:33:43] iteration 31/ 6855 | consumed samples: 3968 | elapsed time per iteration (ms): 86738.4 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.51% | FLOPs estimator: workload-aware | logical tokens/s: 193423.2 | valid tokens/s: 193423.2 | physical tokens/s: 193835.0 | loss tokens/s: 144809.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.070284E-01 | lm-global-supervised-tokens: 1.256049E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.315502E-02 | lm-token-buffer-mib: 5.804317E+01 | global_load_balancing_loss: 1.000984E+00 | loss scale: 1.0 | grad norm: 0.034 | params norm: 11678.394 | params rms: 0.063235 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 20:25:02.576284 | finish at 2026-10-10 12:58:46 + [2026-10-03 16:35:11] iteration 32/ 6855 | consumed samples: 4096 | elapsed time per iteration (ms): 88369.4 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.51% | FLOPs estimator: workload-aware | logical tokens/s: 189852.5 | valid tokens/s: 189852.5 | physical tokens/s: 190237.7 | loss tokens/s: 137845.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.674609E-01 | lm-global-supervised-tokens: 1.218136E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.390550E-02 | lm-token-buffer-mib: 5.812625E+01 | global_load_balancing_loss: 1.001045E+00 | loss scale: 1.0 | grad norm: 0.084 | params norm: 11678.274 | params rms: 0.063234 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 23:29:04.248813 | finish at 2026-10-10 16:04:16 + [2026-10-03 16:36:40] iteration 33/ 6855 | consumed samples: 4224 | elapsed time per iteration (ms): 88315.4 | throughput per GPU (TFLOP/s/GPU): 43.1 | MFU 13.82% | FLOPs estimator: workload-aware | logical tokens/s: 189968.9 | valid tokens/s: 189968.9 | physical tokens/s: 190339.2 | loss tokens/s: 139519.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.956165E-01 | lm-global-supervised-tokens: 1.232168E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.394073E-02 | lm-token-buffer-mib: 5.812135E+01 | global_load_balancing_loss: 9.999436E-01 | loss scale: 1.0 | grad norm: 0.069 | params norm: 11678.155 | params rms: 0.063233 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 23:21:27.718197 | finish at 2026-10-10 15:58:07 + [2026-10-03 16:38:07] iteration 34/ 6855 | consumed samples: 4352 | elapsed time per iteration (ms): 87516.4 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.56% | FLOPs estimator: workload-aware | logical tokens/s: 191702.6 | valid tokens/s: 191702.6 | physical tokens/s: 192161.3 | loss tokens/s: 141457.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.585505E-01 | lm-global-supervised-tokens: 1.237984E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.141731E-02 | lm-token-buffer-mib: 5.813584E+01 | global_load_balancing_loss: 9.990388E-01 | loss scale: 1.0 | grad norm: 0.069 | params norm: 11678.037 | params rms: 0.063233 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 21:49:09.220337 | finish at 2026-10-10 14:27:16 + [2026-10-03 16:39:33] iteration 35/ 6855 | consumed samples: 4480 | elapsed time per iteration (ms): 85575.6 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.42% | FLOPs estimator: workload-aware | logical tokens/s: 196050.6 | valid tokens/s: 196050.6 | physical tokens/s: 196576.6 | loss tokens/s: 146351.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.237529E-01 | lm-global-supervised-tokens: 1.252408E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.134401E-02 | lm-token-buffer-mib: 5.814813E+01 | global_load_balancing_loss: 9.988708E-01 | loss scale: 1.0 | grad norm: 0.150 | params norm: 11677.922 | params rms: 0.063232 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 18:07:05.700321 | finish at 2026-10-10 10:46:38 + [2026-10-03 16:40:59] iteration 36/ 6855 | consumed samples: 4608 | elapsed time per iteration (ms): 86288.1 | throughput per GPU (TFLOP/s/GPU): 43.2 | MFU 13.84% | FLOPs estimator: workload-aware | logical tokens/s: 194432.2 | valid tokens/s: 194432.2 | physical tokens/s: 194782.2 | loss tokens/s: 137784.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.299394E-01 | lm-global-supervised-tokens: 1.188920E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.165146E-02 | lm-token-buffer-mib: 5.825486E+01 | global_load_balancing_loss: 9.998893E-01 | loss scale: 1.0 | grad norm: 0.102 | params norm: 11677.809 | params rms: 0.063232 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 19:26:38.724635 | finish at 2026-10-10 12:07:38 + [2026-10-03 16:42:26] iteration 37/ 6855 | consumed samples: 4736 | elapsed time per iteration (ms): 87117.9 | throughput per GPU (TFLOP/s/GPU): 43.1 | MFU 13.80% | FLOPs estimator: workload-aware | logical tokens/s: 192580.5 | valid tokens/s: 192580.5 | physical tokens/s: 192974.5 | loss tokens/s: 140541.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.727605E-01 | lm-global-supervised-tokens: 1.224366E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.135091E-02 | lm-token-buffer-mib: 5.829156E+01 | global_load_balancing_loss: 1.000211E+00 | loss scale: 1.0 | grad norm: 0.046 | params norm: 11677.696 | params rms: 0.063231 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 20:59:29.960146 | finish at 2026-10-10 13:41:56 + [2026-10-03 16:43:53] iteration 38/ 6855 | consumed samples: 4864 | elapsed time per iteration (ms): 86530.6 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.44% | FLOPs estimator: workload-aware | logical tokens/s: 193887.4 | valid tokens/s: 193887.4 | physical tokens/s: 194290.8 | loss tokens/s: 137186.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.395332E-01 | lm-global-supervised-tokens: 1.187085E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.437120E-02 | lm-token-buffer-mib: 5.809835E+01 | global_load_balancing_loss: 1.001512E+00 | loss scale: 1.0 | grad norm: 0.040 | params norm: 11677.584 | params rms: 0.063230 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 19:51:18.967409 | finish at 2026-10-10 12:35:12 + [2026-10-03 16:45:19] iteration 39/ 6855 | consumed samples: 4992 | elapsed time per iteration (ms): 86561.5 | throughput per GPU (TFLOP/s/GPU): 43.7 | MFU 14.01% | FLOPs estimator: workload-aware | logical tokens/s: 193817.7 | valid tokens/s: 193817.7 | physical tokens/s: 194288.9 | loss tokens/s: 138159.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.246742E-01 | lm-global-supervised-tokens: 1.195933E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.457552E-02 | lm-token-buffer-mib: 5.835693E+01 | global_load_balancing_loss: 1.001652E+00 | loss scale: 1.0 | grad norm: 0.058 | params norm: 11677.472 | params rms: 0.063230 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 19:53:23.229996 | finish at 2026-10-10 12:38:43 + [2026-10-03 16:46:54] iteration 40/ 6855 | consumed samples: 5120 | elapsed time per iteration (ms): 94983.6 | throughput per GPU (TFLOP/s/GPU): 38.3 | MFU 12.27% | FLOPs estimator: workload-aware | logical tokens/s: 176631.8 | valid tokens/s: 176631.7 | physical tokens/s: 177140.6 | loss tokens/s: 132655.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.716218E-01 | lm-global-supervised-tokens: 1.260012E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.677078E-02 | lm-token-buffer-mib: 5.808987E+01 | global_load_balancing_loss: 1.001079E+00 | loss scale: 1.0 | grad norm: 0.109 | params norm: 11677.360 | params rms: 0.063229 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 7 days, 11:48:33.257699 | finish at 2026-10-11 04:35:28 +times across ranks (ms): + save-checkpoint: + rank 0: 113110.02 + rank 1: 113110.01 + rank 2: 113110.17 + rank 3: 113110.03 + rank 4: 113110.02 + rank 5: 113110.03 + rank 6: 113110.01 + rank 7: 113110.02 + rank 8: 113101.11 + rank 9: 113101.10 + rank 10: 113101.10 + rank 11: 113101.10 + rank 12: 113101.11 + rank 13: 113101.11 + rank 14: 113101.10 + rank 15: 113101.10 + rank 16: 113103.55 + rank 17: 113103.58 + rank 18: 113103.57 + rank 19: 113103.58 + rank 20: 113103.58 + rank 21: 113103.51 + rank 22: 113103.57 + rank 23: 113103.58 + rank 24: 113101.84 + rank 25: 113101.81 + rank 26: 113101.84 + rank 27: 113101.81 + rank 28: 113101.82 + rank 29: 113101.80 + rank 30: 113101.81 + rank 31: 113101.80 + rank 32: 113100.89 + rank 33: 113100.90 + rank 34: 113100.89 + rank 35: 113100.91 + rank 36: 113100.88 + rank 37: 113100.88 + rank 38: 113100.91 + rank 39: 113100.88 + rank 40: 113102.73 + rank 41: 113102.73 + rank 42: 113102.73 + rank 43: 113102.74 + rank 44: 113102.72 + rank 45: 113102.73 + rank 46: 113102.73 + rank 47: 113102.73 + rank 48: 113102.39 + rank 49: 113102.34 + rank 50: 113102.32 + rank 51: 113102.44 + rank 52: 113102.40 + rank 53: 113102.41 + rank 54: 113102.38 + rank 55: 113102.40 + rank 56: 113098.28 + rank 57: 113098.30 + rank 58: 113098.30 + rank 59: 113098.30 + rank 60: 113098.30 + rank 61: 113098.30 + rank 62: 113098.29 + rank 63: 113098.30 + [2026-10-03 16:50:12] iteration 41/ 6855 | consumed samples: 5248 | elapsed time per iteration (ms): 84186.3 | throughput per GPU (TFLOP/s/GPU): 43.7 | MFU 14.01% | FLOPs estimator: workload-aware | logical tokens/s: 199286.6 | valid tokens/s: 199286.6 | physical tokens/s: 199597.8 | loss tokens/s: 151275.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.914068E-01 | lm-global-supervised-tokens: 1.273531E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.904765E-02 | lm-token-buffer-mib: 5.805824E+01 | global_load_balancing_loss: 1.000504E+00 | loss scale: 1.0 | grad norm: 0.123 | params norm: 11677.248 | params rms: 0.063228 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 15:20:45.786381 | finish at 2026-10-10 08:10:57 + [2026-10-03 16:51:47] iteration 42/ 6855 | consumed samples: 5376 | elapsed time per iteration (ms): 95310.9 | throughput per GPU (TFLOP/s/GPU): 39.6 | MFU 12.69% | FLOPs estimator: workload-aware | logical tokens/s: 176025.4 | valid tokens/s: 176025.4 | physical tokens/s: 176342.7 | loss tokens/s: 130473.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.225893E-01 | lm-global-supervised-tokens: 1.243560E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.416941E-02 | lm-token-buffer-mib: 5.804226E+01 | global_load_balancing_loss: 9.985629E-01 | loss scale: 1.0 | grad norm: 0.065 | params norm: 11677.135 | params rms: 0.063228 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 7 days, 12:22:33.468517 | finish at 2026-10-11 05:14:20 + [2026-10-03 16:53:11] iteration 43/ 6855 | consumed samples: 5504 | elapsed time per iteration (ms): 84142.5 | throughput per GPU (TFLOP/s/GPU): 43.1 | MFU 13.80% | FLOPs estimator: workload-aware | logical tokens/s: 199388.8 | valid tokens/s: 199388.8 | physical tokens/s: 200023.7 | loss tokens/s: 144468.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.291756E-01 | lm-global-supervised-tokens: 1.215596E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.249354E-02 | lm-token-buffer-mib: 5.818741E+01 | global_load_balancing_loss: 1.000042E+00 | loss scale: 1.0 | grad norm: 0.375 | params norm: 11677.022 | params rms: 0.063227 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 15:12:58.631523 | finish at 2026-10-10 08:06:10 + [2026-10-03 16:54:37] iteration 44/ 6855 | consumed samples: 5632 | elapsed time per iteration (ms): 86158.6 | throughput per GPU (TFLOP/s/GPU): 43.5 | MFU 13.94% | FLOPs estimator: workload-aware | logical tokens/s: 194723.9 | valid tokens/s: 194723.9 | physical tokens/s: 195316.1 | loss tokens/s: 142849.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.851609E-01 | lm-global-supervised-tokens: 1.230771E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.646941E-02 | lm-token-buffer-mib: 5.826642E+01 | global_load_balancing_loss: 9.999058E-01 | loss scale: 1.0 | grad norm: 0.503 | params norm: 11676.909 | params rms: 0.063227 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 19:00:26.015747 | finish at 2026-10-10 11:55:03 + [2026-10-03 16:56:03] iteration 45/ 6855 | consumed samples: 5760 | elapsed time per iteration (ms): 85563.0 | throughput per GPU (TFLOP/s/GPU): 42.8 | MFU 13.70% | FLOPs estimator: workload-aware | logical tokens/s: 196080.2 | valid tokens/s: 196080.2 | physical tokens/s: 196684.6 | loss tokens/s: 140428.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.311920E-01 | lm-global-supervised-tokens: 1.201546E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.212317E-02 | lm-token-buffer-mib: 5.829821E+01 | global_load_balancing_loss: 1.000879E+00 | loss scale: 1.0 | grad norm: 0.086 | params norm: 11676.798 | params rms: 0.063226 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 17:51:24.226213 | finish at 2026-10-10 10:47:27 + [2026-10-03 16:57:27] iteration 46/ 6855 | consumed samples: 5888 | elapsed time per iteration (ms): 84339.0 | throughput per GPU (TFLOP/s/GPU): 43.4 | MFU 13.91% | FLOPs estimator: workload-aware | logical tokens/s: 198925.5 | valid tokens/s: 198925.5 | physical tokens/s: 199346.6 | loss tokens/s: 148560.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.795720E-01 | lm-global-supervised-tokens: 1.252947E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.577638E-02 | lm-token-buffer-mib: 5.810792E+01 | global_load_balancing_loss: 9.999204E-01 | loss scale: 1.0 | grad norm: 0.405 | params norm: 11676.688 | params rms: 0.063225 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 15:31:04.518769 | finish at 2026-10-10 08:28:32 + [2026-10-03 16:58:54] iteration 47/ 6855 | consumed samples: 6016 | elapsed time per iteration (ms): 86848.5 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.64% | FLOPs estimator: workload-aware | logical tokens/s: 193177.5 | valid tokens/s: 193177.5 | physical tokens/s: 193656.6 | loss tokens/s: 137452.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.876001E-01 | lm-global-supervised-tokens: 1.193758E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.527531E-02 | lm-token-buffer-mib: 5.809621E+01 | global_load_balancing_loss: 9.980349E-01 | loss scale: 1.0 | grad norm: 0.047 | params norm: 11676.576 | params rms: 0.063225 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 20:14:24.524788 | finish at 2026-10-10 13:13:18 + [2026-10-03 17:00:25] iteration 48/ 6855 | consumed samples: 6144 | elapsed time per iteration (ms): 90980.6 | throughput per GPU (TFLOP/s/GPU): 40.9 | MFU 13.09% | FLOPs estimator: workload-aware | logical tokens/s: 184403.7 | valid tokens/s: 184403.7 | physical tokens/s: 184866.5 | loss tokens/s: 137367.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.345768E-01 | lm-global-supervised-tokens: 1.249773E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.833894E-02 | lm-token-buffer-mib: 5.816721E+01 | global_load_balancing_loss: 1.000117E+00 | loss scale: 1.0 | grad norm: 0.163 | params norm: 11676.462 | params rms: 0.063224 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 7 days, 4:01:44.763241 | finish at 2026-10-10 21:02:10 + [2026-10-03 17:01:50] iteration 49/ 6855 | consumed samples: 6272 | elapsed time per iteration (ms): 85200.5 | throughput per GPU (TFLOP/s/GPU): 43.9 | MFU 14.09% | FLOPs estimator: workload-aware | logical tokens/s: 196914.0 | valid tokens/s: 196914.0 | physical tokens/s: 197436.7 | loss tokens/s: 138953.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.865407E-01 | lm-global-supervised-tokens: 1.183888E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.382338E-02 | lm-token-buffer-mib: 5.820102E+01 | global_load_balancing_loss: 1.001689E+00 | loss scale: 1.0 | grad norm: 0.182 | params norm: 11676.349 | params rms: 0.063224 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 17:04:34.283411 | finish at 2026-10-10 10:06:24 + [2026-10-03 17:03:22] iteration 50/ 6855 | consumed samples: 6400 | elapsed time per iteration (ms): 92139.8 | throughput per GPU (TFLOP/s/GPU): 39.9 | MFU 12.78% | FLOPs estimator: workload-aware | logical tokens/s: 182083.7 | valid tokens/s: 182083.7 | physical tokens/s: 182552.1 | loss tokens/s: 135398.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.373349E-01 | lm-global-supervised-tokens: 1.247555E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.472054E-02 | lm-token-buffer-mib: 5.806545E+01 | global_load_balancing_loss: 1.000872E+00 | loss scale: 1.0 | grad norm: 0.077 | params norm: 11676.237 | params rms: 0.063223 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 7 days, 6:10:11.573119 | finish at 2026-10-10 23:13:34 + [2026-10-03 17:04:51] iteration 51/ 6855 | consumed samples: 6528 | elapsed time per iteration (ms): 88268.3 | throughput per GPU (TFLOP/s/GPU): 41.0 | MFU 13.15% | FLOPs estimator: workload-aware | logical tokens/s: 190070.5 | valid tokens/s: 190070.5 | physical tokens/s: 190652.1 | loss tokens/s: 138978.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.024138E-01 | lm-global-supervised-tokens: 1.226738E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.634058E-02 | lm-token-buffer-mib: 5.817985E+01 | global_load_balancing_loss: 1.000706E+00 | loss scale: 1.0 | grad norm: 0.090 | params norm: 11676.126 | params rms: 0.063222 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 22:49:37.834780 | finish at 2026-10-10 15:54:28 + [2026-10-03 17:06:18] iteration 52/ 6855 | consumed samples: 6656 | elapsed time per iteration (ms): 87022.8 | throughput per GPU (TFLOP/s/GPU): 43.3 | MFU 13.89% | FLOPs estimator: workload-aware | logical tokens/s: 192790.6 | valid tokens/s: 192790.6 | physical tokens/s: 193267.0 | loss tokens/s: 143685.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.526962E-01 | lm-global-supervised-tokens: 1.250394E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.157178E-02 | lm-token-buffer-mib: 5.817005E+01 | global_load_balancing_loss: 9.994399E-01 | loss scale: 1.0 | grad norm: 0.120 | params norm: 11676.015 | params rms: 0.063222 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 20:26:56.223346 | finish at 2026-10-10 13:33:14 + [2026-10-03 17:07:47] iteration 53/ 6855 | consumed samples: 6784 | elapsed time per iteration (ms): 89557.2 | throughput per GPU (TFLOP/s/GPU): 41.0 | MFU 13.13% | FLOPs estimator: workload-aware | logical tokens/s: 187334.7 | valid tokens/s: 187334.6 | physical tokens/s: 187911.7 | loss tokens/s: 139817.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.839869E-01 | lm-global-supervised-tokens: 1.252166E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.087910E-02 | lm-token-buffer-mib: 5.811797E+01 | global_load_balancing_loss: 1.000745E+00 | loss scale: 1.0 | grad norm: 0.117 | params norm: 11675.904 | params rms: 0.063221 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 7 days, 1:12:48.069229 | finish at 2026-10-10 18:20:35 + [2026-10-03 17:09:14] iteration 54/ 6855 | consumed samples: 6912 | elapsed time per iteration (ms): 87014.9 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.49% | FLOPs estimator: workload-aware | logical tokens/s: 192808.5 | valid tokens/s: 192808.5 | physical tokens/s: 193179.8 | loss tokens/s: 146504.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.393138E-01 | lm-global-supervised-tokens: 1.274810E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.748093E-02 | lm-token-buffer-mib: 5.815152E+01 | global_load_balancing_loss: 1.000999E+00 | loss scale: 1.0 | grad norm: 0.067 | params norm: 11675.793 | params rms: 0.063221 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 20:23:08.441708 | finish at 2026-10-10 13:32:23 + [2026-10-03 17:10:39] iteration 55/ 6855 | consumed samples: 7040 | elapsed time per iteration (ms): 84779.0 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.58% | FLOPs estimator: workload-aware | logical tokens/s: 197892.6 | valid tokens/s: 197892.6 | physical tokens/s: 198335.4 | loss tokens/s: 148195.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.467128E-01 | lm-global-supervised-tokens: 1.256386E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.608741E-02 | lm-token-buffer-mib: 5.807524E+01 | global_load_balancing_loss: 1.000228E+00 | loss scale: 1.0 | grad norm: 0.087 | params norm: 11675.684 | params rms: 0.063220 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 16:08:17.169495 | finish at 2026-10-10 09:18:56 + [2026-10-03 17:12:09] iteration 56/ 6855 | consumed samples: 7168 | elapsed time per iteration (ms): 89675.8 | throughput per GPU (TFLOP/s/GPU): 40.9 | MFU 13.12% | FLOPs estimator: workload-aware | logical tokens/s: 187087.3 | valid tokens/s: 187087.3 | physical tokens/s: 187495.9 | loss tokens/s: 136021.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.014665E-01 | lm-global-supervised-tokens: 1.219786E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.677052E-02 | lm-token-buffer-mib: 5.813933E+01 | global_load_balancing_loss: 1.000303E+00 | loss scale: 1.0 | grad norm: 0.033 | params norm: 11675.574 | params rms: 0.063219 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 7 days, 1:21:46.051697 | finish at 2026-10-10 18:33:55 + [2026-10-03 17:13:32] iteration 57/ 6855 | consumed samples: 7296 | elapsed time per iteration (ms): 83847.8 | throughput per GPU (TFLOP/s/GPU): 43.1 | MFU 13.80% | FLOPs estimator: workload-aware | logical tokens/s: 200090.2 | valid tokens/s: 200090.2 | physical tokens/s: 200546.1 | loss tokens/s: 143553.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.487665E-01 | lm-global-supervised-tokens: 1.203663E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.907606E-02 | lm-token-buffer-mib: 5.825159E+01 | global_load_balancing_loss: 9.991975E-01 | loss scale: 1.0 | grad norm: 0.101 | params norm: 11675.466 | params rms: 0.063219 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 14:19:57.603835 | finish at 2026-10-10 07:33:30 + [2026-10-03 17:14:59] iteration 58/ 6855 | consumed samples: 7424 | elapsed time per iteration (ms): 86435.3 | throughput per GPU (TFLOP/s/GPU): 43.3 | MFU 13.87% | FLOPs estimator: workload-aware | logical tokens/s: 194101.4 | valid tokens/s: 194101.4 | physical tokens/s: 194449.2 | loss tokens/s: 141698.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.896660E-01 | lm-global-supervised-tokens: 1.224778E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.190518E-02 | lm-token-buffer-mib: 5.813446E+01 | global_load_balancing_loss: 9.985434E-01 | loss scale: 1.0 | grad norm: 0.047 | params norm: 11675.358 | params rms: 0.063218 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 19:11:40.840194 | finish at 2026-10-10 12:26:40 + [2026-10-03 17:16:20] iteration 59/ 6855 | consumed samples: 7552 | elapsed time per iteration (ms): 81399.9 | throughput per GPU (TFLOP/s/GPU): 43.2 | MFU 13.85% | FLOPs estimator: workload-aware | logical tokens/s: 206108.4 | valid tokens/s: 206108.4 | physical tokens/s: 206878.9 | loss tokens/s: 163615.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.963448E-01 | lm-global-supervised-tokens: 1.331828E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.646090E-02 | lm-token-buffer-mib: 5.818249E+01 | global_load_balancing_loss: 9.990563E-01 | loss scale: 1.0 | grad norm: 0.052 | params norm: 11675.250 | params rms: 0.063218 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 9:39:53.530551 | finish at 2026-10-10 02:56:14 + [2026-10-03 17:17:58] iteration 60/ 6855 | consumed samples: 7680 | elapsed time per iteration (ms): 98072.1 | throughput per GPU (TFLOP/s/GPU): 38.4 | MFU 12.30% | FLOPs estimator: workload-aware | logical tokens/s: 171070.3 | valid tokens/s: 171070.3 | physical tokens/s: 171555.6 | loss tokens/s: 122968.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.053422E-01 | lm-global-supervised-tokens: 1.205974E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.447193E-02 | lm-token-buffer-mib: 5.806406E+01 | global_load_balancing_loss: 1.000648E+00 | loss scale: 1.0 | grad norm: 0.060 | params norm: 11675.142 | params rms: 0.063217 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 7 days, 17:06:39.748878 | finish at 2026-10-11 10:24:38 + [2026-10-03 17:19:22] iteration 61/ 6855 | consumed samples: 7808 | elapsed time per iteration (ms): 83431.9 | throughput per GPU (TFLOP/s/GPU): 43.1 | MFU 13.82% | FLOPs estimator: workload-aware | logical tokens/s: 201088.4 | valid tokens/s: 201088.4 | physical tokens/s: 201632.5 | loss tokens/s: 150603.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.896516E-01 | lm-global-supervised-tokens: 1.256518E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.154793E-02 | lm-token-buffer-mib: 5.806738E+01 | global_load_balancing_loss: 1.002231E+00 | loss scale: 1.0 | grad norm: 0.054 | params norm: 11675.033 | params rms: 0.063217 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 13:27:16.659208 | finish at 2026-10-10 06:46:38 + [2026-10-03 17:20:47] iteration 62/ 6855 | consumed samples: 7936 | elapsed time per iteration (ms): 85658.3 | throughput per GPU (TFLOP/s/GPU): 43.5 | MFU 13.94% | FLOPs estimator: workload-aware | logical tokens/s: 195862.0 | valid tokens/s: 195862.0 | physical tokens/s: 196345.1 | loss tokens/s: 139031.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.977134E-01 | lm-global-supervised-tokens: 1.190918E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.449681E-02 | lm-token-buffer-mib: 5.807662E+01 | global_load_balancing_loss: 1.002551E+00 | loss scale: 1.0 | grad norm: 0.049 | params norm: 11674.925 | params rms: 0.063216 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 17:37:56.863768 | finish at 2026-10-10 10:58:44 + [2026-10-03 17:22:12] iteration 63/ 6855 | consumed samples: 8064 | elapsed time per iteration (ms): 84988.4 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.79% | FLOPs estimator: workload-aware | logical tokens/s: 197405.0 | valid tokens/s: 197405.0 | physical tokens/s: 197995.3 | loss tokens/s: 145418.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.622961E-01 | lm-global-supervised-tokens: 1.235889E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.882812E-02 | lm-token-buffer-mib: 5.825050E+01 | global_load_balancing_loss: 1.001113E+00 | loss scale: 1.0 | grad norm: 0.067 | params norm: 11674.817 | params rms: 0.063215 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 16:20:41.487968 | finish at 2026-10-10 09:42:54 + [2026-10-03 17:23:35] iteration 64/ 6855 | consumed samples: 8192 | elapsed time per iteration (ms): 82375.8 | throughput per GPU (TFLOP/s/GPU): 44.1 | MFU 14.14% | FLOPs estimator: workload-aware | logical tokens/s: 203666.7 | valid tokens/s: 203666.7 | physical tokens/s: 204140.5 | loss tokens/s: 154530.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.849192E-01 | lm-global-supervised-tokens: 1.272959E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.480983E-02 | lm-token-buffer-mib: 5.815009E+01 | global_load_balancing_loss: 9.991371E-01 | loss scale: 1.0 | grad norm: 0.035 | params norm: 11674.710 | params rms: 0.063215 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 11:23:34.189849 | finish at 2026-10-10 04:47:09 + [2026-10-03 17:25:03] iteration 65/ 6855 | consumed samples: 8320 | elapsed time per iteration (ms): 88617.4 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.48% | FLOPs estimator: workload-aware | logical tokens/s: 189321.4 | valid tokens/s: 189321.4 | physical tokens/s: 189694.4 | loss tokens/s: 149404.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.606055E-01 | lm-global-supervised-tokens: 1.323984E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.236515E-02 | lm-token-buffer-mib: 5.820187E+01 | global_load_balancing_loss: 9.979938E-01 | loss scale: 1.0 | grad norm: 0.047 | params norm: 11674.602 | params rms: 0.063214 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 23:08:32.266946 | finish at 2026-10-10 16:33:36 + [2026-10-03 17:26:26] iteration 66/ 6855 | consumed samples: 8448 | elapsed time per iteration (ms): 83038.6 | throughput per GPU (TFLOP/s/GPU): 43.6 | MFU 13.98% | FLOPs estimator: workload-aware | logical tokens/s: 202040.6 | valid tokens/s: 202040.6 | physical tokens/s: 202480.0 | loss tokens/s: 155817.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.753908E-01 | lm-global-supervised-tokens: 1.293888E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.297155E-02 | lm-token-buffer-mib: 5.805128E+01 | global_load_balancing_loss: 9.987180E-01 | loss scale: 1.0 | grad norm: 0.066 | params norm: 11674.495 | params rms: 0.063214 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 12:35:48.875515 | finish at 2026-10-10 06:02:15 + [2026-10-03 17:27:51] iteration 67/ 6855 | consumed samples: 8576 | elapsed time per iteration (ms): 84998.5 | throughput per GPU (TFLOP/s/GPU): 42.8 | MFU 13.72% | FLOPs estimator: workload-aware | logical tokens/s: 197381.6 | valid tokens/s: 197381.6 | physical tokens/s: 198000.9 | loss tokens/s: 152789.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.466067E-01 | lm-global-supervised-tokens: 1.298686E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.560353E-02 | lm-token-buffer-mib: 5.804687E+01 | global_load_balancing_loss: 9.997393E-01 | loss scale: 1.0 | grad norm: 0.036 | params norm: 11674.389 | params rms: 0.063213 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 16:16:09.702214 | finish at 2026-10-10 09:44:01 + [2026-10-03 17:29:21] iteration 68/ 6855 | consumed samples: 8704 | elapsed time per iteration (ms): 89508.5 | throughput per GPU (TFLOP/s/GPU): 41.5 | MFU 13.31% | FLOPs estimator: workload-aware | logical tokens/s: 187436.2 | valid tokens/s: 187436.1 | physical tokens/s: 187734.5 | loss tokens/s: 131312.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.468135E-01 | lm-global-supervised-tokens: 1.175361E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.473308E-02 | lm-token-buffer-mib: 5.812321E+01 | global_load_balancing_loss: 1.000349E+00 | loss scale: 1.0 | grad norm: 0.114 | params norm: 11674.283 | params rms: 0.063212 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 7 days, 0:44:54.431277 | finish at 2026-10-10 18:14:15 + [2026-10-03 17:30:49] iteration 69/ 6855 | consumed samples: 8832 | elapsed time per iteration (ms): 88232.6 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.45% | FLOPs estimator: workload-aware | logical tokens/s: 190147.2 | valid tokens/s: 190147.2 | physical tokens/s: 190638.4 | loss tokens/s: 139885.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.358140E-01 | lm-global-supervised-tokens: 1.234247E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.206607E-02 | lm-token-buffer-mib: 5.820260E+01 | global_load_balancing_loss: 1.001465E+00 | loss scale: 1.0 | grad norm: 0.064 | params norm: 11674.175 | params rms: 0.063212 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 22:19:06.675815 | finish at 2026-10-10 15:49:56 + [2026-10-03 17:32:32] iteration 70/ 6855 | consumed samples: 8960 | elapsed time per iteration (ms): 102807.1 | throughput per GPU (TFLOP/s/GPU): 37.0 | MFU 11.85% | FLOPs estimator: workload-aware | logical tokens/s: 163191.1 | valid tokens/s: 163191.1 | physical tokens/s: 163786.5 | loss tokens/s: 118944.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.730622E-01 | lm-global-supervised-tokens: 1.222834E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.602901E-02 | lm-token-buffer-mib: 5.815475E+01 | global_load_balancing_loss: 1.001294E+00 | loss scale: 1.0 | grad norm: 0.053 | params norm: 11674.069 | params rms: 0.063211 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 8 days, 1:45:46.044477 | finish at 2026-10-11 19:18:18 + [2026-10-03 17:34:00] iteration 71/ 6855 | consumed samples: 9088 | elapsed time per iteration (ms): 88449.6 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.67% | FLOPs estimator: workload-aware | logical tokens/s: 189679.8 | valid tokens/s: 189679.8 | physical tokens/s: 190146.9 | loss tokens/s: 144908.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.776041E-01 | lm-global-supervised-tokens: 1.281709E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.325138E-02 | lm-token-buffer-mib: 5.805245E+01 | global_load_balancing_loss: 1.000542E+00 | loss scale: 1.0 | grad norm: 0.059 | params norm: 11673.963 | params rms: 0.063211 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 22:40:41.905121 | finish at 2026-10-10 16:14:42 + [2026-10-03 17:35:28] iteration 72/ 6855 | consumed samples: 9216 | elapsed time per iteration (ms): 87742.0 | throughput per GPU (TFLOP/s/GPU): 42.8 | MFU 13.72% | FLOPs estimator: workload-aware | logical tokens/s: 191210.4 | valid tokens/s: 191210.4 | physical tokens/s: 191755.5 | loss tokens/s: 135688.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.858379E-01 | lm-global-supervised-tokens: 1.190556E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.182042E-02 | lm-token-buffer-mib: 5.812787E+01 | global_load_balancing_loss: 1.001102E+00 | loss scale: 1.0 | grad norm: 0.030 | params norm: 11673.859 | params rms: 0.063210 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 21:19:13.748971 | finish at 2026-10-10 14:54:42 + [2026-10-03 17:36:57] iteration 73/ 6855 | consumed samples: 9344 | elapsed time per iteration (ms): 88967.3 | throughput per GPU (TFLOP/s/GPU): 41.6 | MFU 13.33% | FLOPs estimator: workload-aware | logical tokens/s: 188576.9 | valid tokens/s: 188576.9 | physical tokens/s: 189052.9 | loss tokens/s: 146192.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.342477E-01 | lm-global-supervised-tokens: 1.300636E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.455995E-02 | lm-token-buffer-mib: 5.823373E+01 | global_load_balancing_loss: 1.000301E+00 | loss scale: 1.0 | grad norm: 0.073 | params norm: 11673.758 | params rms: 0.063210 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 23:36:16.470724 | finish at 2026-10-10 17:13:14 + [2026-10-03 17:38:25] iteration 74/ 6855 | consumed samples: 9472 | elapsed time per iteration (ms): 87890.4 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.41% | FLOPs estimator: workload-aware | logical tokens/s: 190887.8 | valid tokens/s: 190887.8 | physical tokens/s: 191332.2 | loss tokens/s: 146923.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.042548E-01 | lm-global-supervised-tokens: 1.291315E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.075613E-02 | lm-token-buffer-mib: 5.808304E+01 | global_load_balancing_loss: 9.992722E-01 | loss scale: 1.0 | grad norm: 0.106 | params norm: 11673.657 | params rms: 0.063209 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 21:33:04.556204 | finish at 2026-10-10 15:11:30 + [2026-10-03 17:39:52] iteration 75/ 6855 | consumed samples: 9600 | elapsed time per iteration (ms): 87354.0 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.55% | FLOPs estimator: workload-aware | logical tokens/s: 192059.1 | valid tokens/s: 192059.1 | physical tokens/s: 192440.8 | loss tokens/s: 136851.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.243629E-01 | lm-global-supervised-tokens: 1.195450E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.180425E-02 | lm-token-buffer-mib: 5.808648E+01 | global_load_balancing_loss: 9.988816E-01 | loss scale: 1.0 | grad norm: 0.060 | params norm: 11673.555 | params rms: 0.063209 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 20:31:00.414820 | finish at 2026-10-10 14:10:53 + [2026-10-03 17:41:20] iteration 76/ 6855 | consumed samples: 9728 | elapsed time per iteration (ms): 88102.9 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.59% | FLOPs estimator: workload-aware | logical tokens/s: 190427.0 | valid tokens/s: 190427.0 | physical tokens/s: 190765.5 | loss tokens/s: 134157.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.543988E-01 | lm-global-supervised-tokens: 1.181965E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.396692E-02 | lm-token-buffer-mib: 5.810877E+01 | global_load_balancing_loss: 9.999314E-01 | loss scale: 1.0 | grad norm: 0.047 | params norm: 11673.454 | params rms: 0.063208 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 21:54:09.402516 | finish at 2026-10-10 15:35:30 + [2026-10-03 17:42:49] iteration 77/ 6855 | consumed samples: 9856 | elapsed time per iteration (ms): 88875.1 | throughput per GPU (TFLOP/s/GPU): 42.8 | MFU 13.72% | FLOPs estimator: workload-aware | logical tokens/s: 188772.5 | valid tokens/s: 188772.5 | physical tokens/s: 189098.8 | loss tokens/s: 132494.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.820156E-01 | lm-global-supervised-tokens: 1.177542E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.482810E-02 | lm-token-buffer-mib: 5.818298E+01 | global_load_balancing_loss: 1.000334E+00 | loss scale: 1.0 | grad norm: 0.047 | params norm: 11673.355 | params rms: 0.063207 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 23:19:55.336608 | finish at 2026-10-10 17:02:45 + [2026-10-03 17:44:15] iteration 78/ 6855 | consumed samples: 9984 | elapsed time per iteration (ms): 85748.5 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.76% | FLOPs estimator: workload-aware | logical tokens/s: 195656.1 | valid tokens/s: 195656.1 | physical tokens/s: 196156.8 | loss tokens/s: 139897.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.702053E-01 | lm-global-supervised-tokens: 1.199598E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.617839E-02 | lm-token-buffer-mib: 5.812958E+01 | global_load_balancing_loss: 1.000420E+00 | loss scale: 1.0 | grad norm: 0.035 | params norm: 11673.253 | params rms: 0.063207 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 17:25:17.383266 | finish at 2026-10-10 11:09:33 + [2026-10-03 17:45:40] iteration 79/ 6855 | consumed samples: 10112 | elapsed time per iteration (ms): 84704.5 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.63% | FLOPs estimator: workload-aware | logical tokens/s: 198067.7 | valid tokens/s: 198067.7 | physical tokens/s: 198562.9 | loss tokens/s: 147204.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.660535E-01 | lm-global-supervised-tokens: 1.246886E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.427687E-02 | lm-token-buffer-mib: 5.826199E+01 | global_load_balancing_loss: 1.001348E+00 | loss scale: 1.0 | grad norm: 0.043 | params norm: 11673.150 | params rms: 0.063206 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 15:25:57.467171 | finish at 2026-10-10 09:11:37 + [2026-10-03 17:47:20] iteration 80/ 6855 | consumed samples: 10240 | elapsed time per iteration (ms): 100541.5 | throughput per GPU (TFLOP/s/GPU): 37.3 | MFU 11.96% | FLOPs estimator: workload-aware | logical tokens/s: 166868.0 | valid tokens/s: 166868.0 | physical tokens/s: 167143.2 | loss tokens/s: 121503.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.591104E-01 | lm-global-supervised-tokens: 1.221611E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.288499E-02 | lm-token-buffer-mib: 5.811504E+01 | global_load_balancing_loss: 1.000567E+00 | loss scale: 1.0 | grad norm: 0.038 | params norm: 11673.048 | params rms: 0.063206 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 7 days, 21:12:48.806881 | finish at 2026-10-11 15:00:09 +times across ranks (ms): + save-checkpoint: + rank 0: 24316.01 + rank 1: 24316.03 + rank 2: 24316.05 + rank 3: 24316.04 + rank 4: 24316.02 + rank 5: 24316.03 + rank 6: 24316.00 + rank 7: 24316.00 + rank 8: 24316.36 + rank 9: 24316.34 + rank 10: 24316.33 + rank 11: 24316.33 + rank 12: 24316.33 + rank 13: 24316.35 + rank 14: 24316.33 + rank 15: 24316.33 + rank 16: 24316.05 + rank 17: 24316.04 + rank 18: 24315.98 + rank 19: 24316.02 + rank 20: 24315.98 + rank 21: 24316.05 + rank 22: 24315.98 + rank 23: 24316.00 + rank 24: 24316.12 + rank 25: 24316.08 + rank 26: 24316.14 + rank 27: 24316.21 + rank 28: 24316.12 + rank 29: 24316.11 + rank 30: 24316.10 + rank 31: 24316.11 + rank 32: 24315.08 + rank 33: 24315.07 + rank 34: 24315.06 + rank 35: 24315.06 + rank 36: 24315.08 + rank 37: 24315.03 + rank 38: 24315.04 + rank 39: 24315.04 + rank 40: 24315.93 + rank 41: 24315.96 + rank 42: 24315.90 + rank 43: 24315.92 + rank 44: 24315.89 + rank 45: 24315.99 + rank 46: 24315.89 + rank 47: 24315.89 + rank 48: 24316.34 + rank 49: 24316.33 + rank 50: 24316.32 + rank 51: 24316.32 + rank 52: 24316.37 + rank 53: 24316.28 + rank 54: 24316.32 + rank 55: 24316.30 + rank 56: 24316.08 + rank 57: 24316.06 + rank 58: 24316.04 + rank 59: 24316.04 + rank 60: 24316.05 + rank 61: 24316.06 + rank 62: 24316.05 + rank 63: 24316.06 + [2026-10-03 17:49:32] iteration 81/ 6855 | consumed samples: 10368 | elapsed time per iteration (ms): 107442.3 | throughput per GPU (TFLOP/s/GPU): 33.9 | MFU 10.87% | FLOPs estimator: workload-aware | logical tokens/s: 156149.7 | valid tokens/s: 156149.7 | physical tokens/s: 156593.3 | loss tokens/s: 112866.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.575190E-01 | lm-global-supervised-tokens: 1.212668E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.491406E+00 | lm-token-buffer-mib: 5.835317E+01 | global_load_balancing_loss: 1.001273E+00 | loss scale: 1.0 | grad norm: 0.079 | params norm: 11672.945 | params rms: 0.063205 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 8 days, 10:10:14.412078 | finish at 2026-10-12 03:59:47 + [2026-10-03 17:51:01] iteration 82/ 6855 | consumed samples: 10496 | elapsed time per iteration (ms): 88882.3 | throughput per GPU (TFLOP/s/GPU): 41.5 | MFU 13.31% | FLOPs estimator: workload-aware | logical tokens/s: 188757.3 | valid tokens/s: 188757.3 | physical tokens/s: 189191.1 | loss tokens/s: 145520.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.666715E-01 | lm-global-supervised-tokens: 1.293418E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.539358E-02 | lm-token-buffer-mib: 5.806823E+01 | global_load_balancing_loss: 1.001045E+00 | loss scale: 1.0 | grad norm: 0.101 | params norm: 11672.841 | params rms: 0.063205 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 23:13:19.944793 | finish at 2026-10-10 17:04:21 + [2026-10-03 17:52:30] iteration 83/ 6855 | consumed samples: 10624 | elapsed time per iteration (ms): 88558.6 | throughput per GPU (TFLOP/s/GPU): 41.5 | MFU 13.29% | FLOPs estimator: workload-aware | logical tokens/s: 189446.6 | valid tokens/s: 189446.6 | physical tokens/s: 190029.5 | loss tokens/s: 136845.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.567503E-01 | lm-global-supervised-tokens: 1.211883E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.430238E-02 | lm-token-buffer-mib: 5.815871E+01 | global_load_balancing_loss: 9.998055E-01 | loss scale: 1.0 | grad norm: 0.065 | params norm: 11672.739 | params rms: 0.063204 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 22:35:19.132706 | finish at 2026-10-10 16:27:49 + [2026-10-03 17:53:54] iteration 84/ 6855 | consumed samples: 10752 | elapsed time per iteration (ms): 84370.6 | throughput per GPU (TFLOP/s/GPU): 43.1 | MFU 13.82% | FLOPs estimator: workload-aware | logical tokens/s: 198850.0 | valid tokens/s: 198850.0 | physical tokens/s: 199511.3 | loss tokens/s: 157227.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.334265E-01 | lm-global-supervised-tokens: 1.326533E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 3.448495E-02 | lm-token-buffer-mib: 5.811121E+01 | global_load_balancing_loss: 9.976641E-01 | loss scale: 1.0 | grad norm: 0.039 | params norm: 11672.638 | params rms: 0.063204 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 14:41:13.095192 | finish at 2026-10-10 08:35:07 + [2026-10-03 17:55:18] iteration 85/ 6855 | consumed samples: 10880 | elapsed time per iteration (ms): 84229.5 | throughput per GPU (TFLOP/s/GPU): 43.4 | MFU 13.91% | FLOPs estimator: workload-aware | logical tokens/s: 199184.4 | valid tokens/s: 199184.4 | physical tokens/s: 199709.9 | loss tokens/s: 152582.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.713643E-01 | lm-global-supervised-tokens: 1.285191E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.602825E-02 | lm-token-buffer-mib: 5.823905E+01 | global_load_balancing_loss: 9.985069E-01 | loss scale: 1.0 | grad norm: 0.064 | params norm: 11672.536 | params rms: 0.063203 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 14:23:53.838046 | finish at 2026-10-10 08:19:12 + [2026-10-03 17:56:43] iteration 86/ 6855 | consumed samples: 11008 | elapsed time per iteration (ms): 85005.7 | throughput per GPU (TFLOP/s/GPU): 42.8 | MFU 13.72% | FLOPs estimator: workload-aware | logical tokens/s: 197365.2 | valid tokens/s: 197365.2 | physical tokens/s: 197968.0 | loss tokens/s: 152513.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.061211E-01 | lm-global-supervised-tokens: 1.296452E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.103204E-02 | lm-token-buffer-mib: 5.811311E+01 | global_load_balancing_loss: 9.988737E-01 | loss scale: 1.0 | grad norm: 0.056 | params norm: 11672.434 | params rms: 0.063202 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 15:50:03.258055 | finish at 2026-10-10 09:46:46 + [2026-10-03 17:58:12] iteration 87/ 6855 | consumed samples: 11136 | elapsed time per iteration (ms): 88482.7 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.44% | FLOPs estimator: workload-aware | logical tokens/s: 189609.7 | valid tokens/s: 189609.7 | physical tokens/s: 190050.8 | loss tokens/s: 141568.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.859061E-01 | lm-global-supervised-tokens: 1.252634E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 3.259740E-02 | lm-token-buffer-mib: 5.822473E+01 | global_load_balancing_loss: 1.002334E+00 | loss scale: 1.0 | grad norm: 0.062 | params norm: 11672.333 | params rms: 0.063202 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 22:20:50.875614 | finish at 2026-10-10 16:19:03 + [2026-10-03 17:59:39] iteration 88/ 6855 | consumed samples: 11264 | elapsed time per iteration (ms): 87102.0 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.61% | FLOPs estimator: workload-aware | logical tokens/s: 192615.6 | valid tokens/s: 192615.6 | physical tokens/s: 193012.4 | loss tokens/s: 143892.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.711793E-01 | lm-global-supervised-tokens: 1.253329E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.175016E-02 | lm-token-buffer-mib: 5.823666E+01 | global_load_balancing_loss: 1.001568E+00 | loss scale: 1.0 | grad norm: 0.054 | params norm: 11672.231 | params rms: 0.063201 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 19:43:38.971006 | finish at 2026-10-10 13:43:18 + [2026-10-03 18:01:06] iteration 89/ 6855 | consumed samples: 11392 | elapsed time per iteration (ms): 87458.1 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.64% | FLOPs estimator: workload-aware | logical tokens/s: 191830.7 | valid tokens/s: 191830.7 | physical tokens/s: 192222.4 | loss tokens/s: 139996.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.446519E-01 | lm-global-supervised-tokens: 1.224379E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.100315E-02 | lm-token-buffer-mib: 5.805658E+01 | global_load_balancing_loss: 1.002158E+00 | loss scale: 1.0 | grad norm: 0.042 | params norm: 11672.128 | params rms: 0.063201 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 20:22:21.556765 | finish at 2026-10-10 14:23:28 + [2026-10-03 18:02:44] iteration 90/ 6855 | consumed samples: 11520 | elapsed time per iteration (ms): 98204.9 | throughput per GPU (TFLOP/s/GPU): 38.1 | MFU 12.23% | FLOPs estimator: workload-aware | logical tokens/s: 170838.9 | valid tokens/s: 170838.9 | physical tokens/s: 171233.1 | loss tokens/s: 124573.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.155669E-01 | lm-global-supervised-tokens: 1.223376E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.132372E-02 | lm-token-buffer-mib: 5.812285E+01 | global_load_balancing_loss: 1.001809E+00 | loss scale: 1.0 | grad norm: 0.089 | params norm: 11672.027 | params rms: 0.063200 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 7 days, 16:32:35.848678 | finish at 2026-10-11 10:35:20 + [2026-10-03 18:04:14] iteration 91/ 6855 | consumed samples: 11648 | elapsed time per iteration (ms): 89900.6 | throughput per GPU (TFLOP/s/GPU): 41.5 | MFU 13.31% | FLOPs estimator: workload-aware | logical tokens/s: 186619.1 | valid tokens/s: 186619.1 | physical tokens/s: 186966.5 | loss tokens/s: 139579.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.201875E-01 | lm-global-supervised-tokens: 1.254829E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.289712E-02 | lm-token-buffer-mib: 5.829297E+01 | global_load_balancing_loss: 9.989300E-01 | loss scale: 1.0 | grad norm: 0.062 | params norm: 11671.925 | params rms: 0.063200 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 7 days, 0:54:47.579085 | finish at 2026-10-10 18:59:02 + [2026-10-03 18:05:41] iteration 92/ 6855 | consumed samples: 11776 | elapsed time per iteration (ms): 87101.4 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.38% | FLOPs estimator: workload-aware | logical tokens/s: 192616.9 | valid tokens/s: 192616.9 | physical tokens/s: 193081.5 | loss tokens/s: 145882.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.682619E-01 | lm-global-supervised-tokens: 1.270652E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.423577E-02 | lm-token-buffer-mib: 5.814795E+01 | global_load_balancing_loss: 9.984322E-01 | loss scale: 1.0 | grad norm: 0.056 | params norm: 11671.822 | params rms: 0.063199 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 19:37:46.491789 | finish at 2026-10-10 13:43:28 + [2026-10-03 18:07:10] iteration 93/ 6855 | consumed samples: 11904 | elapsed time per iteration (ms): 88094.8 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.45% | FLOPs estimator: workload-aware | logical tokens/s: 190444.1 | valid tokens/s: 190444.1 | physical tokens/s: 191029.3 | loss tokens/s: 141835.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.630993E-01 | lm-global-supervised-tokens: 1.249498E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.190122E-02 | lm-token-buffer-mib: 5.810264E+01 | global_load_balancing_loss: 9.980683E-01 | loss scale: 1.0 | grad norm: 0.130 | params norm: 11671.719 | params rms: 0.063199 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 21:28:17.002101 | finish at 2026-10-10 15:35:27 + [2026-10-03 18:08:38] iteration 94/ 6855 | consumed samples: 12032 | elapsed time per iteration (ms): 88739.9 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.46% | FLOPs estimator: workload-aware | logical tokens/s: 189059.9 | valid tokens/s: 189059.9 | physical tokens/s: 189355.3 | loss tokens/s: 137264.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.297603E-01 | lm-global-supervised-tokens: 1.218080E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.048087E-02 | lm-token-buffer-mib: 5.815169E+01 | global_load_balancing_loss: 9.993807E-01 | loss scale: 1.0 | grad norm: 0.071 | params norm: 11671.616 | params rms: 0.063198 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 22:39:30.656480 | finish at 2026-10-10 16:48:09 + [2026-10-03 18:10:08] iteration 95/ 6855 | consumed samples: 12160 | elapsed time per iteration (ms): 89590.4 | throughput per GPU (TFLOP/s/GPU): 41.2 | MFU 13.22% | FLOPs estimator: workload-aware | logical tokens/s: 187265.6 | valid tokens/s: 187265.6 | physical tokens/s: 187788.2 | loss tokens/s: 135344.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.676252E-01 | lm-global-supervised-tokens: 1.212557E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.441611E-02 | lm-token-buffer-mib: 5.812093E+01 | global_load_balancing_loss: 1.000561E+00 | loss scale: 1.0 | grad norm: 0.157 | params norm: 11671.513 | params rms: 0.063197 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 7 days, 0:13:50.902405 | finish at 2026-10-10 18:23:59 + [2026-10-03 18:11:36] iteration 96/ 6855 | consumed samples: 12288 | elapsed time per iteration (ms): 88158.1 | throughput per GPU (TFLOP/s/GPU): 41.7 | MFU 13.37% | FLOPs estimator: workload-aware | logical tokens/s: 190307.3 | valid tokens/s: 190307.3 | physical tokens/s: 190748.7 | loss tokens/s: 140187.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.207875E-01 | lm-global-supervised-tokens: 1.235862E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.165927E-02 | lm-token-buffer-mib: 5.811512E+01 | global_load_balancing_loss: 1.001693E+00 | loss scale: 1.0 | grad norm: 0.297 | params norm: 11671.410 | params rms: 0.063197 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 21:31:00.671283 | finish at 2026-10-10 15:42:37 + [2026-10-03 18:13:05] iteration 97/ 6855 | consumed samples: 12416 | elapsed time per iteration (ms): 88718.5 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.49% | FLOPs estimator: workload-aware | logical tokens/s: 189105.6 | valid tokens/s: 189105.6 | physical tokens/s: 189471.7 | loss tokens/s: 134632.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.540148E-01 | lm-global-supervised-tokens: 1.194442E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.267998E-02 | lm-token-buffer-mib: 5.804741E+01 | global_load_balancing_loss: 1.001522E+00 | loss scale: 1.0 | grad norm: 0.074 | params norm: 11671.308 | params rms: 0.063196 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 22:32:39.701267 | finish at 2026-10-10 16:45:44 + [2026-10-03 18:14:32] iteration 98/ 6855 | consumed samples: 12544 | elapsed time per iteration (ms): 87621.9 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.51% | FLOPs estimator: workload-aware | logical tokens/s: 191472.7 | valid tokens/s: 191472.7 | physical tokens/s: 191952.9 | loss tokens/s: 138269.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.946957E-01 | lm-global-supervised-tokens: 1.211541E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.311754E-02 | lm-token-buffer-mib: 5.808457E+01 | global_load_balancing_loss: 1.001149E+00 | loss scale: 1.0 | grad norm: 0.257 | params norm: 11671.206 | params rms: 0.063196 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 20:27:41.249735 | finish at 2026-10-10 14:42:14 + [2026-10-03 18:16:00] iteration 99/ 6855 | consumed samples: 12672 | elapsed time per iteration (ms): 87534.1 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.74% | FLOPs estimator: workload-aware | logical tokens/s: 191664.8 | valid tokens/s: 191664.8 | physical tokens/s: 192283.8 | loss tokens/s: 142888.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.801100E-01 | lm-global-supervised-tokens: 1.250757E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.774909E-02 | lm-token-buffer-mib: 5.822046E+01 | global_load_balancing_loss: 1.001308E+00 | loss scale: 1.0 | grad norm: 0.163 | params norm: 11671.103 | params rms: 0.063195 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 20:16:20.481454 | finish at 2026-10-10 14:32:20 + [2026-10-03 18:17:43] iteration 100/ 6855 | consumed samples: 12800 | elapsed time per iteration (ms): 102713.7 | throughput per GPU (TFLOP/s/GPU): 36.4 | MFU 11.68% | FLOPs estimator: workload-aware | logical tokens/s: 163339.6 | valid tokens/s: 163339.6 | physical tokens/s: 163778.3 | loss tokens/s: 113386.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.428754E-01 | lm-global-supervised-tokens: 1.164635E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.851664E-02 | lm-token-buffer-mib: 5.816994E+01 | global_load_balancing_loss: 9.999835E-01 | loss scale: 1.0 | grad norm: 0.188 | params norm: 11671.001 | params rms: 0.063195 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 8 days, 0:43:50.968184 | finish at 2026-10-11 19:01:34 + [2026-10-03 18:19:11] iteration 101/ 6855 | consumed samples: 12928 | elapsed time per iteration (ms): 88280.6 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.51% | FLOPs estimator: workload-aware | logical tokens/s: 190043.0 | valid tokens/s: 190043.0 | physical tokens/s: 190361.1 | loss tokens/s: 141004.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.503632E-01 | lm-global-supervised-tokens: 1.244794E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.461763E-02 | lm-token-buffer-mib: 5.809507E+01 | global_load_balancing_loss: 9.983826E-01 | loss scale: 1.0 | grad norm: 0.047 | params norm: 11670.899 | params rms: 0.063194 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 21:37:27.348400 | finish at 2026-10-10 15:56:38 + [2026-10-03 18:20:40] iteration 102/ 6855 | consumed samples: 13056 | elapsed time per iteration (ms): 88650.9 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.49% | FLOPs estimator: workload-aware | logical tokens/s: 189249.4 | valid tokens/s: 189249.3 | physical tokens/s: 189635.4 | loss tokens/s: 138988.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.765324E-01 | lm-global-supervised-tokens: 1.232149E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.126045E-02 | lm-token-buffer-mib: 5.815170E+01 | global_load_balancing_loss: 9.995626E-01 | loss scale: 1.0 | grad norm: 0.131 | params norm: 11670.797 | params rms: 0.063194 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 22:17:39.493127 | finish at 2026-10-10 16:38:19 + [2026-10-03 18:22:07] iteration 103/ 6855 | consumed samples: 13184 | elapsed time per iteration (ms): 87912.5 | throughput per GPU (TFLOP/s/GPU): 41.7 | MFU 13.36% | FLOPs estimator: workload-aware | logical tokens/s: 190839.8 | valid tokens/s: 190839.8 | physical tokens/s: 191230.0 | loss tokens/s: 136698.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.479347E-01 | lm-global-supervised-tokens: 1.201747E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.664471E-02 | lm-token-buffer-mib: 5.806999E+01 | global_load_balancing_loss: 9.989901E-01 | loss scale: 1.0 | grad norm: 0.052 | params norm: 11670.694 | params rms: 0.063193 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 20:53:05.118866 | finish at 2026-10-10 15:15:13 + [2026-10-03 18:23:35] iteration 104/ 6855 | consumed samples: 13312 | elapsed time per iteration (ms): 87410.6 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.73% | FLOPs estimator: workload-aware | logical tokens/s: 191935.5 | valid tokens/s: 191935.5 | physical tokens/s: 192244.1 | loss tokens/s: 141388.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.959695E-01 | lm-global-supervised-tokens: 1.235886E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.136308E-02 | lm-token-buffer-mib: 5.803809E+01 | global_load_balancing_loss: 9.993410E-01 | loss scale: 1.0 | grad norm: 0.050 | params norm: 11670.593 | params rms: 0.063192 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 19:55:08.960242 | finish at 2026-10-10 14:18:44 + [2026-10-03 18:25:04] iteration 105/ 6855 | consumed samples: 13440 | elapsed time per iteration (ms): 89401.1 | throughput per GPU (TFLOP/s/GPU): 41.6 | MFU 13.33% | FLOPs estimator: workload-aware | logical tokens/s: 187661.7 | valid tokens/s: 187661.7 | physical tokens/s: 188292.7 | loss tokens/s: 135212.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.203774E-01 | lm-global-supervised-tokens: 1.208811E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.143740E-02 | lm-token-buffer-mib: 5.803996E+01 | global_load_balancing_loss: 1.000926E+00 | loss scale: 1.0 | grad norm: 0.129 | params norm: 11670.491 | params rms: 0.063192 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 23:37:37.624018 | finish at 2026-10-10 18:02:42 + [2026-10-03 18:26:38] iteration 106/ 6855 | consumed samples: 13568 | elapsed time per iteration (ms): 93332.7 | throughput per GPU (TFLOP/s/GPU): 40.6 | MFU 13.02% | FLOPs estimator: workload-aware | logical tokens/s: 179757.1 | valid tokens/s: 179757.1 | physical tokens/s: 180078.9 | loss tokens/s: 127045.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.767884E-01 | lm-global-supervised-tokens: 1.185747E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.470591E-02 | lm-token-buffer-mib: 5.809775E+01 | global_load_balancing_loss: 1.001938E+00 | loss scale: 1.0 | grad norm: 0.063 | params norm: 11670.390 | params rms: 0.063191 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 7 days, 6:58:22.408486 | finish at 2026-10-11 01:25:00 + [2026-10-03 18:28:06] iteration 107/ 6855 | consumed samples: 13696 | elapsed time per iteration (ms): 88179.1 | throughput per GPU (TFLOP/s/GPU): 40.5 | MFU 12.98% | FLOPs estimator: workload-aware | logical tokens/s: 190262.7 | valid tokens/s: 190262.7 | physical tokens/s: 190713.5 | loss tokens/s: 143150.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.754997E-01 | lm-global-supervised-tokens: 1.262290E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.568479E-02 | lm-token-buffer-mib: 5.812599E+01 | global_load_balancing_loss: 1.001031E+00 | loss scale: 1.0 | grad norm: 0.150 | params norm: 11670.288 | params rms: 0.063191 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 21:17:12.615313 | finish at 2026-10-10 15:45:18 + [2026-10-03 18:29:32] iteration 108/ 6855 | consumed samples: 13824 | elapsed time per iteration (ms): 86173.5 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.70% | FLOPs estimator: workload-aware | logical tokens/s: 194690.4 | valid tokens/s: 194690.4 | physical tokens/s: 195211.7 | loss tokens/s: 145120.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.775801E-01 | lm-global-supervised-tokens: 1.250555E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.866215E-02 | lm-token-buffer-mib: 5.825975E+01 | global_load_balancing_loss: 1.000991E+00 | loss scale: 1.0 | grad norm: 0.047 | params norm: 11670.185 | params rms: 0.063190 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 17:30:12.661213 | finish at 2026-10-10 11:59:45 + [2026-10-03 18:30:59] iteration 109/ 6855 | consumed samples: 13952 | elapsed time per iteration (ms): 87044.4 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.55% | FLOPs estimator: workload-aware | logical tokens/s: 192742.8 | valid tokens/s: 192742.8 | physical tokens/s: 193252.3 | loss tokens/s: 150107.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.636693E-01 | lm-global-supervised-tokens: 1.306605E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.139325E-02 | lm-token-buffer-mib: 5.813088E+01 | global_load_balancing_loss: 1.000425E+00 | loss scale: 1.0 | grad norm: 0.188 | params norm: 11670.082 | params rms: 0.063190 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 19:06:41.676647 | finish at 2026-10-10 13:37:41 + [2026-10-03 18:32:38] iteration 110/ 6855 | consumed samples: 14080 | elapsed time per iteration (ms): 98879.5 | throughput per GPU (TFLOP/s/GPU): 37.8 | MFU 12.13% | FLOPs estimator: workload-aware | logical tokens/s: 169673.3 | valid tokens/s: 169673.3 | physical tokens/s: 169997.3 | loss tokens/s: 123146.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.857123E-01 | lm-global-supervised-tokens: 1.217669E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.230575E-02 | lm-token-buffer-mib: 5.809622E+01 | global_load_balancing_loss: 1.000658E+00 | loss scale: 1.0 | grad norm: 0.039 | params norm: 11669.979 | params rms: 0.063189 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 7 days, 17:15:42.228516 | finish at 2026-10-11 11:48:20 + [2026-10-03 18:34:06] iteration 111/ 6855 | consumed samples: 14208 | elapsed time per iteration (ms): 88594.9 | throughput per GPU (TFLOP/s/GPU): 41.3 | MFU 13.25% | FLOPs estimator: workload-aware | logical tokens/s: 189369.8 | valid tokens/s: 189369.8 | physical tokens/s: 189732.6 | loss tokens/s: 141149.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.596819E-01 | lm-global-supervised-tokens: 1.250511E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.133340E-02 | lm-token-buffer-mib: 5.808588E+01 | global_load_balancing_loss: 9.990994E-01 | loss scale: 1.0 | grad norm: 0.106 | params norm: 11669.875 | params rms: 0.063189 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 21:58:04.061153 | finish at 2026-10-10 16:32:11 + [2026-10-03 18:35:32] iteration 112/ 6855 | consumed samples: 14336 | elapsed time per iteration (ms): 85300.5 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.74% | FLOPs estimator: workload-aware | logical tokens/s: 196683.0 | valid tokens/s: 196683.0 | physical tokens/s: 197151.8 | loss tokens/s: 149989.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.902936E-01 | lm-global-supervised-tokens: 1.279418E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.159412E-02 | lm-token-buffer-mib: 5.838383E+01 | global_load_balancing_loss: 9.994006E-01 | loss scale: 1.0 | grad norm: 0.123 | params norm: 11669.773 | params rms: 0.063188 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 15:46:21.352925 | finish at 2026-10-10 10:21:53 + [2026-10-03 18:37:03] iteration 113/ 6855 | consumed samples: 14464 | elapsed time per iteration (ms): 91198.7 | throughput per GPU (TFLOP/s/GPU): 41.4 | MFU 13.26% | FLOPs estimator: workload-aware | logical tokens/s: 183963.3 | valid tokens/s: 183963.3 | physical tokens/s: 184425.4 | loss tokens/s: 130507.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.361738E-01 | lm-global-supervised-tokens: 1.190210E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.154097E-02 | lm-token-buffer-mib: 5.816307E+01 | global_load_balancing_loss: 9.992607E-01 | loss scale: 1.0 | grad norm: 0.097 | params norm: 11669.672 | params rms: 0.063187 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 7 days, 2:47:41.655967 | finish at 2026-10-10 21:24:45 + [2026-10-03 18:38:32] iteration 114/ 6855 | consumed samples: 14592 | elapsed time per iteration (ms): 89025.1 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.68% | FLOPs estimator: workload-aware | logical tokens/s: 188454.3 | valid tokens/s: 188454.3 | physical tokens/s: 188884.9 | loss tokens/s: 131758.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.560869E-01 | lm-global-supervised-tokens: 1.172985E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.524581E-02 | lm-token-buffer-mib: 5.804630E+01 | global_load_balancing_loss: 1.000814E+00 | loss scale: 1.0 | grad norm: 0.112 | params norm: 11669.570 | params rms: 0.063187 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 22:41:58.534952 | finish at 2026-10-10 17:20:31 + [2026-10-03 18:40:00] iteration 115/ 6855 | consumed samples: 14720 | elapsed time per iteration (ms): 87792.4 | throughput per GPU (TFLOP/s/GPU): 43.7 | MFU 14.02% | FLOPs estimator: workload-aware | logical tokens/s: 191100.6 | valid tokens/s: 191100.6 | physical tokens/s: 191454.2 | loss tokens/s: 133004.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.346882E-01 | lm-global-supervised-tokens: 1.167677E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.446396E-02 | lm-token-buffer-mib: 5.806325E+01 | global_load_balancing_loss: 1.000273E+00 | loss scale: 1.0 | grad norm: 0.048 | params norm: 11669.467 | params rms: 0.063186 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 20:22:00.834670 | finish at 2026-10-10 15:02:01 + [2026-10-03 18:41:25] iteration 116/ 6855 | consumed samples: 14848 | elapsed time per iteration (ms): 85193.7 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.64% | FLOPs estimator: workload-aware | logical tokens/s: 196930.3 | valid tokens/s: 196930.3 | physical tokens/s: 197334.3 | loss tokens/s: 150081.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.582876E-01 | lm-global-supervised-tokens: 1.278596E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.843553E-02 | lm-token-buffer-mib: 5.810596E+01 | global_load_balancing_loss: 9.998168E-01 | loss scale: 1.0 | grad norm: 0.088 | params norm: 11669.365 | params rms: 0.063186 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 15:28:40.213057 | finish at 2026-10-10 10:10:05 + [2026-10-03 18:42:51] iteration 117/ 6855 | consumed samples: 14976 | elapsed time per iteration (ms): 85511.1 | throughput per GPU (TFLOP/s/GPU): 43.4 | MFU 13.92% | FLOPs estimator: workload-aware | logical tokens/s: 196198.3 | valid tokens/s: 196198.3 | physical tokens/s: 196624.5 | loss tokens/s: 143396.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.716778E-01 | lm-global-supervised-tokens: 1.226202E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.124946E-02 | lm-token-buffer-mib: 5.815218E+01 | global_load_balancing_loss: 1.001270E+00 | loss scale: 1.0 | grad norm: 0.069 | params norm: 11669.263 | params rms: 0.063185 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 16:02:53.917467 | finish at 2026-10-10 10:45:44 + [2026-10-03 18:44:19] iteration 118/ 6855 | consumed samples: 15104 | elapsed time per iteration (ms): 88152.1 | throughput per GPU (TFLOP/s/GPU): 41.7 | MFU 13.37% | FLOPs estimator: workload-aware | logical tokens/s: 190319.9 | valid tokens/s: 190319.9 | physical tokens/s: 190668.0 | loss tokens/s: 139972.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.609653E-01 | lm-global-supervised-tokens: 1.233886E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.146129E-02 | lm-token-buffer-mib: 5.816848E+01 | global_load_balancing_loss: 1.000852E+00 | loss scale: 1.0 | grad norm: 0.092 | params norm: 11669.160 | params rms: 0.063185 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 20:58:00.817142 | finish at 2026-10-10 15:42:19 + [2026-10-03 18:45:45] iteration 119/ 6855 | consumed samples: 15232 | elapsed time per iteration (ms): 86657.8 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.68% | FLOPs estimator: workload-aware | logical tokens/s: 193603.0 | valid tokens/s: 193602.9 | physical tokens/s: 194077.3 | loss tokens/s: 141956.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.982668E-01 | lm-global-supervised-tokens: 1.230163E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.165633E-02 | lm-token-buffer-mib: 5.821824E+01 | global_load_balancing_loss: 1.000243E+00 | loss scale: 1.0 | grad norm: 0.113 | params norm: 11669.058 | params rms: 0.063184 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 18:08:47.227997 | finish at 2026-10-10 12:54:33 + [2026-10-03 18:47:33] iteration 120/ 6855 | consumed samples: 15360 | elapsed time per iteration (ms): 107602.9 | throughput per GPU (TFLOP/s/GPU): 35.2 | MFU 11.30% | FLOPs estimator: workload-aware | logical tokens/s: 155917.8 | valid tokens/s: 155917.8 | physical tokens/s: 156415.4 | loss tokens/s: 110249.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.727154E-01 | lm-global-supervised-tokens: 1.186318E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.131799E-02 | lm-token-buffer-mib: 5.814670E+01 | global_load_balancing_loss: 9.996844E-01 | loss scale: 1.0 | grad norm: 0.100 | params norm: 11668.957 | params rms: 0.063184 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 8 days, 9:18:25.385567 | finish at 2026-10-12 04:05:58 +times across ranks (ms): + save-checkpoint: + rank 0: 24127.59 + rank 1: 24127.58 + rank 2: 24127.59 + rank 3: 24127.55 + rank 4: 24127.58 + rank 5: 24127.55 + rank 6: 24127.54 + rank 7: 24127.57 + rank 8: 24127.88 + rank 9: 24127.80 + rank 10: 24127.83 + rank 11: 24127.81 + rank 12: 24127.81 + rank 13: 24127.80 + rank 14: 24127.81 + rank 15: 24127.80 + rank 16: 24128.08 + rank 17: 24128.12 + rank 18: 24128.05 + rank 19: 24128.14 + rank 20: 24128.07 + rank 21: 24128.12 + rank 22: 24128.05 + rank 23: 24128.06 + rank 24: 24128.00 + rank 25: 24127.80 + rank 26: 24127.78 + rank 27: 24127.79 + rank 28: 24127.78 + rank 29: 24177.92 + rank 30: 24127.86 + rank 31: 24127.85 + rank 32: 24127.65 + rank 33: 24127.57 + rank 34: 24127.59 + rank 35: 24127.54 + rank 36: 24127.61 + rank 37: 24127.51 + rank 38: 24127.62 + rank 39: 24127.65 + rank 40: 24127.79 + rank 41: 24127.76 + rank 42: 24127.75 + rank 43: 24127.76 + rank 44: 24127.74 + rank 45: 24127.74 + rank 46: 24127.75 + rank 47: 24127.72 + rank 48: 24127.38 + rank 49: 24127.36 + rank 50: 24127.31 + rank 51: 24127.26 + rank 52: 24127.35 + rank 53: 24127.25 + rank 54: 24127.30 + rank 55: 24127.34 + rank 56: 24128.02 + rank 57: 24127.99 + rank 58: 24127.96 + rank 59: 24127.98 + rank 60: 24127.95 + rank 61: 24127.98 + rank 62: 24127.99 + rank 63: 24128.00 + [2026-10-03 18:49:43] iteration 121/ 6855 | consumed samples: 15488 | elapsed time per iteration (ms): 105901.1 | throughput per GPU (TFLOP/s/GPU): 35.2 | MFU 11.28% | FLOPs estimator: workload-aware | logical tokens/s: 158423.1 | valid tokens/s: 158423.1 | physical tokens/s: 158785.6 | loss tokens/s: 118910.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.074334E-01 | lm-global-supervised-tokens: 1.259278E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 6.857152E-01 | lm-token-buffer-mib: 5.829089E+01 | global_load_balancing_loss: 9.995534E-01 | loss scale: 1.0 | grad norm: 0.059 | params norm: 11668.854 | params rms: 0.063183 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 8 days, 6:05:38.050212 | finish at 2026-10-12 00:55:21 + [2026-10-03 18:51:17] iteration 122/ 6855 | consumed samples: 15616 | elapsed time per iteration (ms): 94304.9 | throughput per GPU (TFLOP/s/GPU): 39.5 | MFU 12.65% | FLOPs estimator: workload-aware | logical tokens/s: 177903.1 | valid tokens/s: 177903.1 | physical tokens/s: 178287.8 | loss tokens/s: 132696.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.289728E-01 | lm-global-supervised-tokens: 1.251395E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.528283E-02 | lm-token-buffer-mib: 5.802917E+01 | global_load_balancing_loss: 9.999072E-01 | loss scale: 1.0 | grad norm: 0.081 | params norm: 11668.751 | params rms: 0.063182 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 7 days, 8:22:35.225132 | finish at 2026-10-11 03:13:53 + [2026-10-03 18:52:44] iteration 123/ 6855 | consumed samples: 15744 | elapsed time per iteration (ms): 86492.6 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.75% | FLOPs estimator: workload-aware | logical tokens/s: 193972.3 | valid tokens/s: 193972.3 | physical tokens/s: 194409.6 | loss tokens/s: 142613.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.300982E-01 | lm-global-supervised-tokens: 1.233502E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.190228E-02 | lm-token-buffer-mib: 5.807641E+01 | global_load_balancing_loss: 9.995269E-01 | loss scale: 1.0 | grad norm: 0.064 | params norm: 11668.649 | params rms: 0.063182 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 17:44:28.173328 | finish at 2026-10-10 12:37:12 + [2026-10-03 18:54:10] iteration 124/ 6855 | consumed samples: 15872 | elapsed time per iteration (ms): 86223.0 | throughput per GPU (TFLOP/s/GPU): 42.8 | MFU 13.71% | FLOPs estimator: workload-aware | logical tokens/s: 194579.3 | valid tokens/s: 194579.3 | physical tokens/s: 195092.6 | loss tokens/s: 144102.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.799745E-01 | lm-global-supervised-tokens: 1.242494E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.158769E-02 | lm-token-buffer-mib: 5.832362E+01 | global_load_balancing_loss: 1.000323E+00 | loss scale: 1.0 | grad norm: 0.079 | params norm: 11668.546 | params rms: 0.063181 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 17:12:47.247634 | finish at 2026-10-10 12:06:57 + [2026-10-03 18:55:35] iteration 125/ 6855 | consumed samples: 16000 | elapsed time per iteration (ms): 85295.2 | throughput per GPU (TFLOP/s/GPU): 43.7 | MFU 14.02% | FLOPs estimator: workload-aware | logical tokens/s: 196694.8 | valid tokens/s: 196694.8 | physical tokens/s: 197101.5 | loss tokens/s: 146270.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.223656E-01 | lm-global-supervised-tokens: 1.247620E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.322147E-02 | lm-token-buffer-mib: 5.817720E+01 | global_load_balancing_loss: 1.000398E+00 | loss scale: 1.0 | grad norm: 0.052 | params norm: 11668.443 | params rms: 0.063181 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 15:27:16.629345 | finish at 2026-10-10 10:22:52 + [2026-10-03 18:57:05] iteration 126/ 6855 | consumed samples: 16128 | elapsed time per iteration (ms): 89349.6 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.43% | FLOPs estimator: workload-aware | logical tokens/s: 187769.9 | valid tokens/s: 187769.9 | physical tokens/s: 188159.0 | loss tokens/s: 136165.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.423867E-01 | lm-global-supervised-tokens: 1.216629E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.569757E-02 | lm-token-buffer-mib: 5.805504E+01 | global_load_balancing_loss: 1.002172E+00 | loss scale: 1.0 | grad norm: 0.064 | params norm: 11668.340 | params rms: 0.063180 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 23:00:33.491002 | finish at 2026-10-10 17:57:38 + [2026-10-03 18:58:33] iteration 127/ 6855 | consumed samples: 16256 | elapsed time per iteration (ms): 88561.7 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.67% | FLOPs estimator: workload-aware | logical tokens/s: 189440.6 | valid tokens/s: 189440.5 | physical tokens/s: 189834.3 | loss tokens/s: 143813.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.048760E-01 | lm-global-supervised-tokens: 1.273640E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.149794E-02 | lm-token-buffer-mib: 5.806847E+01 | global_load_balancing_loss: 1.001809E+00 | loss scale: 1.0 | grad norm: 0.107 | params norm: 11668.238 | params rms: 0.063180 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 21:30:43.079813 | finish at 2026-10-10 16:29:16 + [2026-10-03 18:59:59] iteration 128/ 6855 | consumed samples: 16384 | elapsed time per iteration (ms): 85465.1 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.46% | FLOPs estimator: workload-aware | logical tokens/s: 196304.1 | valid tokens/s: 196304.1 | physical tokens/s: 196929.5 | loss tokens/s: 148469.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.846234E-01 | lm-global-supervised-tokens: 1.268895E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.930468E-02 | lm-token-buffer-mib: 5.812666E+01 | global_load_balancing_loss: 9.997028E-01 | loss scale: 1.0 | grad norm: 0.066 | params norm: 11668.138 | params rms: 0.063179 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 15:42:03.899647 | finish at 2026-10-10 10:42:03 + [2026-10-03 19:01:26] iteration 129/ 6855 | consumed samples: 16512 | elapsed time per iteration (ms): 87575.5 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.75% | FLOPs estimator: workload-aware | logical tokens/s: 191573.1 | valid tokens/s: 191573.1 | physical tokens/s: 192020.0 | loss tokens/s: 139846.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.259551E-01 | lm-global-supervised-tokens: 1.224713E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.323899E-02 | lm-token-buffer-mib: 5.808355E+01 | global_load_balancing_loss: 9.983993E-01 | loss scale: 1.0 | grad norm: 0.147 | params norm: 11668.039 | params rms: 0.063179 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 19:37:12.538861 | finish at 2026-10-10 14:38:39 + [2026-10-03 19:03:07] iteration 130/ 6855 | consumed samples: 16640 | elapsed time per iteration (ms): 100606.8 | throughput per GPU (TFLOP/s/GPU): 37.3 | MFU 11.95% | FLOPs estimator: workload-aware | logical tokens/s: 166759.5 | valid tokens/s: 166759.5 | physical tokens/s: 167260.3 | loss tokens/s: 125970.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.778910E-01 | lm-global-supervised-tokens: 1.267345E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.237179E-02 | lm-token-buffer-mib: 5.832497E+01 | global_load_balancing_loss: 9.996833E-01 | loss scale: 1.0 | grad norm: 0.174 | params norm: 11667.939 | params rms: 0.063178 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 7 days, 19:56:20.405051 | finish at 2026-10-11 14:59:27 + [2026-10-03 19:04:33] iteration 131/ 6855 | consumed samples: 16768 | elapsed time per iteration (ms): 85994.8 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.66% | FLOPs estimator: workload-aware | logical tokens/s: 195095.3 | valid tokens/s: 195095.3 | physical tokens/s: 195484.3 | loss tokens/s: 144777.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.672066E-01 | lm-global-supervised-tokens: 1.245006E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.149246E-02 | lm-token-buffer-mib: 5.814948E+01 | global_load_balancing_loss: 1.000194E+00 | loss scale: 1.0 | grad norm: 0.062 | params norm: 11667.839 | params rms: 0.063178 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 16:37:08.841832 | finish at 2026-10-10 11:41:42 + [2026-10-03 19:06:00] iteration 132/ 6855 | consumed samples: 16896 | elapsed time per iteration (ms): 86858.7 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.67% | FLOPs estimator: workload-aware | logical tokens/s: 193155.0 | valid tokens/s: 193155.0 | physical tokens/s: 193686.9 | loss tokens/s: 144329.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.145224E-01 | lm-global-supervised-tokens: 1.253628E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.341056E-02 | lm-token-buffer-mib: 5.806236E+01 | global_load_balancing_loss: 1.000762E+00 | loss scale: 1.0 | grad norm: 0.118 | params norm: 11667.740 | params rms: 0.063177 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 18:12:31.008291 | finish at 2026-10-10 13:18:31 + [2026-10-03 19:07:27] iteration 133/ 6855 | consumed samples: 17024 | elapsed time per iteration (ms): 87446.0 | throughput per GPU (TFLOP/s/GPU): 42.8 | MFU 13.71% | FLOPs estimator: workload-aware | logical tokens/s: 191857.7 | valid tokens/s: 191857.7 | physical tokens/s: 192398.0 | loss tokens/s: 142059.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.420788E-01 | lm-global-supervised-tokens: 1.242250E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.234414E-02 | lm-token-buffer-mib: 5.802249E+01 | global_load_balancing_loss: 1.000095E+00 | loss scale: 1.0 | grad norm: 0.124 | params norm: 11667.641 | params rms: 0.063176 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 19:16:51.900481 | finish at 2026-10-10 14:24:19 + [2026-10-03 19:08:56] iteration 134/ 6855 | consumed samples: 17152 | elapsed time per iteration (ms): 89140.1 | throughput per GPU (TFLOP/s/GPU): 41.7 | MFU 13.36% | FLOPs estimator: workload-aware | logical tokens/s: 188211.4 | valid tokens/s: 188211.4 | physical tokens/s: 188762.0 | loss tokens/s: 132536.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.585694E-01 | lm-global-supervised-tokens: 1.181431E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.330530E-02 | lm-token-buffer-mib: 5.812550E+01 | global_load_balancing_loss: 1.000415E+00 | loss scale: 1.0 | grad norm: 0.077 | params norm: 11667.542 | params rms: 0.063176 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 22:25:10.919778 | finish at 2026-10-10 17:34:07 + [2026-10-03 19:10:22] iteration 135/ 6855 | consumed samples: 17280 | elapsed time per iteration (ms): 85652.2 | throughput per GPU (TFLOP/s/GPU): 43.1 | MFU 13.82% | FLOPs estimator: workload-aware | logical tokens/s: 195875.7 | valid tokens/s: 195875.7 | physical tokens/s: 196318.3 | loss tokens/s: 139457.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.826590E-01 | lm-global-supervised-tokens: 1.194484E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.373611E-02 | lm-token-buffer-mib: 5.816690E+01 | global_load_balancing_loss: 1.000976E+00 | loss scale: 1.0 | grad norm: 0.166 | params norm: 11667.442 | params rms: 0.063175 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 15:53:02.541962 | finish at 2026-10-10 11:03:25 + [2026-10-03 19:11:50] iteration 136/ 6855 | consumed samples: 17408 | elapsed time per iteration (ms): 87873.0 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.38% | FLOPs estimator: workload-aware | logical tokens/s: 190925.8 | valid tokens/s: 190925.8 | physical tokens/s: 191424.5 | loss tokens/s: 139557.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.042531E-01 | lm-global-supervised-tokens: 1.226329E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.380911E-02 | lm-token-buffer-mib: 5.811594E+01 | global_load_balancing_loss: 1.000224E+00 | loss scale: 1.0 | grad norm: 0.049 | params norm: 11667.344 | params rms: 0.063175 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 20:00:18.492538 | finish at 2026-10-10 15:12:08 + [2026-10-03 19:13:14] iteration 137/ 6855 | consumed samples: 17536 | elapsed time per iteration (ms): 83946.7 | throughput per GPU (TFLOP/s/GPU): 43.4 | MFU 13.91% | FLOPs estimator: workload-aware | logical tokens/s: 199854.7 | valid tokens/s: 199854.7 | physical tokens/s: 200475.7 | loss tokens/s: 151587.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.658873E-01 | lm-global-supervised-tokens: 1.272524E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.750229E-02 | lm-token-buffer-mib: 5.805360E+01 | global_load_balancing_loss: 9.997567E-01 | loss scale: 1.0 | grad norm: 0.189 | params norm: 11667.248 | params rms: 0.063174 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 12:39:14.121848 | finish at 2026-10-10 07:52:28 + [2026-10-03 19:14:41] iteration 138/ 6855 | consumed samples: 17664 | elapsed time per iteration (ms): 87492.3 | throughput per GPU (TFLOP/s/GPU): 43.4 | MFU 13.90% | FLOPs estimator: workload-aware | logical tokens/s: 191755.7 | valid tokens/s: 191755.7 | physical tokens/s: 192330.2 | loss tokens/s: 134380.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.053559E-01 | lm-global-supervised-tokens: 1.175725E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.142144E-02 | lm-token-buffer-mib: 5.830865E+01 | global_load_balancing_loss: 9.999312E-01 | loss scale: 1.0 | grad norm: 0.035 | params norm: 11667.152 | params rms: 0.063174 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 19:14:46.086805 | finish at 2026-10-10 14:29:27 + [2026-10-03 19:16:07] iteration 139/ 6855 | consumed samples: 17792 | elapsed time per iteration (ms): 85781.9 | throughput per GPU (TFLOP/s/GPU): 42.8 | MFU 13.72% | FLOPs estimator: workload-aware | logical tokens/s: 195579.6 | valid tokens/s: 195579.6 | physical tokens/s: 196068.7 | loss tokens/s: 146251.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.715540E-01 | lm-global-supervised-tokens: 1.254576E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.153961E-02 | lm-token-buffer-mib: 5.807439E+01 | global_load_balancing_loss: 1.001053E+00 | loss scale: 1.0 | grad norm: 0.177 | params norm: 11667.053 | params rms: 0.063173 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 16:01:51.041372 | finish at 2026-10-10 11:17:58 + [2026-10-03 19:17:50] iteration 140/ 6855 | consumed samples: 17920 | elapsed time per iteration (ms): 102452.8 | throughput per GPU (TFLOP/s/GPU): 35.2 | MFU 11.27% | FLOPs estimator: workload-aware | logical tokens/s: 163754.5 | valid tokens/s: 163754.5 | physical tokens/s: 164301.1 | loss tokens/s: 125055.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.654450E-01 | lm-global-supervised-tokens: 1.281232E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.131221E-02 | lm-token-buffer-mib: 5.812253E+01 | global_load_balancing_loss: 9.999957E-01 | loss scale: 1.0 | grad norm: 0.074 | params norm: 11666.957 | params rms: 0.063173 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 7 days, 23:06:10.882040 | finish at 2026-10-11 18:24:01 + [2026-10-03 19:19:17] iteration 141/ 6855 | consumed samples: 18048 | elapsed time per iteration (ms): 86963.5 | throughput per GPU (TFLOP/s/GPU): 43.6 | MFU 13.99% | FLOPs estimator: workload-aware | logical tokens/s: 192922.0 | valid tokens/s: 192922.0 | physical tokens/s: 193248.4 | loss tokens/s: 136333.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.431726E-01 | lm-global-supervised-tokens: 1.185601E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.508306E-02 | lm-token-buffer-mib: 5.809360E+01 | global_load_balancing_loss: 1.000164E+00 | loss scale: 1.0 | grad norm: 0.042 | params norm: 11666.861 | params rms: 0.063172 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 18:11:12.855915 | finish at 2026-10-10 13:30:29 + [2026-10-03 19:20:44] iteration 142/ 6855 | consumed samples: 18176 | elapsed time per iteration (ms): 87681.3 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.42% | FLOPs estimator: workload-aware | logical tokens/s: 191342.9 | valid tokens/s: 191342.9 | physical tokens/s: 191743.2 | loss tokens/s: 144598.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.550843E-01 | lm-global-supervised-tokens: 1.267862E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.722820E-02 | lm-token-buffer-mib: 5.803197E+01 | global_load_balancing_loss: 9.994913E-01 | loss scale: 1.0 | grad norm: 0.078 | params norm: 11666.764 | params rms: 0.063172 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 19:30:04.267101 | finish at 2026-10-10 14:50:49 + [2026-10-03 19:22:10] iteration 143/ 6855 | consumed samples: 18304 | elapsed time per iteration (ms): 85974.6 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.41% | FLOPs estimator: workload-aware | logical tokens/s: 195141.3 | valid tokens/s: 195141.1 | physical tokens/s: 195645.1 | loss tokens/s: 146652.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.809582E-01 | lm-global-supervised-tokens: 1.260836E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.347546E-02 | lm-token-buffer-mib: 5.813319E+01 | global_load_balancing_loss: 1.000504E+00 | loss scale: 1.0 | grad norm: 0.057 | params norm: 11666.667 | params rms: 0.063171 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 16:17:41.182859 | finish at 2026-10-10 11:39:51 + [2026-10-03 19:23:39] iteration 144/ 6855 | consumed samples: 18432 | elapsed time per iteration (ms): 89219.6 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.42% | FLOPs estimator: workload-aware | logical tokens/s: 188044.0 | valid tokens/s: 188043.9 | physical tokens/s: 188509.9 | loss tokens/s: 137486.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.995054E-01 | lm-global-supervised-tokens: 1.226651E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.218812E-02 | lm-token-buffer-mib: 5.804739E+01 | global_load_balancing_loss: 1.000955E+00 | loss scale: 1.0 | grad norm: 0.104 | params norm: 11666.572 | params rms: 0.063171 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 22:19:12.924150 | finish at 2026-10-10 17:42:52 + [2026-10-03 19:25:10] iteration 145/ 6855 | consumed samples: 18560 | elapsed time per iteration (ms): 90629.1 | throughput per GPU (TFLOP/s/GPU): 40.6 | MFU 13.03% | FLOPs estimator: workload-aware | logical tokens/s: 185118.6 | valid tokens/s: 185118.3 | physical tokens/s: 185801.5 | loss tokens/s: 140993.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.080453E-01 | lm-global-supervised-tokens: 1.277812E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.162191E-02 | lm-token-buffer-mib: 5.844415E+01 | global_load_balancing_loss: 9.996411E-01 | loss scale: 1.0 | grad norm: 0.059 | params norm: 11666.475 | params rms: 0.063170 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 7 days, 0:55:21.413546 | finish at 2026-10-10 20:20:32 + [2026-10-03 19:26:39] iteration 146/ 6855 | consumed samples: 18688 | elapsed time per iteration (ms): 89155.9 | throughput per GPU (TFLOP/s/GPU): 41.3 | MFU 13.23% | FLOPs estimator: workload-aware | logical tokens/s: 188177.6 | valid tokens/s: 188177.6 | physical tokens/s: 188741.5 | loss tokens/s: 143861.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.874164E-01 | lm-global-supervised-tokens: 1.282606E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.130933E-02 | lm-token-buffer-mib: 5.822511E+01 | global_load_balancing_loss: 9.994904E-01 | loss scale: 1.0 | grad norm: 0.086 | params norm: 11666.377 | params rms: 0.063170 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 22:09:07.102663 | finish at 2026-10-10 17:35:46 + [2026-10-03 19:28:12] iteration 147/ 6855 | consumed samples: 18816 | elapsed time per iteration (ms): 92503.8 | throughput per GPU (TFLOP/s/GPU): 41.6 | MFU 13.33% | FLOPs estimator: workload-aware | logical tokens/s: 181367.8 | valid tokens/s: 181367.8 | physical tokens/s: 181596.9 | loss tokens/s: 129205.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.875824E-01 | lm-global-supervised-tokens: 1.195201E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.300569E-02 | lm-token-buffer-mib: 5.810738E+01 | global_load_balancing_loss: 1.000842E+00 | loss scale: 1.0 | grad norm: 0.041 | params norm: 11666.279 | params rms: 0.063169 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 7 days, 4:21:55.259531 | finish at 2026-10-10 23:50:07 + [2026-10-03 19:29:41] iteration 148/ 6855 | consumed samples: 18944 | elapsed time per iteration (ms): 89374.7 | throughput per GPU (TFLOP/s/GPU): 41.6 | MFU 13.34% | FLOPs estimator: workload-aware | logical tokens/s: 187717.0 | valid tokens/s: 187717.0 | physical tokens/s: 188107.2 | loss tokens/s: 134497.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.137862E-01 | lm-global-supervised-tokens: 1.202070E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.181580E-02 | lm-token-buffer-mib: 5.810745E+01 | global_load_balancing_loss: 1.001193E+00 | loss scale: 1.0 | grad norm: 0.060 | params norm: 11666.181 | params rms: 0.063169 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 22:30:36.239692 | finish at 2026-10-10 18:00:17 + [2026-10-03 19:31:08] iteration 149/ 6855 | consumed samples: 19072 | elapsed time per iteration (ms): 87066.5 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.49% | FLOPs estimator: workload-aware | logical tokens/s: 192694.3 | valid tokens/s: 192694.3 | physical tokens/s: 193096.4 | loss tokens/s: 142070.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.377513E-01 | lm-global-supervised-tokens: 1.236958E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.601177E-02 | lm-token-buffer-mib: 5.810029E+01 | global_load_balancing_loss: 9.997823E-01 | loss scale: 1.0 | grad norm: 0.042 | params norm: 11666.083 | params rms: 0.063168 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 18:11:07.785573 | finish at 2026-10-10 13:42:16 + [2026-10-03 19:32:46] iteration 150/ 6855 | consumed samples: 19200 | elapsed time per iteration (ms): 97895.0 | throughput per GPU (TFLOP/s/GPU): 36.7 | MFU 11.75% | FLOPs estimator: workload-aware | logical tokens/s: 171379.6 | valid tokens/s: 171379.6 | physical tokens/s: 171952.0 | loss tokens/s: 130811.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.394553E-01 | lm-global-supervised-tokens: 1.280580E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.555138E-02 | lm-token-buffer-mib: 5.807834E+01 | global_load_balancing_loss: 9.995544E-01 | loss scale: 1.0 | grad norm: 0.059 | params norm: 11665.986 | params rms: 0.063168 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 7 days, 14:19:45.653554 | finish at 2026-10-11 09:52:32 + [2026-10-03 19:34:15] iteration 151/ 6855 | consumed samples: 19328 | elapsed time per iteration (ms): 89065.9 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.48% | FLOPs estimator: workload-aware | logical tokens/s: 188368.5 | valid tokens/s: 188368.5 | physical tokens/s: 188855.5 | loss tokens/s: 137290.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.184455E-01 | lm-global-supervised-tokens: 1.222794E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.354622E-02 | lm-token-buffer-mib: 5.810513E+01 | global_load_balancing_loss: 9.996560E-01 | loss scale: 1.0 | grad norm: 0.058 | params norm: 11665.888 | params rms: 0.063167 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 21:51:38.054718 | finish at 2026-10-10 17:25:53 + [2026-10-03 19:35:42] iteration 152/ 6855 | consumed samples: 19456 | elapsed time per iteration (ms): 86754.1 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.78% | FLOPs estimator: workload-aware | logical tokens/s: 193387.4 | valid tokens/s: 193387.4 | physical tokens/s: 193847.3 | loss tokens/s: 141408.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.733987E-01 | lm-global-supervised-tokens: 1.226780E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.211120E-02 | lm-token-buffer-mib: 5.826845E+01 | global_load_balancing_loss: 1.000597E+00 | loss scale: 1.0 | grad norm: 0.052 | params norm: 11665.790 | params rms: 0.063166 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 17:31:52.977377 | finish at 2026-10-10 13:07:35 + [2026-10-03 19:37:06] iteration 153/ 6855 | consumed samples: 19584 | elapsed time per iteration (ms): 84225.8 | throughput per GPU (TFLOP/s/GPU): 43.7 | MFU 13.99% | FLOPs estimator: workload-aware | logical tokens/s: 199192.7 | valid tokens/s: 199192.6 | physical tokens/s: 199533.8 | loss tokens/s: 144147.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.737400E-01 | lm-global-supervised-tokens: 1.214092E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.127120E-02 | lm-token-buffer-mib: 5.810403E+01 | global_load_balancing_loss: 9.998702E-01 | loss scale: 1.0 | grad norm: 0.044 | params norm: 11665.691 | params rms: 0.063166 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 12:48:01.217576 | finish at 2026-10-10 08:25:07 + [2026-10-03 19:38:33] iteration 154/ 6855 | consumed samples: 19712 | elapsed time per iteration (ms): 86716.0 | throughput per GPU (TFLOP/s/GPU): 43.2 | MFU 13.86% | FLOPs estimator: workload-aware | logical tokens/s: 193473.0 | valid tokens/s: 193473.0 | physical tokens/s: 193879.7 | loss tokens/s: 140858.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.733922E-01 | lm-global-supervised-tokens: 1.221464E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.142651E-02 | lm-token-buffer-mib: 5.804448E+01 | global_load_balancing_loss: 1.001124E+00 | loss scale: 1.0 | grad norm: 0.075 | params norm: 11665.593 | params rms: 0.063165 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 17:24:43.745589 | finish at 2026-10-10 13:03:17 + [2026-10-03 19:40:01] iteration 155/ 6855 | consumed samples: 19840 | elapsed time per iteration (ms): 87836.1 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.62% | FLOPs estimator: workload-aware | logical tokens/s: 191005.6 | valid tokens/s: 191005.6 | physical tokens/s: 191499.6 | loss tokens/s: 140125.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.180639E-01 | lm-global-supervised-tokens: 1.230811E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.268854E-02 | lm-token-buffer-mib: 5.805672E+01 | global_load_balancing_loss: 1.000754E+00 | loss scale: 1.0 | grad norm: 0.112 | params norm: 11665.496 | params rms: 0.063165 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 19:28:21.739693 | finish at 2026-10-10 15:08:22 + [2026-10-03 19:41:28] iteration 156/ 6855 | consumed samples: 19968 | elapsed time per iteration (ms): 86883.8 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.79% | FLOPs estimator: workload-aware | logical tokens/s: 193098.3 | valid tokens/s: 193098.3 | physical tokens/s: 193495.3 | loss tokens/s: 135374.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.718810E-01 | lm-global-supervised-tokens: 1.176182E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.599174E-02 | lm-token-buffer-mib: 5.812878E+01 | global_load_balancing_loss: 1.000273E+00 | loss scale: 1.0 | grad norm: 0.077 | params norm: 11665.398 | params rms: 0.063164 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 17:40:34.306478 | finish at 2026-10-10 13:22:02 + [2026-10-03 19:42:56] iteration 157/ 6855 | consumed samples: 20096 | elapsed time per iteration (ms): 88607.8 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.62% | FLOPs estimator: workload-aware | logical tokens/s: 189341.8 | valid tokens/s: 189341.8 | physical tokens/s: 189856.8 | loss tokens/s: 141597.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.040536E-01 | lm-global-supervised-tokens: 1.254666E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.309219E-02 | lm-token-buffer-mib: 5.806534E+01 | global_load_balancing_loss: 1.000996E+00 | loss scale: 1.0 | grad norm: 0.029 | params norm: 11665.301 | params rms: 0.063164 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 20:51:34.728254 | finish at 2026-10-10 16:34:31 + [2026-10-03 19:44:24] iteration 158/ 6855 | consumed samples: 20224 | elapsed time per iteration (ms): 87790.6 | throughput per GPU (TFLOP/s/GPU): 43.6 | MFU 13.97% | FLOPs estimator: workload-aware | logical tokens/s: 191104.7 | valid tokens/s: 191104.7 | physical tokens/s: 191523.0 | loss tokens/s: 133942.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.147675E-01 | lm-global-supervised-tokens: 1.175888E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.295965E-02 | lm-token-buffer-mib: 5.802869E+01 | global_load_balancing_loss: 9.999350E-01 | loss scale: 1.0 | grad norm: 0.051 | params norm: 11665.205 | params rms: 0.063163 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 19:18:53.822650 | finish at 2026-10-10 15:03:18 + [2026-10-03 19:45:51] iteration 159/ 6855 | consumed samples: 20352 | elapsed time per iteration (ms): 87030.7 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.74% | FLOPs estimator: workload-aware | logical tokens/s: 192772.9 | valid tokens/s: 192772.9 | physical tokens/s: 193140.7 | loss tokens/s: 142452.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.823774E-01 | lm-global-supervised-tokens: 1.239771E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.238851E-02 | lm-token-buffer-mib: 5.807170E+01 | global_load_balancing_loss: 9.994878E-01 | loss scale: 1.0 | grad norm: 0.048 | params norm: 11665.110 | params rms: 0.063163 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 17:52:37.412132 | finish at 2026-10-10 13:38:28 + [2026-10-03 19:47:37] iteration 160/ 6855 | consumed samples: 20480 | elapsed time per iteration (ms): 105991.1 | throughput per GPU (TFLOP/s/GPU): 35.8 | MFU 11.48% | FLOPs estimator: workload-aware | logical tokens/s: 158288.6 | valid tokens/s: 158288.6 | physical tokens/s: 158735.4 | loss tokens/s: 114078.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.154168E-01 | lm-global-supervised-tokens: 1.209130E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.125151E-02 | lm-token-buffer-mib: 5.810471E+01 | global_load_balancing_loss: 1.000464E+00 | loss scale: 1.0 | grad norm: 0.068 | params norm: 11665.017 | params rms: 0.063162 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 8 days, 5:06:50.427758 | finish at 2026-10-12 00:54:27 +times across ranks (ms): + save-checkpoint: + rank 0: 24254.91 + rank 1: 24254.91 + rank 2: 24254.91 + rank 3: 24254.89 + rank 4: 24254.92 + rank 5: 24254.88 + rank 6: 24254.88 + rank 7: 24254.90 + rank 8: 24256.07 + rank 9: 24256.06 + rank 10: 24256.08 + rank 11: 24256.05 + rank 12: 24256.07 + rank 13: 24256.07 + rank 14: 24256.05 + rank 15: 24256.06 + rank 16: 24255.24 + rank 17: 24255.25 + rank 18: 24255.20 + rank 19: 24255.22 + rank 20: 24255.21 + rank 21: 24255.27 + rank 22: 24255.22 + rank 23: 24255.21 + rank 24: 24255.76 + rank 25: 24255.75 + rank 26: 24255.77 + rank 27: 24255.76 + rank 28: 24255.73 + rank 29: 24255.75 + rank 30: 24255.74 + rank 31: 24255.78 + rank 32: 24255.04 + rank 33: 24255.04 + rank 34: 24255.01 + rank 35: 24255.01 + rank 36: 24255.01 + rank 37: 24255.00 + rank 38: 24255.01 + rank 39: 24255.00 + rank 40: 24255.04 + rank 41: 24255.04 + rank 42: 24255.04 + rank 43: 24255.05 + rank 44: 24255.02 + rank 45: 24255.10 + rank 46: 24255.04 + rank 47: 24255.09 + rank 48: 24255.73 + rank 49: 24255.70 + rank 50: 24255.72 + rank 51: 24255.70 + rank 52: 24255.72 + rank 53: 24255.71 + rank 54: 24255.70 + rank 55: 24255.70 + rank 56: 24255.60 + rank 57: 24255.58 + rank 58: 24255.55 + rank 59: 24255.57 + rank 60: 24255.55 + rank 61: 24255.59 + rank 62: 24255.56 + rank 63: 24255.57 + [2026-10-03 19:49:25] iteration 161/ 6855 | consumed samples: 20608 | elapsed time per iteration (ms): 83876.8 | throughput per GPU (TFLOP/s/GPU): 44.0 | MFU 14.10% | FLOPs estimator: workload-aware | logical tokens/s: 200021.9 | valid tokens/s: 200021.9 | physical tokens/s: 200460.4 | loss tokens/s: 151336.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.801457E-01 | lm-global-supervised-tokens: 1.269361E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.761854E-01 | lm-token-buffer-mib: 5.815447E+01 | global_load_balancing_loss: 9.998522E-01 | loss scale: 1.0 | grad norm: 0.132 | params norm: 11664.922 | params rms: 0.063162 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 11:57:51.133622 | finish at 2026-10-10 07:47:16 + [2026-10-03 19:50:58] iteration 162/ 6855 | consumed samples: 20736 | elapsed time per iteration (ms): 93083.3 | throughput per GPU (TFLOP/s/GPU): 40.8 | MFU 13.06% | FLOPs estimator: workload-aware | logical tokens/s: 180238.3 | valid tokens/s: 180238.3 | physical tokens/s: 180743.7 | loss tokens/s: 129207.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.141951E-01 | lm-global-supervised-tokens: 1.202702E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.641440E-02 | lm-token-buffer-mib: 5.808960E+01 | global_load_balancing_loss: 1.000683E+00 | loss scale: 1.0 | grad norm: 0.054 | params norm: 11664.829 | params rms: 0.063161 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 7 days, 5:03:26.586703 | finish at 2026-10-11 00:54:25 + [2026-10-03 19:52:22] iteration 163/ 6855 | consumed samples: 20864 | elapsed time per iteration (ms): 83746.3 | throughput per GPU (TFLOP/s/GPU): 43.2 | MFU 13.85% | FLOPs estimator: workload-aware | logical tokens/s: 200333.6 | valid tokens/s: 200333.6 | physical tokens/s: 200814.9 | loss tokens/s: 151289.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.964725E-01 | lm-global-supervised-tokens: 1.266997E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.332510E-02 | lm-token-buffer-mib: 5.815120E+01 | global_load_balancing_loss: 1.000219E+00 | loss scale: 1.0 | grad norm: 0.115 | params norm: 11664.735 | params rms: 0.063161 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 11:40:30.269794 | finish at 2026-10-10 07:32:52 + [2026-10-03 19:53:45] iteration 164/ 6855 | consumed samples: 20992 | elapsed time per iteration (ms): 83128.2 | throughput per GPU (TFLOP/s/GPU): 43.9 | MFU 14.07% | FLOPs estimator: workload-aware | logical tokens/s: 201821.9 | valid tokens/s: 201821.9 | physical tokens/s: 202196.5 | loss tokens/s: 150843.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.633695E-01 | lm-global-supervised-tokens: 1.253936E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.165726E-02 | lm-token-buffer-mib: 5.806903E+01 | global_load_balancing_loss: 1.000141E+00 | loss scale: 1.0 | grad norm: 0.112 | params norm: 11664.641 | params rms: 0.063160 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 10:30:10.665323 | finish at 2026-10-10 06:23:56 + [2026-10-03 19:55:13] iteration 165/ 6855 | consumed samples: 21120 | elapsed time per iteration (ms): 87519.0 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.58% | FLOPs estimator: workload-aware | logical tokens/s: 191698.0 | valid tokens/s: 191698.0 | physical tokens/s: 192214.3 | loss tokens/s: 140472.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.599715E-01 | lm-global-supervised-tokens: 1.229402E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.827677E-02 | lm-token-buffer-mib: 5.814785E+01 | global_load_balancing_loss: 1.000109E+00 | loss scale: 1.0 | grad norm: 0.049 | params norm: 11664.546 | params rms: 0.063160 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 18:38:21.989136 | finish at 2026-10-10 14:33:35 + [2026-10-03 19:56:41] iteration 166/ 6855 | consumed samples: 21248 | elapsed time per iteration (ms): 88521.2 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.63% | FLOPs estimator: workload-aware | logical tokens/s: 189527.4 | valid tokens/s: 189527.4 | physical tokens/s: 189923.6 | loss tokens/s: 142193.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.529560E-01 | lm-global-supervised-tokens: 1.258715E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.139856E-02 | lm-token-buffer-mib: 5.839220E+01 | global_load_balancing_loss: 1.000998E+00 | loss scale: 1.0 | grad norm: 0.088 | params norm: 11664.451 | params rms: 0.063159 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 20:28:38.004996 | finish at 2026-10-10 16:25:19 + [2026-10-03 19:58:09] iteration 167/ 6855 | consumed samples: 21376 | elapsed time per iteration (ms): 87329.1 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.69% | FLOPs estimator: workload-aware | logical tokens/s: 192114.4 | valid tokens/s: 192114.4 | physical tokens/s: 192502.7 | loss tokens/s: 145077.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.199993E-01 | lm-global-supervised-tokens: 1.266947E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.386230E-02 | lm-token-buffer-mib: 5.807605E+01 | global_load_balancing_loss: 9.994255E-01 | loss scale: 1.0 | grad norm: 0.057 | params norm: 11664.357 | params rms: 0.063159 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 18:14:16.916443 | finish at 2026-10-10 14:12:25 + [2026-10-03 19:59:35] iteration 168/ 6855 | consumed samples: 21504 | elapsed time per iteration (ms): 86574.8 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.42% | FLOPs estimator: workload-aware | logical tokens/s: 193788.2 | valid tokens/s: 193788.2 | physical tokens/s: 194217.2 | loss tokens/s: 147951.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.957372E-01 | lm-global-supervised-tokens: 1.280884E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.477290E-02 | lm-token-buffer-mib: 5.813430E+01 | global_load_balancing_loss: 9.995443E-01 | loss scale: 1.0 | grad norm: 0.138 | params norm: 11664.263 | params rms: 0.063158 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 16:48:45.925248 | finish at 2026-10-10 12:48:21 + [2026-10-03 20:01:02] iteration 169/ 6855 | consumed samples: 21632 | elapsed time per iteration (ms): 87155.8 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.64% | FLOPs estimator: workload-aware | logical tokens/s: 192496.4 | valid tokens/s: 192496.4 | physical tokens/s: 192811.3 | loss tokens/s: 142919.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.057856E-01 | lm-global-supervised-tokens: 1.245626E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.554465E-02 | lm-token-buffer-mib: 5.803080E+01 | global_load_balancing_loss: 9.999175E-01 | loss scale: 1.0 | grad norm: 0.048 | params norm: 11664.170 | params rms: 0.063158 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 17:52:03.786299 | finish at 2026-10-10 13:53:06 + [2026-10-03 20:02:45] iteration 170/ 6855 | consumed samples: 21760 | elapsed time per iteration (ms): 103149.5 | throughput per GPU (TFLOP/s/GPU): 35.2 | MFU 11.27% | FLOPs estimator: workload-aware | logical tokens/s: 162648.5 | valid tokens/s: 162648.5 | physical tokens/s: 163208.9 | loss tokens/s: 120863.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.533022E-01 | lm-global-supervised-tokens: 1.246703E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.140760E-02 | lm-token-buffer-mib: 5.814680E+01 | global_load_balancing_loss: 1.001536E+00 | loss scale: 1.0 | grad norm: 0.156 | params norm: 11664.076 | params rms: 0.063157 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 7 days, 23:32:34.427506 | finish at 2026-10-11 19:35:20 + [2026-10-03 20:04:16] iteration 171/ 6855 | consumed samples: 21888 | elapsed time per iteration (ms): 91044.3 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.39% | FLOPs estimator: workload-aware | logical tokens/s: 184274.8 | valid tokens/s: 184274.8 | physical tokens/s: 184611.5 | loss tokens/s: 129700.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.698949E-01 | lm-global-supervised-tokens: 1.180845E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.132457E-02 | lm-token-buffer-mib: 5.824341E+01 | global_load_balancing_loss: 1.000961E+00 | loss scale: 1.0 | grad norm: 0.119 | params norm: 11663.983 | params rms: 0.063157 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 7 days, 1:02:20.237185 | finish at 2026-10-10 21:06:37 + [2026-10-03 20:05:45] iteration 172/ 6855 | consumed samples: 22016 | elapsed time per iteration (ms): 88078.5 | throughput per GPU (TFLOP/s/GPU): 41.5 | MFU 13.29% | FLOPs estimator: workload-aware | logical tokens/s: 190480.3 | valid tokens/s: 190480.3 | physical tokens/s: 190845.9 | loss tokens/s: 147036.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.594803E-01 | lm-global-supervised-tokens: 1.295073E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.323047E-02 | lm-token-buffer-mib: 5.826033E+01 | global_load_balancing_loss: 9.999883E-01 | loss scale: 1.0 | grad norm: 0.138 | params norm: 11663.890 | params rms: 0.063156 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 19:30:28.300233 | finish at 2026-10-10 15:36:13 + [2026-10-03 20:07:14] iteration 173/ 6855 | consumed samples: 22144 | elapsed time per iteration (ms): 89296.5 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.41% | FLOPs estimator: workload-aware | logical tokens/s: 187882.0 | valid tokens/s: 187882.0 | physical tokens/s: 188436.3 | loss tokens/s: 138594.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.323656E-01 | lm-global-supervised-tokens: 1.237602E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.158985E-02 | lm-token-buffer-mib: 5.822533E+01 | global_load_balancing_loss: 9.997371E-01 | loss scale: 1.0 | grad norm: 0.115 | params norm: 11663.794 | params rms: 0.063156 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 21:44:39.164990 | finish at 2026-10-10 17:51:53 + [2026-10-03 20:08:44] iteration 174/ 6855 | consumed samples: 22272 | elapsed time per iteration (ms): 89836.2 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.46% | FLOPs estimator: workload-aware | logical tokens/s: 186753.2 | valid tokens/s: 186753.2 | physical tokens/s: 187206.7 | loss tokens/s: 133328.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.201192E-01 | lm-global-supervised-tokens: 1.197770E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.704786E-02 | lm-token-buffer-mib: 5.819193E+01 | global_load_balancing_loss: 1.000687E+00 | loss scale: 1.0 | grad norm: 0.214 | params norm: 11663.698 | params rms: 0.063155 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 22:43:15.532727 | finish at 2026-10-10 18:51:59 + [2026-10-03 20:10:10] iteration 175/ 6855 | consumed samples: 22400 | elapsed time per iteration (ms): 86560.0 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.66% | FLOPs estimator: workload-aware | logical tokens/s: 193821.6 | valid tokens/s: 193821.6 | physical tokens/s: 194242.5 | loss tokens/s: 142122.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.106686E-01 | lm-global-supervised-tokens: 1.230210E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.391809E-02 | lm-token-buffer-mib: 5.807077E+01 | global_load_balancing_loss: 9.997253E-01 | loss scale: 1.0 | grad norm: 0.059 | params norm: 11663.603 | params rms: 0.063155 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 16:37:00.739098 | finish at 2026-10-10 12:47:11 + [2026-10-03 20:11:41] iteration 176/ 6855 | consumed samples: 22528 | elapsed time per iteration (ms): 90812.3 | throughput per GPU (TFLOP/s/GPU): 41.6 | MFU 13.32% | FLOPs estimator: workload-aware | logical tokens/s: 184745.4 | valid tokens/s: 184745.4 | physical tokens/s: 185101.2 | loss tokens/s: 140070.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.361662E-01 | lm-global-supervised-tokens: 1.272013E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.546445E-02 | lm-token-buffer-mib: 5.820628E+01 | global_load_balancing_loss: 9.995347E-01 | loss scale: 1.0 | grad norm: 0.178 | params norm: 11663.511 | params rms: 0.063154 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 7 days, 0:28:55.568044 | finish at 2026-10-10 20:40:37 + [2026-10-03 20:13:12] iteration 177/ 6855 | consumed samples: 22656 | elapsed time per iteration (ms): 90625.5 | throughput per GPU (TFLOP/s/GPU): 41.2 | MFU 13.20% | FLOPs estimator: workload-aware | logical tokens/s: 185126.0 | valid tokens/s: 185126.0 | physical tokens/s: 185615.0 | loss tokens/s: 134964.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.798651E-01 | lm-global-supervised-tokens: 1.223118E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.218355E-02 | lm-token-buffer-mib: 5.804230E+01 | global_load_balancing_loss: 9.996916E-01 | loss scale: 1.0 | grad norm: 0.070 | params norm: 11663.417 | params rms: 0.063154 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 7 days, 0:06:37.087166 | finish at 2026-10-10 20:19:49 + [2026-10-03 20:14:39] iteration 178/ 6855 | consumed samples: 22784 | elapsed time per iteration (ms): 87191.4 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.49% | FLOPs estimator: workload-aware | logical tokens/s: 192417.9 | valid tokens/s: 192417.9 | physical tokens/s: 192869.4 | loss tokens/s: 143027.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.335720E-01 | lm-global-supervised-tokens: 1.247079E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.705034E-02 | lm-token-buffer-mib: 5.810567E+01 | global_load_balancing_loss: 1.001477E+00 | loss scale: 1.0 | grad norm: 0.178 | params norm: 11663.321 | params rms: 0.063153 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 17:42:57.172356 | finish at 2026-10-10 13:57:36 + [2026-10-03 20:16:07] iteration 179/ 6855 | consumed samples: 22912 | elapsed time per iteration (ms): 87754.9 | throughput per GPU (TFLOP/s/GPU): 43.3 | MFU 13.87% | FLOPs estimator: workload-aware | logical tokens/s: 191181.6 | valid tokens/s: 191181.6 | physical tokens/s: 191785.5 | loss tokens/s: 136642.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.105642E-01 | lm-global-supervised-tokens: 1.199109E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.216180E-02 | lm-token-buffer-mib: 5.813163E+01 | global_load_balancing_loss: 1.001389E+00 | loss scale: 1.0 | grad norm: 0.081 | params norm: 11663.225 | params rms: 0.063153 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 18:44:11.952601 | finish at 2026-10-10 15:00:19 + [2026-10-03 20:17:49] iteration 180/ 6855 | consumed samples: 23040 | elapsed time per iteration (ms): 102273.8 | throughput per GPU (TFLOP/s/GPU): 36.2 | MFU 11.61% | FLOPs estimator: workload-aware | logical tokens/s: 164041.8 | valid tokens/s: 164041.7 | physical tokens/s: 164470.0 | loss tokens/s: 116905.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.520887E-01 | lm-global-supervised-tokens: 1.195635E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.137400E-02 | lm-token-buffer-mib: 5.804685E+01 | global_load_balancing_loss: 1.000194E+00 | loss scale: 1.0 | grad norm: 0.071 | params norm: 11663.130 | params rms: 0.063152 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 7 days, 21:37:57.292836 | finish at 2026-10-11 17:55:46 + [2026-10-03 20:19:18] iteration 181/ 6855 | consumed samples: 23168 | elapsed time per iteration (ms): 89535.9 | throughput per GPU (TFLOP/s/GPU): 41.1 | MFU 13.17% | FLOPs estimator: workload-aware | logical tokens/s: 187379.5 | valid tokens/s: 187379.5 | physical tokens/s: 187818.9 | loss tokens/s: 139525.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.186550E-01 | lm-global-supervised-tokens: 1.249252E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.509675E-02 | lm-token-buffer-mib: 5.821257E+01 | global_load_balancing_loss: 1.000314E+00 | loss scale: 1.0 | grad norm: 0.137 | params norm: 11663.035 | params rms: 0.063152 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 21:59:22.452574 | finish at 2026-10-10 18:18:41 + [2026-10-03 20:20:45] iteration 182/ 6855 | consumed samples: 23296 | elapsed time per iteration (ms): 86308.1 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.51% | FLOPs estimator: workload-aware | logical tokens/s: 194386.8 | valid tokens/s: 194386.8 | physical tokens/s: 194947.6 | loss tokens/s: 143725.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.768382E-01 | lm-global-supervised-tokens: 1.240465E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.596966E-02 | lm-token-buffer-mib: 5.806155E+01 | global_load_balancing_loss: 1.000347E+00 | loss scale: 1.0 | grad norm: 0.050 | params norm: 11662.936 | params rms: 0.063151 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 15:58:53.804832 | finish at 2026-10-10 12:19:39 + [2026-10-03 20:22:10] iteration 183/ 6855 | consumed samples: 23424 | elapsed time per iteration (ms): 85002.0 | throughput per GPU (TFLOP/s/GPU): 42.8 | MFU 13.70% | FLOPs estimator: workload-aware | logical tokens/s: 197374.0 | valid tokens/s: 197374.0 | physical tokens/s: 197899.6 | loss tokens/s: 147809.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.771085E-01 | lm-global-supervised-tokens: 1.256411E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.388283E-02 | lm-token-buffer-mib: 5.822804E+01 | global_load_balancing_loss: 9.992791E-01 | loss scale: 1.0 | grad norm: 0.108 | params norm: 11662.838 | params rms: 0.063150 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 13:32:13.085335 | finish at 2026-10-10 09:54:23 + [2026-10-03 20:23:38] iteration 184/ 6855 | consumed samples: 23552 | elapsed time per iteration (ms): 88520.6 | throughput per GPU (TFLOP/s/GPU): 41.5 | MFU 13.31% | FLOPs estimator: workload-aware | logical tokens/s: 189528.5 | valid tokens/s: 189528.5 | physical tokens/s: 189917.8 | loss tokens/s: 141958.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.445370E-01 | lm-global-supervised-tokens: 1.256627E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.288595E-02 | lm-token-buffer-mib: 5.820652E+01 | global_load_balancing_loss: 9.999279E-01 | loss scale: 1.0 | grad norm: 0.045 | params norm: 11662.743 | params rms: 0.063150 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 20:02:00.818165 | finish at 2026-10-10 16:25:39 + [2026-10-03 20:25:05] iteration 185/ 6855 | consumed samples: 23680 | elapsed time per iteration (ms): 86805.7 | throughput per GPU (TFLOP/s/GPU): 41.3 | MFU 13.22% | FLOPs estimator: workload-aware | logical tokens/s: 193273.2 | valid tokens/s: 193273.2 | physical tokens/s: 193738.4 | loss tokens/s: 149719.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.666114E-01 | lm-global-supervised-tokens: 1.299648E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.140320E-02 | lm-token-buffer-mib: 5.830133E+01 | global_load_balancing_loss: 9.999302E-01 | loss scale: 1.0 | grad norm: 0.072 | params norm: 11662.646 | params rms: 0.063149 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 16:49:54.113250 | finish at 2026-10-10 13:14:59 + [2026-10-03 20:26:34] iteration 186/ 6855 | consumed samples: 23808 | elapsed time per iteration (ms): 88689.3 | throughput per GPU (TFLOP/s/GPU): 41.7 | MFU 13.36% | FLOPs estimator: workload-aware | logical tokens/s: 189167.7 | valid tokens/s: 189167.7 | physical tokens/s: 189537.5 | loss tokens/s: 137871.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.671106E-01 | lm-global-supervised-tokens: 1.222769E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.825066E-02 | lm-token-buffer-mib: 5.810427E+01 | global_load_balancing_loss: 1.000449E+00 | loss scale: 1.0 | grad norm: 0.070 | params norm: 11662.549 | params rms: 0.063149 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 20:17:49.102693 | finish at 2026-10-10 16:44:23 + [2026-10-03 20:28:03] iteration 187/ 6855 | consumed samples: 23936 | elapsed time per iteration (ms): 88935.0 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.49% | FLOPs estimator: workload-aware | logical tokens/s: 188645.1 | valid tokens/s: 188645.1 | physical tokens/s: 189017.6 | loss tokens/s: 136997.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.551652E-01 | lm-global-supervised-tokens: 1.218383E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.158140E-02 | lm-token-buffer-mib: 5.802410E+01 | global_load_balancing_loss: 1.000878E+00 | loss scale: 1.0 | grad norm: 0.049 | params norm: 11662.449 | params rms: 0.063148 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 20:43:38.377717 | finish at 2026-10-10 17:11:41 + [2026-10-03 20:29:29] iteration 188/ 6855 | consumed samples: 24064 | elapsed time per iteration (ms): 85908.5 | throughput per GPU (TFLOP/s/GPU): 43.1 | MFU 13.80% | FLOPs estimator: workload-aware | logical tokens/s: 195291.1 | valid tokens/s: 195291.1 | physical tokens/s: 195625.2 | loss tokens/s: 148179.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.061743E-01 | lm-global-supervised-tokens: 1.272988E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.471197E-02 | lm-token-buffer-mib: 5.812192E+01 | global_load_balancing_loss: 9.998574E-01 | loss scale: 1.0 | grad norm: 0.069 | params norm: 11662.350 | params rms: 0.063148 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 15:05:52.228302 | finish at 2026-10-10 11:35:21 + [2026-10-03 20:30:53] iteration 189/ 6855 | consumed samples: 24192 | elapsed time per iteration (ms): 84094.9 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.74% | FLOPs estimator: workload-aware | logical tokens/s: 199503.2 | valid tokens/s: 199503.2 | physical tokens/s: 199975.4 | loss tokens/s: 157648.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.922719E-01 | lm-global-supervised-tokens: 1.325746E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.194716E-02 | lm-token-buffer-mib: 5.808718E+01 | global_load_balancing_loss: 9.989747E-01 | loss scale: 1.0 | grad norm: 0.045 | params norm: 11662.253 | params rms: 0.063147 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 11:42:56.698043 | finish at 2026-10-10 08:13:49 + [2026-10-03 20:32:34] iteration 190/ 6855 | consumed samples: 24320 | elapsed time per iteration (ms): 100993.1 | throughput per GPU (TFLOP/s/GPU): 36.5 | MFU 11.69% | FLOPs estimator: workload-aware | logical tokens/s: 166121.7 | valid tokens/s: 166121.7 | physical tokens/s: 166477.7 | loss tokens/s: 122824.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.065052E-01 | lm-global-supervised-tokens: 1.240445E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.371400E-02 | lm-token-buffer-mib: 5.816135E+01 | global_load_balancing_loss: 1.000261E+00 | loss scale: 1.0 | grad norm: 0.102 | params norm: 11662.153 | params rms: 0.063147 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 7 days, 18:58:39.036444 | finish at 2026-10-11 15:31:13 + [2026-10-03 20:34:02] iteration 191/ 6855 | consumed samples: 24448 | elapsed time per iteration (ms): 88786.3 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.67% | FLOPs estimator: workload-aware | logical tokens/s: 188961.2 | valid tokens/s: 188961.2 | physical tokens/s: 189385.7 | loss tokens/s: 137478.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.926616E-01 | lm-global-supervised-tokens: 1.220621E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.131650E-02 | lm-token-buffer-mib: 5.813416E+01 | global_load_balancing_loss: 1.001378E+00 | loss scale: 1.0 | grad norm: 0.043 | params norm: 11662.051 | params rms: 0.063146 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 20:21:11.751888 | finish at 2026-10-10 16:55:14 + [2026-10-03 20:35:29] iteration 192/ 6855 | consumed samples: 24576 | elapsed time per iteration (ms): 86662.0 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.70% | FLOPs estimator: workload-aware | logical tokens/s: 193593.4 | valid tokens/s: 193593.4 | physical tokens/s: 194021.7 | loss tokens/s: 142738.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.285021E-01 | lm-global-supervised-tokens: 1.237003E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.287342E-02 | lm-token-buffer-mib: 5.816062E+01 | global_load_balancing_loss: 9.999530E-01 | loss scale: 1.0 | grad norm: 0.082 | params norm: 11661.950 | params rms: 0.063146 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 16:23:48.618073 | finish at 2026-10-10 12:59:18 + [2026-10-03 20:36:56] iteration 193/ 6855 | consumed samples: 24704 | elapsed time per iteration (ms): 87319.5 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.55% | FLOPs estimator: workload-aware | logical tokens/s: 192135.7 | valid tokens/s: 192135.6 | physical tokens/s: 192511.0 | loss tokens/s: 140291.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.881802E-01 | lm-global-supervised-tokens: 1.225015E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.149625E-02 | lm-token-buffer-mib: 5.810139E+01 | global_load_balancing_loss: 1.001103E+00 | loss scale: 1.0 | grad norm: 0.095 | params norm: 11661.850 | params rms: 0.063145 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 17:35:22.607274 | finish at 2026-10-10 14:12:19 + [2026-10-03 20:38:23] iteration 194/ 6855 | consumed samples: 24832 | elapsed time per iteration (ms): 86276.6 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.64% | FLOPs estimator: workload-aware | logical tokens/s: 194458.1 | valid tokens/s: 194458.1 | physical tokens/s: 194976.4 | loss tokens/s: 142138.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.844810E-01 | lm-global-supervised-tokens: 1.226321E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.767868E-02 | lm-token-buffer-mib: 5.829736E+01 | global_load_balancing_loss: 1.000838E+00 | loss scale: 1.0 | grad norm: 0.057 | params norm: 11661.749 | params rms: 0.063145 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 15:38:08.112618 | finish at 2026-10-10 12:16:31 + [2026-10-03 20:39:52] iteration 195/ 6855 | consumed samples: 24960 | elapsed time per iteration (ms): 89170.8 | throughput per GPU (TFLOP/s/GPU): 41.7 | MFU 13.38% | FLOPs estimator: workload-aware | logical tokens/s: 188145.8 | valid tokens/s: 188145.8 | physical tokens/s: 188765.9 | loss tokens/s: 142368.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.967399E-01 | lm-global-supervised-tokens: 1.269514E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.150049E-02 | lm-token-buffer-mib: 5.831802E+01 | global_load_balancing_loss: 9.991494E-01 | loss scale: 1.0 | grad norm: 0.072 | params norm: 11661.649 | params rms: 0.063144 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 20:57:57.832675 | finish at 2026-10-10 17:37:50 + [2026-10-03 20:41:20] iteration 196/ 6855 | consumed samples: 25088 | elapsed time per iteration (ms): 87656.6 | throughput per GPU (TFLOP/s/GPU): 41.5 | MFU 13.30% | FLOPs estimator: workload-aware | logical tokens/s: 191396.3 | valid tokens/s: 191396.3 | physical tokens/s: 191841.2 | loss tokens/s: 145631.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.688633E-01 | lm-global-supervised-tokens: 1.276556E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.483689E-02 | lm-token-buffer-mib: 5.819445E+01 | global_load_balancing_loss: 9.999409E-01 | loss scale: 1.0 | grad norm: 0.111 | params norm: 11661.550 | params rms: 0.063143 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 18:08:25.161266 | finish at 2026-10-10 14:49:45 + [2026-10-03 20:42:45] iteration 197/ 6855 | consumed samples: 25216 | elapsed time per iteration (ms): 85881.9 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.56% | FLOPs estimator: workload-aware | logical tokens/s: 195351.9 | valid tokens/s: 195351.9 | physical tokens/s: 195945.3 | loss tokens/s: 144065.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.167341E-01 | lm-global-supervised-tokens: 1.237258E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.535373E-02 | lm-token-buffer-mib: 5.823492E+01 | global_load_balancing_loss: 9.997711E-01 | loss scale: 1.0 | grad norm: 0.110 | params norm: 11661.451 | params rms: 0.063143 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 14:50:01.938313 | finish at 2026-10-10 11:32:47 + [2026-10-03 20:44:13] iteration 198/ 6855 | consumed samples: 25344 | elapsed time per iteration (ms): 87562.1 | throughput per GPU (TFLOP/s/GPU): 41.7 | MFU 13.37% | FLOPs estimator: workload-aware | logical tokens/s: 191603.2 | valid tokens/s: 191603.2 | physical tokens/s: 192033.7 | loss tokens/s: 144278.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.315550E-01 | lm-global-supervised-tokens: 1.263329E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.155358E-02 | lm-token-buffer-mib: 5.812824E+01 | global_load_balancing_loss: 1.000803E+00 | loss scale: 1.0 | grad norm: 0.117 | params norm: 11661.353 | params rms: 0.063142 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 17:55:01.081781 | finish at 2026-10-10 14:39:14 + [2026-10-03 20:45:37] iteration 199/ 6855 | consumed samples: 25472 | elapsed time per iteration (ms): 84335.4 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.66% | FLOPs estimator: workload-aware | logical tokens/s: 198933.2 | valid tokens/s: 198933.2 | physical tokens/s: 199439.5 | loss tokens/s: 149768.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.618420E-01 | lm-global-supervised-tokens: 1.263075E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.276039E-02 | lm-token-buffer-mib: 5.814817E+01 | global_load_balancing_loss: 1.000779E+00 | loss scale: 1.0 | grad norm: 0.053 | params norm: 11661.253 | params rms: 0.063142 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 11:55:36.307251 | finish at 2026-10-10 08:41:14 + [2026-10-03 20:47:21] iteration 200/ 6855 | consumed samples: 25600 | elapsed time per iteration (ms): 103731.2 | throughput per GPU (TFLOP/s/GPU): 34.5 | MFU 11.07% | FLOPs estimator: workload-aware | logical tokens/s: 161736.5 | valid tokens/s: 161736.5 | physical tokens/s: 162096.6 | loss tokens/s: 120810.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.184747E-01 | lm-global-supervised-tokens: 1.253177E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.340617E-02 | lm-token-buffer-mib: 5.822741E+01 | global_load_balancing_loss: 1.000904E+00 | loss scale: 1.0 | grad norm: 0.173 | params norm: 11661.153 | params rms: 0.063141 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 7 days, 23:45:31.259626 | finish at 2026-10-11 20:32:52 +times across ranks (ms): + save-checkpoint: + rank 0: 25757.87 + rank 1: 25757.86 + rank 2: 25757.90 + rank 3: 25757.86 + rank 4: 25757.87 + rank 5: 25757.86 + rank 6: 25757.84 + rank 7: 25757.87 + rank 8: 25757.86 + rank 9: 25757.81 + rank 10: 25757.82 + rank 11: 25757.82 + rank 12: 25757.82 + rank 13: 25757.80 + rank 14: 25757.82 + rank 15: 25757.80 + rank 16: 25757.39 + rank 17: 25757.38 + rank 18: 25757.36 + rank 19: 25757.38 + rank 20: 25757.37 + rank 21: 25757.41 + rank 22: 25757.35 + rank 23: 25757.35 + rank 24: 25757.94 + rank 25: 25757.72 + rank 26: 25757.95 + rank 27: 25757.79 + rank 28: 25757.88 + rank 29: 25757.89 + rank 30: 25757.84 + rank 31: 25757.96 + rank 32: 25757.32 + rank 33: 25757.61 + rank 34: 25757.58 + rank 35: 25757.57 + rank 36: 25757.58 + rank 37: 25757.51 + rank 38: 25757.60 + rank 39: 25757.50 + rank 40: 25756.73 + rank 41: 25756.70 + rank 42: 25756.72 + rank 43: 25756.74 + rank 44: 25756.69 + rank 45: 25756.70 + rank 46: 25756.71 + rank 47: 25756.73 + rank 48: 25757.60 + rank 49: 25757.59 + rank 50: 25757.56 + rank 51: 25757.62 + rank 52: 25757.64 + rank 53: 25757.58 + rank 54: 25757.63 + rank 55: 25757.71 + rank 56: 25758.64 + rank 57: 25758.59 + rank 58: 25758.54 + rank 59: 25758.54 + rank 60: 25758.54 + rank 61: 25758.56 + rank 62: 25758.60 + rank 63: 25758.57 + [2026-10-03 20:49:17] iteration 201/ 6855 | consumed samples: 25728 | elapsed time per iteration (ms): 90339.8 | throughput per GPU (TFLOP/s/GPU): 40.7 | MFU 13.04% | FLOPs estimator: workload-aware | logical tokens/s: 185711.8 | valid tokens/s: 185711.8 | physical tokens/s: 186106.6 | loss tokens/s: 138290.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.745442E-01 | lm-global-supervised-tokens: 1.249313E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.577307E+00 | lm-token-buffer-mib: 5.805148E+01 | global_load_balancing_loss: 1.001081E+00 | loss scale: 1.0 | grad norm: 0.203 | params norm: 11661.054 | params rms: 0.063141 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 22:58:41.025235 | finish at 2026-10-10 19:47:58 + [2026-10-03 20:50:47] iteration 202/ 6855 | consumed samples: 25856 | elapsed time per iteration (ms): 89997.0 | throughput per GPU (TFLOP/s/GPU): 39.7 | MFU 12.73% | FLOPs estimator: workload-aware | logical tokens/s: 186418.7 | valid tokens/s: 186418.6 | physical tokens/s: 186935.0 | loss tokens/s: 141942.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.856143E-01 | lm-global-supervised-tokens: 1.277440E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.493302E-02 | lm-token-buffer-mib: 5.824689E+01 | global_load_balancing_loss: 9.998135E-01 | loss scale: 1.0 | grad norm: 0.066 | params norm: 11660.956 | params rms: 0.063140 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 22:19:09.975826 | finish at 2026-10-10 19:09:57 + [2026-10-03 20:52:16] iteration 203/ 6855 | consumed samples: 25984 | elapsed time per iteration (ms): 88706.3 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.40% | FLOPs estimator: workload-aware | logical tokens/s: 189131.9 | valid tokens/s: 189131.9 | physical tokens/s: 189519.5 | loss tokens/s: 146793.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.962812E-01 | lm-global-supervised-tokens: 1.302148E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.372496E-02 | lm-token-buffer-mib: 5.807304E+01 | global_load_balancing_loss: 1.000182E+00 | loss scale: 1.0 | grad norm: 0.161 | params norm: 11660.858 | params rms: 0.063140 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 19:54:34.001644 | finish at 2026-10-10 16:46:50 + [2026-10-03 20:53:40] iteration 204/ 6855 | consumed samples: 26112 | elapsed time per iteration (ms): 84256.8 | throughput per GPU (TFLOP/s/GPU): 43.2 | MFU 13.86% | FLOPs estimator: workload-aware | logical tokens/s: 199119.1 | valid tokens/s: 199119.1 | physical tokens/s: 199691.7 | loss tokens/s: 159634.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.079076E-01 | lm-global-supervised-tokens: 1.345031E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.325600E-02 | lm-token-buffer-mib: 5.808593E+01 | global_load_balancing_loss: 9.986907E-01 | loss scale: 1.0 | grad norm: 0.072 | params norm: 11660.759 | params rms: 0.063139 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 11:39:51.789261 | finish at 2026-10-10 08:33:32 + [2026-10-03 20:55:06] iteration 205/ 6855 | consumed samples: 26240 | elapsed time per iteration (ms): 86316.2 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.76% | FLOPs estimator: workload-aware | logical tokens/s: 194369.1 | valid tokens/s: 194369.1 | physical tokens/s: 194698.4 | loss tokens/s: 143695.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.500218E-01 | lm-global-supervised-tokens: 1.240324E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.165285E-02 | lm-token-buffer-mib: 5.804353E+01 | global_load_balancing_loss: 9.988394E-01 | loss scale: 1.0 | grad norm: 0.084 | params norm: 11660.661 | params rms: 0.063139 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 15:26:42.847445 | finish at 2026-10-10 12:21:49 + [2026-10-03 20:56:32] iteration 206/ 6855 | consumed samples: 26368 | elapsed time per iteration (ms): 85876.6 | throughput per GPU (TFLOP/s/GPU): 43.3 | MFU 13.88% | FLOPs estimator: workload-aware | logical tokens/s: 195363.8 | valid tokens/s: 195363.8 | physical tokens/s: 195844.3 | loss tokens/s: 141569.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.738003E-01 | lm-global-supervised-tokens: 1.215750E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.181026E-02 | lm-token-buffer-mib: 5.808176E+01 | global_load_balancing_loss: 1.001025E+00 | loss scale: 1.0 | grad norm: 0.175 | params norm: 11660.565 | params rms: 0.063138 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 14:36:33.596013 | finish at 2026-10-10 11:33:06 + [2026-10-03 20:58:01] iteration 207/ 6855 | consumed samples: 26496 | elapsed time per iteration (ms): 88769.9 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.54% | FLOPs estimator: workload-aware | logical tokens/s: 188996.2 | valid tokens/s: 188996.2 | physical tokens/s: 189518.8 | loss tokens/s: 138395.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.413686E-01 | lm-global-supervised-tokens: 1.228533E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.180184E-02 | lm-token-buffer-mib: 5.807112E+01 | global_load_balancing_loss: 1.000910E+00 | loss scale: 1.0 | grad norm: 0.087 | params norm: 11660.470 | params rms: 0.063138 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 19:55:42.495466 | finish at 2026-10-10 16:53:44 + [2026-10-03 20:59:28] iteration 208/ 6855 | consumed samples: 26624 | elapsed time per iteration (ms): 86518.3 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.47% | FLOPs estimator: workload-aware | logical tokens/s: 193913.9 | valid tokens/s: 193913.9 | physical tokens/s: 194321.5 | loss tokens/s: 136060.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.516885E-01 | lm-global-supervised-tokens: 1.177174E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.407171E-02 | lm-token-buffer-mib: 5.810511E+01 | global_load_balancing_loss: 9.999039E-01 | loss scale: 1.0 | grad norm: 0.163 | params norm: 11660.374 | params rms: 0.063137 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 15:44:47.092932 | finish at 2026-10-10 12:44:15 + [2026-10-03 21:00:54] iteration 209/ 6855 | consumed samples: 26752 | elapsed time per iteration (ms): 86695.0 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.61% | FLOPs estimator: workload-aware | logical tokens/s: 193520.0 | valid tokens/s: 193520.0 | physical tokens/s: 194260.5 | loss tokens/s: 150251.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.141954E-01 | lm-global-supervised-tokens: 1.302603E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.147584E-02 | lm-token-buffer-mib: 5.818430E+01 | global_load_balancing_loss: 9.995605E-01 | loss scale: 1.0 | grad norm: 0.038 | params norm: 11660.279 | params rms: 0.063137 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 16:02:54.850717 | finish at 2026-10-10 13:03:49 + [2026-10-03 21:02:37] iteration 210/ 6855 | consumed samples: 26880 | elapsed time per iteration (ms): 103091.9 | throughput per GPU (TFLOP/s/GPU): 36.1 | MFU 11.58% | FLOPs estimator: workload-aware | logical tokens/s: 162740.3 | valid tokens/s: 162740.3 | physical tokens/s: 163082.0 | loss tokens/s: 122959.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.251990E-01 | lm-global-supervised-tokens: 1.267612E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.411234E-02 | lm-token-buffer-mib: 5.819756E+01 | global_load_balancing_loss: 9.994701E-01 | loss scale: 1.0 | grad norm: 0.088 | params norm: 11660.184 | params rms: 0.063136 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 7 days, 22:17:25.684154 | finish at 2026-10-11 19:20:03 + [2026-10-03 21:04:04] iteration 211/ 6855 | consumed samples: 27008 | elapsed time per iteration (ms): 86838.7 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.40% | FLOPs estimator: workload-aware | logical tokens/s: 193199.7 | valid tokens/s: 193199.7 | physical tokens/s: 193682.0 | loss tokens/s: 151478.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.729715E-01 | lm-global-supervised-tokens: 1.315422E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.835624E-02 | lm-token-buffer-mib: 5.810393E+01 | global_load_balancing_loss: 1.001982E+00 | loss scale: 1.0 | grad norm: 0.165 | params norm: 11660.087 | params rms: 0.063136 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 16:15:56.150511 | finish at 2026-10-10 13:20:00 + [2026-10-03 21:05:34] iteration 212/ 6855 | consumed samples: 27136 | elapsed time per iteration (ms): 89237.9 | throughput per GPU (TFLOP/s/GPU): 41.6 | MFU 13.34% | FLOPs estimator: workload-aware | logical tokens/s: 188005.1 | valid tokens/s: 188005.1 | physical tokens/s: 188429.8 | loss tokens/s: 138867.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.750196E-01 | lm-global-supervised-tokens: 1.239221E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.340352E-02 | lm-token-buffer-mib: 5.816657E+01 | global_load_balancing_loss: 1.000328E+00 | loss scale: 1.0 | grad norm: 0.046 | params norm: 11659.993 | params rms: 0.063135 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 20:40:07.555130 | finish at 2026-10-10 17:45:41 + [2026-10-03 21:07:03] iteration 213/ 6855 | consumed samples: 27264 | elapsed time per iteration (ms): 89283.7 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.54% | FLOPs estimator: workload-aware | logical tokens/s: 187909.0 | valid tokens/s: 187909.0 | physical tokens/s: 188288.1 | loss tokens/s: 133191.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.451711E-01 | lm-global-supervised-tokens: 1.189180E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.309375E-02 | lm-token-buffer-mib: 5.817751E+01 | global_load_balancing_loss: 1.000768E+00 | loss scale: 1.0 | grad norm: 0.165 | params norm: 11659.899 | params rms: 0.063135 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 20:43:42.316326 | finish at 2026-10-10 17:50:45 + [2026-10-03 21:08:30] iteration 214/ 6855 | consumed samples: 27392 | elapsed time per iteration (ms): 87411.3 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.58% | FLOPs estimator: workload-aware | logical tokens/s: 191933.8 | valid tokens/s: 191933.8 | physical tokens/s: 192242.9 | loss tokens/s: 141612.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.976238E-01 | lm-global-supervised-tokens: 1.237857E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.445230E-02 | lm-token-buffer-mib: 5.805880E+01 | global_load_balancing_loss: 9.994641E-01 | loss scale: 1.0 | grad norm: 0.068 | params norm: 11659.804 | params rms: 0.063134 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 17:14:58.737428 | finish at 2026-10-10 14:23:29 + [2026-10-03 21:09:59] iteration 215/ 6855 | consumed samples: 27520 | elapsed time per iteration (ms): 88632.3 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.44% | FLOPs estimator: workload-aware | logical tokens/s: 189289.2 | valid tokens/s: 189289.2 | physical tokens/s: 189724.4 | loss tokens/s: 139308.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.938961E-01 | lm-global-supervised-tokens: 1.234719E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.524692E-02 | lm-token-buffer-mib: 5.810821E+01 | global_load_balancing_loss: 1.000218E+00 | loss scale: 1.0 | grad norm: 0.079 | params norm: 11659.707 | params rms: 0.063134 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 19:28:38.597984 | finish at 2026-10-10 16:38:37 + [2026-10-03 21:11:25] iteration 216/ 6855 | consumed samples: 27648 | elapsed time per iteration (ms): 86134.5 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.75% | FLOPs estimator: workload-aware | logical tokens/s: 194777.8 | valid tokens/s: 194777.8 | physical tokens/s: 195293.6 | loss tokens/s: 144856.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.793525E-01 | lm-global-supervised-tokens: 1.247717E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.221533E-02 | lm-token-buffer-mib: 5.819385E+01 | global_load_balancing_loss: 9.990512E-01 | loss scale: 1.0 | grad norm: 0.105 | params norm: 11659.610 | params rms: 0.063133 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 14:50:47.035900 | finish at 2026-10-10 12:02:12 + [2026-10-03 21:12:51] iteration 217/ 6855 | consumed samples: 27776 | elapsed time per iteration (ms): 86202.1 | throughput per GPU (TFLOP/s/GPU): 43.3 | MFU 13.87% | FLOPs estimator: workload-aware | logical tokens/s: 194626.6 | valid tokens/s: 194626.6 | physical tokens/s: 195011.8 | loss tokens/s: 138306.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.525006E-01 | lm-global-supervised-tokens: 1.192233E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.188919E-02 | lm-token-buffer-mib: 5.805410E+01 | global_load_balancing_loss: 1.000020E+00 | loss scale: 1.0 | grad norm: 0.039 | params norm: 11659.514 | params rms: 0.063132 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 14:56:49.286836 | finish at 2026-10-10 12:09:40 + [2026-10-03 21:14:19] iteration 218/ 6855 | consumed samples: 27904 | elapsed time per iteration (ms): 87960.7 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.56% | FLOPs estimator: workload-aware | logical tokens/s: 190734.8 | valid tokens/s: 190734.8 | physical tokens/s: 191210.8 | loss tokens/s: 141938.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.101183E-01 | lm-global-supervised-tokens: 1.248498E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.509756E-02 | lm-token-buffer-mib: 5.829768E+01 | global_load_balancing_loss: 1.001046E+00 | loss scale: 1.0 | grad norm: 0.078 | params norm: 11659.418 | params rms: 0.063132 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 18:09:55.398743 | finish at 2026-10-10 15:24:15 + [2026-10-03 21:15:47] iteration 219/ 6855 | consumed samples: 28032 | elapsed time per iteration (ms): 87888.8 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.42% | FLOPs estimator: workload-aware | logical tokens/s: 190890.8 | valid tokens/s: 190890.8 | physical tokens/s: 191599.7 | loss tokens/s: 141731.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.735343E-01 | lm-global-supervised-tokens: 1.245660E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.867765E-02 | lm-token-buffer-mib: 5.807915E+01 | global_load_balancing_loss: 1.001078E+00 | loss scale: 1.0 | grad norm: 0.163 | params norm: 11659.323 | params rms: 0.063131 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 18:00:30.364125 | finish at 2026-10-10 15:16:17 + [2026-10-03 21:17:30] iteration 220/ 6855 | consumed samples: 28160 | elapsed time per iteration (ms): 102989.1 | throughput per GPU (TFLOP/s/GPU): 35.3 | MFU 11.32% | FLOPs estimator: workload-aware | logical tokens/s: 162902.5 | valid tokens/s: 162902.5 | physical tokens/s: 163335.3 | loss tokens/s: 125468.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.629966E-01 | lm-global-supervised-tokens: 1.292186E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.208501E-02 | lm-token-buffer-mib: 5.818060E+01 | global_load_balancing_loss: 1.000685E+00 | loss scale: 1.0 | grad norm: 0.093 | params norm: 11659.229 | params rms: 0.063131 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 7 days, 21:48:52.841300 | finish at 2026-10-11 19:06:23 + [2026-10-03 21:18:57] iteration 221/ 6855 | consumed samples: 28288 | elapsed time per iteration (ms): 86599.1 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.44% | FLOPs estimator: workload-aware | logical tokens/s: 193733.9 | valid tokens/s: 193733.9 | physical tokens/s: 194282.0 | loss tokens/s: 143579.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.932988E-01 | lm-global-supervised-tokens: 1.243384E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.141140E-02 | lm-token-buffer-mib: 5.819270E+01 | global_load_balancing_loss: 1.000848E+00 | loss scale: 1.0 | grad norm: 0.174 | params norm: 11659.135 | params rms: 0.063130 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 15:34:58.202389 | finish at 2026-10-10 12:53:55 + [2026-10-03 21:20:23] iteration 222/ 6855 | consumed samples: 28416 | elapsed time per iteration (ms): 86553.4 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.49% | FLOPs estimator: workload-aware | logical tokens/s: 193836.3 | valid tokens/s: 193836.3 | physical tokens/s: 194314.9 | loss tokens/s: 143205.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.850245E-01 | lm-global-supervised-tokens: 1.239495E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.810647E-02 | lm-token-buffer-mib: 5.812131E+01 | global_load_balancing_loss: 1.000741E+00 | loss scale: 1.0 | grad norm: 0.071 | params norm: 11659.043 | params rms: 0.063130 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 15:28:28.395666 | finish at 2026-10-10 12:48:52 + [2026-10-03 21:21:52] iteration 223/ 6855 | consumed samples: 28544 | elapsed time per iteration (ms): 88628.2 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.62% | FLOPs estimator: workload-aware | logical tokens/s: 189298.0 | valid tokens/s: 189298.0 | physical tokens/s: 189662.9 | loss tokens/s: 140182.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.906332E-01 | lm-global-supervised-tokens: 1.242414E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.133471E-02 | lm-token-buffer-mib: 5.806359E+01 | global_load_balancing_loss: 9.992756E-01 | loss scale: 1.0 | grad norm: 0.117 | params norm: 11658.949 | params rms: 0.063129 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 19:16:22.344511 | finish at 2026-10-10 16:38:14 + [2026-10-03 21:23:20] iteration 224/ 6855 | consumed samples: 28672 | elapsed time per iteration (ms): 88351.2 | throughput per GPU (TFLOP/s/GPU): 43.2 | MFU 13.83% | FLOPs estimator: workload-aware | logical tokens/s: 189891.8 | valid tokens/s: 189891.8 | physical tokens/s: 190116.8 | loss tokens/s: 134777.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.730831E-01 | lm-global-supervised-tokens: 1.190773E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.733609E-02 | lm-token-buffer-mib: 5.805872E+01 | global_load_balancing_loss: 9.990398E-01 | loss scale: 1.0 | grad norm: 0.046 | params norm: 11658.855 | params rms: 0.063129 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 18:44:17.073488 | finish at 2026-10-10 16:07:37 + [2026-10-03 21:24:47] iteration 225/ 6855 | consumed samples: 28800 | elapsed time per iteration (ms): 87305.3 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.62% | FLOPs estimator: workload-aware | logical tokens/s: 192167.2 | valid tokens/s: 192167.2 | physical tokens/s: 192614.6 | loss tokens/s: 140481.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.461595E-01 | lm-global-supervised-tokens: 1.226479E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.311350E-02 | lm-token-buffer-mib: 5.829589E+01 | global_load_balancing_loss: 9.990356E-01 | loss scale: 1.0 | grad norm: 0.041 | params norm: 11658.761 | params rms: 0.063128 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 16:47:14.034655 | finish at 2026-10-10 14:12:02 + [2026-10-03 21:26:15] iteration 226/ 6855 | consumed samples: 28928 | elapsed time per iteration (ms): 87695.8 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.62% | FLOPs estimator: workload-aware | logical tokens/s: 191311.5 | valid tokens/s: 191311.5 | physical tokens/s: 191658.5 | loss tokens/s: 144972.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.357687E-01 | lm-global-supervised-tokens: 1.271347E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.439114E-02 | lm-token-buffer-mib: 5.806022E+01 | global_load_balancing_loss: 9.985802E-01 | loss scale: 1.0 | grad norm: 0.054 | params norm: 11658.667 | params rms: 0.063128 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 17:28:55.468120 | finish at 2026-10-10 14:55:11 + [2026-10-03 21:27:44] iteration 227/ 6855 | consumed samples: 29056 | elapsed time per iteration (ms): 89164.2 | throughput per GPU (TFLOP/s/GPU): 41.4 | MFU 13.28% | FLOPs estimator: workload-aware | logical tokens/s: 188160.5 | valid tokens/s: 188160.5 | physical tokens/s: 188484.9 | loss tokens/s: 138403.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.243575E-01 | lm-global-supervised-tokens: 1.234066E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.152642E-02 | lm-token-buffer-mib: 5.802337E+01 | global_load_balancing_loss: 1.000879E+00 | loss scale: 1.0 | grad norm: 0.049 | params norm: 11658.574 | params rms: 0.063127 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 20:09:40.077851 | finish at 2026-10-10 17:37:24 + [2026-10-03 21:29:12] iteration 228/ 6855 | consumed samples: 29184 | elapsed time per iteration (ms): 87287.3 | throughput per GPU (TFLOP/s/GPU): 42.8 | MFU 13.73% | FLOPs estimator: workload-aware | logical tokens/s: 192206.7 | valid tokens/s: 192206.7 | physical tokens/s: 192799.4 | loss tokens/s: 140073.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.308257E-01 | lm-global-supervised-tokens: 1.222661E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.293993E-02 | lm-token-buffer-mib: 5.814097E+01 | global_load_balancing_loss: 1.002053E+00 | loss scale: 1.0 | grad norm: 0.044 | params norm: 11658.482 | params rms: 0.063127 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 16:40:53.179568 | finish at 2026-10-10 14:10:05 + [2026-10-03 21:30:39] iteration 229/ 6855 | consumed samples: 29312 | elapsed time per iteration (ms): 87518.6 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.75% | FLOPs estimator: workload-aware | logical tokens/s: 191698.8 | valid tokens/s: 191698.8 | physical tokens/s: 192245.6 | loss tokens/s: 143582.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.829093E-01 | lm-global-supervised-tokens: 1.256616E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.759898E-02 | lm-token-buffer-mib: 5.808500E+01 | global_load_balancing_loss: 1.002119E+00 | loss scale: 1.0 | grad norm: 0.039 | params norm: 11658.392 | params rms: 0.063126 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 17:04:58.422027 | finish at 2026-10-10 14:35:38 + [2026-10-03 21:32:25] iteration 230/ 6855 | consumed samples: 29440 | elapsed time per iteration (ms): 105803.3 | throughput per GPU (TFLOP/s/GPU): 35.5 | MFU 11.38% | FLOPs estimator: workload-aware | logical tokens/s: 158569.9 | valid tokens/s: 158569.9 | physical tokens/s: 158887.6 | loss tokens/s: 120689.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.893107E-01 | lm-global-supervised-tokens: 1.276937E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.152669E-02 | lm-token-buffer-mib: 5.811878E+01 | global_load_balancing_loss: 1.002137E+00 | loss scale: 1.0 | grad norm: 0.039 | params norm: 11658.302 | params rms: 0.063126 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 8 days, 2:42:26.767092 | finish at 2026-10-12 00:14:52 + [2026-10-03 21:33:51] iteration 231/ 6855 | consumed samples: 29568 | elapsed time per iteration (ms): 86017.6 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.76% | FLOPs estimator: workload-aware | logical tokens/s: 195044.0 | valid tokens/s: 195044.0 | physical tokens/s: 195418.5 | loss tokens/s: 141965.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.955698E-01 | lm-global-supervised-tokens: 1.221153E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.413276E-02 | lm-token-buffer-mib: 5.804588E+01 | global_load_balancing_loss: 1.000845E+00 | loss scale: 1.0 | grad norm: 0.045 | params norm: 11658.212 | params rms: 0.063125 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 14:16:20.344322 | finish at 2026-10-10 11:50:11 + [2026-10-03 21:35:16] iteration 232/ 6855 | consumed samples: 29696 | elapsed time per iteration (ms): 85388.7 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.78% | FLOPs estimator: workload-aware | logical tokens/s: 196480.2 | valid tokens/s: 196480.2 | physical tokens/s: 197088.8 | loss tokens/s: 147204.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.172506E-01 | lm-global-supervised-tokens: 1.256959E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.317162E-02 | lm-token-buffer-mib: 5.848278E+01 | global_load_balancing_loss: 9.989814E-01 | loss scale: 1.0 | grad norm: 0.049 | params norm: 11658.123 | params rms: 0.063125 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 13:05:29.308047 | finish at 2026-10-10 10:40:46 + [2026-10-03 21:36:44] iteration 233/ 6855 | consumed samples: 29824 | elapsed time per iteration (ms): 87210.8 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.60% | FLOPs estimator: workload-aware | logical tokens/s: 192375.5 | valid tokens/s: 192375.5 | physical tokens/s: 192994.1 | loss tokens/s: 142075.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.772000E-01 | lm-global-supervised-tokens: 1.239048E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.294580E-02 | lm-token-buffer-mib: 5.813055E+01 | global_load_balancing_loss: 9.990501E-01 | loss scale: 1.0 | grad norm: 0.046 | params norm: 11658.033 | params rms: 0.063124 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 16:25:09.732432 | finish at 2026-10-10 14:01:53 + [2026-10-03 21:38:15] iteration 234/ 6855 | consumed samples: 29952 | elapsed time per iteration (ms): 91842.7 | throughput per GPU (TFLOP/s/GPU): 41.0 | MFU 13.13% | FLOPs estimator: workload-aware | logical tokens/s: 182672.9 | valid tokens/s: 182672.9 | physical tokens/s: 183036.0 | loss tokens/s: 136618.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.308894E-01 | lm-global-supervised-tokens: 1.254744E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.160889E-02 | lm-token-buffer-mib: 5.816236E+01 | global_load_balancing_loss: 9.994366E-01 | loss scale: 1.0 | grad norm: 0.059 | params norm: 11657.943 | params rms: 0.063124 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 7 days, 0:54:50.360452 | finish at 2026-10-10 22:33:06 + [2026-10-03 21:39:42] iteration 235/ 6855 | consumed samples: 30080 | elapsed time per iteration (ms): 86421.6 | throughput per GPU (TFLOP/s/GPU): 43.5 | MFU 13.94% | FLOPs estimator: workload-aware | logical tokens/s: 194131.8 | valid tokens/s: 194131.8 | physical tokens/s: 194519.7 | loss tokens/s: 141047.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.137638E-01 | lm-global-supervised-tokens: 1.218955E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.179471E-02 | lm-token-buffer-mib: 5.815220E+01 | global_load_balancing_loss: 9.997903E-01 | loss scale: 1.0 | grad norm: 0.060 | params norm: 11657.853 | params rms: 0.063123 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 14:55:11.038456 | finish at 2026-10-10 12:34:53 + [2026-10-03 21:41:10] iteration 236/ 6855 | consumed samples: 30208 | elapsed time per iteration (ms): 87732.5 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.61% | FLOPs estimator: workload-aware | logical tokens/s: 191231.5 | valid tokens/s: 191231.5 | physical tokens/s: 191647.3 | loss tokens/s: 140073.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.480176E-01 | lm-global-supervised-tokens: 1.228900E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.175619E-02 | lm-token-buffer-mib: 5.810914E+01 | global_load_balancing_loss: 9.997796E-01 | loss scale: 1.0 | grad norm: 0.051 | params norm: 11657.763 | params rms: 0.063123 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 17:18:21.302804 | finish at 2026-10-10 14:59:31 + [2026-10-03 21:42:38] iteration 237/ 6855 | consumed samples: 30336 | elapsed time per iteration (ms): 88041.2 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.39% | FLOPs estimator: workload-aware | logical tokens/s: 190560.8 | valid tokens/s: 190560.8 | physical tokens/s: 191000.5 | loss tokens/s: 139271.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.127623E-01 | lm-global-supervised-tokens: 1.226161E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.147517E-02 | lm-token-buffer-mib: 5.813438E+01 | global_load_balancing_loss: 1.000654E+00 | loss scale: 1.0 | grad norm: 0.041 | params norm: 11657.672 | params rms: 0.063123 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 17:50:56.869364 | finish at 2026-10-10 15:33:34 + [2026-10-03 21:44:05] iteration 238/ 6855 | consumed samples: 30464 | elapsed time per iteration (ms): 87694.3 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.62% | FLOPs estimator: workload-aware | logical tokens/s: 191314.7 | valid tokens/s: 191314.7 | physical tokens/s: 191721.2 | loss tokens/s: 146436.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.357761E-01 | lm-global-supervised-tokens: 1.284169E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.121229E-02 | lm-token-buffer-mib: 5.805293E+01 | global_load_balancing_loss: 9.987321E-01 | loss scale: 1.0 | grad norm: 0.040 | params norm: 11657.581 | params rms: 0.063122 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 17:11:13.318353 | finish at 2026-10-10 14:55:19 + [2026-10-03 21:45:37] iteration 239/ 6855 | consumed samples: 30592 | elapsed time per iteration (ms): 91301.8 | throughput per GPU (TFLOP/s/GPU): 40.8 | MFU 13.07% | FLOPs estimator: workload-aware | logical tokens/s: 183755.0 | valid tokens/s: 183755.0 | physical tokens/s: 184202.8 | loss tokens/s: 138775.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.721757E-01 | lm-global-supervised-tokens: 1.267050E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.507532E-02 | lm-token-buffer-mib: 5.806151E+01 | global_load_balancing_loss: 1.001044E+00 | loss scale: 1.0 | grad norm: 0.035 | params norm: 11657.490 | params rms: 0.063122 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 23:47:32.858734 | finish at 2026-10-10 21:33:09 + [2026-10-03 21:47:25] iteration 240/ 6855 | consumed samples: 30720 | elapsed time per iteration (ms): 108851.6 | throughput per GPU (TFLOP/s/GPU): 34.3 | MFU 10.99% | FLOPs estimator: workload-aware | logical tokens/s: 154128.3 | valid tokens/s: 154128.3 | physical tokens/s: 154564.7 | loss tokens/s: 112613.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.345055E-01 | lm-global-supervised-tokens: 1.225820E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.251227E-02 | lm-token-buffer-mib: 5.820457E+01 | global_load_balancing_loss: 1.001182E+00 | loss scale: 1.0 | grad norm: 0.059 | params norm: 11657.399 | params rms: 0.063121 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 8 days, 8:00:53.656862 | finish at 2026-10-12 05:48:19 +times across ranks (ms): + save-checkpoint: + rank 0: 23478.66 + rank 1: 23478.64 + rank 2: 23478.72 + rank 3: 23478.65 + rank 4: 23478.65 + rank 5: 23478.63 + rank 6: 23478.63 + rank 7: 23478.68 + rank 8: 23478.66 + rank 9: 23478.60 + rank 10: 23478.62 + rank 11: 23478.61 + rank 12: 23478.61 + rank 13: 23478.61 + rank 14: 23478.63 + rank 15: 23478.60 + rank 16: 23478.54 + rank 17: 23478.50 + rank 18: 23478.48 + rank 19: 23478.54 + rank 20: 23478.50 + rank 21: 23478.57 + rank 22: 23478.48 + rank 23: 23478.47 + rank 24: 23478.67 + rank 25: 23478.65 + rank 26: 23478.66 + rank 27: 23478.68 + rank 28: 23478.66 + rank 29: 23478.65 + rank 30: 23478.62 + rank 31: 23478.67 + rank 32: 23479.39 + rank 33: 23479.42 + rank 34: 23479.39 + rank 35: 23479.37 + rank 36: 23479.34 + rank 37: 23479.36 + rank 38: 23479.39 + rank 39: 23479.36 + rank 40: 23478.96 + rank 41: 23478.98 + rank 42: 23479.00 + rank 43: 23479.02 + rank 44: 23478.96 + rank 45: 23478.96 + rank 46: 23479.00 + rank 47: 23478.93 + rank 48: 23479.22 + rank 49: 23479.21 + rank 50: 23479.18 + rank 51: 23479.20 + rank 52: 23479.21 + rank 53: 23479.20 + rank 54: 23479.23 + rank 55: 23479.22 + rank 56: 23478.73 + rank 57: 23478.71 + rank 58: 23478.71 + rank 59: 23478.71 + rank 60: 23478.72 + rank 61: 23478.72 + rank 62: 23478.74 + rank 63: 23478.73 + [2026-10-03 21:49:20] iteration 241/ 6855 | consumed samples: 30848 | elapsed time per iteration (ms): 91371.6 | throughput per GPU (TFLOP/s/GPU): 39.4 | MFU 12.62% | FLOPs estimator: workload-aware | logical tokens/s: 183614.8 | valid tokens/s: 183614.8 | physical tokens/s: 184132.6 | loss tokens/s: 140218.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.568043E-01 | lm-global-supervised-tokens: 1.281199E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 3.595553E-01 | lm-token-buffer-mib: 5.813068E+01 | global_load_balancing_loss: 1.001042E+00 | loss scale: 1.0 | grad norm: 0.102 | params norm: 11657.305 | params rms: 0.063121 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 23:52:11.922666 | finish at 2026-10-10 21:41:32 + [2026-10-03 21:50:51] iteration 242/ 6855 | consumed samples: 30976 | elapsed time per iteration (ms): 90772.1 | throughput per GPU (TFLOP/s/GPU): 39.9 | MFU 12.80% | FLOPs estimator: workload-aware | logical tokens/s: 184827.4 | valid tokens/s: 184827.4 | physical tokens/s: 185352.9 | loss tokens/s: 138501.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.431090E-01 | lm-global-supervised-tokens: 1.257206E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.508380E-02 | lm-token-buffer-mib: 5.829574E+01 | global_load_balancing_loss: 1.001680E+00 | loss scale: 1.0 | grad norm: 0.173 | params norm: 11657.213 | params rms: 0.063120 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 22:44:35.742600 | finish at 2026-10-10 20:35:27 + [2026-10-03 21:52:19] iteration 243/ 6855 | consumed samples: 31104 | elapsed time per iteration (ms): 88089.7 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.62% | FLOPs estimator: workload-aware | logical tokens/s: 190456.0 | valid tokens/s: 190456.0 | physical tokens/s: 190725.4 | loss tokens/s: 138405.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.417522E-01 | lm-global-supervised-tokens: 1.219209E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.747165E-02 | lm-token-buffer-mib: 5.803187E+01 | global_load_balancing_loss: 1.001035E+00 | loss scale: 1.0 | grad norm: 0.119 | params norm: 11657.122 | params rms: 0.063120 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 17:47:29.134126 | finish at 2026-10-10 15:39:48 + [2026-10-03 21:53:45] iteration 244/ 6855 | consumed samples: 31232 | elapsed time per iteration (ms): 86242.7 | throughput per GPU (TFLOP/s/GPU): 43.2 | MFU 13.84% | FLOPs estimator: workload-aware | logical tokens/s: 194534.0 | valid tokens/s: 194534.0 | physical tokens/s: 194869.1 | loss tokens/s: 142486.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.631415E-01 | lm-global-supervised-tokens: 1.228845E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.148254E-02 | lm-token-buffer-mib: 5.805225E+01 | global_load_balancing_loss: 9.987010E-01 | loss scale: 1.0 | grad norm: 0.113 | params norm: 11657.030 | params rms: 0.063119 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 14:22:30.261814 | finish at 2026-10-10 12:16:16 + [2026-10-03 21:55:12] iteration 245/ 6855 | consumed samples: 31360 | elapsed time per iteration (ms): 86255.0 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.43% | FLOPs estimator: workload-aware | logical tokens/s: 194507.2 | valid tokens/s: 194507.2 | physical tokens/s: 194900.3 | loss tokens/s: 145956.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.708706E-01 | lm-global-supervised-tokens: 1.258948E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.432688E-02 | lm-token-buffer-mib: 5.819798E+01 | global_load_balancing_loss: 1.000039E+00 | loss scale: 1.0 | grad norm: 0.098 | params norm: 11656.937 | params rms: 0.063119 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 14:22:25.390010 | finish at 2026-10-10 12:17:37 + [2026-10-03 21:56:38] iteration 246/ 6855 | consumed samples: 31488 | elapsed time per iteration (ms): 86215.2 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.75% | FLOPs estimator: workload-aware | logical tokens/s: 194596.1 | valid tokens/s: 194596.1 | physical tokens/s: 195142.6 | loss tokens/s: 144293.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.555651E-01 | lm-global-supervised-tokens: 1.244026E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.623974E-02 | lm-token-buffer-mib: 5.813189E+01 | global_load_balancing_loss: 9.993588E-01 | loss scale: 1.0 | grad norm: 0.188 | params norm: 11656.845 | params rms: 0.063118 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 14:16:36.287966 | finish at 2026-10-10 12:13:14 + [2026-10-03 21:58:05] iteration 247/ 6855 | consumed samples: 31616 | elapsed time per iteration (ms): 87051.6 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.45% | FLOPs estimator: workload-aware | logical tokens/s: 192727.2 | valid tokens/s: 192727.1 | physical tokens/s: 193243.9 | loss tokens/s: 144154.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.060229E-01 | lm-global-supervised-tokens: 1.254887E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.351888E-02 | lm-token-buffer-mib: 5.817441E+01 | global_load_balancing_loss: 9.992016E-01 | loss scale: 1.0 | grad norm: 0.239 | params norm: 11656.753 | params rms: 0.063118 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 15:47:17.016777 | finish at 2026-10-10 13:45:22 + [2026-10-03 21:59:31] iteration 248/ 6855 | consumed samples: 31744 | elapsed time per iteration (ms): 85915.1 | throughput per GPU (TFLOP/s/GPU): 43.3 | MFU 13.88% | FLOPs estimator: workload-aware | logical tokens/s: 195276.5 | valid tokens/s: 195276.5 | physical tokens/s: 195653.4 | loss tokens/s: 140946.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.422130E-01 | lm-global-supervised-tokens: 1.210942E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.148088E-02 | lm-token-buffer-mib: 5.812154E+01 | global_load_balancing_loss: 1.000207E+00 | loss scale: 1.0 | grad norm: 0.059 | params norm: 11656.659 | params rms: 0.063117 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 13:40:41.129354 | finish at 2026-10-10 11:40:12 + [2026-10-03 22:01:03] iteration 249/ 6855 | consumed samples: 31872 | elapsed time per iteration (ms): 92179.4 | throughput per GPU (TFLOP/s/GPU): 41.2 | MFU 13.22% | FLOPs estimator: workload-aware | logical tokens/s: 182005.7 | valid tokens/s: 182005.7 | physical tokens/s: 182396.1 | loss tokens/s: 136733.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.414821E-01 | lm-global-supervised-tokens: 1.260400E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.519071E-02 | lm-token-buffer-mib: 5.824506E+01 | global_load_balancing_loss: 9.994586E-01 | loss scale: 1.0 | grad norm: 0.069 | params norm: 11656.564 | params rms: 0.063117 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 7 days, 1:08:57.352432 | finish at 2026-10-10 23:10:00 + [2026-10-03 22:02:45] iteration 250/ 6855 | consumed samples: 32000 | elapsed time per iteration (ms): 102341.5 | throughput per GPU (TFLOP/s/GPU): 35.8 | MFU 11.48% | FLOPs estimator: workload-aware | logical tokens/s: 163933.0 | valid tokens/s: 163933.0 | physical tokens/s: 164279.1 | loss tokens/s: 124842.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.894247E-01 | lm-global-supervised-tokens: 1.277653E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.571254E-02 | lm-token-buffer-mib: 5.805625E+01 | global_load_balancing_loss: 1.000239E+00 | loss scale: 1.0 | grad norm: 0.104 | params norm: 11656.472 | params rms: 0.063116 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 7 days, 19:46:05.708574 | finish at 2026-10-11 17:48:51 + [2026-10-03 22:04:16] iteration 251/ 6855 | consumed samples: 32128 | elapsed time per iteration (ms): 91052.7 | throughput per GPU (TFLOP/s/GPU): 40.8 | MFU 13.07% | FLOPs estimator: workload-aware | logical tokens/s: 184258.3 | valid tokens/s: 184258.3 | physical tokens/s: 184776.1 | loss tokens/s: 138274.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.864635E-01 | lm-global-supervised-tokens: 1.259022E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.150478E-02 | lm-token-buffer-mib: 5.809113E+01 | global_load_balancing_loss: 1.001506E+00 | loss scale: 1.0 | grad norm: 0.092 | params norm: 11656.380 | params rms: 0.063116 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 23:01:51.804352 | finish at 2026-10-10 21:06:08 + [2026-10-03 22:05:44] iteration 252/ 6855 | consumed samples: 32256 | elapsed time per iteration (ms): 88043.3 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.60% | FLOPs estimator: workload-aware | logical tokens/s: 190556.0 | valid tokens/s: 190556.0 | physical tokens/s: 191150.5 | loss tokens/s: 144720.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.152415E-01 | lm-global-supervised-tokens: 1.274165E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.127648E-02 | lm-token-buffer-mib: 5.816260E+01 | global_load_balancing_loss: 1.000075E+00 | loss scale: 1.0 | grad norm: 0.068 | params norm: 11656.285 | params rms: 0.063115 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 17:29:09.951834 | finish at 2026-10-10 15:34:54 + [2026-10-03 22:07:15] iteration 253/ 6855 | consumed samples: 32384 | elapsed time per iteration (ms): 90652.9 | throughput per GPU (TFLOP/s/GPU): 41.1 | MFU 13.17% | FLOPs estimator: workload-aware | logical tokens/s: 185070.6 | valid tokens/s: 185070.6 | physical tokens/s: 185520.9 | loss tokens/s: 132175.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.144693E-01 | lm-global-supervised-tokens: 1.198204E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.228753E-02 | lm-token-buffer-mib: 5.813343E+01 | global_load_balancing_loss: 1.001291E+00 | loss scale: 1.0 | grad norm: 0.147 | params norm: 11656.190 | params rms: 0.063114 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 22:14:50.143456 | finish at 2026-10-10 20:22:05 + [2026-10-03 22:08:41] iteration 254/ 6855 | consumed samples: 32512 | elapsed time per iteration (ms): 86003.2 | throughput per GPU (TFLOP/s/GPU): 43.1 | MFU 13.83% | FLOPs estimator: workload-aware | logical tokens/s: 195076.3 | valid tokens/s: 195076.2 | physical tokens/s: 195522.3 | loss tokens/s: 145479.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.922179E-01 | lm-global-supervised-tokens: 1.251174E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.278121E-02 | lm-token-buffer-mib: 5.807523E+01 | global_load_balancing_loss: 1.000347E+00 | loss scale: 1.0 | grad norm: 0.059 | params norm: 11656.096 | params rms: 0.063114 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 13:41:47.438318 | finish at 2026-10-10 11:50:29 + [2026-10-03 22:10:11] iteration 255/ 6855 | consumed samples: 32640 | elapsed time per iteration (ms): 89493.0 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.45% | FLOPs estimator: workload-aware | logical tokens/s: 187469.4 | valid tokens/s: 187469.4 | physical tokens/s: 187828.5 | loss tokens/s: 135139.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.586113E-01 | lm-global-supervised-tokens: 1.209403E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.757492E-02 | lm-token-buffer-mib: 5.810246E+01 | global_load_balancing_loss: 1.001350E+00 | loss scale: 1.0 | grad norm: 0.074 | params norm: 11656.001 | params rms: 0.063113 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 20:04:14.085016 | finish at 2026-10-10 18:14:25 + [2026-10-03 22:11:41] iteration 256/ 6855 | consumed samples: 32768 | elapsed time per iteration (ms): 90724.8 | throughput per GPU (TFLOP/s/GPU): 40.6 | MFU 13.00% | FLOPs estimator: workload-aware | logical tokens/s: 184924.3 | valid tokens/s: 184924.3 | physical tokens/s: 185336.3 | loss tokens/s: 135993.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.938503E-01 | lm-global-supervised-tokens: 1.233795E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.242116E-02 | lm-token-buffer-mib: 5.817378E+01 | global_load_balancing_loss: 1.000344E+00 | loss scale: 1.0 | grad norm: 0.085 | params norm: 11655.905 | params rms: 0.063113 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 22:18:12.784433 | finish at 2026-10-10 20:29:54 + [2026-10-03 22:13:09] iteration 257/ 6855 | consumed samples: 32896 | elapsed time per iteration (ms): 87732.4 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.45% | FLOPs estimator: workload-aware | logical tokens/s: 191231.7 | valid tokens/s: 191231.7 | physical tokens/s: 191720.9 | loss tokens/s: 143505.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.338167E-01 | lm-global-supervised-tokens: 1.259011E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.492068E-02 | lm-token-buffer-mib: 5.815625E+01 | global_load_balancing_loss: 9.996252E-01 | loss scale: 1.0 | grad norm: 0.040 | params norm: 11655.809 | params rms: 0.063112 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 16:47:38.208060 | finish at 2026-10-10 15:00:47 + [2026-10-03 22:14:39] iteration 258/ 6855 | consumed samples: 33024 | elapsed time per iteration (ms): 89529.0 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.59% | FLOPs estimator: workload-aware | logical tokens/s: 187393.8 | valid tokens/s: 187393.8 | physical tokens/s: 187830.1 | loss tokens/s: 135912.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.914076E-01 | lm-global-supervised-tokens: 1.216809E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.059847E-02 | lm-token-buffer-mib: 5.812659E+01 | global_load_balancing_loss: 9.990095E-01 | loss scale: 1.0 | grad norm: 0.072 | params norm: 11655.714 | params rms: 0.063112 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 20:03:42.877776 | finish at 2026-10-10 18:18:21 + [2026-10-03 22:16:02] iteration 259/ 6855 | consumed samples: 33152 | elapsed time per iteration (ms): 83642.1 | throughput per GPU (TFLOP/s/GPU): 43.3 | MFU 13.87% | FLOPs estimator: workload-aware | logical tokens/s: 200582.6 | valid tokens/s: 200582.5 | physical tokens/s: 201135.3 | loss tokens/s: 152992.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.346981E-01 | lm-global-supervised-tokens: 1.279662E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.253237E-02 | lm-token-buffer-mib: 5.819542E+01 | global_load_balancing_loss: 9.982349E-01 | loss scale: 1.0 | grad norm: 0.091 | params norm: 11655.619 | params rms: 0.063111 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 9:15:03.303240 | finish at 2026-10-10 07:31:06 + [2026-10-03 22:17:52] iteration 260/ 6855 | consumed samples: 33280 | elapsed time per iteration (ms): 109343.1 | throughput per GPU (TFLOP/s/GPU): 35.4 | MFU 11.33% | FLOPs estimator: workload-aware | logical tokens/s: 153436.4 | valid tokens/s: 153436.4 | physical tokens/s: 153675.7 | loss tokens/s: 109091.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.691675E-01 | lm-global-supervised-tokens: 1.192841E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.456830E-02 | lm-token-buffer-mib: 5.804944E+01 | global_load_balancing_loss: 1.000215E+00 | loss scale: 1.0 | grad norm: 0.038 | params norm: 11655.521 | params rms: 0.063111 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 8 days, 8:18:37.850317 | finish at 2026-10-12 06:36:29 + [2026-10-03 22:19:21] iteration 261/ 6855 | consumed samples: 33408 | elapsed time per iteration (ms): 89218.3 | throughput per GPU (TFLOP/s/GPU): 41.2 | MFU 13.19% | FLOPs estimator: workload-aware | logical tokens/s: 188046.2 | valid tokens/s: 188046.2 | physical tokens/s: 188484.5 | loss tokens/s: 142873.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.043220E-01 | lm-global-supervised-tokens: 1.274691E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.531838E-02 | lm-token-buffer-mib: 5.813068E+01 | global_load_balancing_loss: 1.000720E+00 | loss scale: 1.0 | grad norm: 0.071 | params norm: 11655.421 | params rms: 0.063110 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 19:25:05.178470 | finish at 2026-10-10 17:44:26 + [2026-10-03 22:20:48] iteration 262/ 6855 | consumed samples: 33536 | elapsed time per iteration (ms): 87215.6 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.53% | FLOPs estimator: workload-aware | logical tokens/s: 192364.3 | valid tokens/s: 192364.3 | physical tokens/s: 192797.7 | loss tokens/s: 142418.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.640618E-01 | lm-global-supervised-tokens: 1.242111E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.496417E-02 | lm-token-buffer-mib: 5.815224E+01 | global_load_balancing_loss: 1.001363E+00 | loss scale: 1.0 | grad norm: 0.140 | params norm: 11655.324 | params rms: 0.063110 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 15:43:32.681959 | finish at 2026-10-10 14:04:21 + [2026-10-03 22:22:17] iteration 263/ 6855 | consumed samples: 33664 | elapsed time per iteration (ms): 88523.0 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.40% | FLOPs estimator: workload-aware | logical tokens/s: 189523.7 | valid tokens/s: 189523.7 | physical tokens/s: 189897.1 | loss tokens/s: 138618.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.691971E-01 | lm-global-supervised-tokens: 1.227095E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.139854E-02 | lm-token-buffer-mib: 5.811360E+01 | global_load_balancing_loss: 1.001927E+00 | loss scale: 1.0 | grad norm: 0.242 | params norm: 11655.227 | params rms: 0.063109 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 18:05:43.303253 | finish at 2026-10-10 16:28:00 + [2026-10-03 22:23:47] iteration 264/ 6855 | consumed samples: 33792 | elapsed time per iteration (ms): 90436.4 | throughput per GPU (TFLOP/s/GPU): 40.8 | MFU 13.08% | FLOPs estimator: workload-aware | logical tokens/s: 185513.8 | valid tokens/s: 185513.8 | physical tokens/s: 185968.2 | loss tokens/s: 141702.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.245699E-01 | lm-global-supervised-tokens: 1.281511E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.464172E-02 | lm-token-buffer-mib: 5.820415E+01 | global_load_balancing_loss: 1.000738E+00 | loss scale: 1.0 | grad norm: 0.297 | params norm: 11655.129 | params rms: 0.063109 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 21:34:26.549267 | finish at 2026-10-10 19:58:14 + [2026-10-03 22:25:15] iteration 265/ 6855 | consumed samples: 33920 | elapsed time per iteration (ms): 88259.8 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.47% | FLOPs estimator: workload-aware | logical tokens/s: 190088.9 | valid tokens/s: 190088.9 | physical tokens/s: 190556.3 | loss tokens/s: 137404.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.169282E-01 | lm-global-supervised-tokens: 1.212732E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.130246E-02 | lm-token-buffer-mib: 5.812179E+01 | global_load_balancing_loss: 1.000224E+00 | loss scale: 1.0 | grad norm: 0.125 | params norm: 11655.030 | params rms: 0.063108 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 17:33:52.177560 | finish at 2026-10-10 15:59:07 + [2026-10-03 22:26:42] iteration 266/ 6855 | consumed samples: 34048 | elapsed time per iteration (ms): 87039.7 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.44% | FLOPs estimator: workload-aware | logical tokens/s: 192753.5 | valid tokens/s: 192753.5 | physical tokens/s: 193261.2 | loss tokens/s: 142456.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.641429E-01 | lm-global-supervised-tokens: 1.239941E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.348202E-02 | lm-token-buffer-mib: 5.831201E+01 | global_load_balancing_loss: 9.993686E-01 | loss scale: 1.0 | grad norm: 0.152 | params norm: 11654.932 | params rms: 0.063108 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 15:18:24.500246 | finish at 2026-10-10 13:45:07 + [2026-10-03 22:28:07] iteration 267/ 6855 | consumed samples: 34176 | elapsed time per iteration (ms): 84372.9 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.58% | FLOPs estimator: workload-aware | logical tokens/s: 198845.6 | valid tokens/s: 198845.6 | physical tokens/s: 199329.1 | loss tokens/s: 151506.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.474558E-01 | lm-global-supervised-tokens: 1.278308E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.784225E-02 | lm-token-buffer-mib: 5.814713E+01 | global_load_balancing_loss: 9.993840E-01 | loss scale: 1.0 | grad norm: 0.124 | params norm: 11654.834 | params rms: 0.063107 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 10:24:08.891441 | finish at 2026-10-10 08:52:16 + [2026-10-03 22:29:33] iteration 268/ 6855 | consumed samples: 34304 | elapsed time per iteration (ms): 86764.3 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.57% | FLOPs estimator: workload-aware | logical tokens/s: 193365.2 | valid tokens/s: 193365.2 | physical tokens/s: 193712.9 | loss tokens/s: 142153.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.224343E-01 | lm-global-supervised-tokens: 1.233381E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.574505E-02 | lm-token-buffer-mib: 5.808321E+01 | global_load_balancing_loss: 9.993072E-01 | loss scale: 1.0 | grad norm: 0.051 | params norm: 11654.739 | params rms: 0.063107 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 14:45:16.336449 | finish at 2026-10-10 13:14:50 + [2026-10-03 22:30:59] iteration 269/ 6855 | consumed samples: 34432 | elapsed time per iteration (ms): 85663.7 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.75% | FLOPs estimator: workload-aware | logical tokens/s: 195849.5 | valid tokens/s: 195849.5 | physical tokens/s: 196212.6 | loss tokens/s: 152968.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.339116E-01 | lm-global-supervised-tokens: 1.310383E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.438743E-02 | lm-token-buffer-mib: 5.822306E+01 | global_load_balancing_loss: 9.995566E-01 | loss scale: 1.0 | grad norm: 0.093 | params norm: 11654.644 | params rms: 0.063106 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 12:43:01.420945 | finish at 2026-10-10 11:14:01 + [2026-10-03 22:32:45] iteration 270/ 6855 | consumed samples: 34560 | elapsed time per iteration (ms): 105684.8 | throughput per GPU (TFLOP/s/GPU): 35.4 | MFU 11.35% | FLOPs estimator: workload-aware | logical tokens/s: 158747.1 | valid tokens/s: 158747.1 | physical tokens/s: 159048.9 | loss tokens/s: 114151.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.831372E-01 | lm-global-supervised-tokens: 1.206407E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.571882E-02 | lm-token-buffer-mib: 5.814643E+01 | global_load_balancing_loss: 1.000284E+00 | loss scale: 1.0 | grad norm: 0.117 | params norm: 11654.550 | params rms: 0.063106 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 8 days, 1:18:54.631913 | finish at 2026-10-11 23:51:39 + [2026-10-03 22:34:14] iteration 271/ 6855 | consumed samples: 34688 | elapsed time per iteration (ms): 89034.2 | throughput per GPU (TFLOP/s/GPU): 41.3 | MFU 13.25% | FLOPs estimator: workload-aware | logical tokens/s: 188435.2 | valid tokens/s: 188435.2 | physical tokens/s: 189024.5 | loss tokens/s: 135868.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.161579E-01 | lm-global-supervised-tokens: 1.209694E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.250107E-02 | lm-token-buffer-mib: 5.807706E+01 | global_load_balancing_loss: 1.000772E+00 | loss scale: 1.0 | grad norm: 0.562 | params norm: 11654.456 | params rms: 0.063105 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 18:50:01.390686 | finish at 2026-10-10 17:24:15 + [2026-10-03 22:35:42] iteration 272/ 6855 | consumed samples: 34816 | elapsed time per iteration (ms): 87837.7 | throughput per GPU (TFLOP/s/GPU): 43.3 | MFU 13.88% | FLOPs estimator: workload-aware | logical tokens/s: 191001.9 | valid tokens/s: 191001.9 | physical tokens/s: 191380.2 | loss tokens/s: 142668.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.145344E-01 | lm-global-supervised-tokens: 1.253163E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.219179E-02 | lm-token-buffer-mib: 5.816932E+01 | global_load_balancing_loss: 1.000831E+00 | loss scale: 1.0 | grad norm: 0.386 | params norm: 11654.361 | params rms: 0.063105 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 16:37:15.670978 | finish at 2026-10-10 15:12:57 + [2026-10-03 22:37:12] iteration 273/ 6855 | consumed samples: 34944 | elapsed time per iteration (ms): 90787.7 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.41% | FLOPs estimator: workload-aware | logical tokens/s: 184795.8 | valid tokens/s: 184795.8 | physical tokens/s: 185319.0 | loss tokens/s: 128527.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.506274E-01 | lm-global-supervised-tokens: 1.166875E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.981823E-02 | lm-token-buffer-mib: 5.810880E+01 | global_load_balancing_loss: 1.000682E+00 | loss scale: 1.0 | grad norm: 0.394 | params norm: 11654.266 | params rms: 0.063104 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 21:59:24.807333 | finish at 2026-10-10 20:36:37 + [2026-10-03 22:38:38] iteration 274/ 6855 | consumed samples: 35072 | elapsed time per iteration (ms): 85277.1 | throughput per GPU (TFLOP/s/GPU): 42.8 | MFU 13.73% | FLOPs estimator: workload-aware | logical tokens/s: 196736.4 | valid tokens/s: 196736.4 | physical tokens/s: 197244.7 | loss tokens/s: 148245.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.532894E-01 | lm-global-supervised-tokens: 1.264193E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.328976E-02 | lm-token-buffer-mib: 5.809043E+01 | global_load_balancing_loss: 1.000429E+00 | loss scale: 1.0 | grad norm: 0.197 | params norm: 11654.172 | params rms: 0.063104 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 11:53:28.802780 | finish at 2026-10-10 10:32:07 + [2026-10-03 22:40:05] iteration 275/ 6855 | consumed samples: 35200 | elapsed time per iteration (ms): 87344.7 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.74% | FLOPs estimator: workload-aware | logical tokens/s: 192080.0 | valid tokens/s: 192080.0 | physical tokens/s: 192488.4 | loss tokens/s: 150884.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.075114E-01 | lm-global-supervised-tokens: 1.317897E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.631466E-02 | lm-token-buffer-mib: 5.816432E+01 | global_load_balancing_loss: 9.987041E-01 | loss scale: 1.0 | grad norm: 0.391 | params norm: 11654.079 | params rms: 0.063103 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 15:38:48.352551 | finish at 2026-10-10 14:18:53 + [2026-10-03 22:41:31] iteration 276/ 6855 | consumed samples: 35328 | elapsed time per iteration (ms): 86099.5 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.59% | FLOPs estimator: workload-aware | logical tokens/s: 194858.5 | valid tokens/s: 194858.4 | physical tokens/s: 195192.8 | loss tokens/s: 141387.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.511004E-01 | lm-global-supervised-tokens: 1.217338E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.572535E-02 | lm-token-buffer-mib: 5.808450E+01 | global_load_balancing_loss: 1.000685E+00 | loss scale: 1.0 | grad norm: 0.056 | params norm: 11653.985 | params rms: 0.063103 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 13:20:48.563054 | finish at 2026-10-10 12:02:20 + [2026-10-03 22:43:00] iteration 277/ 6855 | consumed samples: 35456 | elapsed time per iteration (ms): 89305.1 | throughput per GPU (TFLOP/s/GPU): 41.0 | MFU 13.15% | FLOPs estimator: workload-aware | logical tokens/s: 187863.6 | valid tokens/s: 187863.6 | physical tokens/s: 188413.1 | loss tokens/s: 139792.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.890550E-01 | lm-global-supervised-tokens: 1.248414E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.480498E-02 | lm-token-buffer-mib: 5.818433E+01 | global_load_balancing_loss: 1.000715E+00 | loss scale: 1.0 | grad norm: 0.226 | params norm: 11653.894 | params rms: 0.063102 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 19:10:49.112237 | finish at 2026-10-10 17:53:50 + [2026-10-03 22:44:28] iteration 278/ 6855 | consumed samples: 35584 | elapsed time per iteration (ms): 87537.1 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.55% | FLOPs estimator: workload-aware | logical tokens/s: 191657.6 | valid tokens/s: 191657.6 | physical tokens/s: 192214.4 | loss tokens/s: 139643.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.550030E-01 | lm-global-supervised-tokens: 1.222398E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.676315E-02 | lm-token-buffer-mib: 5.814189E+01 | global_load_balancing_loss: 1.000491E+00 | loss scale: 1.0 | grad norm: 0.390 | params norm: 11653.805 | params rms: 0.063102 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 15:55:31.309626 | finish at 2026-10-10 14:39:59 + [2026-10-03 22:45:54] iteration 279/ 6855 | consumed samples: 35712 | elapsed time per iteration (ms): 86418.8 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.63% | FLOPs estimator: workload-aware | logical tokens/s: 194138.0 | valid tokens/s: 194138.0 | physical tokens/s: 194594.4 | loss tokens/s: 147820.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.252128E-01 | lm-global-supervised-tokens: 1.277447E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.327053E-02 | lm-token-buffer-mib: 5.809061E+01 | global_load_balancing_loss: 9.999855E-01 | loss scale: 1.0 | grad norm: 0.124 | params norm: 11653.718 | params rms: 0.063101 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 13:51:30.267872 | finish at 2026-10-10 12:37:25 + [2026-10-03 22:47:40] iteration 280/ 6855 | consumed samples: 35840 | elapsed time per iteration (ms): 105998.4 | throughput per GPU (TFLOP/s/GPU): 35.3 | MFU 11.33% | FLOPs estimator: workload-aware | logical tokens/s: 158276.9 | valid tokens/s: 158276.9 | physical tokens/s: 158667.1 | loss tokens/s: 114955.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.018795E-01 | lm-global-supervised-tokens: 1.218513E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.134395E-02 | lm-token-buffer-mib: 5.813198E+01 | global_load_balancing_loss: 9.988974E-01 | loss scale: 1.0 | grad norm: 0.317 | params norm: 11653.631 | params rms: 0.063101 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 8 days, 1:35:39.506471 | finish at 2026-10-12 00:23:20 +times across ranks (ms): + save-checkpoint: + rank 0: 25699.38 + rank 1: 25699.38 + rank 2: 25699.40 + rank 3: 25699.38 + rank 4: 25699.39 + rank 5: 25699.39 + rank 6: 25699.38 + rank 7: 25699.38 + rank 8: 25699.27 + rank 9: 25699.27 + rank 10: 25699.27 + rank 11: 25699.28 + rank 12: 25699.27 + rank 13: 25699.25 + rank 14: 25699.29 + rank 15: 25699.27 + rank 16: 25700.71 + rank 17: 25700.73 + rank 18: 25700.70 + rank 19: 25700.74 + rank 20: 25700.70 + rank 21: 25700.71 + rank 22: 25700.71 + rank 23: 25700.70 + rank 24: 25699.51 + rank 25: 25699.54 + rank 26: 25699.50 + rank 27: 25699.51 + rank 28: 25699.51 + rank 29: 25699.50 + rank 30: 25699.56 + rank 31: 25699.52 + rank 32: 25699.37 + rank 33: 25699.36 + rank 34: 25699.38 + rank 35: 25699.34 + rank 36: 25699.34 + rank 37: 25699.32 + rank 38: 25699.34 + rank 39: 25699.36 + rank 40: 25699.56 + rank 41: 25699.50 + rank 42: 25699.51 + rank 43: 25699.52 + rank 44: 25699.50 + rank 45: 25699.51 + rank 46: 25699.51 + rank 47: 25699.49 + rank 48: 25699.49 + rank 49: 25699.54 + rank 50: 25699.36 + rank 51: 25699.41 + rank 52: 25699.37 + rank 53: 25699.40 + rank 54: 25699.54 + rank 55: 25699.39 + rank 56: 25699.79 + rank 57: 25699.77 + rank 58: 25699.76 + rank 59: 25699.77 + rank 60: 25699.76 + rank 61: 25699.77 + rank 62: 25699.77 + rank 63: 25699.79 + [2026-10-03 22:49:52] iteration 281/ 6855 | consumed samples: 35968 | elapsed time per iteration (ms): 105456.2 | throughput per GPU (TFLOP/s/GPU): 35.1 | MFU 11.26% | FLOPs estimator: workload-aware | logical tokens/s: 159091.5 | valid tokens/s: 159091.5 | physical tokens/s: 159321.3 | loss tokens/s: 119588.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.360775E-01 | lm-global-supervised-tokens: 1.261135E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 7.421041E-01 | lm-token-buffer-mib: 5.805946E+01 | global_load_balancing_loss: 9.990979E-01 | loss scale: 1.0 | grad norm: 0.093 | params norm: 11653.542 | params rms: 0.063100 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 8 days, 0:34:29.341733 | finish at 2026-10-11 23:24:21 + [2026-10-03 22:51:21] iteration 282/ 6855 | consumed samples: 36096 | elapsed time per iteration (ms): 89764.9 | throughput per GPU (TFLOP/s/GPU): 40.1 | MFU 12.84% | FLOPs estimator: workload-aware | logical tokens/s: 186901.1 | valid tokens/s: 186901.1 | physical tokens/s: 187327.8 | loss tokens/s: 142761.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.858026E-01 | lm-global-supervised-tokens: 1.281498E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.661575E-02 | lm-token-buffer-mib: 5.812605E+01 | global_load_balancing_loss: 1.001176E+00 | loss scale: 1.0 | grad norm: 0.063 | params norm: 11653.452 | params rms: 0.063100 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 19:53:44.955475 | finish at 2026-10-10 18:45:06 + [2026-10-03 22:52:49] iteration 283/ 6855 | consumed samples: 36224 | elapsed time per iteration (ms): 87143.5 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.39% | FLOPs estimator: workload-aware | logical tokens/s: 192523.3 | valid tokens/s: 192523.3 | physical tokens/s: 192938.5 | loss tokens/s: 145462.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.436465E-01 | lm-global-supervised-tokens: 1.267610E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.479812E-02 | lm-token-buffer-mib: 5.816826E+01 | global_load_balancing_loss: 1.000451E+00 | loss scale: 1.0 | grad norm: 0.099 | params norm: 11653.363 | params rms: 0.063099 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 15:05:07.298820 | finish at 2026-10-10 13:57:56 + [2026-10-03 22:54:14] iteration 284/ 6855 | consumed samples: 36352 | elapsed time per iteration (ms): 85813.1 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.64% | FLOPs estimator: workload-aware | logical tokens/s: 195508.3 | valid tokens/s: 195508.3 | physical tokens/s: 195919.7 | loss tokens/s: 152589.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.382745E-01 | lm-global-supervised-tokens: 1.309421E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.391504E-02 | lm-token-buffer-mib: 5.811676E+01 | global_load_balancing_loss: 1.001326E+00 | loss scale: 1.0 | grad norm: 0.170 | params norm: 11653.273 | params rms: 0.063099 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 12:37:57.943420 | finish at 2026-10-10 11:32:12 + [2026-10-03 22:55:42] iteration 285/ 6855 | consumed samples: 36480 | elapsed time per iteration (ms): 87400.7 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.67% | FLOPs estimator: workload-aware | logical tokens/s: 191957.0 | valid tokens/s: 191957.0 | physical tokens/s: 192318.1 | loss tokens/s: 142005.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.517538E-01 | lm-global-supervised-tokens: 1.241139E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.158088E-02 | lm-token-buffer-mib: 5.814232E+01 | global_load_balancing_loss: 1.001393E+00 | loss scale: 1.0 | grad norm: 0.078 | params norm: 11653.184 | params rms: 0.063098 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 15:30:22.681060 | finish at 2026-10-10 14:26:04 + [2026-10-03 22:57:10] iteration 286/ 6855 | consumed samples: 36608 | elapsed time per iteration (ms): 88207.7 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.49% | FLOPs estimator: workload-aware | logical tokens/s: 190201.0 | valid tokens/s: 190200.9 | physical tokens/s: 190647.6 | loss tokens/s: 143885.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.594840E-01 | lm-global-supervised-tokens: 1.269181E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.111721E-02 | lm-token-buffer-mib: 5.810815E+01 | global_load_balancing_loss: 9.992378E-01 | loss scale: 1.0 | grad norm: 0.104 | params norm: 11653.095 | params rms: 0.063098 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 16:57:16.154298 | finish at 2026-10-10 15:54:26 + [2026-10-03 22:58:36] iteration 287/ 6855 | consumed samples: 36736 | elapsed time per iteration (ms): 85622.9 | throughput per GPU (TFLOP/s/GPU): 43.3 | MFU 13.89% | FLOPs estimator: workload-aware | logical tokens/s: 195942.5 | valid tokens/s: 195942.5 | physical tokens/s: 196385.4 | loss tokens/s: 144730.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.275553E-01 | lm-global-supervised-tokens: 1.239221E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.355139E-02 | lm-token-buffer-mib: 5.816991E+01 | global_load_balancing_loss: 9.994298E-01 | loss scale: 1.0 | grad norm: 0.058 | params norm: 11653.004 | params rms: 0.063097 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 12:12:51.274595 | finish at 2026-10-10 11:11:27 + [2026-10-03 23:00:02] iteration 288/ 6855 | consumed samples: 36864 | elapsed time per iteration (ms): 86118.4 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.77% | FLOPs estimator: workload-aware | logical tokens/s: 194815.5 | valid tokens/s: 194815.5 | physical tokens/s: 195248.9 | loss tokens/s: 139792.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.822897E-01 | lm-global-supervised-tokens: 1.203870E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.352470E-02 | lm-token-buffer-mib: 5.813617E+01 | global_load_balancing_loss: 1.000447E+00 | loss scale: 1.0 | grad norm: 0.083 | params norm: 11652.914 | params rms: 0.063097 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 13:05:39.647732 | finish at 2026-10-10 12:05:41 + [2026-10-03 23:01:32] iteration 289/ 6855 | consumed samples: 36992 | elapsed time per iteration (ms): 89934.7 | throughput per GPU (TFLOP/s/GPU): 41.5 | MFU 13.29% | FLOPs estimator: workload-aware | logical tokens/s: 186548.1 | valid tokens/s: 186548.1 | physical tokens/s: 186936.1 | loss tokens/s: 133361.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.184674E-01 | lm-global-supervised-tokens: 1.199381E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.502291E-02 | lm-token-buffer-mib: 5.807065E+01 | global_load_balancing_loss: 9.996615E-01 | loss scale: 1.0 | grad norm: 0.061 | params norm: 11652.822 | params rms: 0.063096 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 20:01:51.456313 | finish at 2026-10-10 19:03:23 + [2026-10-03 23:03:22] iteration 290/ 6855 | consumed samples: 37120 | elapsed time per iteration (ms): 110249.7 | throughput per GPU (TFLOP/s/GPU): 34.3 | MFU 11.00% | FLOPs estimator: workload-aware | logical tokens/s: 152174.5 | valid tokens/s: 152174.5 | physical tokens/s: 152484.8 | loss tokens/s: 111477.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.665428E-01 | lm-global-supervised-tokens: 1.229031E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.158107E-02 | lm-token-buffer-mib: 5.806447E+01 | global_load_balancing_loss: 9.993224E-01 | loss scale: 1.0 | grad norm: 0.061 | params norm: 11652.729 | params rms: 0.063096 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 8 days, 9:03:09.376434 | finish at 2026-10-12 08:06:31 + [2026-10-03 23:04:51] iteration 291/ 6855 | consumed samples: 37248 | elapsed time per iteration (ms): 88723.6 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.56% | FLOPs estimator: workload-aware | logical tokens/s: 189094.9 | valid tokens/s: 189094.9 | physical tokens/s: 189399.8 | loss tokens/s: 139919.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.406535E-01 | lm-global-supervised-tokens: 1.241412E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.543071E-02 | lm-token-buffer-mib: 5.802165E+01 | global_load_balancing_loss: 9.992175E-01 | loss scale: 1.0 | grad norm: 0.047 | params norm: 11652.637 | params rms: 0.063095 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 17:46:22.007160 | finish at 2026-10-10 16:51:13 + [2026-10-03 23:06:17] iteration 292/ 6855 | consumed samples: 37376 | elapsed time per iteration (ms): 86564.2 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.49% | FLOPs estimator: workload-aware | logical tokens/s: 193812.1 | valid tokens/s: 193812.0 | physical tokens/s: 194158.3 | loss tokens/s: 148271.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.082119E-01 | lm-global-supervised-tokens: 1.283500E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.138778E-02 | lm-token-buffer-mib: 5.805660E+01 | global_load_balancing_loss: 1.001471E+00 | loss scale: 1.0 | grad norm: 0.040 | params norm: 11652.542 | params rms: 0.063095 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 13:48:40.637558 | finish at 2026-10-10 12:54:58 + [2026-10-03 23:07:47] iteration 293/ 6855 | consumed samples: 37504 | elapsed time per iteration (ms): 89738.1 | throughput per GPU (TFLOP/s/GPU): 41.1 | MFU 13.18% | FLOPs estimator: workload-aware | logical tokens/s: 186957.4 | valid tokens/s: 186957.4 | physical tokens/s: 187483.9 | loss tokens/s: 145686.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.015694E-01 | lm-global-supervised-tokens: 1.307366E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.175594E-02 | lm-token-buffer-mib: 5.808429E+01 | global_load_balancing_loss: 1.001298E+00 | loss scale: 1.0 | grad norm: 0.058 | params norm: 11652.448 | params rms: 0.063094 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 19:34:21.471992 | finish at 2026-10-10 18:42:08 + [2026-10-03 23:09:13] iteration 294/ 6855 | consumed samples: 37632 | elapsed time per iteration (ms): 85668.7 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.68% | FLOPs estimator: workload-aware | logical tokens/s: 195837.9 | valid tokens/s: 195837.9 | physical tokens/s: 196490.4 | loss tokens/s: 152112.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.048515E-01 | lm-global-supervised-tokens: 1.303128E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.427029E-02 | lm-token-buffer-mib: 5.809715E+01 | global_load_balancing_loss: 1.001248E+00 | loss scale: 1.0 | grad norm: 0.055 | params norm: 11652.353 | params rms: 0.063094 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 12:07:52.274868 | finish at 2026-10-10 11:17:05 + [2026-10-03 23:10:42] iteration 295/ 6855 | consumed samples: 37760 | elapsed time per iteration (ms): 89622.4 | throughput per GPU (TFLOP/s/GPU): 41.5 | MFU 13.32% | FLOPs estimator: workload-aware | logical tokens/s: 187198.5 | valid tokens/s: 187198.5 | physical tokens/s: 187591.3 | loss tokens/s: 138885.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.168389E-01 | lm-global-supervised-tokens: 1.244727E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.552696E-02 | lm-token-buffer-mib: 5.806406E+01 | global_load_balancing_loss: 1.001803E+00 | loss scale: 1.0 | grad norm: 0.041 | params norm: 11652.258 | params rms: 0.063093 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 19:18:43.008423 | finish at 2026-10-10 18:29:25 + [2026-10-03 23:12:09] iteration 296/ 6855 | consumed samples: 37888 | elapsed time per iteration (ms): 86405.4 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.63% | FLOPs estimator: workload-aware | logical tokens/s: 194168.1 | valid tokens/s: 194168.1 | physical tokens/s: 194577.4 | loss tokens/s: 151461.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.016944E-01 | lm-global-supervised-tokens: 1.308711E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.135604E-02 | lm-token-buffer-mib: 5.814401E+01 | global_load_balancing_loss: 1.000073E+00 | loss scale: 1.0 | grad norm: 0.071 | params norm: 11652.161 | params rms: 0.063093 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 13:25:32.759259 | finish at 2026-10-10 12:37:41 + [2026-10-03 23:13:35] iteration 297/ 6855 | consumed samples: 38016 | elapsed time per iteration (ms): 86743.9 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.60% | FLOPs estimator: workload-aware | logical tokens/s: 193410.4 | valid tokens/s: 193410.4 | physical tokens/s: 193785.0 | loss tokens/s: 147161.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.776260E-01 | lm-global-supervised-tokens: 1.276535E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.509126E-02 | lm-token-buffer-mib: 5.807224E+01 | global_load_balancing_loss: 9.992568E-01 | loss scale: 1.0 | grad norm: 0.032 | params norm: 11652.067 | params rms: 0.063092 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 14:01:06.806180 | finish at 2026-10-10 13:14:42 + [2026-10-03 23:15:00] iteration 298/ 6855 | consumed samples: 38144 | elapsed time per iteration (ms): 85002.3 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.78% | FLOPs estimator: workload-aware | logical tokens/s: 197373.3 | valid tokens/s: 197373.3 | physical tokens/s: 197828.4 | loss tokens/s: 142918.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.667376E-01 | lm-global-supervised-tokens: 1.214842E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.513750E-02 | lm-token-buffer-mib: 5.809885E+01 | global_load_balancing_loss: 1.000192E+00 | loss scale: 1.0 | grad norm: 0.062 | params norm: 11651.972 | params rms: 0.063092 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 10:49:20.115650 | finish at 2026-10-10 10:04:20 + [2026-10-03 23:16:31] iteration 299/ 6855 | consumed samples: 38272 | elapsed time per iteration (ms): 90218.6 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.50% | FLOPs estimator: workload-aware | logical tokens/s: 185961.5 | valid tokens/s: 185961.5 | physical tokens/s: 186371.9 | loss tokens/s: 138492.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.017755E-01 | lm-global-supervised-tokens: 1.249455E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.180503E-02 | lm-token-buffer-mib: 5.835495E+01 | global_load_balancing_loss: 9.994631E-01 | loss scale: 1.0 | grad norm: 0.046 | params norm: 11651.876 | params rms: 0.063091 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 20:17:52.935502 | finish at 2026-10-10 19:34:24 + [2026-10-03 23:18:18] iteration 300/ 6855 | consumed samples: 38400 | elapsed time per iteration (ms): 107059.5 | throughput per GPU (TFLOP/s/GPU): 34.5 | MFU 11.04% | FLOPs estimator: workload-aware | logical tokens/s: 156709.0 | valid tokens/s: 156709.0 | physical tokens/s: 157133.2 | loss tokens/s: 114303.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.157195E-01 | lm-global-supervised-tokens: 1.223729E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.407305E-02 | lm-token-buffer-mib: 5.823203E+01 | global_load_balancing_loss: 1.000121E+00 | loss scale: 1.0 | grad norm: 0.049 | params norm: 11651.779 | params rms: 0.063091 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 8 days, 2:56:15.166143 | finish at 2026-10-12 02:14:33 + [2026-10-03 23:19:42] iteration 301/ 6855 | consumed samples: 38528 | elapsed time per iteration (ms): 84754.5 | throughput per GPU (TFLOP/s/GPU): 43.5 | MFU 13.93% | FLOPs estimator: workload-aware | logical tokens/s: 197950.3 | valid tokens/s: 197950.3 | physical tokens/s: 198278.3 | loss tokens/s: 146534.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.482532E-01 | lm-global-supervised-tokens: 1.241947E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.826669E-02 | lm-token-buffer-mib: 5.813185E+01 | global_load_balancing_loss: 1.000174E+00 | loss scale: 1.0 | grad norm: 0.042 | params norm: 11651.682 | params rms: 0.063090 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 10:18:01.000238 | finish at 2026-10-10 09:37:43 + [2026-10-03 23:21:11] iteration 302/ 6855 | consumed samples: 38656 | elapsed time per iteration (ms): 88259.2 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.43% | FLOPs estimator: workload-aware | logical tokens/s: 190089.7 | valid tokens/s: 190089.7 | physical tokens/s: 190656.1 | loss tokens/s: 138368.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.764072E-01 | lm-global-supervised-tokens: 1.221226E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.630759E-02 | lm-token-buffer-mib: 5.816381E+01 | global_load_balancing_loss: 1.000852E+00 | loss scale: 1.0 | grad norm: 0.050 | params norm: 11651.584 | params rms: 0.063090 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 16:39:22.294502 | finish at 2026-10-10 16:00:33 + [2026-10-03 23:22:36] iteration 303/ 6855 | consumed samples: 38784 | elapsed time per iteration (ms): 85189.6 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.49% | FLOPs estimator: workload-aware | logical tokens/s: 196939.2 | valid tokens/s: 196939.2 | physical tokens/s: 197334.0 | loss tokens/s: 149755.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.235960E-01 | lm-global-supervised-tokens: 1.275758E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.251244E-02 | lm-token-buffer-mib: 5.813639E+01 | global_load_balancing_loss: 1.001602E+00 | loss scale: 1.0 | grad norm: 0.048 | params norm: 11651.488 | params rms: 0.063089 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 11:02:42.066999 | finish at 2026-10-10 10:25:18 + [2026-10-03 23:24:04] iteration 304/ 6855 | consumed samples: 38912 | elapsed time per iteration (ms): 87769.4 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.62% | FLOPs estimator: workload-aware | logical tokens/s: 191151.2 | valid tokens/s: 191151.2 | physical tokens/s: 191537.3 | loss tokens/s: 140121.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.073921E-01 | lm-global-supervised-tokens: 1.229838E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.641153E-02 | lm-token-buffer-mib: 5.807506E+01 | global_load_balancing_loss: 1.000064E+00 | loss scale: 1.0 | grad norm: 0.069 | params norm: 11651.394 | params rms: 0.063089 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 15:42:57.018437 | finish at 2026-10-10 15:07:01 + [2026-10-03 23:25:36] iteration 305/ 6855 | consumed samples: 39040 | elapsed time per iteration (ms): 92038.2 | throughput per GPU (TFLOP/s/GPU): 40.8 | MFU 13.08% | FLOPs estimator: workload-aware | logical tokens/s: 182285.1 | valid tokens/s: 182285.1 | physical tokens/s: 182536.5 | loss tokens/s: 127253.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.402637E-01 | lm-global-supervised-tokens: 1.171216E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.540424E-02 | lm-token-buffer-mib: 5.804298E+01 | global_load_balancing_loss: 9.999214E-01 | loss scale: 1.0 | grad norm: 0.034 | params norm: 11651.298 | params rms: 0.063088 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 23:27:30.321794 | finish at 2026-10-10 22:53:06 + [2026-10-03 23:27:01] iteration 306/ 6855 | consumed samples: 39168 | elapsed time per iteration (ms): 85431.9 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.57% | FLOPs estimator: workload-aware | logical tokens/s: 196381.1 | valid tokens/s: 196381.1 | physical tokens/s: 196937.2 | loss tokens/s: 146612.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.874602E-01 | lm-global-supervised-tokens: 1.252542E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.749998E-02 | lm-token-buffer-mib: 5.815556E+01 | global_load_balancing_loss: 9.995472E-01 | loss scale: 1.0 | grad norm: 0.050 | params norm: 11651.203 | params rms: 0.063087 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 11:24:53.592891 | finish at 2026-10-10 10:51:55 + [2026-10-03 23:28:31] iteration 307/ 6855 | consumed samples: 39296 | elapsed time per iteration (ms): 90408.2 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.43% | FLOPs estimator: workload-aware | logical tokens/s: 185571.5 | valid tokens/s: 185571.5 | physical tokens/s: 185987.3 | loss tokens/s: 136595.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.740125E-01 | lm-global-supervised-tokens: 1.234939E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.339258E-02 | lm-token-buffer-mib: 5.809591E+01 | global_load_balancing_loss: 1.000139E+00 | loss scale: 1.0 | grad norm: 0.047 | params norm: 11651.108 | params rms: 0.063087 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 20:26:32.576851 | finish at 2026-10-10 19:55:04 + [2026-10-03 23:29:57] iteration 308/ 6855 | consumed samples: 39424 | elapsed time per iteration (ms): 85906.9 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.53% | FLOPs estimator: workload-aware | logical tokens/s: 195294.9 | valid tokens/s: 195294.9 | physical tokens/s: 195664.2 | loss tokens/s: 148915.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.962689E-01 | lm-global-supervised-tokens: 1.279286E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.331031E-02 | lm-token-buffer-mib: 5.804933E+01 | global_load_balancing_loss: 9.994279E-01 | loss scale: 1.0 | grad norm: 0.050 | params norm: 11651.015 | params rms: 0.063086 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 12:13:52.659586 | finish at 2026-10-10 11:43:50 + [2026-10-03 23:31:20] iteration 309/ 6855 | consumed samples: 39552 | elapsed time per iteration (ms): 82693.2 | throughput per GPU (TFLOP/s/GPU): 43.5 | MFU 13.93% | FLOPs estimator: workload-aware | logical tokens/s: 202884.5 | valid tokens/s: 202884.5 | physical tokens/s: 203339.5 | loss tokens/s: 153078.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.409749E-01 | lm-global-supervised-tokens: 1.265854E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.268850E-02 | lm-token-buffer-mib: 5.810264E+01 | global_load_balancing_loss: 1.000292E+00 | loss scale: 1.0 | grad norm: 0.054 | params norm: 11650.922 | params rms: 0.063086 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 6:21:50.006310 | finish at 2026-10-10 05:53:10 + [2026-10-03 23:33:12] iteration 310/ 6855 | consumed samples: 39680 | elapsed time per iteration (ms): 111570.8 | throughput per GPU (TFLOP/s/GPU): 34.2 | MFU 10.95% | FLOPs estimator: workload-aware | logical tokens/s: 150372.8 | valid tokens/s: 150372.8 | physical tokens/s: 150589.3 | loss tokens/s: 106081.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.278454E-01 | lm-global-supervised-tokens: 1.183559E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.583651E-02 | lm-token-buffer-mib: 5.803154E+01 | global_load_balancing_loss: 1.000647E+00 | loss scale: 1.0 | grad norm: 0.031 | params norm: 11650.827 | params rms: 0.063085 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 8 days, 10:50:30.719464 | finish at 2026-10-12 10:23:42 + [2026-10-03 23:34:43] iteration 311/ 6855 | consumed samples: 39808 | elapsed time per iteration (ms): 91558.1 | throughput per GPU (TFLOP/s/GPU): 41.2 | MFU 13.22% | FLOPs estimator: workload-aware | logical tokens/s: 183241.2 | valid tokens/s: 183241.2 | physical tokens/s: 183647.7 | loss tokens/s: 131146.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.197208E-01 | lm-global-supervised-tokens: 1.200756E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.486920E-02 | lm-token-buffer-mib: 5.807343E+01 | global_load_balancing_loss: 1.001204E+00 | loss scale: 1.0 | grad norm: 0.053 | params norm: 11650.732 | params rms: 0.063085 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 22:25:56.143894 | finish at 2026-10-10 22:00:39 + [2026-10-03 23:36:12] iteration 312/ 6855 | consumed samples: 39936 | elapsed time per iteration (ms): 88407.5 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.50% | FLOPs estimator: workload-aware | logical tokens/s: 189770.8 | valid tokens/s: 189770.8 | physical tokens/s: 190122.8 | loss tokens/s: 143927.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.312704E-01 | lm-global-supervised-tokens: 1.272430E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.198494E-02 | lm-token-buffer-mib: 5.815553E+01 | global_load_balancing_loss: 1.000756E+00 | loss scale: 1.0 | grad norm: 0.045 | params norm: 11650.637 | params rms: 0.063084 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 16:40:50.562842 | finish at 2026-10-10 16:17:02 + [2026-10-03 23:37:38] iteration 313/ 6855 | consumed samples: 40064 | elapsed time per iteration (ms): 86823.1 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.47% | FLOPs estimator: workload-aware | logical tokens/s: 193234.1 | valid tokens/s: 193234.1 | physical tokens/s: 193734.3 | loss tokens/s: 147588.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.990646E-01 | lm-global-supervised-tokens: 1.281412E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.174754E-02 | lm-token-buffer-mib: 5.816359E+01 | global_load_balancing_loss: 1.000355E+00 | loss scale: 1.0 | grad norm: 0.037 | params norm: 11650.544 | params rms: 0.063084 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 13:46:36.711431 | finish at 2026-10-10 13:24:15 + [2026-10-03 23:39:06] iteration 314/ 6855 | consumed samples: 40192 | elapsed time per iteration (ms): 87501.6 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.55% | FLOPs estimator: workload-aware | logical tokens/s: 191735.7 | valid tokens/s: 191735.7 | physical tokens/s: 192199.9 | loss tokens/s: 134410.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 5.893157E-01 | lm-global-supervised-tokens: 1.176111E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.848457E-02 | lm-token-buffer-mib: 5.803163E+01 | global_load_balancing_loss: 1.000180E+00 | loss scale: 1.0 | grad norm: 0.043 | params norm: 11650.450 | params rms: 0.063083 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 14:59:07.741210 | finish at 2026-10-10 14:38:14 + [2026-10-03 23:40:32] iteration 315/ 6855 | consumed samples: 40320 | elapsed time per iteration (ms): 86398.8 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.60% | FLOPs estimator: workload-aware | logical tokens/s: 194183.2 | valid tokens/s: 194183.2 | physical tokens/s: 194949.2 | loss tokens/s: 140292.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.704253E-01 | lm-global-supervised-tokens: 1.212113E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.155845E-02 | lm-token-buffer-mib: 5.809074E+01 | global_load_balancing_loss: 9.997950E-01 | loss scale: 1.0 | grad norm: 0.039 | params norm: 11650.356 | params rms: 0.063083 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 12:57:28.109221 | finish at 2026-10-10 12:38:00 + [2026-10-03 23:42:01] iteration 316/ 6855 | consumed samples: 40448 | elapsed time per iteration (ms): 88803.1 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.64% | FLOPs estimator: workload-aware | logical tokens/s: 188925.9 | valid tokens/s: 188925.9 | physical tokens/s: 189446.0 | loss tokens/s: 145322.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.363133E-01 | lm-global-supervised-tokens: 1.290512E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.403333E-02 | lm-token-buffer-mib: 5.816335E+01 | global_load_balancing_loss: 9.999838E-01 | loss scale: 1.0 | grad norm: 0.062 | params norm: 11650.263 | params rms: 0.063082 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 17:18:03.360923 | finish at 2026-10-10 17:00:05 + [2026-10-03 23:43:32] iteration 317/ 6855 | consumed samples: 40576 | elapsed time per iteration (ms): 91121.2 | throughput per GPU (TFLOP/s/GPU): 40.6 | MFU 13.02% | FLOPs estimator: workload-aware | logical tokens/s: 184119.8 | valid tokens/s: 184119.8 | physical tokens/s: 184529.8 | loss tokens/s: 129935.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.714143E-01 | lm-global-supervised-tokens: 1.183984E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.126077E-02 | lm-token-buffer-mib: 5.809711E+01 | global_load_balancing_loss: 9.993035E-01 | loss scale: 1.0 | grad norm: 0.044 | params norm: 11650.170 | params rms: 0.063082 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 21:29:10.437329 | finish at 2026-10-10 21:12:43 + [2026-10-03 23:44:59] iteration 318/ 6855 | consumed samples: 40704 | elapsed time per iteration (ms): 86498.5 | throughput per GPU (TFLOP/s/GPU): 42.8 | MFU 13.73% | FLOPs estimator: workload-aware | logical tokens/s: 193959.5 | valid tokens/s: 193959.5 | physical tokens/s: 194406.1 | loss tokens/s: 140547.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.610368E-01 | lm-global-supervised-tokens: 1.215713E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.178087E-02 | lm-token-buffer-mib: 5.821019E+01 | global_load_balancing_loss: 9.993182E-01 | loss scale: 1.0 | grad norm: 0.032 | params norm: 11650.076 | params rms: 0.063081 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 13:04:00.701445 | finish at 2026-10-10 12:48:59 + [2026-10-03 23:46:24] iteration 319/ 6855 | consumed samples: 40832 | elapsed time per iteration (ms): 85123.3 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.60% | FLOPs estimator: workload-aware | logical tokens/s: 197092.3 | valid tokens/s: 197092.3 | physical tokens/s: 197617.1 | loss tokens/s: 152261.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.571911E-01 | lm-global-supervised-tokens: 1.296098E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.164830E-02 | lm-token-buffer-mib: 5.803994E+01 | global_load_balancing_loss: 1.000217E+00 | loss scale: 1.0 | grad norm: 0.068 | params norm: 11649.984 | params rms: 0.063081 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 10:32:45.931368 | finish at 2026-10-10 10:19:10 + [2026-10-03 23:48:14] iteration 320/ 6855 | consumed samples: 40960 | elapsed time per iteration (ms): 109950.6 | throughput per GPU (TFLOP/s/GPU): 33.5 | MFU 10.74% | FLOPs estimator: workload-aware | logical tokens/s: 152588.3 | valid tokens/s: 152588.3 | physical tokens/s: 152928.1 | loss tokens/s: 110671.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.567593E-01 | lm-global-supervised-tokens: 1.216842E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.159898E-02 | lm-token-buffer-mib: 5.804653E+01 | global_load_balancing_loss: 1.001305E+00 | loss scale: 1.0 | grad norm: 0.122 | params norm: 11649.891 | params rms: 0.063080 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 8 days, 7:35:27.421253 | finish at 2026-10-12 07:23:41 +times across ranks (ms): + save-checkpoint: + rank 0: 24564.42 + rank 1: 24564.41 + rank 2: 24564.46 + rank 3: 24564.43 + rank 4: 24564.43 + rank 5: 24564.46 + rank 6: 24564.42 + rank 7: 24564.42 + rank 8: 24564.11 + rank 9: 24563.99 + rank 10: 24563.98 + rank 11: 24563.98 + rank 12: 24563.98 + rank 13: 24563.96 + rank 14: 24563.98 + rank 15: 24563.99 + rank 16: 24565.17 + rank 17: 24565.13 + rank 18: 24564.67 + rank 19: 24565.27 + rank 20: 24565.07 + rank 21: 24565.12 + rank 22: 24565.09 + rank 23: 24565.08 + rank 24: 24564.94 + rank 25: 24565.30 + rank 26: 24564.99 + rank 27: 24564.98 + rank 28: 24564.89 + rank 29: 24564.93 + rank 30: 24564.91 + rank 31: 24564.94 + rank 32: 24565.26 + rank 33: 24565.25 + rank 34: 24565.28 + rank 35: 24565.25 + rank 36: 24565.18 + rank 37: 24565.17 + rank 38: 24565.24 + rank 39: 24565.18 + rank 40: 24564.81 + rank 41: 24564.79 + rank 42: 24564.79 + rank 43: 24564.80 + rank 44: 24564.77 + rank 45: 24564.77 + rank 46: 24564.79 + rank 47: 24564.77 + rank 48: 24566.02 + rank 49: 24565.89 + rank 50: 24565.70 + rank 51: 24566.05 + rank 52: 24565.96 + rank 53: 24565.95 + rank 54: 24565.88 + rank 55: 24566.02 + rank 56: 24564.72 + rank 57: 24564.74 + rank 58: 24564.77 + rank 59: 24564.74 + rank 60: 24564.72 + rank 61: 24564.75 + rank 62: 24564.72 + rank 63: 24564.74 + [2026-10-03 23:50:21] iteration 321/ 6855 | consumed samples: 41088 | elapsed time per iteration (ms): 102217.2 | throughput per GPU (TFLOP/s/GPU): 36.8 | MFU 11.80% | FLOPs estimator: workload-aware | logical tokens/s: 164132.6 | valid tokens/s: 164132.6 | physical tokens/s: 164394.8 | loss tokens/s: 118162.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.811218E-01 | lm-global-supervised-tokens: 1.207827E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.961092E+00 | lm-token-buffer-mib: 5.810911E+01 | global_load_balancing_loss: 1.001125E+00 | loss scale: 1.0 | grad norm: 0.053 | params norm: 11649.800 | params rms: 0.063080 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 7 days, 17:31:27.482612 | finish at 2026-10-11 17:21:48 + [2026-10-03 23:51:54] iteration 322/ 6855 | consumed samples: 41216 | elapsed time per iteration (ms): 93009.9 | throughput per GPU (TFLOP/s/GPU): 39.7 | MFU 12.73% | FLOPs estimator: workload-aware | logical tokens/s: 180380.5 | valid tokens/s: 180380.5 | physical tokens/s: 180629.1 | loss tokens/s: 134442.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.589993E-01 | lm-global-supervised-tokens: 1.250450E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.264479E-02 | lm-token-buffer-mib: 5.802890E+01 | global_load_balancing_loss: 1.001320E+00 | loss scale: 1.0 | grad norm: 0.151 | params norm: 11649.705 | params rms: 0.063079 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 7 days, 0:47:13.583853 | finish at 2026-10-11 00:39:07 + [2026-10-03 23:53:19] iteration 323/ 6855 | consumed samples: 41344 | elapsed time per iteration (ms): 85460.8 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.46% | FLOPs estimator: workload-aware | logical tokens/s: 196314.1 | valid tokens/s: 196314.1 | physical tokens/s: 196961.1 | loss tokens/s: 148282.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.436613E-01 | lm-global-supervised-tokens: 1.267237E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.101126E-02 | lm-token-buffer-mib: 5.823721E+01 | global_load_balancing_loss: 1.000113E+00 | loss scale: 1.0 | grad norm: 0.191 | params norm: 11649.612 | params rms: 0.063079 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 11:03:49.672623 | finish at 2026-10-10 10:57:09 + [2026-10-03 23:54:47] iteration 324/ 6855 | consumed samples: 41472 | elapsed time per iteration (ms): 87517.6 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.52% | FLOPs estimator: workload-aware | logical tokens/s: 191699.8 | valid tokens/s: 191699.8 | physical tokens/s: 192193.5 | loss tokens/s: 146056.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.675345E-01 | lm-global-supervised-tokens: 1.278251E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.448527E-02 | lm-token-buffer-mib: 5.808990E+01 | global_load_balancing_loss: 9.994614E-01 | loss scale: 1.0 | grad norm: 0.139 | params norm: 11649.518 | params rms: 0.063078 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 14:46:17.520730 | finish at 2026-10-10 14:41:04 + [2026-10-03 23:56:11] iteration 325/ 6855 | consumed samples: 41600 | elapsed time per iteration (ms): 83880.0 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.79% | FLOPs estimator: workload-aware | logical tokens/s: 200014.0 | valid tokens/s: 200014.0 | physical tokens/s: 200557.1 | loss tokens/s: 153518.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.193553E-01 | lm-global-supervised-tokens: 1.287715E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.385803E-02 | lm-token-buffer-mib: 5.810005E+01 | global_load_balancing_loss: 9.996903E-01 | loss scale: 1.0 | grad norm: 0.033 | params norm: 11649.426 | params rms: 0.063078 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 8:08:56.515956 | finish at 2026-10-10 08:05:07 + [2026-10-03 23:57:36] iteration 326/ 6855 | consumed samples: 41728 | elapsed time per iteration (ms): 85653.0 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.77% | FLOPs estimator: workload-aware | logical tokens/s: 195873.9 | valid tokens/s: 195873.9 | physical tokens/s: 196290.7 | loss tokens/s: 153152.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.716483E-01 | lm-global-supervised-tokens: 1.311794E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.316824E-02 | lm-token-buffer-mib: 5.814533E+01 | global_load_balancing_loss: 9.993857E-01 | loss scale: 1.0 | grad norm: 0.110 | params norm: 11649.334 | params rms: 0.063077 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 11:20:28.697718 | finish at 2026-10-10 11:18:05 + [2026-10-03 23:59:04] iteration 327/ 6855 | consumed samples: 41856 | elapsed time per iteration (ms): 87874.1 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.53% | FLOPs estimator: workload-aware | logical tokens/s: 190922.5 | valid tokens/s: 190922.4 | physical tokens/s: 191344.9 | loss tokens/s: 138590.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.887596E-01 | lm-global-supervised-tokens: 1.217854E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.094184E-02 | lm-token-buffer-mib: 5.811582E+01 | global_load_balancing_loss: 1.000172E+00 | loss scale: 1.0 | grad norm: 0.077 | params norm: 11649.243 | params rms: 0.063077 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 15:20:42.404755 | finish at 2026-10-10 15:19:46 + [2026-10-04 00:00:32] iteration 328/ 6855 | consumed samples: 41984 | elapsed time per iteration (ms): 88080.0 | throughput per GPU (TFLOP/s/GPU): 41.4 | MFU 13.28% | FLOPs estimator: workload-aware | logical tokens/s: 190477.0 | valid tokens/s: 190477.0 | physical tokens/s: 191109.2 | loss tokens/s: 143767.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.573172E-01 | lm-global-supervised-tokens: 1.266304E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.289341E-02 | lm-token-buffer-mib: 5.808075E+01 | global_load_balancing_loss: 9.994462E-01 | loss scale: 1.0 | grad norm: 0.075 | params norm: 11649.150 | params rms: 0.063076 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 15:41:38.075469 | finish at 2026-10-10 15:42:10 + [2026-10-04 00:02:01] iteration 329/ 6855 | consumed samples: 42112 | elapsed time per iteration (ms): 89083.0 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.51% | FLOPs estimator: workload-aware | logical tokens/s: 188332.5 | valid tokens/s: 188332.5 | physical tokens/s: 188624.2 | loss tokens/s: 135896.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.617351E-01 | lm-global-supervised-tokens: 1.210606E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.350615E-02 | lm-token-buffer-mib: 5.803076E+01 | global_load_balancing_loss: 1.000544E+00 | loss scale: 1.0 | grad norm: 0.052 | params norm: 11649.057 | params rms: 0.063076 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 17:29:15.365126 | finish at 2026-10-10 17:31:17 + [2026-10-04 00:03:50] iteration 330/ 6855 | consumed samples: 42240 | elapsed time per iteration (ms): 108445.0 | throughput per GPU (TFLOP/s/GPU): 34.1 | MFU 10.93% | FLOPs estimator: workload-aware | logical tokens/s: 154706.3 | valid tokens/s: 154706.3 | physical tokens/s: 155096.7 | loss tokens/s: 112990.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.726995E-01 | lm-global-supervised-tokens: 1.225321E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.115562E-02 | lm-token-buffer-mib: 5.804340E+01 | global_load_balancing_loss: 1.001555E+00 | loss scale: 1.0 | grad norm: 0.119 | params norm: 11648.963 | params rms: 0.063075 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 8 days, 4:33:23.384990 | finish at 2026-10-12 04:37:13 + [2026-10-04 00:05:17] iteration 331/ 6855 | consumed samples: 42368 | elapsed time per iteration (ms): 86846.0 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.57% | FLOPs estimator: workload-aware | logical tokens/s: 193183.2 | valid tokens/s: 193183.2 | physical tokens/s: 193511.1 | loss tokens/s: 142121.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.431536E-01 | lm-global-supervised-tokens: 1.234272E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.131834E-02 | lm-token-buffer-mib: 5.812737E+01 | global_load_balancing_loss: 1.000589E+00 | loss scale: 1.0 | grad norm: 0.085 | params norm: 11648.871 | params rms: 0.063075 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 13:23:03.404818 | finish at 2026-10-10 13:28:20 + [2026-10-04 00:06:44] iteration 332/ 6855 | consumed samples: 42496 | elapsed time per iteration (ms): 87196.0 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.40% | FLOPs estimator: workload-aware | logical tokens/s: 192407.8 | valid tokens/s: 192407.8 | physical tokens/s: 193071.2 | loss tokens/s: 138587.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.981311E-01 | lm-global-supervised-tokens: 1.208430E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.878810E-02 | lm-token-buffer-mib: 5.850768E+01 | global_load_balancing_loss: 1.001121E+00 | loss scale: 1.0 | grad norm: 0.212 | params norm: 11648.778 | params rms: 0.063074 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 13:59:39.681275 | finish at 2026-10-10 14:06:23 + [2026-10-04 00:08:09] iteration 333/ 6855 | consumed samples: 42624 | elapsed time per iteration (ms): 85527.0 | throughput per GPU (TFLOP/s/GPU): 43.2 | MFU 13.83% | FLOPs estimator: workload-aware | logical tokens/s: 196162.6 | valid tokens/s: 196162.6 | physical tokens/s: 196581.8 | loss tokens/s: 151979.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.912831E-01 | lm-global-supervised-tokens: 1.299834E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.138744E-02 | lm-token-buffer-mib: 5.814268E+01 | global_load_balancing_loss: 9.994369E-01 | loss scale: 1.0 | grad norm: 0.132 | params norm: 11648.680 | params rms: 0.063074 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 10:56:47.302042 | finish at 2026-10-10 11:04:57 + [2026-10-04 00:09:41] iteration 334/ 6855 | consumed samples: 42752 | elapsed time per iteration (ms): 92077.7 | throughput per GPU (TFLOP/s/GPU): 40.8 | MFU 13.08% | FLOPs estimator: workload-aware | logical tokens/s: 182206.8 | valid tokens/s: 182206.8 | physical tokens/s: 182675.9 | loss tokens/s: 133811.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.866685E-01 | lm-global-supervised-tokens: 1.232105E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.189283E-02 | lm-token-buffer-mib: 5.816232E+01 | global_load_balancing_loss: 9.991326E-01 | loss scale: 1.0 | grad norm: 0.270 | params norm: 11648.586 | params rms: 0.063073 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 22:47:18.531792 | finish at 2026-10-10 22:57:00 + [2026-10-04 00:11:09] iteration 335/ 6855 | consumed samples: 42880 | elapsed time per iteration (ms): 88155.3 | throughput per GPU (TFLOP/s/GPU): 42.8 | MFU 13.71% | FLOPs estimator: workload-aware | logical tokens/s: 190313.2 | valid tokens/s: 190313.2 | physical tokens/s: 190715.3 | loss tokens/s: 142158.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.833798E-01 | lm-global-supervised-tokens: 1.253206E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.151413E-02 | lm-token-buffer-mib: 5.809443E+01 | global_load_balancing_loss: 9.997111E-01 | loss scale: 1.0 | grad norm: 0.190 | params norm: 11648.495 | params rms: 0.063073 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 15:39:32.493181 | finish at 2026-10-10 15:50:42 + [2026-10-04 00:12:38] iteration 336/ 6855 | consumed samples: 43008 | elapsed time per iteration (ms): 88980.0 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.50% | FLOPs estimator: workload-aware | logical tokens/s: 188549.9 | valid tokens/s: 188549.9 | physical tokens/s: 188880.4 | loss tokens/s: 135693.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.557435E-01 | lm-global-supervised-tokens: 1.207404E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.051786E-02 | lm-token-buffer-mib: 5.803677E+01 | global_load_balancing_loss: 1.000559E+00 | loss scale: 1.0 | grad norm: 0.211 | params norm: 11648.405 | params rms: 0.063072 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 17:07:40.599919 | finish at 2026-10-10 17:20:19 + [2026-10-04 00:14:04] iteration 337/ 6855 | consumed samples: 43136 | elapsed time per iteration (ms): 85785.1 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.64% | FLOPs estimator: workload-aware | logical tokens/s: 195572.4 | valid tokens/s: 195572.3 | physical tokens/s: 195932.3 | loss tokens/s: 143328.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.524163E-01 | lm-global-supervised-tokens: 1.229548E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.162449E-02 | lm-token-buffer-mib: 5.817300E+01 | global_load_balancing_loss: 9.994460E-01 | loss scale: 1.0 | grad norm: 0.096 | params norm: 11648.315 | params rms: 0.063072 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 11:19:07.123181 | finish at 2026-10-10 11:33:11 + [2026-10-04 00:15:30] iteration 338/ 6855 | consumed samples: 43264 | elapsed time per iteration (ms): 86018.7 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.69% | FLOPs estimator: workload-aware | logical tokens/s: 195041.4 | valid tokens/s: 195041.4 | physical tokens/s: 195534.5 | loss tokens/s: 146277.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.236488E-01 | lm-global-supervised-tokens: 1.258261E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.116260E-02 | lm-token-buffer-mib: 5.809275E+01 | global_load_balancing_loss: 1.001194E+00 | loss scale: 1.0 | grad norm: 0.283 | params norm: 11648.224 | params rms: 0.063071 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 11:43:04.084675 | finish at 2026-10-10 11:58:34 + [2026-10-04 00:16:55] iteration 339/ 6855 | consumed samples: 43392 | elapsed time per iteration (ms): 84568.7 | throughput per GPU (TFLOP/s/GPU): 42.8 | MFU 13.71% | FLOPs estimator: workload-aware | logical tokens/s: 198385.4 | valid tokens/s: 198385.4 | physical tokens/s: 198892.2 | loss tokens/s: 148803.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.969252E-01 | lm-global-supervised-tokens: 1.258412E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.164771E-02 | lm-token-buffer-mib: 5.820443E+01 | global_load_balancing_loss: 1.000550E+00 | loss scale: 1.0 | grad norm: 0.083 | params norm: 11648.135 | params rms: 0.063071 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 9:04:09.438409 | finish at 2026-10-10 09:21:04 + [2026-10-04 00:18:39] iteration 340/ 6855 | consumed samples: 43520 | elapsed time per iteration (ms): 104151.1 | throughput per GPU (TFLOP/s/GPU): 35.2 | MFU 11.28% | FLOPs estimator: workload-aware | logical tokens/s: 161084.9 | valid tokens/s: 161084.9 | physical tokens/s: 161335.4 | loss tokens/s: 119721.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.788486E-01 | lm-global-supervised-tokens: 1.246910E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.249881E-02 | lm-token-buffer-mib: 5.803816E+01 | global_load_balancing_loss: 1.000587E+00 | loss scale: 1.0 | grad norm: 0.267 | params norm: 11648.044 | params rms: 0.063070 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 7 days, 20:29:04.541798 | finish at 2026-10-11 20:47:44 + [2026-10-04 00:20:08] iteration 341/ 6855 | consumed samples: 43648 | elapsed time per iteration (ms): 88622.0 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.58% | FLOPs estimator: workload-aware | logical tokens/s: 189311.6 | valid tokens/s: 189311.6 | physical tokens/s: 189810.7 | loss tokens/s: 139965.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.141770E-01 | lm-global-supervised-tokens: 1.240399E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.158497E-02 | lm-token-buffer-mib: 5.812329E+01 | global_load_balancing_loss: 1.000118E+00 | loss scale: 1.0 | grad norm: 0.081 | params norm: 11647.954 | params rms: 0.063070 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 16:21:23.436078 | finish at 2026-10-10 16:41:31 + [2026-10-04 00:21:35] iteration 342/ 6855 | consumed samples: 43776 | elapsed time per iteration (ms): 87508.4 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.61% | FLOPs estimator: workload-aware | logical tokens/s: 191720.8 | valid tokens/s: 191720.8 | physical tokens/s: 192264.7 | loss tokens/s: 137410.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.557737E-01 | lm-global-supervised-tokens: 1.202457E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.221049E-02 | lm-token-buffer-mib: 5.812292E+01 | global_load_balancing_loss: 9.994483E-01 | loss scale: 1.0 | grad norm: 0.162 | params norm: 11647.865 | params rms: 0.063069 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 14:19:01.972933 | finish at 2026-10-10 14:40:37 + [2026-10-04 00:23:04] iteration 343/ 6855 | consumed samples: 43904 | elapsed time per iteration (ms): 88833.9 | throughput per GPU (TFLOP/s/GPU): 41.0 | MFU 13.14% | FLOPs estimator: workload-aware | logical tokens/s: 188860.1 | valid tokens/s: 188860.0 | physical tokens/s: 189308.5 | loss tokens/s: 141066.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.615366E-01 | lm-global-supervised-tokens: 1.253144E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.843339E-02 | lm-token-buffer-mib: 5.812597E+01 | global_load_balancing_loss: 9.994043E-01 | loss scale: 1.0 | grad norm: 0.121 | params norm: 11647.776 | params rms: 0.063069 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 16:41:26.064751 | finish at 2026-10-10 17:04:30 + [2026-10-04 00:24:32] iteration 344/ 6855 | consumed samples: 44032 | elapsed time per iteration (ms): 87636.3 | throughput per GPU (TFLOP/s/GPU): 41.4 | MFU 13.26% | FLOPs estimator: workload-aware | logical tokens/s: 191441.1 | valid tokens/s: 191441.0 | physical tokens/s: 192021.7 | loss tokens/s: 148038.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.131685E-01 | lm-global-supervised-tokens: 1.297357E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.564546E-02 | lm-token-buffer-mib: 5.813581E+01 | global_load_balancing_loss: 1.000117E+00 | loss scale: 1.0 | grad norm: 0.060 | params norm: 11647.688 | params rms: 0.063068 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 14:29:59.844307 | finish at 2026-10-10 14:54:31 + [2026-10-04 00:25:59] iteration 345/ 6855 | consumed samples: 44160 | elapsed time per iteration (ms): 87874.8 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.55% | FLOPs estimator: workload-aware | logical tokens/s: 190921.2 | valid tokens/s: 190921.2 | physical tokens/s: 191287.3 | loss tokens/s: 141775.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.984595E-01 | lm-global-supervised-tokens: 1.245845E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.634362E-02 | lm-token-buffer-mib: 5.812630E+01 | global_load_balancing_loss: 1.000602E+00 | loss scale: 1.0 | grad norm: 0.069 | params norm: 11647.598 | params rms: 0.063068 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 14:54:24.764636 | finish at 2026-10-10 15:20:24 + [2026-10-04 00:27:28] iteration 346/ 6855 | consumed samples: 44288 | elapsed time per iteration (ms): 88733.8 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.48% | FLOPs estimator: workload-aware | logical tokens/s: 189072.6 | valid tokens/s: 189072.6 | physical tokens/s: 189499.2 | loss tokens/s: 134375.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.371225E-01 | lm-global-supervised-tokens: 1.192368E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.086077E-02 | lm-token-buffer-mib: 5.809924E+01 | global_load_balancing_loss: 1.000739E+00 | loss scale: 1.0 | grad norm: 0.150 | params norm: 11647.508 | params rms: 0.063067 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 16:26:08.112893 | finish at 2026-10-10 16:53:36 + [2026-10-04 00:28:57] iteration 347/ 6855 | consumed samples: 44416 | elapsed time per iteration (ms): 88750.4 | throughput per GPU (TFLOP/s/GPU): 41.6 | MFU 13.32% | FLOPs estimator: workload-aware | logical tokens/s: 189037.3 | valid tokens/s: 189037.3 | physical tokens/s: 189503.2 | loss tokens/s: 140204.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.706749E-01 | lm-global-supervised-tokens: 1.244316E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.145458E-02 | lm-token-buffer-mib: 5.804593E+01 | global_load_balancing_loss: 1.001324E+00 | loss scale: 1.0 | grad norm: 0.070 | params norm: 11647.419 | params rms: 0.063067 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 16:26:27.487260 | finish at 2026-10-10 16:55:24 + [2026-10-04 00:30:24] iteration 348/ 6855 | consumed samples: 44544 | elapsed time per iteration (ms): 86690.9 | throughput per GPU (TFLOP/s/GPU): 41.5 | MFU 13.32% | FLOPs estimator: workload-aware | logical tokens/s: 193528.5 | valid tokens/s: 193528.5 | physical tokens/s: 194087.5 | loss tokens/s: 144182.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.647201E-01 | lm-global-supervised-tokens: 1.249928E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.460586E-02 | lm-token-buffer-mib: 5.807320E+01 | global_load_balancing_loss: 1.001463E+00 | loss scale: 1.0 | grad norm: 0.110 | params norm: 11647.332 | params rms: 0.063067 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 12:41:37.657392 | finish at 2026-10-10 13:12:01 + [2026-10-04 00:31:51] iteration 349/ 6855 | consumed samples: 44672 | elapsed time per iteration (ms): 86921.2 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.52% | FLOPs estimator: workload-aware | logical tokens/s: 193016.0 | valid tokens/s: 193016.0 | physical tokens/s: 193615.5 | loss tokens/s: 150684.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.534691E-01 | lm-global-supervised-tokens: 1.309769E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.495562E-02 | lm-token-buffer-mib: 5.817500E+01 | global_load_balancing_loss: 9.993302E-01 | loss scale: 1.0 | grad norm: 0.071 | params norm: 11647.244 | params rms: 0.063066 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 13:05:09.082359 | finish at 2026-10-10 13:37:00 + [2026-10-04 00:33:35] iteration 350/ 6855 | consumed samples: 44800 | elapsed time per iteration (ms): 104670.0 | throughput per GPU (TFLOP/s/GPU): 35.2 | MFU 11.27% | FLOPs estimator: workload-aware | logical tokens/s: 160286.5 | valid tokens/s: 160286.5 | physical tokens/s: 160763.5 | loss tokens/s: 118716.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.788274E-01 | lm-global-supervised-tokens: 1.242607E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.547954E-02 | lm-token-buffer-mib: 5.809297E+01 | global_load_balancing_loss: 9.999442E-01 | loss scale: 1.0 | grad norm: 0.047 | params norm: 11647.154 | params rms: 0.063066 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 7 days, 21:07:58.327233 | finish at 2026-10-11 21:41:34 + [2026-10-04 00:35:03] iteration 351/ 6855 | consumed samples: 44928 | elapsed time per iteration (ms): 87326.4 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.73% | FLOPs estimator: workload-aware | logical tokens/s: 192119.9 | valid tokens/s: 192119.9 | physical tokens/s: 192666.8 | loss tokens/s: 143485.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.782556E-01 | lm-global-supervised-tokens: 1.253006E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.741509E-02 | lm-token-buffer-mib: 5.819971E+01 | global_load_balancing_loss: 9.994984E-01 | loss scale: 1.0 | grad norm: 0.090 | params norm: 11647.066 | params rms: 0.063065 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 13:46:10.819033 | finish at 2026-10-10 14:21:13 + [2026-10-04 00:36:32] iteration 352/ 6855 | consumed samples: 45056 | elapsed time per iteration (ms): 89487.6 | throughput per GPU (TFLOP/s/GPU): 41.6 | MFU 13.32% | FLOPs estimator: workload-aware | logical tokens/s: 187480.8 | valid tokens/s: 187480.8 | physical tokens/s: 187884.0 | loss tokens/s: 137692.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.632085E-01 | lm-global-supervised-tokens: 1.232180E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.374933E-02 | lm-token-buffer-mib: 5.815548E+01 | global_load_balancing_loss: 9.985210E-01 | loss scale: 1.0 | grad norm: 0.032 | params norm: 11646.978 | params rms: 0.063065 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 17:38:58.052526 | finish at 2026-10-10 18:15:30 + [2026-10-04 00:37:56] iteration 353/ 6855 | consumed samples: 45184 | elapsed time per iteration (ms): 84216.1 | throughput per GPU (TFLOP/s/GPU): 43.3 | MFU 13.88% | FLOPs estimator: workload-aware | logical tokens/s: 199215.4 | valid tokens/s: 199215.4 | physical tokens/s: 199570.5 | loss tokens/s: 143923.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.125433E-01 | lm-global-supervised-tokens: 1.212068E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.339662E-02 | lm-token-buffer-mib: 5.813976E+01 | global_load_balancing_loss: 9.999576E-01 | loss scale: 1.0 | grad norm: 0.054 | params norm: 11646.890 | params rms: 0.063064 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 8:06:13.350238 | finish at 2026-10-10 08:44:10 + [2026-10-04 00:39:24] iteration 354/ 6855 | consumed samples: 45312 | elapsed time per iteration (ms): 87583.1 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.39% | FLOPs estimator: workload-aware | logical tokens/s: 191557.6 | valid tokens/s: 191557.6 | physical tokens/s: 192052.0 | loss tokens/s: 144564.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.932630E-01 | lm-global-supervised-tokens: 1.266144E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.141306E-02 | lm-token-buffer-mib: 5.814316E+01 | global_load_balancing_loss: 1.000245E+00 | loss scale: 1.0 | grad norm: 0.070 | params norm: 11646.801 | params rms: 0.063064 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 14:09:37.932018 | finish at 2026-10-10 14:49:02 + [2026-10-04 00:40:51] iteration 355/ 6855 | consumed samples: 45440 | elapsed time per iteration (ms): 86865.0 | throughput per GPU (TFLOP/s/GPU): 43.6 | MFU 13.98% | FLOPs estimator: workload-aware | logical tokens/s: 193140.5 | valid tokens/s: 193140.5 | physical tokens/s: 193495.4 | loss tokens/s: 141869.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.951789E-01 | lm-global-supervised-tokens: 1.232346E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.589364E-02 | lm-token-buffer-mib: 5.810107E+01 | global_load_balancing_loss: 1.001739E+00 | loss scale: 1.0 | grad norm: 0.078 | params norm: 11646.714 | params rms: 0.063063 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 12:50:22.717023 | finish at 2026-10-10 13:31:13 + [2026-10-04 00:42:16] iteration 356/ 6855 | consumed samples: 45568 | elapsed time per iteration (ms): 85279.8 | throughput per GPU (TFLOP/s/GPU): 42.8 | MFU 13.73% | FLOPs estimator: workload-aware | logical tokens/s: 196731.1 | valid tokens/s: 196731.1 | physical tokens/s: 197076.3 | loss tokens/s: 147450.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.698087E-01 | lm-global-supervised-tokens: 1.257456E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.127619E-02 | lm-token-buffer-mib: 5.811769E+01 | global_load_balancing_loss: 1.001805E+00 | loss scale: 1.0 | grad norm: 0.060 | params norm: 11646.626 | params rms: 0.063063 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 9:57:13.269499 | finish at 2026-10-10 10:39:29 + [2026-10-04 00:43:47] iteration 357/ 6855 | consumed samples: 45696 | elapsed time per iteration (ms): 90913.0 | throughput per GPU (TFLOP/s/GPU): 40.7 | MFU 13.04% | FLOPs estimator: workload-aware | logical tokens/s: 184541.4 | valid tokens/s: 184541.4 | physical tokens/s: 185052.8 | loss tokens/s: 136291.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.681762E-01 | lm-global-supervised-tokens: 1.239066E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.284337E-02 | lm-token-buffer-mib: 5.808822E+01 | global_load_balancing_loss: 9.998048E-01 | loss scale: 1.0 | grad norm: 0.076 | params norm: 11646.537 | params rms: 0.063062 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 20:05:52.927221 | finish at 2026-10-10 20:49:40 + [2026-10-04 00:45:14] iteration 358/ 6855 | consumed samples: 45824 | elapsed time per iteration (ms): 87207.1 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.68% | FLOPs estimator: workload-aware | logical tokens/s: 192382.9 | valid tokens/s: 192382.9 | physical tokens/s: 192790.4 | loss tokens/s: 145024.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.808786E-01 | lm-global-supervised-tokens: 1.264714E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.561370E-02 | lm-token-buffer-mib: 5.834634E+01 | global_load_balancing_loss: 9.986275E-01 | loss scale: 1.0 | grad norm: 0.054 | params norm: 11646.446 | params rms: 0.063062 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 13:23:04.320578 | finish at 2026-10-10 14:08:18 + [2026-10-04 00:46:40] iteration 359/ 6855 | consumed samples: 45952 | elapsed time per iteration (ms): 86325.2 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.45% | FLOPs estimator: workload-aware | logical tokens/s: 194348.6 | valid tokens/s: 194348.6 | physical tokens/s: 194687.9 | loss tokens/s: 146705.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.632850E-01 | lm-global-supervised-tokens: 1.266441E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.252945E-02 | lm-token-buffer-mib: 5.807065E+01 | global_load_balancing_loss: 9.998429E-01 | loss scale: 1.0 | grad norm: 0.047 | params norm: 11646.355 | params rms: 0.063061 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 11:46:08.208557 | finish at 2026-10-10 12:32:49 + [2026-10-04 00:48:29] iteration 360/ 6855 | consumed samples: 46080 | elapsed time per iteration (ms): 108188.3 | throughput per GPU (TFLOP/s/GPU): 33.1 | MFU 10.61% | FLOPs estimator: workload-aware | logical tokens/s: 155074.0 | valid tokens/s: 155074.0 | physical tokens/s: 155465.9 | loss tokens/s: 115095.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.341894E-01 | lm-global-supervised-tokens: 1.245201E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.208099E-02 | lm-token-buffer-mib: 5.805981E+01 | global_load_balancing_loss: 1.000998E+00 | loss scale: 1.0 | grad norm: 0.108 | params norm: 11646.264 | params rms: 0.063061 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 8 days, 3:11:23.171958 | finish at 2026-10-12 03:59:52 +times across ranks (ms): + save-checkpoint: + rank 0: 23353.23 + rank 1: 23353.23 + rank 2: 23353.27 + rank 3: 23353.25 + rank 4: 23353.27 + rank 5: 23353.23 + rank 6: 23353.21 + rank 7: 23353.25 + rank 8: 23352.13 + rank 9: 23352.08 + rank 10: 23352.07 + rank 11: 23352.14 + rank 12: 23352.14 + rank 13: 23352.07 + rank 14: 23352.10 + rank 15: 23352.08 + rank 16: 23352.29 + rank 17: 23352.20 + rank 18: 23352.17 + rank 19: 23352.19 + rank 20: 23352.16 + rank 21: 23352.21 + rank 22: 23352.15 + rank 23: 23352.14 + rank 24: 23352.20 + rank 25: 23352.21 + rank 26: 23352.21 + rank 27: 23352.21 + rank 28: 23352.17 + rank 29: 23352.20 + rank 30: 23352.20 + rank 31: 23352.22 + rank 32: 23352.43 + rank 33: 23352.45 + rank 34: 23352.41 + rank 35: 23352.39 + rank 36: 23352.37 + rank 37: 23352.38 + rank 38: 23352.40 + rank 39: 23352.38 + rank 40: 23351.97 + rank 41: 23351.99 + rank 42: 23351.96 + rank 43: 23351.98 + rank 44: 23351.98 + rank 45: 23351.97 + rank 46: 23351.96 + rank 47: 23351.97 + rank 48: 23351.05 + rank 49: 23351.07 + rank 50: 23351.05 + rank 51: 23351.05 + rank 52: 23351.14 + rank 53: 23351.05 + rank 54: 23351.04 + rank 55: 23351.03 + rank 56: 23352.15 + rank 57: 23352.10 + rank 58: 23352.07 + rank 59: 23352.09 + rank 60: 23352.09 + rank 61: 23352.11 + rank 62: 23352.08 + rank 63: 23352.09 + [2026-10-04 00:50:39] iteration 361/ 6855 | consumed samples: 46208 | elapsed time per iteration (ms): 107031.8 | throughput per GPU (TFLOP/s/GPU): 34.9 | MFU 11.18% | FLOPs estimator: workload-aware | logical tokens/s: 156749.1 | valid tokens/s: 156749.1 | physical tokens/s: 157115.0 | loss tokens/s: 114868.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.537386E-01 | lm-global-supervised-tokens: 1.229458E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 4.742560E-01 | lm-token-buffer-mib: 5.805702E+01 | global_load_balancing_loss: 1.000993E+00 | loss scale: 1.0 | grad norm: 0.057 | params norm: 11646.175 | params rms: 0.063060 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 8 days, 1:04:24.586820 | finish at 2026-10-12 01:55:04 + [2026-10-04 00:52:10] iteration 362/ 6855 | consumed samples: 46336 | elapsed time per iteration (ms): 91075.5 | throughput per GPU (TFLOP/s/GPU): 39.5 | MFU 12.66% | FLOPs estimator: workload-aware | logical tokens/s: 184212.1 | valid tokens/s: 184212.1 | physical tokens/s: 184595.5 | loss tokens/s: 140279.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.964904E-01 | lm-global-supervised-tokens: 1.277602E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.211499E-02 | lm-token-buffer-mib: 5.810319E+01 | global_load_balancing_loss: 9.999805E-01 | loss scale: 1.0 | grad norm: 0.082 | params norm: 11646.084 | params rms: 0.063060 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 20:15:53.303621 | finish at 2026-10-10 21:08:04 + [2026-10-04 00:53:32] iteration 363/ 6855 | consumed samples: 46464 | elapsed time per iteration (ms): 81818.8 | throughput per GPU (TFLOP/s/GPU): 43.1 | MFU 13.81% | FLOPs estimator: workload-aware | logical tokens/s: 205052.5 | valid tokens/s: 205052.5 | physical tokens/s: 205655.9 | loss tokens/s: 156386.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.484997E-01 | lm-global-supervised-tokens: 1.279536E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.364419E-02 | lm-token-buffer-mib: 5.813862E+01 | global_load_balancing_loss: 1.001131E+00 | loss scale: 1.0 | grad norm: 0.053 | params norm: 11645.992 | params rms: 0.063059 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 3:32:47.332120 | finish at 2026-10-10 04:26:20 + [2026-10-04 00:55:00] iteration 364/ 6855 | consumed samples: 46592 | elapsed time per iteration (ms): 87672.9 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.49% | FLOPs estimator: workload-aware | logical tokens/s: 191360.9 | valid tokens/s: 191360.9 | physical tokens/s: 191759.6 | loss tokens/s: 138831.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.734146E-01 | lm-global-supervised-tokens: 1.217174E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.165044E-02 | lm-token-buffer-mib: 5.805042E+01 | global_load_balancing_loss: 1.000558E+00 | loss scale: 1.0 | grad norm: 0.081 | params norm: 11645.900 | params rms: 0.063059 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 14:04:44.485682 | finish at 2026-10-10 14:59:44 + [2026-10-04 00:56:25] iteration 365/ 6855 | consumed samples: 46720 | elapsed time per iteration (ms): 84859.0 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.68% | FLOPs estimator: workload-aware | logical tokens/s: 197706.1 | valid tokens/s: 197706.1 | physical tokens/s: 198272.7 | loss tokens/s: 151156.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.134071E-01 | lm-global-supervised-tokens: 1.282696E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.146386E-02 | lm-token-buffer-mib: 5.816953E+01 | global_load_balancing_loss: 9.990806E-01 | loss scale: 1.0 | grad norm: 0.070 | params norm: 11645.809 | params rms: 0.063058 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 8:58:54.753509 | finish at 2026-10-10 09:55:20 + [2026-10-04 00:57:56] iteration 366/ 6855 | consumed samples: 46848 | elapsed time per iteration (ms): 90879.9 | throughput per GPU (TFLOP/s/GPU): 41.4 | MFU 13.26% | FLOPs estimator: workload-aware | logical tokens/s: 184607.6 | valid tokens/s: 184607.6 | physical tokens/s: 185080.7 | loss tokens/s: 130960.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.600254E-01 | lm-global-supervised-tokens: 1.190171E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.349009E-02 | lm-token-buffer-mib: 5.811135E+01 | global_load_balancing_loss: 9.995655E-01 | loss scale: 1.0 | grad norm: 0.046 | params norm: 11645.716 | params rms: 0.063058 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 19:48:39.432708 | finish at 2026-10-10 20:46:35 + [2026-10-04 00:59:23] iteration 367/ 6855 | consumed samples: 46976 | elapsed time per iteration (ms): 86913.9 | throughput per GPU (TFLOP/s/GPU): 43.2 | MFU 13.86% | FLOPs estimator: workload-aware | logical tokens/s: 193032.6 | valid tokens/s: 193032.6 | physical tokens/s: 193409.5 | loss tokens/s: 141845.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.439671E-01 | lm-global-supervised-tokens: 1.232830E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.839248E-02 | lm-token-buffer-mib: 5.810847E+01 | global_load_balancing_loss: 1.000418E+00 | loss scale: 1.0 | grad norm: 0.043 | params norm: 11645.622 | params rms: 0.063057 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 12:38:17.077810 | finish at 2026-10-10 13:37:40 + [2026-10-04 01:00:52] iteration 368/ 6855 | consumed samples: 47104 | elapsed time per iteration (ms): 89259.9 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.52% | FLOPs estimator: workload-aware | logical tokens/s: 187958.9 | valid tokens/s: 187958.9 | physical tokens/s: 188498.8 | loss tokens/s: 142725.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.007909E-01 | lm-global-supervised-tokens: 1.273969E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 3.036113E-02 | lm-token-buffer-mib: 5.826276E+01 | global_load_balancing_loss: 9.988259E-01 | loss scale: 1.0 | grad norm: 0.069 | params norm: 11645.527 | params rms: 0.063057 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 16:50:28.710525 | finish at 2026-10-10 17:51:21 + [2026-10-04 01:02:21] iteration 369/ 6855 | consumed samples: 47232 | elapsed time per iteration (ms): 88961.6 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.53% | FLOPs estimator: workload-aware | logical tokens/s: 188588.7 | valid tokens/s: 188588.7 | physical tokens/s: 189249.2 | loss tokens/s: 140881.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.033306E-01 | lm-global-supervised-tokens: 1.253303E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.155757E-02 | lm-token-buffer-mib: 5.827203E+01 | global_load_balancing_loss: 9.996063E-01 | loss scale: 1.0 | grad norm: 0.067 | params norm: 11645.434 | params rms: 0.063056 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 16:16:44.664313 | finish at 2026-10-10 17:19:05 + [2026-10-04 01:04:10] iteration 370/ 6855 | consumed samples: 47360 | elapsed time per iteration (ms): 109039.6 | throughput per GPU (TFLOP/s/GPU): 34.2 | MFU 10.97% | FLOPs estimator: workload-aware | logical tokens/s: 153863.3 | valid tokens/s: 153863.3 | physical tokens/s: 154236.0 | loss tokens/s: 111432.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.009524E-01 | lm-global-supervised-tokens: 1.215051E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.377108E-02 | lm-token-buffer-mib: 5.816036E+01 | global_load_balancing_loss: 1.000751E+00 | loss scale: 1.0 | grad norm: 0.142 | params norm: 11645.341 | params rms: 0.063056 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 8 days, 4:25:21.564124 | finish at 2026-10-12 05:29:31 + [2026-10-04 01:05:35] iteration 371/ 6855 | consumed samples: 47488 | elapsed time per iteration (ms): 85487.6 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.79% | FLOPs estimator: workload-aware | logical tokens/s: 196252.8 | valid tokens/s: 196252.8 | physical tokens/s: 196887.7 | loss tokens/s: 145810.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.374254E-01 | lm-global-supervised-tokens: 1.246495E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.796765E-02 | lm-token-buffer-mib: 5.824578E+01 | global_load_balancing_loss: 1.001281E+00 | loss scale: 1.0 | grad norm: 0.035 | params norm: 11645.248 | params rms: 0.063055 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 9:58:21.504671 | finish at 2026-10-10 11:03:57 + [2026-10-04 01:07:02] iteration 372/ 6855 | consumed samples: 47616 | elapsed time per iteration (ms): 86250.4 | throughput per GPU (TFLOP/s/GPU): 41.5 | MFU 13.31% | FLOPs estimator: workload-aware | logical tokens/s: 194517.6 | valid tokens/s: 194517.6 | physical tokens/s: 194986.0 | loss tokens/s: 153602.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.133036E-01 | lm-global-supervised-tokens: 1.324828E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.448812E-02 | lm-token-buffer-mib: 5.813516E+01 | global_load_balancing_loss: 1.000636E+00 | loss scale: 1.0 | grad norm: 0.190 | params norm: 11645.155 | params rms: 0.063055 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 11:19:21.142694 | finish at 2026-10-10 12:26:23 + [2026-10-04 01:08:29] iteration 373/ 6855 | consumed samples: 47744 | elapsed time per iteration (ms): 87024.1 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.63% | FLOPs estimator: workload-aware | logical tokens/s: 192788.3 | valid tokens/s: 192788.3 | physical tokens/s: 193097.5 | loss tokens/s: 143345.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.859670E-01 | lm-global-supervised-tokens: 1.247446E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.641436E-02 | lm-token-buffer-mib: 5.812719E+01 | global_load_balancing_loss: 1.000112E+00 | loss scale: 1.0 | grad norm: 0.085 | params norm: 11645.063 | params rms: 0.063054 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 12:41:29.902901 | finish at 2026-10-10 13:49:59 + [2026-10-04 01:09:53] iteration 374/ 6855 | consumed samples: 47872 | elapsed time per iteration (ms): 84350.1 | throughput per GPU (TFLOP/s/GPU): 43.5 | MFU 13.94% | FLOPs estimator: workload-aware | logical tokens/s: 198899.6 | valid tokens/s: 198899.6 | physical tokens/s: 199364.1 | loss tokens/s: 148162.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.074143E-01 | lm-global-supervised-tokens: 1.249748E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.662008E-02 | lm-token-buffer-mib: 5.802633E+01 | global_load_balancing_loss: 1.000818E+00 | loss scale: 1.0 | grad norm: 0.164 | params norm: 11644.969 | params rms: 0.063054 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 7:51:12.996817 | finish at 2026-10-10 09:01:06 + [2026-10-04 01:11:15] iteration 375/ 6855 | consumed samples: 48000 | elapsed time per iteration (ms): 82279.7 | throughput per GPU (TFLOP/s/GPU): 44.0 | MFU 14.11% | FLOPs estimator: workload-aware | logical tokens/s: 203904.5 | valid tokens/s: 203904.5 | physical tokens/s: 204407.8 | loss tokens/s: 153052.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.793366E-01 | lm-global-supervised-tokens: 1.259309E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.406511E-02 | lm-token-buffer-mib: 5.829275E+01 | global_load_balancing_loss: 1.002469E+00 | loss scale: 1.0 | grad norm: 0.107 | params norm: 11644.875 | params rms: 0.063053 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 4:06:12.289410 | finish at 2026-10-10 05:17:28 + [2026-10-04 01:12:42] iteration 376/ 6855 | consumed samples: 48128 | elapsed time per iteration (ms): 86681.5 | throughput per GPU (TFLOP/s/GPU): 42.8 | MFU 13.70% | FLOPs estimator: workload-aware | logical tokens/s: 193550.1 | valid tokens/s: 193550.1 | physical tokens/s: 193939.4 | loss tokens/s: 144047.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.023157E-01 | lm-global-supervised-tokens: 1.248623E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.949011E-02 | lm-token-buffer-mib: 5.810595E+01 | global_load_balancing_loss: 9.998298E-01 | loss scale: 1.0 | grad norm: 0.124 | params norm: 11644.782 | params rms: 0.063053 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 12:00:09.626683 | finish at 2026-10-10 13:12:52 + [2026-10-04 01:14:08] iteration 377/ 6855 | consumed samples: 48256 | elapsed time per iteration (ms): 86137.2 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.70% | FLOPs estimator: workload-aware | logical tokens/s: 194772.5 | valid tokens/s: 194772.5 | physical tokens/s: 195215.0 | loss tokens/s: 149366.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.186793E-01 | lm-global-supervised-tokens: 1.286602E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.393375E-02 | lm-token-buffer-mib: 5.815314E+01 | global_load_balancing_loss: 9.992251E-01 | loss scale: 1.0 | grad norm: 0.099 | params norm: 11644.688 | params rms: 0.063052 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 10:59:57.003219 | finish at 2026-10-10 12:14:05 + [2026-10-04 01:15:35] iteration 378/ 6855 | consumed samples: 48384 | elapsed time per iteration (ms): 87195.9 | throughput per GPU (TFLOP/s/GPU): 42.8 | MFU 13.72% | FLOPs estimator: workload-aware | logical tokens/s: 192408.4 | valid tokens/s: 192408.4 | physical tokens/s: 192743.3 | loss tokens/s: 136197.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.374736E-01 | lm-global-supervised-tokens: 1.187583E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.134375E-02 | lm-token-buffer-mib: 5.812038E+01 | global_load_balancing_loss: 9.995751E-01 | loss scale: 1.0 | grad norm: 0.050 | params norm: 11644.594 | params rms: 0.063052 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 12:52:47.524406 | finish at 2026-10-10 14:08:23 + [2026-10-04 01:17:07] iteration 379/ 6855 | consumed samples: 48512 | elapsed time per iteration (ms): 92104.5 | throughput per GPU (TFLOP/s/GPU): 41.6 | MFU 13.35% | FLOPs estimator: workload-aware | logical tokens/s: 182153.8 | valid tokens/s: 182153.8 | physical tokens/s: 182769.1 | loss tokens/s: 132965.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.485455E-01 | lm-global-supervised-tokens: 1.224670E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.153322E-02 | lm-token-buffer-mib: 5.805486E+01 | global_load_balancing_loss: 9.992875E-01 | loss scale: 1.0 | grad norm: 0.056 | params norm: 11644.500 | params rms: 0.063051 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 21:41:08.662070 | finish at 2026-10-10 22:58:16 + [2026-10-04 01:18:54] iteration 380/ 6855 | consumed samples: 48640 | elapsed time per iteration (ms): 106231.7 | throughput per GPU (TFLOP/s/GPU): 34.8 | MFU 11.15% | FLOPs estimator: workload-aware | logical tokens/s: 157930.0 | valid tokens/s: 157930.0 | physical tokens/s: 158323.0 | loss tokens/s: 118267.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.740443E-01 | lm-global-supervised-tokens: 1.256380E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.669107E-02 | lm-token-buffer-mib: 5.811395E+01 | global_load_balancing_loss: 1.000006E+00 | loss scale: 1.0 | grad norm: 0.096 | params norm: 11644.405 | params rms: 0.063051 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 7 days, 23:04:10.430036 | finish at 2026-10-12 00:23:04 + [2026-10-04 01:20:23] iteration 381/ 6855 | consumed samples: 48768 | elapsed time per iteration (ms): 89809.8 | throughput per GPU (TFLOP/s/GPU): 41.1 | MFU 13.16% | FLOPs estimator: workload-aware | logical tokens/s: 186808.3 | valid tokens/s: 186808.3 | physical tokens/s: 187357.8 | loss tokens/s: 141991.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.394221E-01 | lm-global-supervised-tokens: 1.275218E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.818851E-02 | lm-token-buffer-mib: 5.810152E+01 | global_load_balancing_loss: 9.996399E-01 | loss scale: 1.0 | grad norm: 0.058 | params norm: 11644.309 | params rms: 0.063050 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 17:30:28.355976 | finish at 2026-10-10 18:50:52 + [2026-10-04 01:21:52] iteration 382/ 6855 | consumed samples: 48896 | elapsed time per iteration (ms): 88288.7 | throughput per GPU (TFLOP/s/GPU): 41.5 | MFU 13.31% | FLOPs estimator: workload-aware | logical tokens/s: 190026.4 | valid tokens/s: 190026.4 | physical tokens/s: 190659.9 | loss tokens/s: 145475.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.647221E-01 | lm-global-supervised-tokens: 1.284381E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.124213E-02 | lm-token-buffer-mib: 5.812419E+01 | global_load_balancing_loss: 1.001227E+00 | loss scale: 1.0 | grad norm: 0.078 | params norm: 11644.212 | params rms: 0.063050 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 14:44:52.819046 | finish at 2026-10-10 16:06:45 + [2026-10-04 01:23:18] iteration 383/ 6855 | consumed samples: 49024 | elapsed time per iteration (ms): 85821.9 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.68% | FLOPs estimator: workload-aware | logical tokens/s: 195488.0 | valid tokens/s: 195488.0 | physical tokens/s: 196100.2 | loss tokens/s: 152698.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.792791E-01 | lm-global-supervised-tokens: 1.310488E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.164196E-02 | lm-token-buffer-mib: 5.808488E+01 | global_load_balancing_loss: 1.001521E+00 | loss scale: 1.0 | grad norm: 0.054 | params norm: 11644.117 | params rms: 0.063049 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 10:17:19.465710 | finish at 2026-10-10 11:40:37 + [2026-10-04 01:24:45] iteration 384/ 6855 | consumed samples: 49152 | elapsed time per iteration (ms): 87669.5 | throughput per GPU (TFLOP/s/GPU): 41.2 | MFU 13.21% | FLOPs estimator: workload-aware | logical tokens/s: 191368.6 | valid tokens/s: 191368.6 | physical tokens/s: 191912.5 | loss tokens/s: 142662.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.606066E-01 | lm-global-supervised-tokens: 1.250717E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.396588E-02 | lm-token-buffer-mib: 5.821393E+01 | global_load_balancing_loss: 1.001261E+00 | loss scale: 1.0 | grad norm: 0.044 | params norm: 11644.024 | params rms: 0.063049 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 13:35:09.443225 | finish at 2026-10-10 14:59:55 + [2026-10-04 01:26:11] iteration 385/ 6855 | consumed samples: 49280 | elapsed time per iteration (ms): 86187.1 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.56% | FLOPs estimator: workload-aware | logical tokens/s: 194659.6 | valid tokens/s: 194659.5 | physical tokens/s: 195122.3 | loss tokens/s: 141696.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.593156E-01 | lm-global-supervised-tokens: 1.221240E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.373611E-02 | lm-token-buffer-mib: 5.806013E+01 | global_load_balancing_loss: 1.000289E+00 | loss scale: 1.0 | grad norm: 0.041 | params norm: 11643.930 | params rms: 0.063048 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 10:53:50.419335 | finish at 2026-10-10 12:20:02 + [2026-10-04 01:27:39] iteration 386/ 6855 | consumed samples: 49408 | elapsed time per iteration (ms): 87222.2 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.58% | FLOPs estimator: workload-aware | logical tokens/s: 192349.6 | valid tokens/s: 192349.6 | physical tokens/s: 192716.7 | loss tokens/s: 140723.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.846166E-01 | lm-global-supervised-tokens: 1.227420E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.137024E-02 | lm-token-buffer-mib: 5.803220E+01 | global_load_balancing_loss: 9.998335E-01 | loss scale: 1.0 | grad norm: 0.040 | params norm: 11643.836 | params rms: 0.063048 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 12:44:00.641069 | finish at 2026-10-10 14:11:39 + [2026-10-04 01:29:05] iteration 387/ 6855 | consumed samples: 49536 | elapsed time per iteration (ms): 86800.5 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.58% | FLOPs estimator: workload-aware | logical tokens/s: 193284.7 | valid tokens/s: 193284.7 | physical tokens/s: 193735.4 | loss tokens/s: 142861.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.587768E-01 | lm-global-supervised-tokens: 1.240044E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.703307E-02 | lm-token-buffer-mib: 5.809604E+01 | global_load_balancing_loss: 9.979189E-01 | loss scale: 1.0 | grad norm: 0.031 | params norm: 11643.742 | params rms: 0.063047 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 11:57:05.431443 | finish at 2026-10-10 13:26:11 + [2026-10-04 01:30:31] iteration 388/ 6855 | consumed samples: 49664 | elapsed time per iteration (ms): 85787.6 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.60% | FLOPs estimator: workload-aware | logical tokens/s: 195566.5 | valid tokens/s: 195566.5 | physical tokens/s: 196110.3 | loss tokens/s: 144000.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.769146E-01 | lm-global-supervised-tokens: 1.235349E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.387899E-02 | lm-token-buffer-mib: 5.806496E+01 | global_load_balancing_loss: 9.986236E-01 | loss scale: 1.0 | grad norm: 0.034 | params norm: 11643.647 | params rms: 0.063047 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 10:06:28.697788 | finish at 2026-10-10 11:37:00 + [2026-10-04 01:31:59] iteration 389/ 6855 | consumed samples: 49792 | elapsed time per iteration (ms): 88000.3 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.77% | FLOPs estimator: workload-aware | logical tokens/s: 190649.3 | valid tokens/s: 190649.3 | physical tokens/s: 191095.6 | loss tokens/s: 135671.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.569677E-01 | lm-global-supervised-tokens: 1.193916E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.882303E-02 | lm-token-buffer-mib: 5.813197E+01 | global_load_balancing_loss: 1.000190E+00 | loss scale: 1.0 | grad norm: 0.040 | params norm: 11643.552 | params rms: 0.063046 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 14:03:29.797523 | finish at 2026-10-10 15:35:29 + [2026-10-04 01:33:43] iteration 390/ 6855 | consumed samples: 49920 | elapsed time per iteration (ms): 104048.2 | throughput per GPU (TFLOP/s/GPU): 35.1 | MFU 11.25% | FLOPs estimator: workload-aware | logical tokens/s: 161244.3 | valid tokens/s: 161244.3 | physical tokens/s: 161677.2 | loss tokens/s: 119615.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.726877E-01 | lm-global-supervised-tokens: 1.244581E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.289018E-02 | lm-token-buffer-mib: 5.821095E+01 | global_load_balancing_loss: 1.001550E+00 | loss scale: 1.0 | grad norm: 0.036 | params norm: 11643.456 | params rms: 0.063046 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 7 days, 18:51:11.680123 | finish at 2026-10-11 20:24:55 + [2026-10-04 01:35:10] iteration 391/ 6855 | consumed samples: 50048 | elapsed time per iteration (ms): 86579.6 | throughput per GPU (TFLOP/s/GPU): 41.7 | MFU 13.35% | FLOPs estimator: workload-aware | logical tokens/s: 193777.7 | valid tokens/s: 193777.7 | physical tokens/s: 194305.9 | loss tokens/s: 143327.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.110111E-01 | lm-global-supervised-tokens: 1.240922E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.333185E-02 | lm-token-buffer-mib: 5.806982E+01 | global_load_balancing_loss: 1.002049E+00 | loss scale: 1.0 | grad norm: 0.080 | params norm: 11643.360 | params rms: 0.063045 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 11:27:30.592041 | finish at 2026-10-10 13:02:40 + [2026-10-04 01:36:34] iteration 392/ 6855 | consumed samples: 50176 | elapsed time per iteration (ms): 84229.7 | throughput per GPU (TFLOP/s/GPU): 43.6 | MFU 13.97% | FLOPs estimator: workload-aware | logical tokens/s: 199183.3 | valid tokens/s: 199183.3 | physical tokens/s: 199695.2 | loss tokens/s: 150110.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.037358E-01 | lm-global-supervised-tokens: 1.264374E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.903460E-02 | lm-token-buffer-mib: 5.824172E+01 | global_load_balancing_loss: 9.998944E-01 | loss scale: 1.0 | grad norm: 0.080 | params norm: 11643.265 | params rms: 0.063044 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 7:12:56.337487 | finish at 2026-10-10 08:49:30 + [2026-10-04 01:38:01] iteration 393/ 6855 | consumed samples: 50304 | elapsed time per iteration (ms): 86923.1 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.77% | FLOPs estimator: workload-aware | logical tokens/s: 193011.3 | valid tokens/s: 193011.3 | physical tokens/s: 193348.2 | loss tokens/s: 143787.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.746548E-01 | lm-global-supervised-tokens: 1.249848E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.754964E-02 | lm-token-buffer-mib: 5.812560E+01 | global_load_balancing_loss: 1.000600E+00 | loss scale: 1.0 | grad norm: 0.035 | params norm: 11643.170 | params rms: 0.063044 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 12:01:37.323293 | finish at 2026-10-10 13:39:38 + [2026-10-04 01:39:29] iteration 394/ 6855 | consumed samples: 50432 | elapsed time per iteration (ms): 88187.8 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.50% | FLOPs estimator: workload-aware | logical tokens/s: 190243.9 | valid tokens/s: 190243.8 | physical tokens/s: 190622.7 | loss tokens/s: 139384.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.074289E-01 | lm-global-supervised-tokens: 1.229200E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.169649E-02 | lm-token-buffer-mib: 5.814645E+01 | global_load_balancing_loss: 1.000838E+00 | loss scale: 1.0 | grad norm: 0.103 | params norm: 11643.075 | params rms: 0.063043 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 14:16:21.115020 | finish at 2026-10-10 15:55:50 + [2026-10-04 01:40:55] iteration 395/ 6855 | consumed samples: 50560 | elapsed time per iteration (ms): 86246.4 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.52% | FLOPs estimator: workload-aware | logical tokens/s: 194526.3 | valid tokens/s: 194526.3 | physical tokens/s: 195070.9 | loss tokens/s: 152782.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.670984E-01 | lm-global-supervised-tokens: 1.317690E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.504252E-02 | lm-token-buffer-mib: 5.828215E+01 | global_load_balancing_loss: 9.984320E-01 | loss scale: 1.0 | grad norm: 0.246 | params norm: 11642.980 | params rms: 0.063043 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 10:45:51.667752 | finish at 2026-10-10 12:26:47 + [2026-10-04 01:42:25] iteration 396/ 6855 | consumed samples: 50688 | elapsed time per iteration (ms): 89400.8 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.79% | FLOPs estimator: workload-aware | logical tokens/s: 187662.8 | valid tokens/s: 187662.8 | physical tokens/s: 188009.5 | loss tokens/s: 130886.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.940846E-01 | lm-global-supervised-tokens: 1.170135E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.833855E-02 | lm-token-buffer-mib: 5.802725E+01 | global_load_balancing_loss: 1.000571E+00 | loss scale: 1.0 | grad norm: 0.199 | params norm: 11642.886 | params rms: 0.063042 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 16:23:59.968866 | finish at 2026-10-10 18:06:25 + [2026-10-04 01:43:54] iteration 397/ 6855 | consumed samples: 50816 | elapsed time per iteration (ms): 89247.2 | throughput per GPU (TFLOP/s/GPU): 41.3 | MFU 13.25% | FLOPs estimator: workload-aware | logical tokens/s: 187985.9 | valid tokens/s: 187985.9 | physical tokens/s: 188333.0 | loss tokens/s: 143381.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.142443E-01 | lm-global-supervised-tokens: 1.279637E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.195012E-02 | lm-token-buffer-mib: 5.823684E+01 | global_load_balancing_loss: 9.989185E-01 | loss scale: 1.0 | grad norm: 0.082 | params norm: 11642.792 | params rms: 0.063042 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 16:05:58.317598 | finish at 2026-10-10 17:49:52 + [2026-10-04 01:45:18] iteration 398/ 6855 | consumed samples: 50944 | elapsed time per iteration (ms): 84244.8 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.62% | FLOPs estimator: workload-aware | logical tokens/s: 199148.4 | valid tokens/s: 199148.4 | physical tokens/s: 199667.7 | loss tokens/s: 155554.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.435351E-01 | lm-global-supervised-tokens: 1.310468E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.785799E-02 | lm-token-buffer-mib: 5.808971E+01 | global_load_balancing_loss: 9.996394E-01 | loss scale: 1.0 | grad norm: 0.084 | params norm: 11642.698 | params rms: 0.063041 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 7:06:08.470976 | finish at 2026-10-10 08:51:27 + [2026-10-04 01:46:48] iteration 399/ 6855 | consumed samples: 51072 | elapsed time per iteration (ms): 89599.4 | throughput per GPU (TFLOP/s/GPU): 41.1 | MFU 13.17% | FLOPs estimator: workload-aware | logical tokens/s: 187246.1 | valid tokens/s: 187246.1 | physical tokens/s: 187870.3 | loss tokens/s: 130962.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.819591E-01 | lm-global-supervised-tokens: 1.173412E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.121195E-02 | lm-token-buffer-mib: 5.814634E+01 | global_load_balancing_loss: 1.002568E+00 | loss scale: 1.0 | grad norm: 0.235 | params norm: 11642.606 | params rms: 0.063041 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 16:40:53.908310 | finish at 2026-10-10 18:27:42 + [2026-10-04 01:48:36] iteration 400/ 6855 | consumed samples: 51200 | elapsed time per iteration (ms): 107851.0 | throughput per GPU (TFLOP/s/GPU): 34.8 | MFU 11.14% | FLOPs estimator: workload-aware | logical tokens/s: 155558.9 | valid tokens/s: 155558.9 | physical tokens/s: 155855.9 | loss tokens/s: 119692.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.236187E-01 | lm-global-supervised-tokens: 1.290895E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.419314E-02 | lm-token-buffer-mib: 5.810159E+01 | global_load_balancing_loss: 1.001393E+00 | loss scale: 1.0 | grad norm: 0.135 | params norm: 11642.513 | params rms: 0.063040 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 8 days, 1:22:58.317802 | finish at 2026-10-12 03:11:34 +times across ranks (ms): + save-checkpoint: + rank 0: 24590.82 + rank 1: 24590.82 + rank 2: 24590.85 + rank 3: 24590.81 + rank 4: 24590.81 + rank 5: 24590.80 + rank 6: 24590.79 + rank 7: 24590.82 + rank 8: 24590.90 + rank 9: 24590.88 + rank 10: 24590.88 + rank 11: 24590.72 + rank 12: 24590.89 + rank 13: 24590.89 + rank 14: 24590.89 + rank 15: 24590.90 + rank 16: 24591.29 + rank 17: 24591.27 + rank 18: 24591.24 + rank 19: 24591.28 + rank 20: 24591.25 + rank 21: 24591.31 + rank 22: 24591.24 + rank 23: 24591.24 + rank 24: 24590.92 + rank 25: 24590.71 + rank 26: 24590.90 + rank 27: 24590.86 + rank 28: 24591.01 + rank 29: 24590.87 + rank 30: 24590.85 + rank 31: 24590.72 + rank 32: 24591.79 + rank 33: 24591.76 + rank 34: 24591.56 + rank 35: 24591.64 + rank 36: 24591.58 + rank 37: 24591.56 + rank 38: 24591.55 + rank 39: 24591.52 + rank 40: 24590.85 + rank 41: 24590.83 + rank 42: 24590.82 + rank 43: 24590.85 + rank 44: 24590.81 + rank 45: 24590.81 + rank 46: 24590.81 + rank 47: 24590.81 + rank 48: 24590.91 + rank 49: 24590.87 + rank 50: 24590.82 + rank 51: 24590.92 + rank 52: 24590.93 + rank 53: 24590.85 + rank 54: 24590.87 + rank 55: 24590.91 + rank 56: 24590.85 + rank 57: 24590.84 + rank 58: 24590.79 + rank 59: 24590.80 + rank 60: 24590.84 + rank 61: 24590.84 + rank 62: 24590.81 + rank 63: 24590.81 + [2026-10-04 01:50:25] iteration 401/ 6855 | consumed samples: 51328 | elapsed time per iteration (ms): 84351.7 | throughput per GPU (TFLOP/s/GPU): 43.6 | MFU 13.96% | FLOPs estimator: workload-aware | logical tokens/s: 198895.9 | valid tokens/s: 198895.9 | physical tokens/s: 199295.1 | loss tokens/s: 149276.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.774282E-01 | lm-global-supervised-tokens: 1.259171E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 5.719922E-01 | lm-token-buffer-mib: 5.811139E+01 | global_load_balancing_loss: 1.000335E+00 | loss scale: 1.0 | grad norm: 0.293 | params norm: 11642.421 | params rms: 0.063040 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 7:13:25.924553 | finish at 2026-10-10 09:03:51 + [2026-10-04 01:52:02] iteration 402/ 6855 | consumed samples: 51456 | elapsed time per iteration (ms): 96689.3 | throughput per GPU (TFLOP/s/GPU): 38.9 | MFU 12.46% | FLOPs estimator: workload-aware | logical tokens/s: 173516.4 | valid tokens/s: 173516.4 | physical tokens/s: 173812.5 | loss tokens/s: 121776.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.882430E-01 | lm-global-supervised-tokens: 1.177445E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.704147E-02 | lm-token-buffer-mib: 5.816785E+01 | global_load_balancing_loss: 9.997860E-01 | loss scale: 1.0 | grad norm: 0.115 | params norm: 11642.329 | params rms: 0.063039 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 7 days, 5:18:55.984056 | finish at 2026-10-11 07:10:58 + [2026-10-04 01:53:29] iteration 403/ 6855 | consumed samples: 51584 | elapsed time per iteration (ms): 87127.3 | throughput per GPU (TFLOP/s/GPU): 43.3 | MFU 13.88% | FLOPs estimator: workload-aware | logical tokens/s: 192559.5 | valid tokens/s: 192559.5 | physical tokens/s: 192933.0 | loss tokens/s: 140779.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.889421E-01 | lm-global-supervised-tokens: 1.226577E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.763724E-02 | lm-token-buffer-mib: 5.806393E+01 | global_load_balancing_loss: 1.000030E+00 | loss scale: 1.0 | grad norm: 0.296 | params norm: 11642.236 | params rms: 0.063039 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 12:09:05.116705 | finish at 2026-10-10 14:02:34 + [2026-10-04 01:54:54] iteration 404/ 6855 | consumed samples: 51712 | elapsed time per iteration (ms): 85381.7 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.52% | FLOPs estimator: workload-aware | logical tokens/s: 196496.2 | valid tokens/s: 196496.2 | physical tokens/s: 197074.1 | loss tokens/s: 145748.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.770887E-01 | lm-global-supervised-tokens: 1.244428E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.148699E-02 | lm-token-buffer-mib: 5.808702E+01 | global_load_balancing_loss: 9.989997E-01 | loss scale: 1.0 | grad norm: 0.242 | params norm: 11642.144 | params rms: 0.063038 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 8:59:57.585347 | finish at 2026-10-10 10:54:52 + [2026-10-04 01:56:19] iteration 405/ 6855 | consumed samples: 51840 | elapsed time per iteration (ms): 85048.0 | throughput per GPU (TFLOP/s/GPU): 43.3 | MFU 13.87% | FLOPs estimator: workload-aware | logical tokens/s: 197267.7 | valid tokens/s: 197267.7 | physical tokens/s: 197725.1 | loss tokens/s: 147876.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.669788E-01 | lm-global-supervised-tokens: 1.257663E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.136365E-02 | lm-token-buffer-mib: 5.817174E+01 | global_load_balancing_loss: 9.999797E-01 | loss scale: 1.0 | grad norm: 0.274 | params norm: 11642.052 | params rms: 0.063038 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 8:22:39.283841 | finish at 2026-10-10 10:18:59 + [2026-10-04 01:57:49] iteration 406/ 6855 | consumed samples: 51968 | elapsed time per iteration (ms): 89864.8 | throughput per GPU (TFLOP/s/GPU): 41.1 | MFU 13.18% | FLOPs estimator: workload-aware | logical tokens/s: 186693.5 | valid tokens/s: 186693.4 | physical tokens/s: 187182.0 | loss tokens/s: 136000.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.000963E-01 | lm-global-supervised-tokens: 1.222164E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.353735E-02 | lm-token-buffer-mib: 5.815627E+01 | global_load_balancing_loss: 9.999331E-01 | loss scale: 1.0 | grad norm: 0.153 | params norm: 11641.960 | params rms: 0.063037 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 16:58:57.921303 | finish at 2026-10-10 18:56:47 + [2026-10-04 01:59:17] iteration 407/ 6855 | consumed samples: 52096 | elapsed time per iteration (ms): 87787.7 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.58% | FLOPs estimator: workload-aware | logical tokens/s: 191111.2 | valid tokens/s: 191111.2 | physical tokens/s: 191476.3 | loss tokens/s: 146451.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.111156E-01 | lm-global-supervised-tokens: 1.285667E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.525839E-02 | lm-token-buffer-mib: 5.811007E+01 | global_load_balancing_loss: 9.993052E-01 | loss scale: 1.0 | grad norm: 0.159 | params norm: 11641.869 | params rms: 0.063037 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 13:14:15.129169 | finish at 2026-10-10 15:13:32 + [2026-10-04 02:00:45] iteration 408/ 6855 | consumed samples: 52224 | elapsed time per iteration (ms): 88145.8 | throughput per GPU (TFLOP/s/GPU): 41.5 | MFU 13.30% | FLOPs estimator: workload-aware | logical tokens/s: 190334.5 | valid tokens/s: 190334.5 | physical tokens/s: 190732.7 | loss tokens/s: 139215.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.669984E-01 | lm-global-supervised-tokens: 1.227126E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.228785E-02 | lm-token-buffer-mib: 5.809019E+01 | global_load_balancing_loss: 1.000682E+00 | loss scale: 1.0 | grad norm: 0.105 | params norm: 11641.780 | params rms: 0.063036 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 13:51:15.908775 | finish at 2026-10-10 15:52:01 + [2026-10-04 02:02:11] iteration 409/ 6855 | consumed samples: 52352 | elapsed time per iteration (ms): 85642.4 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.68% | FLOPs estimator: workload-aware | logical tokens/s: 195898.0 | valid tokens/s: 195898.0 | physical tokens/s: 196488.6 | loss tokens/s: 151685.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.787897E-01 | lm-global-supervised-tokens: 1.299070E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.425505E-02 | lm-token-buffer-mib: 5.814279E+01 | global_load_balancing_loss: 1.000199E+00 | loss scale: 1.0 | grad norm: 0.109 | params norm: 11641.689 | params rms: 0.063036 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 9:20:51.122654 | finish at 2026-10-10 11:23:02 + [2026-10-04 02:04:00] iteration 410/ 6855 | consumed samples: 52480 | elapsed time per iteration (ms): 109500.2 | throughput per GPU (TFLOP/s/GPU): 34.0 | MFU 10.91% | FLOPs estimator: workload-aware | logical tokens/s: 153216.1 | valid tokens/s: 153216.1 | physical tokens/s: 153632.6 | loss tokens/s: 109869.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.657349E-01 | lm-global-supervised-tokens: 1.203078E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.495175E-02 | lm-token-buffer-mib: 5.816888E+01 | global_load_balancing_loss: 1.003426E+00 | loss scale: 1.0 | grad norm: 0.084 | params norm: 11641.597 | params rms: 0.063035 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 8 days, 4:02:08.858361 | finish at 2026-10-12 06:06:09 + [2026-10-04 02:05:28] iteration 411/ 6855 | consumed samples: 52608 | elapsed time per iteration (ms): 87614.2 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.67% | FLOPs estimator: workload-aware | logical tokens/s: 191489.7 | valid tokens/s: 191489.7 | physical tokens/s: 192102.1 | loss tokens/s: 145016.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.528323E-01 | lm-global-supervised-tokens: 1.270550E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.465913E-02 | lm-token-buffer-mib: 5.828732E+01 | global_load_balancing_loss: 1.001266E+00 | loss scale: 1.0 | grad norm: 0.094 | params norm: 11641.506 | params rms: 0.063035 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 12:49:45.739614 | finish at 2026-10-10 14:55:14 + [2026-10-04 02:06:59] iteration 412/ 6855 | consumed samples: 52736 | elapsed time per iteration (ms): 91164.1 | throughput per GPU (TFLOP/s/GPU): 41.3 | MFU 13.24% | FLOPs estimator: workload-aware | logical tokens/s: 184033.1 | valid tokens/s: 184033.0 | physical tokens/s: 184522.4 | loss tokens/s: 136682.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.794090E-01 | lm-global-supervised-tokens: 1.246058E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.088087E-02 | lm-token-buffer-mib: 5.815614E+01 | global_load_balancing_loss: 1.000393E+00 | loss scale: 1.0 | grad norm: 0.191 | params norm: 11641.416 | params rms: 0.063034 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 19:09:30.547786 | finish at 2026-10-10 21:16:30 + [2026-10-04 02:08:26] iteration 413/ 6855 | consumed samples: 52864 | elapsed time per iteration (ms): 87317.2 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.56% | FLOPs estimator: workload-aware | logical tokens/s: 192140.5 | valid tokens/s: 192140.5 | physical tokens/s: 192554.0 | loss tokens/s: 149556.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.921081E-01 | lm-global-supervised-tokens: 1.305882E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.264032E-02 | lm-token-buffer-mib: 5.813792E+01 | global_load_balancing_loss: 9.979640E-01 | loss scale: 1.0 | grad norm: 0.055 | params norm: 11641.326 | params rms: 0.063034 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 12:14:57.675437 | finish at 2026-10-10 14:23:24 + [2026-10-04 02:09:53] iteration 414/ 6855 | consumed samples: 52992 | elapsed time per iteration (ms): 86158.3 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.56% | FLOPs estimator: workload-aware | logical tokens/s: 194724.8 | valid tokens/s: 194724.8 | physical tokens/s: 195141.4 | loss tokens/s: 138352.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.282409E-01 | lm-global-supervised-tokens: 1.192025E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.207301E-02 | lm-token-buffer-mib: 5.814468E+01 | global_load_balancing_loss: 9.988981E-01 | loss scale: 1.0 | grad norm: 0.163 | params norm: 11641.235 | params rms: 0.063034 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 10:09:05.798689 | finish at 2026-10-10 12:18:58 + [2026-10-04 02:11:19] iteration 415/ 6855 | consumed samples: 53120 | elapsed time per iteration (ms): 86437.2 | throughput per GPU (TFLOP/s/GPU): 41.7 | MFU 13.37% | FLOPs estimator: workload-aware | logical tokens/s: 194096.9 | valid tokens/s: 194096.8 | physical tokens/s: 194691.5 | loss tokens/s: 146901.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.052118E-01 | lm-global-supervised-tokens: 1.269773E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.156583E-02 | lm-token-buffer-mib: 5.827875E+01 | global_load_balancing_loss: 9.997978E-01 | loss scale: 1.0 | grad norm: 0.102 | params norm: 11641.144 | params rms: 0.063033 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 10:37:35.780334 | finish at 2026-10-10 12:48:55 + [2026-10-04 02:12:44] iteration 416/ 6855 | consumed samples: 53248 | elapsed time per iteration (ms): 84727.8 | throughput per GPU (TFLOP/s/GPU): 42.8 | MFU 13.71% | FLOPs estimator: workload-aware | logical tokens/s: 198012.1 | valid tokens/s: 198012.1 | physical tokens/s: 198574.8 | loss tokens/s: 147146.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.919671E-01 | lm-global-supervised-tokens: 1.246740E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.221140E-02 | lm-token-buffer-mib: 5.829615E+01 | global_load_balancing_loss: 9.993180E-01 | loss scale: 1.0 | grad norm: 0.208 | params norm: 11641.053 | params rms: 0.063033 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 7:32:42.093188 | finish at 2026-10-10 09:45:26 + [2026-10-04 02:14:13] iteration 417/ 6855 | consumed samples: 53376 | elapsed time per iteration (ms): 89460.7 | throughput per GPU (TFLOP/s/GPU): 41.1 | MFU 13.18% | FLOPs estimator: workload-aware | logical tokens/s: 187536.4 | valid tokens/s: 187536.4 | physical tokens/s: 188083.0 | loss tokens/s: 140376.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.601568E-01 | lm-global-supervised-tokens: 1.255821E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.777674E-02 | lm-token-buffer-mib: 5.818609E+01 | global_load_balancing_loss: 1.000518E+00 | loss scale: 1.0 | grad norm: 0.071 | params norm: 11640.962 | params rms: 0.063032 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 15:59:07.903939 | finish at 2026-10-10 18:13:21 + [2026-10-04 02:15:41] iteration 418/ 6855 | consumed samples: 53504 | elapsed time per iteration (ms): 88289.6 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.48% | FLOPs estimator: workload-aware | logical tokens/s: 190024.5 | valid tokens/s: 190024.5 | physical tokens/s: 190469.4 | loss tokens/s: 134085.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.464540E-01 | lm-global-supervised-tokens: 1.183838E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.218127E-02 | lm-token-buffer-mib: 5.822205E+01 | global_load_balancing_loss: 1.001441E+00 | loss scale: 1.0 | grad norm: 0.197 | params norm: 11640.870 | params rms: 0.063032 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 13:51:59.935065 | finish at 2026-10-10 16:07:41 + [2026-10-04 02:17:08] iteration 419/ 6855 | consumed samples: 53632 | elapsed time per iteration (ms): 86526.9 | throughput per GPU (TFLOP/s/GPU): 42.8 | MFU 13.72% | FLOPs estimator: workload-aware | logical tokens/s: 193895.6 | valid tokens/s: 193895.6 | physical tokens/s: 194359.0 | loss tokens/s: 150202.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.653039E-01 | lm-global-supervised-tokens: 1.299651E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.494198E-02 | lm-token-buffer-mib: 5.814202E+01 | global_load_balancing_loss: 1.001224E+00 | loss scale: 1.0 | grad norm: 0.082 | params norm: 11640.777 | params rms: 0.063031 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 10:41:27.059690 | finish at 2026-10-10 12:58:35 + [2026-10-04 02:18:50] iteration 420/ 6855 | consumed samples: 53760 | elapsed time per iteration (ms): 101582.9 | throughput per GPU (TFLOP/s/GPU): 34.8 | MFU 11.15% | FLOPs estimator: workload-aware | logical tokens/s: 165157.6 | valid tokens/s: 165157.6 | physical tokens/s: 165645.6 | loss tokens/s: 125793.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.513625E-01 | lm-global-supervised-tokens: 1.277846E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.244192E-02 | lm-token-buffer-mib: 5.809924E+01 | global_load_balancing_loss: 1.001267E+00 | loss scale: 1.0 | grad norm: 0.090 | params norm: 11640.685 | params rms: 0.063031 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 7 days, 13:34:45.820656 | finish at 2026-10-11 15:53:35 + [2026-10-04 02:20:14] iteration 421/ 6855 | consumed samples: 53888 | elapsed time per iteration (ms): 84282.7 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.66% | FLOPs estimator: workload-aware | logical tokens/s: 199058.2 | valid tokens/s: 199058.2 | physical tokens/s: 199380.2 | loss tokens/s: 148257.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.181584E-01 | lm-global-supervised-tokens: 1.249553E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.313880E-02 | lm-token-buffer-mib: 5.815850E+01 | global_load_balancing_loss: 1.000490E+00 | loss scale: 1.0 | grad norm: 0.097 | params norm: 11640.592 | params rms: 0.063030 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 6:37:55.056334 | finish at 2026-10-10 08:58:09 + [2026-10-04 02:21:43] iteration 422/ 6855 | consumed samples: 54016 | elapsed time per iteration (ms): 89054.1 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.46% | FLOPs estimator: workload-aware | logical tokens/s: 188393.4 | valid tokens/s: 188393.4 | physical tokens/s: 188728.2 | loss tokens/s: 138105.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.795150E-01 | lm-global-supervised-tokens: 1.229888E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.316268E-02 | lm-token-buffer-mib: 5.812552E+01 | global_load_balancing_loss: 1.000579E+00 | loss scale: 1.0 | grad norm: 0.033 | params norm: 11640.499 | params rms: 0.063030 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 15:08:05.270935 | finish at 2026-10-10 17:29:48 + [2026-10-04 02:23:11] iteration 423/ 6855 | consumed samples: 54144 | elapsed time per iteration (ms): 88203.5 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.47% | FLOPs estimator: workload-aware | logical tokens/s: 190210.2 | valid tokens/s: 190210.2 | physical tokens/s: 190542.8 | loss tokens/s: 138048.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.939387E-01 | lm-global-supervised-tokens: 1.217632E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.259826E-02 | lm-token-buffer-mib: 5.821157E+01 | global_load_balancing_loss: 9.996273E-01 | loss scale: 1.0 | grad norm: 0.165 | params norm: 11640.405 | params rms: 0.063029 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 13:35:25.025688 | finish at 2026-10-10 15:58:36 + [2026-10-04 02:24:42] iteration 424/ 6855 | consumed samples: 54272 | elapsed time per iteration (ms): 90548.7 | throughput per GPU (TFLOP/s/GPU): 41.3 | MFU 13.23% | FLOPs estimator: workload-aware | logical tokens/s: 185283.5 | valid tokens/s: 185283.5 | physical tokens/s: 185621.4 | loss tokens/s: 141766.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.197537E-01 | lm-global-supervised-tokens: 1.283675E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.833807E-02 | lm-token-buffer-mib: 5.813367E+01 | global_load_balancing_loss: 9.986805E-01 | loss scale: 1.0 | grad norm: 0.062 | params norm: 11640.311 | params rms: 0.063028 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 17:45:18.679573 | finish at 2026-10-10 20:10:00 + [2026-10-04 02:26:14] iteration 425/ 6855 | consumed samples: 54400 | elapsed time per iteration (ms): 92293.4 | throughput per GPU (TFLOP/s/GPU): 40.7 | MFU 13.04% | FLOPs estimator: workload-aware | logical tokens/s: 181781.0 | valid tokens/s: 181781.0 | physical tokens/s: 182306.3 | loss tokens/s: 136836.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.254450E-01 | lm-global-supervised-tokens: 1.262907E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.149019E-02 | lm-token-buffer-mib: 5.841349E+01 | global_load_balancing_loss: 9.983814E-01 | loss scale: 1.0 | grad norm: 0.116 | params norm: 11640.219 | params rms: 0.063028 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 20:50:46.352291 | finish at 2026-10-10 23:17:00 + [2026-10-04 02:27:42] iteration 426/ 6855 | consumed samples: 54528 | elapsed time per iteration (ms): 88408.8 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.56% | FLOPs estimator: workload-aware | logical tokens/s: 189767.3 | valid tokens/s: 189767.3 | physical tokens/s: 190228.6 | loss tokens/s: 138317.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.307578E-01 | lm-global-supervised-tokens: 1.222848E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.190220E-02 | lm-token-buffer-mib: 5.805754E+01 | global_load_balancing_loss: 9.995168E-01 | loss scale: 1.0 | grad norm: 0.089 | params norm: 11640.127 | params rms: 0.063028 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 13:53:00.280234 | finish at 2026-10-10 16:20:43 + [2026-10-04 02:29:10] iteration 427/ 6855 | consumed samples: 54656 | elapsed time per iteration (ms): 87419.6 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.41% | FLOPs estimator: workload-aware | logical tokens/s: 191914.7 | valid tokens/s: 191914.7 | physical tokens/s: 192578.5 | loss tokens/s: 135450.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.293518E-01 | lm-global-supervised-tokens: 1.184101E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.688052E-02 | lm-token-buffer-mib: 5.808222E+01 | global_load_balancing_loss: 1.001269E+00 | loss scale: 1.0 | grad norm: 0.154 | params norm: 11640.035 | params rms: 0.063027 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 12:05:33.126029 | finish at 2026-10-10 14:34:43 + [2026-10-04 02:30:39] iteration 428/ 6855 | consumed samples: 54784 | elapsed time per iteration (ms): 89658.8 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.58% | FLOPs estimator: workload-aware | logical tokens/s: 187122.0 | valid tokens/s: 187122.0 | physical tokens/s: 187543.1 | loss tokens/s: 137604.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.125802E-01 | lm-global-supervised-tokens: 1.233743E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.262530E-02 | lm-token-buffer-mib: 5.819977E+01 | global_load_balancing_loss: 1.000745E+00 | loss scale: 1.0 | grad norm: 0.103 | params norm: 11639.943 | params rms: 0.063027 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 16:03:57.387286 | finish at 2026-10-10 18:34:37 + [2026-10-04 02:32:13] iteration 429/ 6855 | consumed samples: 54912 | elapsed time per iteration (ms): 93163.7 | throughput per GPU (TFLOP/s/GPU): 40.4 | MFU 12.95% | FLOPs estimator: workload-aware | logical tokens/s: 180083.2 | valid tokens/s: 180083.2 | physical tokens/s: 180564.6 | loss tokens/s: 129838.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.934754E-01 | lm-global-supervised-tokens: 1.209622E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.353085E-02 | lm-token-buffer-mib: 5.815097E+01 | global_load_balancing_loss: 1.001401E+00 | loss scale: 1.0 | grad norm: 0.112 | params norm: 11639.851 | params rms: 0.063026 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 22:17:49.730036 | finish at 2026-10-11 00:50:02 + [2026-10-04 02:33:58] iteration 430/ 6855 | consumed samples: 55040 | elapsed time per iteration (ms): 104895.4 | throughput per GPU (TFLOP/s/GPU): 34.4 | MFU 11.01% | FLOPs estimator: workload-aware | logical tokens/s: 159942.3 | valid tokens/s: 159942.3 | physical tokens/s: 160266.3 | loss tokens/s: 119201.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.734515E-01 | lm-global-supervised-tokens: 1.250366E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.653047E-02 | lm-token-buffer-mib: 5.813794E+01 | global_load_balancing_loss: 1.000791E+00 | loss scale: 1.0 | grad norm: 0.045 | params norm: 11639.759 | params rms: 0.063026 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 7 days, 19:12:33.017300 | finish at 2026-10-11 21:46:31 + [2026-10-04 02:35:22] iteration 431/ 6855 | consumed samples: 55168 | elapsed time per iteration (ms): 84370.7 | throughput per GPU (TFLOP/s/GPU): 43.1 | MFU 13.80% | FLOPs estimator: workload-aware | logical tokens/s: 198851.2 | valid tokens/s: 198851.1 | physical tokens/s: 199279.8 | loss tokens/s: 153457.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.693709E-01 | lm-global-supervised-tokens: 1.294727E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.186289E-02 | lm-token-buffer-mib: 5.821297E+01 | global_load_balancing_loss: 1.000226E+00 | loss scale: 1.0 | grad norm: 0.086 | params norm: 11639.667 | params rms: 0.063025 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 6:33:17.323971 | finish at 2026-10-10 09:08:39 + [2026-10-04 02:36:49] iteration 432/ 6855 | consumed samples: 55296 | elapsed time per iteration (ms): 87441.7 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.60% | FLOPs estimator: workload-aware | logical tokens/s: 191867.2 | valid tokens/s: 191867.2 | physical tokens/s: 192400.2 | loss tokens/s: 138349.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.974321E-01 | lm-global-supervised-tokens: 1.209749E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.140010E-02 | lm-token-buffer-mib: 5.817015E+01 | global_load_balancing_loss: 9.995700E-01 | loss scale: 1.0 | grad norm: 0.105 | params norm: 11639.574 | params rms: 0.063025 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 12:00:37.813872 | finish at 2026-10-10 14:37:27 + [2026-10-04 02:38:18] iteration 433/ 6855 | consumed samples: 55424 | elapsed time per iteration (ms): 88542.3 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.46% | FLOPs estimator: workload-aware | logical tokens/s: 189481.6 | valid tokens/s: 189481.6 | physical tokens/s: 189990.1 | loss tokens/s: 138334.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.436746E-01 | lm-global-supervised-tokens: 1.224848E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.194852E-02 | lm-token-buffer-mib: 5.817482E+01 | global_load_balancing_loss: 9.996352E-01 | loss scale: 1.0 | grad norm: 0.046 | params norm: 11639.484 | params rms: 0.063024 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 13:56:58.776062 | finish at 2026-10-10 16:35:17 + [2026-10-04 02:39:47] iteration 434/ 6855 | consumed samples: 55552 | elapsed time per iteration (ms): 88640.7 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.40% | FLOPs estimator: workload-aware | logical tokens/s: 189271.7 | valid tokens/s: 189271.7 | physical tokens/s: 189714.5 | loss tokens/s: 136245.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.949176E-01 | lm-global-supervised-tokens: 1.207687E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.309951E-02 | lm-token-buffer-mib: 5.815687E+01 | global_load_balancing_loss: 9.999003E-01 | loss scale: 1.0 | grad norm: 0.134 | params norm: 11639.395 | params rms: 0.063024 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 14:06:02.007306 | finish at 2026-10-10 16:45:49 + [2026-10-04 02:41:15] iteration 435/ 6855 | consumed samples: 55680 | elapsed time per iteration (ms): 88297.5 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.53% | FLOPs estimator: workload-aware | logical tokens/s: 190007.0 | valid tokens/s: 190007.0 | physical tokens/s: 190538.1 | loss tokens/s: 139460.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.429263E-01 | lm-global-supervised-tokens: 1.231398E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.343136E-02 | lm-token-buffer-mib: 5.816187E+01 | global_load_balancing_loss: 1.001042E+00 | loss scale: 1.0 | grad norm: 0.114 | params norm: 11639.307 | params rms: 0.063023 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 13:27:50.195761 | finish at 2026-10-10 16:09:05 + [2026-10-04 02:42:41] iteration 436/ 6855 | consumed samples: 55808 | elapsed time per iteration (ms): 86003.6 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.56% | FLOPs estimator: workload-aware | logical tokens/s: 195075.3 | valid tokens/s: 195075.3 | physical tokens/s: 195460.7 | loss tokens/s: 149250.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.062755E-01 | lm-global-supervised-tokens: 1.283605E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.309428E-02 | lm-token-buffer-mib: 5.815052E+01 | global_load_balancing_loss: 9.997461E-01 | loss scale: 1.0 | grad norm: 0.043 | params norm: 11639.218 | params rms: 0.063023 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 9:20:57.419847 | finish at 2026-10-10 12:03:38 + [2026-10-04 02:44:10] iteration 437/ 6855 | consumed samples: 55936 | elapsed time per iteration (ms): 89463.6 | throughput per GPU (TFLOP/s/GPU): 41.6 | MFU 13.33% | FLOPs estimator: workload-aware | logical tokens/s: 187530.7 | valid tokens/s: 187530.7 | physical tokens/s: 188051.6 | loss tokens/s: 139182.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.961505E-01 | lm-global-supervised-tokens: 1.245181E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.326267E-02 | lm-token-buffer-mib: 5.821233E+01 | global_load_balancing_loss: 1.000198E+00 | loss scale: 1.0 | grad norm: 0.137 | params norm: 11639.127 | params rms: 0.063022 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 15:29:37.596982 | finish at 2026-10-10 18:13:48 + [2026-10-04 02:45:38] iteration 438/ 6855 | consumed samples: 56064 | elapsed time per iteration (ms): 87654.5 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.56% | FLOPs estimator: workload-aware | logical tokens/s: 191401.3 | valid tokens/s: 191401.3 | physical tokens/s: 191860.6 | loss tokens/s: 145118.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.805388E-01 | lm-global-supervised-tokens: 1.272031E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.602293E-02 | lm-token-buffer-mib: 5.815956E+01 | global_load_balancing_loss: 1.000303E+00 | loss scale: 1.0 | grad norm: 0.149 | params norm: 11639.038 | params rms: 0.063022 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 12:14:38.687880 | finish at 2026-10-10 15:00:17 + [2026-10-04 02:47:02] iteration 439/ 6855 | consumed samples: 56192 | elapsed time per iteration (ms): 84346.6 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.79% | FLOPs estimator: workload-aware | logical tokens/s: 198907.8 | valid tokens/s: 198907.8 | physical tokens/s: 199549.1 | loss tokens/s: 149760.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.025245E-01 | lm-global-supervised-tokens: 1.263180E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.442434E-02 | lm-token-buffer-mib: 5.861620E+01 | global_load_balancing_loss: 1.001379E+00 | loss scale: 1.0 | grad norm: 0.058 | params norm: 11638.949 | params rms: 0.063021 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 6:19:28.087307 | finish at 2026-10-10 09:06:30 + [2026-10-04 02:48:51] iteration 440/ 6855 | consumed samples: 56320 | elapsed time per iteration (ms): 108175.7 | throughput per GPU (TFLOP/s/GPU): 34.0 | MFU 10.90% | FLOPs estimator: workload-aware | logical tokens/s: 155092.0 | valid tokens/s: 155091.9 | physical tokens/s: 155507.1 | loss tokens/s: 116837.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.848698E-01 | lm-global-supervised-tokens: 1.263894E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.135133E-02 | lm-token-buffer-mib: 5.813771E+01 | global_load_balancing_loss: 9.998150E-01 | loss scale: 1.0 | grad norm: 0.105 | params norm: 11638.861 | params rms: 0.063021 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 8 days, 0:45:47.283415 | finish at 2026-10-12 03:34:38 +times across ranks (ms): + save-checkpoint: + rank 0: 24192.41 + rank 1: 24192.39 + rank 2: 24192.44 + rank 3: 24192.43 + rank 4: 24192.42 + rank 5: 24192.39 + rank 6: 24192.38 + rank 7: 24192.44 + rank 8: 24192.91 + rank 9: 24192.77 + rank 10: 24192.77 + rank 11: 24192.77 + rank 12: 24192.78 + rank 13: 24192.80 + rank 14: 24192.80 + rank 15: 24192.65 + rank 16: 24192.53 + rank 17: 24192.53 + rank 18: 24192.49 + rank 19: 24192.52 + rank 20: 24192.50 + rank 21: 24192.53 + rank 22: 24192.48 + rank 23: 24192.47 + rank 24: 24192.95 + rank 25: 24192.84 + rank 26: 24192.92 + rank 27: 24193.01 + rank 28: 24192.88 + rank 29: 24192.90 + rank 30: 24192.89 + rank 31: 24192.84 + rank 32: 24192.69 + rank 33: 24192.70 + rank 34: 24192.69 + rank 35: 24192.70 + rank 36: 24192.61 + rank 37: 24192.49 + rank 38: 24192.70 + rank 39: 24192.61 + rank 40: 24192.67 + rank 41: 24192.62 + rank 42: 24192.64 + rank 43: 24192.63 + rank 44: 24192.61 + rank 45: 24192.61 + rank 46: 24192.64 + rank 47: 24192.62 + rank 48: 24192.88 + rank 49: 24192.88 + rank 50: 24192.82 + rank 51: 24192.86 + rank 52: 24192.84 + rank 53: 24192.82 + rank 54: 24192.86 + rank 55: 24192.83 + rank 56: 24192.43 + rank 57: 24192.38 + rank 58: 24192.35 + rank 59: 24192.36 + rank 60: 24192.35 + rank 61: 24192.40 + rank 62: 24192.36 + rank 63: 24192.21 + [2026-10-04 02:50:43] iteration 441/ 6855 | consumed samples: 56448 | elapsed time per iteration (ms): 88120.0 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.43% | FLOPs estimator: workload-aware | logical tokens/s: 190390.2 | valid tokens/s: 190390.2 | physical tokens/s: 190915.7 | loss tokens/s: 141917.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.261987E-01 | lm-global-supervised-tokens: 1.250576E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.050213E+00 | lm-token-buffer-mib: 5.804178E+01 | global_load_balancing_loss: 9.997474E-01 | loss scale: 1.0 | grad norm: 0.038 | params norm: 11638.772 | params rms: 0.063020 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 13:00:01.469763 | finish at 2026-10-10 15:50:45 + [2026-10-04 02:52:14] iteration 442/ 6855 | consumed samples: 56576 | elapsed time per iteration (ms): 90670.7 | throughput per GPU (TFLOP/s/GPU): 40.6 | MFU 13.02% | FLOPs estimator: workload-aware | logical tokens/s: 185033.5 | valid tokens/s: 185033.5 | physical tokens/s: 185522.6 | loss tokens/s: 138652.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.907905E-01 | lm-global-supervised-tokens: 1.257173E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.388873E-02 | lm-token-buffer-mib: 5.804817E+01 | global_load_balancing_loss: 1.000540E+00 | loss scale: 1.0 | grad norm: 0.096 | params norm: 11638.684 | params rms: 0.063020 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 17:31:11.476315 | finish at 2026-10-10 20:23:25 + [2026-10-04 02:53:40] iteration 443/ 6855 | consumed samples: 56704 | elapsed time per iteration (ms): 86486.8 | throughput per GPU (TFLOP/s/GPU): 43.5 | MFU 13.93% | FLOPs estimator: workload-aware | logical tokens/s: 193985.2 | valid tokens/s: 193985.2 | physical tokens/s: 194404.6 | loss tokens/s: 145235.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.633141E-01 | lm-global-supervised-tokens: 1.256093E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.346974E-02 | lm-token-buffer-mib: 5.805054E+01 | global_load_balancing_loss: 1.000689E+00 | loss scale: 1.0 | grad norm: 0.099 | params norm: 11638.596 | params rms: 0.063019 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 10:02:33.168693 | finish at 2026-10-10 12:56:13 + [2026-10-04 02:55:10] iteration 444/ 6855 | consumed samples: 56832 | elapsed time per iteration (ms): 89468.8 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.58% | FLOPs estimator: workload-aware | logical tokens/s: 187519.9 | valid tokens/s: 187519.9 | physical tokens/s: 187819.9 | loss tokens/s: 136654.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.680350E-01 | lm-global-supervised-tokens: 1.222631E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.292746E-02 | lm-token-buffer-mib: 5.813707E+01 | global_load_balancing_loss: 1.000723E+00 | loss scale: 1.0 | grad norm: 0.057 | params norm: 11638.508 | params rms: 0.063019 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 15:19:44.406924 | finish at 2026-10-10 18:14:54 + [2026-10-04 02:56:35] iteration 445/ 6855 | consumed samples: 56960 | elapsed time per iteration (ms): 85650.4 | throughput per GPU (TFLOP/s/GPU): 43.1 | MFU 13.82% | FLOPs estimator: workload-aware | logical tokens/s: 195880.2 | valid tokens/s: 195880.2 | physical tokens/s: 196413.0 | loss tokens/s: 142373.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.771783E-01 | lm-global-supervised-tokens: 1.219432E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.835809E-02 | lm-token-buffer-mib: 5.806595E+01 | global_load_balancing_loss: 1.000190E+00 | loss scale: 1.0 | grad norm: 0.166 | params norm: 11638.420 | params rms: 0.063018 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 8:30:18.942776 | finish at 2026-10-10 11:26:54 + [2026-10-04 02:57:59] iteration 446/ 6855 | consumed samples: 57088 | elapsed time per iteration (ms): 83389.1 | throughput per GPU (TFLOP/s/GPU): 43.9 | MFU 14.07% | FLOPs estimator: workload-aware | logical tokens/s: 201191.7 | valid tokens/s: 201191.7 | physical tokens/s: 201542.2 | loss tokens/s: 153045.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.064678E-01 | lm-global-supervised-tokens: 1.276235E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.390293E-02 | lm-token-buffer-mib: 5.812874E+01 | global_load_balancing_loss: 9.988690E-01 | loss scale: 1.0 | grad norm: 0.085 | params norm: 11638.332 | params rms: 0.063018 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 4:27:20.931854 | finish at 2026-10-10 07:25:20 + [2026-10-04 02:59:29] iteration 447/ 6855 | consumed samples: 57216 | elapsed time per iteration (ms): 89914.2 | throughput per GPU (TFLOP/s/GPU): 41.3 | MFU 13.25% | FLOPs estimator: workload-aware | logical tokens/s: 186590.7 | valid tokens/s: 186590.7 | physical tokens/s: 186948.2 | loss tokens/s: 141803.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.817694E-01 | lm-global-supervised-tokens: 1.275013E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.144074E-02 | lm-token-buffer-mib: 5.822829E+01 | global_load_balancing_loss: 9.996957E-01 | loss scale: 1.0 | grad norm: 0.097 | params norm: 11638.245 | params rms: 0.063017 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 16:02:50.212366 | finish at 2026-10-10 19:02:19 + [2026-10-04 03:00:55] iteration 448/ 6855 | consumed samples: 57344 | elapsed time per iteration (ms): 86679.4 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.42% | FLOPs estimator: workload-aware | logical tokens/s: 193554.9 | valid tokens/s: 193554.9 | physical tokens/s: 194003.3 | loss tokens/s: 146743.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.326731E-01 | lm-global-supervised-tokens: 1.271967E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.153465E-02 | lm-token-buffer-mib: 5.823140E+01 | global_load_balancing_loss: 9.989341E-01 | loss scale: 1.0 | grad norm: 0.091 | params norm: 11638.157 | params rms: 0.063017 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 10:15:54.721591 | finish at 2026-10-10 13:16:50 + [2026-10-04 03:02:21] iteration 449/ 6855 | consumed samples: 57472 | elapsed time per iteration (ms): 85494.9 | throughput per GPU (TFLOP/s/GPU): 43.2 | MFU 13.85% | FLOPs estimator: workload-aware | logical tokens/s: 196235.9 | valid tokens/s: 196235.9 | physical tokens/s: 196706.7 | loss tokens/s: 150830.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.968790E-01 | lm-global-supervised-tokens: 1.289526E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.604975E-02 | lm-token-buffer-mib: 5.822167E+01 | global_load_balancing_loss: 9.997756E-01 | loss scale: 1.0 | grad norm: 0.082 | params norm: 11638.068 | params rms: 0.063016 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 8:08:00.440818 | finish at 2026-10-10 11:10:21 + [2026-10-04 03:04:04] iteration 450/ 6855 | consumed samples: 57600 | elapsed time per iteration (ms): 103160.8 | throughput per GPU (TFLOP/s/GPU): 35.1 | MFU 11.25% | FLOPs estimator: workload-aware | logical tokens/s: 162631.5 | valid tokens/s: 162631.5 | physical tokens/s: 163038.2 | loss tokens/s: 122313.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.967552E-01 | lm-global-supervised-tokens: 1.261798E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.150466E-02 | lm-token-buffer-mib: 5.827835E+01 | global_load_balancing_loss: 1.000388E+00 | loss scale: 1.0 | grad norm: 0.083 | params norm: 11637.978 | params rms: 0.063016 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 7 days, 15:32:24.960523 | finish at 2026-10-11 18:36:29 + [2026-10-04 03:05:29] iteration 451/ 6855 | consumed samples: 57728 | elapsed time per iteration (ms): 85223.9 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.63% | FLOPs estimator: workload-aware | logical tokens/s: 196859.6 | valid tokens/s: 196859.6 | physical tokens/s: 197399.0 | loss tokens/s: 144492.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.513166E-01 | lm-global-supervised-tokens: 1.231420E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.172306E-02 | lm-token-buffer-mib: 5.819808E+01 | global_load_balancing_loss: 1.002828E+00 | loss scale: 1.0 | grad norm: 0.111 | params norm: 11637.888 | params rms: 0.063015 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 7:36:13.630139 | finish at 2026-10-10 10:41:43 + [2026-10-04 03:06:57] iteration 452/ 6855 | consumed samples: 57856 | elapsed time per iteration (ms): 88003.6 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.47% | FLOPs estimator: workload-aware | logical tokens/s: 190642.3 | valid tokens/s: 190642.3 | physical tokens/s: 191096.5 | loss tokens/s: 141528.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.440157E-01 | lm-global-supervised-tokens: 1.245500E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.808127E-02 | lm-token-buffer-mib: 5.813708E+01 | global_load_balancing_loss: 1.001333E+00 | loss scale: 1.0 | grad norm: 0.085 | params norm: 11637.799 | params rms: 0.063015 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 12:31:27.111109 | finish at 2026-10-10 15:38:24 + [2026-10-04 03:08:23] iteration 453/ 6855 | consumed samples: 57984 | elapsed time per iteration (ms): 85693.7 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.65% | FLOPs estimator: workload-aware | logical tokens/s: 195780.4 | valid tokens/s: 195780.4 | physical tokens/s: 196267.7 | loss tokens/s: 142011.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.897630E-01 | lm-global-supervised-tokens: 1.216945E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.316028E-02 | lm-token-buffer-mib: 5.811943E+01 | global_load_balancing_loss: 9.997764E-01 | loss scale: 1.0 | grad norm: 0.101 | params norm: 11637.706 | params rms: 0.063014 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 8:23:31.238833 | finish at 2026-10-10 11:31:54 + [2026-10-04 03:09:53] iteration 454/ 6855 | consumed samples: 58112 | elapsed time per iteration (ms): 90385.3 | throughput per GPU (TFLOP/s/GPU): 41.2 | MFU 13.21% | FLOPs estimator: workload-aware | logical tokens/s: 185618.8 | valid tokens/s: 185618.8 | physical tokens/s: 186078.0 | loss tokens/s: 131130.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.945438E-01 | lm-global-supervised-tokens: 1.185228E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.124090E-02 | lm-token-buffer-mib: 5.804873E+01 | global_load_balancing_loss: 1.000388E+00 | loss scale: 1.0 | grad norm: 0.051 | params norm: 11637.613 | params rms: 0.063014 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 16:42:36.350578 | finish at 2026-10-10 19:52:30 + [2026-10-04 03:11:21] iteration 455/ 6855 | consumed samples: 58240 | elapsed time per iteration (ms): 87544.1 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.57% | FLOPs estimator: workload-aware | logical tokens/s: 191642.4 | valid tokens/s: 191642.4 | physical tokens/s: 192198.3 | loss tokens/s: 139002.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.726919E-01 | lm-global-supervised-tokens: 1.216887E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.154128E-02 | lm-token-buffer-mib: 5.814799E+01 | global_load_balancing_loss: 9.991420E-01 | loss scale: 1.0 | grad norm: 0.065 | params norm: 11637.520 | params rms: 0.063013 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 11:38:02.231140 | finish at 2026-10-10 14:49:23 + [2026-10-04 03:12:49] iteration 456/ 6855 | consumed samples: 58368 | elapsed time per iteration (ms): 88584.4 | throughput per GPU (TFLOP/s/GPU): 41.3 | MFU 13.24% | FLOPs estimator: workload-aware | logical tokens/s: 189391.9 | valid tokens/s: 189391.9 | physical tokens/s: 190063.4 | loss tokens/s: 141485.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.870015E-01 | lm-global-supervised-tokens: 1.253337E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.084038E-02 | lm-token-buffer-mib: 5.810248E+01 | global_load_balancing_loss: 1.000021E+00 | loss scale: 1.0 | grad norm: 0.141 | params norm: 11637.427 | params rms: 0.063013 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 13:27:31.585886 | finish at 2026-10-10 16:40:21 + [2026-10-04 03:14:18] iteration 457/ 6855 | consumed samples: 58496 | elapsed time per iteration (ms): 89034.7 | throughput per GPU (TFLOP/s/GPU): 43.3 | MFU 13.88% | FLOPs estimator: workload-aware | logical tokens/s: 188434.5 | valid tokens/s: 188434.4 | physical tokens/s: 188797.1 | loss tokens/s: 133388.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.867434E-01 | lm-global-supervised-tokens: 1.187616E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.209050E-02 | lm-token-buffer-mib: 5.810590E+01 | global_load_balancing_loss: 1.001692E+00 | loss scale: 1.0 | grad norm: 0.134 | params norm: 11637.334 | params rms: 0.063012 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 14:14:03.947526 | finish at 2026-10-10 17:28:22 + [2026-10-04 03:15:43] iteration 458/ 6855 | consumed samples: 58624 | elapsed time per iteration (ms): 84597.7 | throughput per GPU (TFLOP/s/GPU): 43.7 | MFU 14.01% | FLOPs estimator: workload-aware | logical tokens/s: 198317.6 | valid tokens/s: 198317.6 | physical tokens/s: 198810.7 | loss tokens/s: 146937.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.648513E-01 | lm-global-supervised-tokens: 1.243060E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.157006E-02 | lm-token-buffer-mib: 5.807748E+01 | global_load_balancing_loss: 1.000278E+00 | loss scale: 1.0 | grad norm: 0.162 | params norm: 11637.243 | params rms: 0.063012 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 6:19:31.454108 | finish at 2026-10-10 09:35:14 + [2026-10-04 03:17:12] iteration 459/ 6855 | consumed samples: 58752 | elapsed time per iteration (ms): 89358.8 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.47% | FLOPs estimator: workload-aware | logical tokens/s: 187748.8 | valid tokens/s: 187748.8 | physical tokens/s: 188321.4 | loss tokens/s: 132778.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.755029E-01 | lm-global-supervised-tokens: 1.186493E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.866891E-02 | lm-token-buffer-mib: 5.814442E+01 | global_load_balancing_loss: 9.986244E-01 | loss scale: 1.0 | grad norm: 0.089 | params norm: 11637.151 | params rms: 0.063011 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 14:45:39.030773 | finish at 2026-10-10 18:02:51 + [2026-10-04 03:18:58] iteration 460/ 6855 | consumed samples: 58880 | elapsed time per iteration (ms): 105616.5 | throughput per GPU (TFLOP/s/GPU): 34.5 | MFU 11.07% | FLOPs estimator: workload-aware | logical tokens/s: 158849.7 | valid tokens/s: 158849.7 | physical tokens/s: 159249.0 | loss tokens/s: 117823.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.395519E-01 | lm-global-supervised-tokens: 1.244416E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.297535E-02 | lm-token-buffer-mib: 5.821843E+01 | global_load_balancing_loss: 9.981725E-01 | loss scale: 1.0 | grad norm: 0.257 | params norm: 11637.060 | params rms: 0.063011 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 7 days, 19:36:57.733371 | finish at 2026-10-11 22:55:56 + [2026-10-04 03:20:27] iteration 461/ 6855 | consumed samples: 59008 | elapsed time per iteration (ms): 89060.8 | throughput per GPU (TFLOP/s/GPU): 41.6 | MFU 13.33% | FLOPs estimator: workload-aware | logical tokens/s: 188379.4 | valid tokens/s: 188379.4 | physical tokens/s: 188882.5 | loss tokens/s: 135431.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.482648E-01 | lm-global-supervised-tokens: 1.206159E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.241132E-02 | lm-token-buffer-mib: 5.821706E+01 | global_load_balancing_loss: 1.000341E+00 | loss scale: 1.0 | grad norm: 0.055 | params norm: 11636.970 | params rms: 0.063010 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 14:10:54.496527 | finish at 2026-10-10 17:31:22 + [2026-10-04 03:21:55] iteration 462/ 6855 | consumed samples: 59136 | elapsed time per iteration (ms): 87722.6 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.50% | FLOPs estimator: workload-aware | logical tokens/s: 191252.8 | valid tokens/s: 191252.8 | physical tokens/s: 191761.1 | loss tokens/s: 137057.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.680761E-01 | lm-global-supervised-tokens: 1.202301E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.247380E-02 | lm-token-buffer-mib: 5.818719E+01 | global_load_balancing_loss: 1.000522E+00 | loss scale: 1.0 | grad norm: 0.211 | params norm: 11636.882 | params rms: 0.063010 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 11:46:50.822518 | finish at 2026-10-10 15:08:46 + [2026-10-04 03:23:19] iteration 463/ 6855 | consumed samples: 59264 | elapsed time per iteration (ms): 84537.9 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.47% | FLOPs estimator: workload-aware | logical tokens/s: 198457.3 | valid tokens/s: 198457.3 | physical tokens/s: 198945.8 | loss tokens/s: 146521.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.356671E-01 | lm-global-supervised-tokens: 1.238663E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.136463E-02 | lm-token-buffer-mib: 5.807310E+01 | global_load_balancing_loss: 1.000878E+00 | loss scale: 1.0 | grad norm: 0.116 | params norm: 11636.792 | params rms: 0.063009 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 6:06:05.992968 | finish at 2026-10-10 09:29:25 + [2026-10-04 03:24:47] iteration 464/ 6855 | consumed samples: 59392 | elapsed time per iteration (ms): 87761.3 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.55% | FLOPs estimator: workload-aware | logical tokens/s: 191168.1 | valid tokens/s: 191168.1 | physical tokens/s: 191477.2 | loss tokens/s: 146546.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.679991E-01 | lm-global-supervised-tokens: 1.286108E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.552114E-02 | lm-token-buffer-mib: 5.818089E+01 | global_load_balancing_loss: 1.000388E+00 | loss scale: 1.0 | grad norm: 0.063 | params norm: 11636.700 | params rms: 0.063009 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 11:48:02.458190 | finish at 2026-10-10 15:12:50 + [2026-10-04 03:26:15] iteration 465/ 6855 | consumed samples: 59520 | elapsed time per iteration (ms): 88308.3 | throughput per GPU (TFLOP/s/GPU): 43.2 | MFU 13.83% | FLOPs estimator: workload-aware | logical tokens/s: 189984.3 | valid tokens/s: 189984.3 | physical tokens/s: 190436.5 | loss tokens/s: 139720.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.080615E-01 | lm-global-supervised-tokens: 1.233845E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.404174E-02 | lm-token-buffer-mib: 5.812897E+01 | global_load_balancing_loss: 1.001398E+00 | loss scale: 1.0 | grad norm: 0.159 | params norm: 11636.607 | params rms: 0.063008 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 12:44:50.338769 | finish at 2026-10-10 16:11:06 + [2026-10-04 03:27:48] iteration 466/ 6855 | consumed samples: 59648 | elapsed time per iteration (ms): 92988.5 | throughput per GPU (TFLOP/s/GPU): 41.0 | MFU 13.14% | FLOPs estimator: workload-aware | logical tokens/s: 180421.9 | valid tokens/s: 180421.9 | physical tokens/s: 181018.3 | loss tokens/s: 135156.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.895348E-01 | lm-global-supervised-tokens: 1.256798E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.209588E-02 | lm-token-buffer-mib: 5.810880E+01 | global_load_balancing_loss: 1.000920E+00 | loss scale: 1.0 | grad norm: 0.089 | params norm: 11636.516 | params rms: 0.063008 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 21:01:43.649116 | finish at 2026-10-11 00:29:32 + [2026-10-04 03:29:14] iteration 467/ 6855 | consumed samples: 59776 | elapsed time per iteration (ms): 85806.0 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.53% | FLOPs estimator: workload-aware | logical tokens/s: 195524.6 | valid tokens/s: 195524.6 | physical tokens/s: 196101.7 | loss tokens/s: 145904.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.901319E-01 | lm-global-supervised-tokens: 1.251943E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.463781E-02 | lm-token-buffer-mib: 5.810006E+01 | global_load_balancing_loss: 1.000539E+00 | loss scale: 1.0 | grad norm: 0.124 | params norm: 11636.424 | params rms: 0.063007 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 8:15:28.665520 | finish at 2026-10-10 11:44:43 + [2026-10-04 03:30:45] iteration 468/ 6855 | consumed samples: 59904 | elapsed time per iteration (ms): 90999.7 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.53% | FLOPs estimator: workload-aware | logical tokens/s: 184365.2 | valid tokens/s: 184365.2 | physical tokens/s: 184565.4 | loss tokens/s: 128836.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.811702E-01 | lm-global-supervised-tokens: 1.172404E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.906532E-02 | lm-token-buffer-mib: 5.805460E+01 | global_load_balancing_loss: 1.000570E+00 | loss scale: 1.0 | grad norm: 0.036 | params norm: 11636.333 | params rms: 0.063007 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 17:26:54.900087 | finish at 2026-10-10 20:57:40 + [2026-10-04 03:32:14] iteration 469/ 6855 | consumed samples: 60032 | elapsed time per iteration (ms): 88364.8 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.53% | FLOPs estimator: workload-aware | logical tokens/s: 189862.3 | valid tokens/s: 189862.3 | physical tokens/s: 190365.4 | loss tokens/s: 145027.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.959336E-01 | lm-global-supervised-tokens: 1.281530E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.143759E-02 | lm-token-buffer-mib: 5.805302E+01 | global_load_balancing_loss: 9.996770E-01 | loss scale: 1.0 | grad norm: 0.092 | params norm: 11636.241 | params rms: 0.063006 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 12:44:57.510639 | finish at 2026-10-10 16:17:11 + [2026-10-04 03:33:59] iteration 470/ 6855 | consumed samples: 60160 | elapsed time per iteration (ms): 105579.8 | throughput per GPU (TFLOP/s/GPU): 34.5 | MFU 11.04% | FLOPs estimator: workload-aware | logical tokens/s: 158905.2 | valid tokens/s: 158905.2 | physical tokens/s: 159422.0 | loss tokens/s: 116739.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.757599E-01 | lm-global-supervised-tokens: 1.232528E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.128997E-02 | lm-token-buffer-mib: 5.807281E+01 | global_load_balancing_loss: 9.992602E-01 | loss scale: 1.0 | grad norm: 0.115 | params norm: 11636.149 | params rms: 0.063006 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 7 days, 19:15:27.071015 | finish at 2026-10-11 22:49:26 + [2026-10-04 03:35:28] iteration 471/ 6855 | consumed samples: 60288 | elapsed time per iteration (ms): 89057.9 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.38% | FLOPs estimator: workload-aware | logical tokens/s: 188385.5 | valid tokens/s: 188385.5 | physical tokens/s: 188816.8 | loss tokens/s: 138966.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.940756E-01 | lm-global-supervised-tokens: 1.237605E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.240244E-02 | lm-token-buffer-mib: 5.810857E+01 | global_load_balancing_loss: 9.995541E-01 | loss scale: 1.0 | grad norm: 0.053 | params norm: 11636.058 | params rms: 0.063005 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 13:55:45.386719 | finish at 2026-10-10 17:31:14 + [2026-10-04 03:37:00] iteration 472/ 6855 | consumed samples: 60416 | elapsed time per iteration (ms): 91352.2 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.40% | FLOPs estimator: workload-aware | logical tokens/s: 183653.5 | valid tokens/s: 183653.5 | physical tokens/s: 183958.8 | loss tokens/s: 135242.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.928109E-01 | lm-global-supervised-tokens: 1.235467E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.338277E-02 | lm-token-buffer-mib: 5.811519E+01 | global_load_balancing_loss: 1.000149E+00 | loss scale: 1.0 | grad norm: 0.082 | params norm: 11635.966 | params rms: 0.063005 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 17:58:21.127802 | finish at 2026-10-10 21:35:21 + [2026-10-04 03:39:17] iteration 473/ 6855 | consumed samples: 60544 | elapsed time per iteration (ms): 137591.5 | throughput per GPU (TFLOP/s/GPU): 27.1 | MFU 8.69% | FLOPs estimator: workload-aware | logical tokens/s: 120991.8 | valid tokens/s: 120991.8 | physical tokens/s: 121217.9 | loss tokens/s: 88534.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.742253E-01 | lm-global-supervised-tokens: 1.218165E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.244071E-02 | lm-token-buffer-mib: 5.807024E+01 | global_load_balancing_loss: 1.000412E+00 | loss scale: 1.0 | grad norm: 0.036 | params norm: 11635.877 | params rms: 0.063004 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 10 days, 3:55:09.011100 | finish at 2026-10-14 07:34:26 + [2026-10-04 03:40:41] iteration 474/ 6855 | consumed samples: 60672 | elapsed time per iteration (ms): 84313.0 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.74% | FLOPs estimator: workload-aware | logical tokens/s: 198987.1 | valid tokens/s: 198987.1 | physical tokens/s: 199366.0 | loss tokens/s: 150945.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.782060E-01 | lm-global-supervised-tokens: 1.272666E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.794672E-02 | lm-token-buffer-mib: 5.803860E+01 | global_load_balancing_loss: 9.998229E-01 | loss scale: 1.0 | grad norm: 0.091 | params norm: 11635.789 | params rms: 0.063004 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 5:26:41.071728 | finish at 2026-10-10 09:07:23 + [2026-10-04 03:42:09] iteration 475/ 6855 | consumed samples: 60800 | elapsed time per iteration (ms): 87746.1 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.67% | FLOPs estimator: workload-aware | logical tokens/s: 191201.6 | valid tokens/s: 191201.6 | physical tokens/s: 191754.2 | loss tokens/s: 140558.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.291908E-01 | lm-global-supervised-tokens: 1.233350E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.368595E-02 | lm-token-buffer-mib: 5.824878E+01 | global_load_balancing_loss: 1.000526E+00 | loss scale: 1.0 | grad norm: 0.116 | params norm: 11635.700 | params rms: 0.063004 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 11:30:20.023365 | finish at 2026-10-10 15:12:29 + [2026-10-04 03:43:37] iteration 476/ 6855 | consumed samples: 60928 | elapsed time per iteration (ms): 88055.9 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.44% | FLOPs estimator: workload-aware | logical tokens/s: 190528.9 | valid tokens/s: 190528.9 | physical tokens/s: 190880.4 | loss tokens/s: 138842.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.799819E-01 | lm-global-supervised-tokens: 1.222593E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.149643E-02 | lm-token-buffer-mib: 5.806765E+01 | global_load_balancing_loss: 9.989014E-01 | loss scale: 1.0 | grad norm: 0.043 | params norm: 11635.613 | params rms: 0.063003 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 12:01:48.615615 | finish at 2026-10-10 15:45:26 + [2026-10-04 03:45:02] iteration 477/ 6855 | consumed samples: 61056 | elapsed time per iteration (ms): 84380.4 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.79% | FLOPs estimator: workload-aware | logical tokens/s: 198828.1 | valid tokens/s: 198828.1 | physical tokens/s: 199133.7 | loss tokens/s: 146554.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.592811E-01 | lm-global-supervised-tokens: 1.236629E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.187855E-02 | lm-token-buffer-mib: 5.815116E+01 | global_load_balancing_loss: 1.000682E+00 | loss scale: 1.0 | grad norm: 0.053 | params norm: 11635.528 | params rms: 0.063003 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 5:29:37.977944 | finish at 2026-10-10 09:14:40 + [2026-10-04 03:46:28] iteration 478/ 6855 | consumed samples: 61184 | elapsed time per iteration (ms): 86009.5 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.59% | FLOPs estimator: workload-aware | logical tokens/s: 195061.1 | valid tokens/s: 195061.1 | physical tokens/s: 195481.2 | loss tokens/s: 142302.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.525428E-01 | lm-global-supervised-tokens: 1.223938E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.424674E-02 | lm-token-buffer-mib: 5.815224E+01 | global_load_balancing_loss: 1.000233E+00 | loss scale: 1.0 | grad norm: 0.056 | params norm: 11635.441 | params rms: 0.063002 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 8:21:22.847739 | finish at 2026-10-10 12:07:50 + [2026-10-04 03:47:56] iteration 479/ 6855 | consumed samples: 61312 | elapsed time per iteration (ms): 88581.5 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.59% | FLOPs estimator: workload-aware | logical tokens/s: 189398.2 | valid tokens/s: 189398.2 | physical tokens/s: 189775.6 | loss tokens/s: 141805.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.737232E-01 | lm-global-supervised-tokens: 1.256139E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.160626E-02 | lm-token-buffer-mib: 5.820949E+01 | global_load_balancing_loss: 1.001829E+00 | loss scale: 1.0 | grad norm: 0.045 | params norm: 11635.351 | params rms: 0.063002 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 12:53:15.811558 | finish at 2026-10-10 16:41:12 + [2026-10-04 03:49:43] iteration 480/ 6855 | consumed samples: 61440 | elapsed time per iteration (ms): 106739.8 | throughput per GPU (TFLOP/s/GPU): 34.8 | MFU 11.15% | FLOPs estimator: workload-aware | logical tokens/s: 157178.6 | valid tokens/s: 157178.6 | physical tokens/s: 157562.8 | loss tokens/s: 115397.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.290148E-01 | lm-global-supervised-tokens: 1.231746E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.627124E-02 | lm-token-buffer-mib: 5.819514E+01 | global_load_balancing_loss: 1.001325E+00 | loss scale: 1.0 | grad norm: 0.038 | params norm: 11635.263 | params rms: 0.063001 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 7 days, 21:01:06.237009 | finish at 2026-10-12 00:50:49 +times across ranks (ms): + save-checkpoint: + rank 0: 23707.65 + rank 1: 23707.65 + rank 2: 23707.66 + rank 3: 23707.69 + rank 4: 23707.70 + rank 5: 23707.65 + rank 6: 23707.67 + rank 7: 23707.65 + rank 8: 23707.12 + rank 9: 23707.11 + rank 10: 23707.10 + rank 11: 23707.10 + rank 12: 23707.11 + rank 13: 23707.11 + rank 14: 23707.12 + rank 15: 23707.10 + rank 16: 23706.84 + rank 17: 23706.85 + rank 18: 23706.83 + rank 19: 23706.83 + rank 20: 23706.81 + rank 21: 23706.84 + rank 22: 23706.82 + rank 23: 23706.82 + rank 24: 23706.20 + rank 25: 23706.19 + rank 26: 23706.21 + rank 27: 23706.22 + rank 28: 23706.18 + rank 29: 23706.18 + rank 30: 23706.18 + rank 31: 23706.17 + rank 32: 23707.11 + rank 33: 23707.13 + rank 34: 23707.12 + rank 35: 23707.08 + rank 36: 23707.09 + rank 37: 23707.07 + rank 38: 23707.16 + rank 39: 23707.07 + rank 40: 23707.01 + rank 41: 23706.99 + rank 42: 23707.00 + rank 43: 23706.98 + rank 44: 23706.95 + rank 45: 23706.96 + rank 46: 23706.98 + rank 47: 23706.95 + rank 48: 23707.11 + rank 49: 23707.19 + rank 50: 23707.09 + rank 51: 23707.10 + rank 52: 23707.13 + rank 53: 23707.09 + rank 54: 23707.10 + rank 55: 23707.08 + rank 56: 23706.45 + rank 57: 23706.47 + rank 58: 23706.42 + rank 59: 23706.42 + rank 60: 23706.41 + rank 61: 23706.43 + rank 62: 23706.42 + rank 63: 23706.43 + [2026-10-04 03:51:36] iteration 481/ 6855 | consumed samples: 61568 | elapsed time per iteration (ms): 88560.9 | throughput per GPU (TFLOP/s/GPU): 41.4 | MFU 13.27% | FLOPs estimator: workload-aware | logical tokens/s: 189442.3 | valid tokens/s: 189442.3 | physical tokens/s: 189927.3 | loss tokens/s: 137807.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.843721E-01 | lm-global-supervised-tokens: 1.220440E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 8.652645E-01 | lm-token-buffer-mib: 5.819507E+01 | global_load_balancing_loss: 1.001113E+00 | loss scale: 1.0 | grad norm: 0.039 | params norm: 11635.175 | params rms: 0.063001 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 12:48:07.439332 | finish at 2026-10-10 16:39:43 + [2026-10-04 03:53:09] iteration 482/ 6855 | consumed samples: 61696 | elapsed time per iteration (ms): 93297.4 | throughput per GPU (TFLOP/s/GPU): 39.5 | MFU 12.66% | FLOPs estimator: workload-aware | logical tokens/s: 179824.4 | valid tokens/s: 179824.4 | physical tokens/s: 180136.6 | loss tokens/s: 127488.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.847087E-01 | lm-global-supervised-tokens: 1.189431E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.461034E-02 | lm-token-buffer-mib: 5.805225E+01 | global_load_balancing_loss: 9.997826E-01 | loss scale: 1.0 | grad norm: 0.046 | params norm: 11635.086 | params rms: 0.063000 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 21:09:44.278524 | finish at 2026-10-11 01:02:53 + [2026-10-04 03:54:34] iteration 483/ 6855 | consumed samples: 61824 | elapsed time per iteration (ms): 85514.3 | throughput per GPU (TFLOP/s/GPU): 43.3 | MFU 13.87% | FLOPs estimator: workload-aware | logical tokens/s: 196192.0 | valid tokens/s: 196192.0 | physical tokens/s: 196581.4 | loss tokens/s: 143565.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.951612E-01 | lm-global-supervised-tokens: 1.227688E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.988223E-02 | lm-token-buffer-mib: 5.810630E+01 | global_load_balancing_loss: 9.992307E-01 | loss scale: 1.0 | grad norm: 0.060 | params norm: 11634.995 | params rms: 0.063000 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 7:21:36.947099 | finish at 2026-10-10 11:16:11 + [2026-10-04 03:56:04] iteration 484/ 6855 | consumed samples: 61952 | elapsed time per iteration (ms): 89651.4 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.44% | FLOPs estimator: workload-aware | logical tokens/s: 187137.7 | valid tokens/s: 187137.7 | physical tokens/s: 187490.0 | loss tokens/s: 138507.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.114794E-01 | lm-global-supervised-tokens: 1.241736E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.139874E-02 | lm-token-buffer-mib: 5.814391E+01 | global_load_balancing_loss: 9.975948E-01 | loss scale: 1.0 | grad norm: 0.058 | params norm: 11634.905 | params rms: 0.062999 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 14:39:28.916553 | finish at 2026-10-10 18:35:33 + [2026-10-04 03:57:29] iteration 485/ 6855 | consumed samples: 62080 | elapsed time per iteration (ms): 84783.6 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.58% | FLOPs estimator: workload-aware | logical tokens/s: 197882.4 | valid tokens/s: 197882.4 | physical tokens/s: 198565.2 | loss tokens/s: 149156.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.410469E-01 | lm-global-supervised-tokens: 1.264605E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.235063E-02 | lm-token-buffer-mib: 5.826369E+01 | global_load_balancing_loss: 9.997532E-01 | loss scale: 1.0 | grad norm: 0.058 | params norm: 11634.816 | params rms: 0.062999 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 6:01:11.514361 | finish at 2026-10-10 09:58:40 + [2026-10-04 03:58:58] iteration 486/ 6855 | consumed samples: 62208 | elapsed time per iteration (ms): 89192.2 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.43% | FLOPs estimator: workload-aware | logical tokens/s: 188101.3 | valid tokens/s: 188101.3 | physical tokens/s: 188526.9 | loss tokens/s: 140233.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.953898E-01 | lm-global-supervised-tokens: 1.250770E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.183107E-02 | lm-token-buffer-mib: 5.816415E+01 | global_load_balancing_loss: 9.993647E-01 | loss scale: 1.0 | grad norm: 0.093 | params norm: 11634.728 | params rms: 0.062998 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 13:47:45.211043 | finish at 2026-10-10 17:46:43 + [2026-10-04 04:00:23] iteration 487/ 6855 | consumed samples: 62336 | elapsed time per iteration (ms): 85305.8 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.75% | FLOPs estimator: workload-aware | logical tokens/s: 196671.4 | valid tokens/s: 196671.4 | physical tokens/s: 197119.4 | loss tokens/s: 144399.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.472853E-01 | lm-global-supervised-tokens: 1.231807E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.327079E-02 | lm-token-buffer-mib: 5.807829E+01 | global_load_balancing_loss: 1.001035E+00 | loss scale: 1.0 | grad norm: 0.320 | params norm: 11634.640 | params rms: 0.062998 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 6:53:47.135262 | finish at 2026-10-10 10:54:10 + [2026-10-04 04:01:52] iteration 488/ 6855 | consumed samples: 62464 | elapsed time per iteration (ms): 88588.0 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.57% | FLOPs estimator: workload-aware | logical tokens/s: 189384.5 | valid tokens/s: 189384.5 | physical tokens/s: 189722.4 | loss tokens/s: 141950.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.533707E-01 | lm-global-supervised-tokens: 1.257512E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.277487E-02 | lm-token-buffer-mib: 5.817104E+01 | global_load_balancing_loss: 1.001645E+00 | loss scale: 1.0 | grad norm: 0.360 | params norm: 11634.549 | params rms: 0.062997 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 12:40:40.001308 | finish at 2026-10-10 16:42:32 + [2026-10-04 04:03:23] iteration 489/ 6855 | consumed samples: 62592 | elapsed time per iteration (ms): 90608.3 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.54% | FLOPs estimator: workload-aware | logical tokens/s: 185161.6 | valid tokens/s: 185161.6 | physical tokens/s: 185449.4 | loss tokens/s: 129272.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.480040E-01 | lm-global-supervised-tokens: 1.171318E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.190195E-02 | lm-token-buffer-mib: 5.812341E+01 | global_load_balancing_loss: 1.002145E+00 | loss scale: 1.0 | grad norm: 0.091 | params norm: 11634.458 | params rms: 0.062997 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 16:13:32.352618 | finish at 2026-10-10 20:16:55 + [2026-10-04 04:05:10] iteration 490/ 6855 | consumed samples: 62720 | elapsed time per iteration (ms): 107495.6 | throughput per GPU (TFLOP/s/GPU): 34.3 | MFU 11.00% | FLOPs estimator: workload-aware | logical tokens/s: 156073.3 | valid tokens/s: 156073.3 | physical tokens/s: 156303.0 | loss tokens/s: 113372.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.071304E-01 | lm-global-supervised-tokens: 1.218705E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.132026E-02 | lm-token-buffer-mib: 5.806986E+01 | global_load_balancing_loss: 1.002381E+00 | loss scale: 1.0 | grad norm: 0.212 | params norm: 11634.367 | params rms: 0.062996 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 7 days, 22:03:29.632001 | finish at 2026-10-12 02:08:40 + [2026-10-04 04:06:33] iteration 491/ 6855 | consumed samples: 62848 | elapsed time per iteration (ms): 83129.9 | throughput per GPU (TFLOP/s/GPU): 43.4 | MFU 13.91% | FLOPs estimator: workload-aware | logical tokens/s: 201818.7 | valid tokens/s: 201818.7 | physical tokens/s: 202477.0 | loss tokens/s: 152941.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.921067E-01 | lm-global-supervised-tokens: 1.271398E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.365652E-02 | lm-token-buffer-mib: 5.816381E+01 | global_load_balancing_loss: 9.995549E-01 | loss scale: 1.0 | grad norm: 0.390 | params norm: 11634.279 | params rms: 0.062996 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 2:57:18.924714 | finish at 2026-10-10 07:03:52 + [2026-10-04 04:08:00] iteration 492/ 6855 | consumed samples: 62976 | elapsed time per iteration (ms): 86804.2 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.56% | FLOPs estimator: workload-aware | logical tokens/s: 193275.6 | valid tokens/s: 193275.6 | physical tokens/s: 193659.8 | loss tokens/s: 146293.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.644892E-01 | lm-global-supervised-tokens: 1.269886E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.140584E-02 | lm-token-buffer-mib: 5.820716E+01 | global_load_balancing_loss: 9.996002E-01 | loss scale: 1.0 | grad norm: 0.239 | params norm: 11634.188 | params rms: 0.062995 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 9:25:35.071083 | finish at 2026-10-10 13:33:35 + [2026-10-04 04:09:25] iteration 493/ 6855 | consumed samples: 63104 | elapsed time per iteration (ms): 85422.6 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.78% | FLOPs estimator: workload-aware | logical tokens/s: 196402.2 | valid tokens/s: 196402.2 | physical tokens/s: 196996.5 | loss tokens/s: 141692.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.471962E-01 | lm-global-supervised-tokens: 1.210372E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.166816E-02 | lm-token-buffer-mib: 5.827034E+01 | global_load_balancing_loss: 9.983046E-01 | loss scale: 1.0 | grad norm: 0.511 | params norm: 11634.098 | params rms: 0.062995 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 6:57:38.560161 | finish at 2026-10-10 11:07:04 + [2026-10-04 04:10:52] iteration 494/ 6855 | consumed samples: 63232 | elapsed time per iteration (ms): 86300.0 | throughput per GPU (TFLOP/s/GPU): 43.8 | MFU 14.04% | FLOPs estimator: workload-aware | logical tokens/s: 194405.0 | valid tokens/s: 194405.0 | physical tokens/s: 194735.7 | loss tokens/s: 142004.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.977478E-01 | lm-global-supervised-tokens: 1.225498E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.349813E-02 | lm-token-buffer-mib: 5.811964E+01 | global_load_balancing_loss: 9.985089E-01 | loss scale: 1.0 | grad norm: 0.067 | params norm: 11634.009 | params rms: 0.062994 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 8:29:14.577231 | finish at 2026-10-10 12:40:06 + [2026-10-04 04:12:21] iteration 495/ 6855 | consumed samples: 63360 | elapsed time per iteration (ms): 89362.2 | throughput per GPU (TFLOP/s/GPU): 40.3 | MFU 12.91% | FLOPs estimator: workload-aware | logical tokens/s: 187743.9 | valid tokens/s: 187743.9 | physical tokens/s: 188149.2 | loss tokens/s: 141213.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.032850E-01 | lm-global-supervised-tokens: 1.261912E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.228690E-02 | lm-token-buffer-mib: 5.807464E+01 | global_load_balancing_loss: 9.994001E-01 | loss scale: 1.0 | grad norm: 0.285 | params norm: 11633.921 | params rms: 0.062994 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 13:52:23.557262 | finish at 2026-10-10 18:04:45 + [2026-10-04 04:13:48] iteration 496/ 6855 | consumed samples: 63488 | elapsed time per iteration (ms): 86866.3 | throughput per GPU (TFLOP/s/GPU): 41.6 | MFU 13.34% | FLOPs estimator: workload-aware | logical tokens/s: 193136.9 | valid tokens/s: 193136.9 | physical tokens/s: 193691.3 | loss tokens/s: 152044.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.658963E-01 | lm-global-supervised-tokens: 1.320757E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.362897E-02 | lm-token-buffer-mib: 5.814185E+01 | global_load_balancing_loss: 1.000064E+00 | loss scale: 1.0 | grad norm: 0.439 | params norm: 11633.835 | params rms: 0.062993 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 9:26:22.917598 | finish at 2026-10-10 13:40:11 + [2026-10-04 04:15:13] iteration 497/ 6855 | consumed samples: 63616 | elapsed time per iteration (ms): 85252.0 | throughput per GPU (TFLOP/s/GPU): 41.6 | MFU 13.33% | FLOPs estimator: workload-aware | logical tokens/s: 196795.0 | valid tokens/s: 196795.0 | physical tokens/s: 197341.1 | loss tokens/s: 146115.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.543977E-01 | lm-global-supervised-tokens: 1.245666E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.841009E-02 | lm-token-buffer-mib: 5.811515E+01 | global_load_balancing_loss: 1.002112E+00 | loss scale: 1.0 | grad norm: 0.339 | params norm: 11633.748 | params rms: 0.062993 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 6:33:52.428815 | finish at 2026-10-10 10:49:06 + [2026-10-04 04:16:40] iteration 498/ 6855 | consumed samples: 63744 | elapsed time per iteration (ms): 86948.2 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.61% | FLOPs estimator: workload-aware | logical tokens/s: 192956.1 | valid tokens/s: 192956.0 | physical tokens/s: 193308.2 | loss tokens/s: 143810.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.755721E-01 | lm-global-supervised-tokens: 1.250402E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.500264E-02 | lm-token-buffer-mib: 5.808813E+01 | global_load_balancing_loss: 1.003766E+00 | loss scale: 1.0 | grad norm: 0.116 | params norm: 11633.663 | params rms: 0.062992 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 9:32:09.840695 | finish at 2026-10-10 13:48:50 + [2026-10-04 04:18:04] iteration 499/ 6855 | consumed samples: 63872 | elapsed time per iteration (ms): 84276.0 | throughput per GPU (TFLOP/s/GPU): 43.6 | MFU 13.99% | FLOPs estimator: workload-aware | logical tokens/s: 199073.8 | valid tokens/s: 199073.8 | physical tokens/s: 199571.4 | loss tokens/s: 149065.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.927644E-01 | lm-global-supervised-tokens: 1.256260E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.416265E-02 | lm-token-buffer-mib: 5.811802E+01 | global_load_balancing_loss: 1.000674E+00 | loss scale: 1.0 | grad norm: 0.165 | params norm: 11633.578 | params rms: 0.062992 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 4:47:37.939429 | finish at 2026-10-10 09:05:42 + [2026-10-04 04:19:55] iteration 500/ 6855 | consumed samples: 64000 | elapsed time per iteration (ms): 110582.7 | throughput per GPU (TFLOP/s/GPU): 34.4 | MFU 11.04% | FLOPs estimator: workload-aware | logical tokens/s: 151715.9 | valid tokens/s: 151715.9 | physical tokens/s: 152086.4 | loss tokens/s: 114310.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.393095E-01 | lm-global-supervised-tokens: 1.264073E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.136568E-02 | lm-token-buffer-mib: 5.826765E+01 | global_load_balancing_loss: 9.988142E-01 | loss scale: 1.0 | grad norm: 0.228 | params norm: 11633.492 | params rms: 0.062992 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 8 days, 3:12:33.201014 | finish at 2026-10-12 07:32:28 + [2026-10-04 04:21:22] iteration 501/ 6855 | consumed samples: 64128 | elapsed time per iteration (ms): 87040.8 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.57% | FLOPs estimator: workload-aware | logical tokens/s: 192750.8 | valid tokens/s: 192750.7 | physical tokens/s: 193191.7 | loss tokens/s: 143387.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.596235E-01 | lm-global-supervised-tokens: 1.248053E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.426484E-02 | lm-token-buffer-mib: 5.813274E+01 | global_load_balancing_loss: 9.979195E-01 | loss scale: 1.0 | grad norm: 0.080 | params norm: 11633.405 | params rms: 0.062991 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 9:37:37.034743 | finish at 2026-10-10 13:58:59 + [2026-10-04 04:22:49] iteration 502/ 6855 | consumed samples: 64256 | elapsed time per iteration (ms): 86656.1 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.39% | FLOPs estimator: workload-aware | logical tokens/s: 193606.2 | valid tokens/s: 193606.2 | physical tokens/s: 194133.6 | loss tokens/s: 144457.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.450678E-01 | lm-global-supervised-tokens: 1.251811E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.159227E-02 | lm-token-buffer-mib: 5.837125E+01 | global_load_balancing_loss: 9.995621E-01 | loss scale: 1.0 | grad norm: 0.077 | params norm: 11633.319 | params rms: 0.062991 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 8:55:26.356503 | finish at 2026-10-10 13:18:15 + [2026-10-04 04:24:14] iteration 503/ 6855 | consumed samples: 64384 | elapsed time per iteration (ms): 84811.4 | throughput per GPU (TFLOP/s/GPU): 43.3 | MFU 13.87% | FLOPs estimator: workload-aware | logical tokens/s: 197817.3 | valid tokens/s: 197817.3 | physical tokens/s: 198216.5 | loss tokens/s: 152231.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.396140E-01 | lm-global-supervised-tokens: 1.291096E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.205806E-02 | lm-token-buffer-mib: 5.811034E+01 | global_load_balancing_loss: 9.995574E-01 | loss scale: 1.0 | grad norm: 0.154 | params norm: 11633.235 | params rms: 0.062990 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 5:38:42.153240 | finish at 2026-10-10 10:02:56 + [2026-10-04 04:25:42] iteration 504/ 6855 | consumed samples: 64512 | elapsed time per iteration (ms): 88903.9 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.73% | FLOPs estimator: workload-aware | logical tokens/s: 188711.1 | valid tokens/s: 188711.1 | physical tokens/s: 189057.6 | loss tokens/s: 137680.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.624561E-01 | lm-global-supervised-tokens: 1.224037E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.192403E-02 | lm-token-buffer-mib: 5.821310E+01 | global_load_balancing_loss: 1.000167E+00 | loss scale: 1.0 | grad norm: 0.136 | params norm: 11633.149 | params rms: 0.062990 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 12:50:28.971155 | finish at 2026-10-10 17:16:11 + [2026-10-04 04:27:08] iteration 505/ 6855 | consumed samples: 64640 | elapsed time per iteration (ms): 85712.5 | throughput per GPU (TFLOP/s/GPU): 43.6 | MFU 13.99% | FLOPs estimator: workload-aware | logical tokens/s: 195738.0 | valid tokens/s: 195738.0 | physical tokens/s: 196325.4 | loss tokens/s: 140284.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.921535E-01 | lm-global-supervised-tokens: 1.202416E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.482725E-02 | lm-token-buffer-mib: 5.805428E+01 | global_load_balancing_loss: 1.000841E+00 | loss scale: 1.0 | grad norm: 0.051 | params norm: 11633.063 | params rms: 0.062989 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 7:11:14.248433 | finish at 2026-10-10 11:38:22 + [2026-10-04 04:28:40] iteration 506/ 6855 | consumed samples: 64768 | elapsed time per iteration (ms): 91541.9 | throughput per GPU (TFLOP/s/GPU): 40.0 | MFU 12.81% | FLOPs estimator: workload-aware | logical tokens/s: 183273.2 | valid tokens/s: 183273.2 | physical tokens/s: 183826.1 | loss tokens/s: 134188.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.645821E-01 | lm-global-supervised-tokens: 1.228388E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.282497E-02 | lm-token-buffer-mib: 5.811263E+01 | global_load_balancing_loss: 1.001552E+00 | loss scale: 1.0 | grad norm: 0.254 | params norm: 11632.976 | params rms: 0.062989 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 17:26:39.469609 | finish at 2026-10-10 21:55:19 + [2026-10-04 04:30:05] iteration 507/ 6855 | consumed samples: 64896 | elapsed time per iteration (ms): 85101.2 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.55% | FLOPs estimator: workload-aware | logical tokens/s: 197143.9 | valid tokens/s: 197143.8 | physical tokens/s: 197670.1 | loss tokens/s: 145741.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.723603E-01 | lm-global-supervised-tokens: 1.240274E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.313256E-02 | lm-token-buffer-mib: 5.813548E+01 | global_load_balancing_loss: 1.001179E+00 | loss scale: 1.0 | grad norm: 0.057 | params norm: 11632.891 | params rms: 0.062988 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 6:03:42.599876 | finish at 2026-10-10 10:33:47 + [2026-10-04 04:31:32] iteration 508/ 6855 | consumed samples: 65024 | elapsed time per iteration (ms): 87196.6 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.76% | FLOPs estimator: workload-aware | logical tokens/s: 192406.5 | valid tokens/s: 192406.5 | physical tokens/s: 192640.1 | loss tokens/s: 136500.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.542279E-01 | lm-global-supervised-tokens: 1.190238E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.475919E-02 | lm-token-buffer-mib: 5.805917E+01 | global_load_balancing_loss: 1.001109E+00 | loss scale: 1.0 | grad norm: 0.056 | params norm: 11632.805 | params rms: 0.062988 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 9:43:56.639872 | finish at 2026-10-10 14:15:29 + [2026-10-04 04:32:59] iteration 509/ 6855 | consumed samples: 65152 | elapsed time per iteration (ms): 86909.5 | throughput per GPU (TFLOP/s/GPU): 43.1 | MFU 13.81% | FLOPs estimator: workload-aware | logical tokens/s: 193041.8 | valid tokens/s: 193041.8 | physical tokens/s: 193550.2 | loss tokens/s: 138992.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.473799E-01 | lm-global-supervised-tokens: 1.207977E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.192701E-02 | lm-token-buffer-mib: 5.821142E+01 | global_load_balancing_loss: 1.000025E+00 | loss scale: 1.0 | grad norm: 0.247 | params norm: 11632.715 | params rms: 0.062987 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 9:12:07.899595 | finish at 2026-10-10 13:45:07 + [2026-10-04 04:34:47] iteration 510/ 6855 | consumed samples: 65280 | elapsed time per iteration (ms): 108275.1 | throughput per GPU (TFLOP/s/GPU): 34.0 | MFU 10.90% | FLOPs estimator: workload-aware | logical tokens/s: 154949.6 | valid tokens/s: 154949.6 | physical tokens/s: 155474.0 | loss tokens/s: 116968.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.775098E-01 | lm-global-supervised-tokens: 1.266476E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.145347E-02 | lm-token-buffer-mib: 5.836431E+01 | global_load_balancing_loss: 9.993067E-01 | loss scale: 1.0 | grad norm: 0.052 | params norm: 11632.626 | params rms: 0.062987 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 7 days, 22:50:05.802931 | finish at 2026-10-12 03:24:53 + [2026-10-04 04:36:16] iteration 511/ 6855 | consumed samples: 65408 | elapsed time per iteration (ms): 88802.3 | throughput per GPU (TFLOP/s/GPU): 41.2 | MFU 13.20% | FLOPs estimator: workload-aware | logical tokens/s: 188926.8 | valid tokens/s: 188926.8 | physical tokens/s: 189412.5 | loss tokens/s: 147430.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.342657E-01 | lm-global-supervised-tokens: 1.309213E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.310932E-02 | lm-token-buffer-mib: 5.815282E+01 | global_load_balancing_loss: 9.979681E-01 | loss scale: 1.0 | grad norm: 0.152 | params norm: 11632.538 | params rms: 0.062986 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 12:29:21.533920 | finish at 2026-10-10 17:05:37 + [2026-10-04 04:37:44] iteration 512/ 6855 | consumed samples: 65536 | elapsed time per iteration (ms): 87762.2 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.41% | FLOPs estimator: workload-aware | logical tokens/s: 191165.6 | valid tokens/s: 191165.6 | physical tokens/s: 191486.1 | loss tokens/s: 143917.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.219653E-01 | lm-global-supervised-tokens: 1.263055E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.335713E-02 | lm-token-buffer-mib: 5.808667E+01 | global_load_balancing_loss: 9.995526E-01 | loss scale: 1.0 | grad norm: 0.114 | params norm: 11632.450 | params rms: 0.062986 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 10:37:55.759351 | finish at 2026-10-10 15:15:39 + [2026-10-04 04:39:12] iteration 513/ 6855 | consumed samples: 65664 | elapsed time per iteration (ms): 87997.1 | throughput per GPU (TFLOP/s/GPU): 41.5 | MFU 13.31% | FLOPs estimator: workload-aware | logical tokens/s: 190656.5 | valid tokens/s: 190656.5 | physical tokens/s: 191099.8 | loss tokens/s: 139763.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.112383E-01 | lm-global-supervised-tokens: 1.229882E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.297694E-02 | lm-token-buffer-mib: 5.817671E+01 | global_load_balancing_loss: 1.000712E+00 | loss scale: 1.0 | grad norm: 0.044 | params norm: 11632.363 | params rms: 0.062985 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 11:01:17.415386 | finish at 2026-10-10 15:40:29 + [2026-10-04 04:40:38] iteration 514/ 6855 | consumed samples: 65792 | elapsed time per iteration (ms): 86534.9 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.48% | FLOPs estimator: workload-aware | logical tokens/s: 193877.7 | valid tokens/s: 193877.7 | physical tokens/s: 194351.6 | loss tokens/s: 144069.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.919062E-01 | lm-global-supervised-tokens: 1.246706E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.411398E-02 | lm-token-buffer-mib: 5.835344E+01 | global_load_balancing_loss: 1.000945E+00 | loss scale: 1.0 | grad norm: 0.070 | params norm: 11632.275 | params rms: 0.062985 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 8:25:17.528457 | finish at 2026-10-10 13:05:56 + [2026-10-04 04:42:04] iteration 515/ 6855 | consumed samples: 65920 | elapsed time per iteration (ms): 85942.7 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.59% | FLOPs estimator: workload-aware | logical tokens/s: 195213.5 | valid tokens/s: 195213.5 | physical tokens/s: 195599.3 | loss tokens/s: 144173.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.508173E-01 | lm-global-supervised-tokens: 1.239064E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.355450E-02 | lm-token-buffer-mib: 5.818221E+01 | global_load_balancing_loss: 9.998249E-01 | loss scale: 1.0 | grad norm: 0.163 | params norm: 11632.186 | params rms: 0.062985 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 7:21:16.916952 | finish at 2026-10-10 12:03:21 + [2026-10-04 04:43:29] iteration 516/ 6855 | consumed samples: 66048 | elapsed time per iteration (ms): 85278.0 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.75% | FLOPs estimator: workload-aware | logical tokens/s: 196735.4 | valid tokens/s: 196735.3 | physical tokens/s: 197223.1 | loss tokens/s: 149300.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.598682E-01 | lm-global-supervised-tokens: 1.273205E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.172018E-02 | lm-token-buffer-mib: 5.817747E+01 | global_load_balancing_loss: 9.992108E-01 | loss scale: 1.0 | grad norm: 0.057 | params norm: 11632.098 | params rms: 0.062984 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 6:09:36.938959 | finish at 2026-10-10 10:53:06 + [2026-10-04 04:44:57] iteration 517/ 6855 | consumed samples: 66176 | elapsed time per iteration (ms): 87092.8 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.65% | FLOPs estimator: workload-aware | logical tokens/s: 192635.8 | valid tokens/s: 192635.8 | physical tokens/s: 193006.5 | loss tokens/s: 138309.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.955097E-01 | lm-global-supervised-tokens: 1.204571E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.164452E-02 | lm-token-buffer-mib: 5.827830E+01 | global_load_balancing_loss: 1.001829E+00 | loss scale: 1.0 | grad norm: 0.101 | params norm: 11632.010 | params rms: 0.062984 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 9:19:53.942136 | finish at 2026-10-10 14:04:51 + [2026-10-04 04:46:21] iteration 518/ 6855 | consumed samples: 66304 | elapsed time per iteration (ms): 84605.1 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.78% | FLOPs estimator: workload-aware | logical tokens/s: 198300.0 | valid tokens/s: 198300.0 | physical tokens/s: 198668.4 | loss tokens/s: 151425.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.827110E-01 | lm-global-supervised-tokens: 1.281138E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.518562E-02 | lm-token-buffer-mib: 5.808342E+01 | global_load_balancing_loss: 1.001819E+00 | loss scale: 1.0 | grad norm: 0.132 | params norm: 11631.922 | params rms: 0.062983 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 4:55:42.250749 | finish at 2026-10-10 09:42:03 + [2026-10-04 04:47:47] iteration 519/ 6855 | consumed samples: 66432 | elapsed time per iteration (ms): 86276.1 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.75% | FLOPs estimator: workload-aware | logical tokens/s: 194459.3 | valid tokens/s: 194459.3 | physical tokens/s: 194908.5 | loss tokens/s: 142455.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.109680E-01 | lm-global-supervised-tokens: 1.229050E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.298293E-02 | lm-token-buffer-mib: 5.812870E+01 | global_load_balancing_loss: 1.000870E+00 | loss scale: 1.0 | grad norm: 0.059 | params norm: 11631.834 | params rms: 0.062983 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 7:50:45.476349 | finish at 2026-10-10 12:38:33 + [2026-10-04 04:49:35] iteration 520/ 6855 | consumed samples: 66560 | elapsed time per iteration (ms): 107710.6 | throughput per GPU (TFLOP/s/GPU): 34.7 | MFU 11.13% | FLOPs estimator: workload-aware | logical tokens/s: 155761.4 | valid tokens/s: 155761.4 | physical tokens/s: 156153.6 | loss tokens/s: 115750.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.274916E-01 | lm-global-supervised-tokens: 1.246758E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.443650E-02 | lm-token-buffer-mib: 5.805726E+01 | global_load_balancing_loss: 1.000348E+00 | loss scale: 1.0 | grad norm: 0.110 | params norm: 11631.746 | params rms: 0.062982 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 7 days, 21:32:26.586926 | finish at 2026-10-12 02:22:02 +times across ranks (ms): + save-checkpoint: + rank 0: 25728.26 + rank 1: 25728.27 + rank 2: 25728.31 + rank 3: 25728.26 + rank 4: 25728.27 + rank 5: 25728.26 + rank 6: 25728.27 + rank 7: 25728.29 + rank 8: 25729.22 + rank 9: 25729.16 + rank 10: 25729.14 + rank 11: 25729.18 + rank 12: 25729.18 + rank 13: 25729.15 + rank 14: 25729.18 + rank 15: 25729.14 + rank 16: 25728.61 + rank 17: 25728.61 + rank 18: 25728.53 + rank 19: 25728.56 + rank 20: 25728.52 + rank 21: 25728.58 + rank 22: 25728.51 + rank 23: 25728.51 + rank 24: 25730.68 + rank 25: 25730.32 + rank 26: 25730.68 + rank 27: 25730.72 + rank 28: 25730.56 + rank 29: 25730.64 + rank 30: 25731.54 + rank 31: 25730.64 + rank 32: 25729.37 + rank 33: 25729.34 + rank 34: 25729.35 + rank 35: 25729.25 + rank 36: 25729.24 + rank 37: 25729.23 + rank 38: 25729.31 + rank 39: 25729.23 + rank 40: 25729.26 + rank 41: 25729.16 + rank 42: 25729.24 + rank 43: 25729.26 + rank 44: 25729.23 + rank 45: 25729.25 + rank 46: 25729.26 + rank 47: 25729.23 + rank 48: 25728.41 + rank 49: 25728.43 + rank 50: 25728.40 + rank 51: 25728.44 + rank 52: 25728.45 + rank 53: 25728.42 + rank 54: 25728.41 + rank 55: 25728.42 + rank 56: 25728.46 + rank 57: 25728.44 + rank 58: 25728.41 + rank 59: 25728.39 + rank 60: 25728.40 + rank 61: 25728.43 + rank 62: 25728.41 + rank 63: 25728.44 + [2026-10-04 04:51:32] iteration 521/ 6855 | consumed samples: 66688 | elapsed time per iteration (ms): 90531.5 | throughput per GPU (TFLOP/s/GPU): 40.7 | MFU 13.04% | FLOPs estimator: workload-aware | logical tokens/s: 185318.7 | valid tokens/s: 185318.7 | physical tokens/s: 185728.4 | loss tokens/s: 136035.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.936730E-01 | lm-global-supervised-tokens: 1.231553E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.272687E-02 | lm-token-buffer-mib: 5.812264E+01 | global_load_balancing_loss: 9.988452E-01 | loss scale: 1.0 | grad norm: 0.117 | params norm: 11631.658 | params rms: 0.062982 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 15:17:06.622820 | finish at 2026-10-10 20:08:38 + [2026-10-04 04:53:03] iteration 522/ 6855 | consumed samples: 66816 | elapsed time per iteration (ms): 91207.5 | throughput per GPU (TFLOP/s/GPU): 41.3 | MFU 13.24% | FLOPs estimator: workload-aware | logical tokens/s: 183945.1 | valid tokens/s: 183945.1 | physical tokens/s: 184283.3 | loss tokens/s: 138361.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.628154E-01 | lm-global-supervised-tokens: 1.261959E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.598035E-02 | lm-token-buffer-mib: 5.805643E+01 | global_load_balancing_loss: 9.976256E-01 | loss scale: 1.0 | grad norm: 0.033 | params norm: 11631.571 | params rms: 0.062981 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 16:26:56.982626 | finish at 2026-10-10 21:20:00 + [2026-10-04 04:54:28] iteration 523/ 6855 | consumed samples: 66944 | elapsed time per iteration (ms): 85578.3 | throughput per GPU (TFLOP/s/GPU): 42.8 | MFU 13.71% | FLOPs estimator: workload-aware | logical tokens/s: 196044.6 | valid tokens/s: 196044.6 | physical tokens/s: 196660.0 | loss tokens/s: 147077.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.181382E-01 | lm-global-supervised-tokens: 1.258666E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.185110E-02 | lm-token-buffer-mib: 5.816917E+01 | global_load_balancing_loss: 9.993018E-01 | loss scale: 1.0 | grad norm: 0.283 | params norm: 11631.483 | params rms: 0.062981 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 6:31:21.593300 | finish at 2026-10-10 11:25:50 + [2026-10-04 04:55:55] iteration 524/ 6855 | consumed samples: 67072 | elapsed time per iteration (ms): 86687.0 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.53% | FLOPs estimator: workload-aware | logical tokens/s: 193537.4 | valid tokens/s: 193537.4 | physical tokens/s: 194116.3 | loss tokens/s: 142112.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.983606E-01 | lm-global-supervised-tokens: 1.231927E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.167440E-02 | lm-token-buffer-mib: 5.805065E+01 | global_load_balancing_loss: 1.000562E+00 | loss scale: 1.0 | grad norm: 0.222 | params norm: 11631.394 | params rms: 0.062980 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 8:26:55.354965 | finish at 2026-10-10 13:22:50 + [2026-10-04 04:57:22] iteration 525/ 6855 | consumed samples: 67200 | elapsed time per iteration (ms): 87200.6 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.69% | FLOPs estimator: workload-aware | logical tokens/s: 192397.2 | valid tokens/s: 192397.2 | physical tokens/s: 192940.7 | loss tokens/s: 148529.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.035521E-01 | lm-global-supervised-tokens: 1.295186E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.335241E-02 | lm-token-buffer-mib: 5.829771E+01 | global_load_balancing_loss: 1.000716E+00 | loss scale: 1.0 | grad norm: 0.129 | params norm: 11631.305 | params rms: 0.062980 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 9:19:39.789877 | finish at 2026-10-10 14:17:02 + [2026-10-04 04:58:46] iteration 526/ 6855 | consumed samples: 67328 | elapsed time per iteration (ms): 83441.2 | throughput per GPU (TFLOP/s/GPU): 43.1 | MFU 13.80% | FLOPs estimator: workload-aware | logical tokens/s: 201066.2 | valid tokens/s: 201066.2 | physical tokens/s: 201608.3 | loss tokens/s: 149592.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.268280E-01 | lm-global-supervised-tokens: 1.248219E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 3.644122E-02 | lm-token-buffer-mib: 5.815649E+01 | global_load_balancing_loss: 1.001328E+00 | loss scale: 1.0 | grad norm: 0.076 | params norm: 11631.216 | params rms: 0.062979 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 2:41:39.597855 | finish at 2026-10-10 07:40:25 + [2026-10-04 05:00:11] iteration 527/ 6855 | consumed samples: 67456 | elapsed time per iteration (ms): 85301.7 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.66% | FLOPs estimator: workload-aware | logical tokens/s: 196680.3 | valid tokens/s: 196680.3 | physical tokens/s: 197137.2 | loss tokens/s: 151092.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.080216E-01 | lm-global-supervised-tokens: 1.288844E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.174331E-02 | lm-token-buffer-mib: 5.807703E+01 | global_load_balancing_loss: 1.001100E+00 | loss scale: 1.0 | grad norm: 0.148 | params norm: 11631.127 | params rms: 0.062979 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 5:56:28.910900 | finish at 2026-10-10 10:56:40 + [2026-10-04 05:01:40] iteration 528/ 6855 | consumed samples: 67584 | elapsed time per iteration (ms): 88777.1 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.41% | FLOPs estimator: workload-aware | logical tokens/s: 188981.1 | valid tokens/s: 188981.1 | physical tokens/s: 189567.2 | loss tokens/s: 133352.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.769492E-01 | lm-global-supervised-tokens: 1.183867E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.661606E-02 | lm-token-buffer-mib: 5.846223E+01 | global_load_balancing_loss: 1.002100E+00 | loss scale: 1.0 | grad norm: 0.060 | params norm: 11631.037 | params rms: 0.062978 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 12:01:32.455805 | finish at 2026-10-10 17:03:12 + [2026-10-04 05:03:03] iteration 529/ 6855 | consumed samples: 67712 | elapsed time per iteration (ms): 83354.2 | throughput per GPU (TFLOP/s/GPU): 43.1 | MFU 13.81% | FLOPs estimator: workload-aware | logical tokens/s: 201276.1 | valid tokens/s: 201276.1 | physical tokens/s: 201868.1 | loss tokens/s: 144583.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.555752E-01 | lm-global-supervised-tokens: 1.205164E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.256646E-02 | lm-token-buffer-mib: 5.804214E+01 | global_load_balancing_loss: 1.001480E+00 | loss scale: 1.0 | grad norm: 0.080 | params norm: 11630.947 | params rms: 0.062978 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 2:28:18.903766 | finish at 2026-10-10 07:31:22 + [2026-10-04 05:04:50] iteration 530/ 6855 | consumed samples: 67840 | elapsed time per iteration (ms): 107175.4 | throughput per GPU (TFLOP/s/GPU): 34.5 | MFU 11.06% | FLOPs estimator: workload-aware | logical tokens/s: 156539.6 | valid tokens/s: 156539.6 | physical tokens/s: 156914.6 | loss tokens/s: 117667.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.427740E-01 | lm-global-supervised-tokens: 1.261101E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.841582E-02 | lm-token-buffer-mib: 5.835876E+01 | global_load_balancing_loss: 9.987321E-01 | loss scale: 1.0 | grad norm: 0.075 | params norm: 11630.857 | params rms: 0.062977 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 7 days, 20:18:04.601158 | finish at 2026-10-12 01:22:55 + [2026-10-04 05:06:17] iteration 531/ 6855 | consumed samples: 67968 | elapsed time per iteration (ms): 86399.6 | throughput per GPU (TFLOP/s/GPU): 41.7 | MFU 13.36% | FLOPs estimator: workload-aware | logical tokens/s: 194181.0 | valid tokens/s: 194181.0 | physical tokens/s: 194804.4 | loss tokens/s: 143503.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.748002E-01 | lm-global-supervised-tokens: 1.239867E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.165698E-02 | lm-token-buffer-mib: 5.823068E+01 | global_load_balancing_loss: 9.997147E-01 | loss scale: 1.0 | grad norm: 0.123 | params norm: 11630.768 | params rms: 0.062977 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 7:46:31.014796 | finish at 2026-10-10 12:52:48 + [2026-10-04 05:07:45] iteration 532/ 6855 | consumed samples: 68096 | elapsed time per iteration (ms): 88308.1 | throughput per GPU (TFLOP/s/GPU): 41.6 | MFU 13.34% | FLOPs estimator: workload-aware | logical tokens/s: 189984.0 | valid tokens/s: 189984.0 | physical tokens/s: 190335.3 | loss tokens/s: 140412.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.858982E-01 | lm-global-supervised-tokens: 1.239954E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.173576E-02 | lm-token-buffer-mib: 5.812440E+01 | global_load_balancing_loss: 9.990311E-01 | loss scale: 1.0 | grad norm: 0.099 | params norm: 11630.677 | params rms: 0.062976 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 11:06:12.089071 | finish at 2026-10-10 16:13:57 + [2026-10-04 05:09:13] iteration 533/ 6855 | consumed samples: 68224 | elapsed time per iteration (ms): 88415.5 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.48% | FLOPs estimator: workload-aware | logical tokens/s: 189753.9 | valid tokens/s: 189753.9 | physical tokens/s: 190138.0 | loss tokens/s: 138289.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.421978E-01 | lm-global-supervised-tokens: 1.222694E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.290819E-02 | lm-token-buffer-mib: 5.804961E+01 | global_load_balancing_loss: 9.989463E-01 | loss scale: 1.0 | grad norm: 0.075 | params norm: 11630.587 | params rms: 0.062976 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 11:16:03.079928 | finish at 2026-10-10 16:25:17 + [2026-10-04 05:10:40] iteration 534/ 6855 | consumed samples: 68352 | elapsed time per iteration (ms): 86467.0 | throughput per GPU (TFLOP/s/GPU): 43.1 | MFU 13.83% | FLOPs estimator: workload-aware | logical tokens/s: 194029.9 | valid tokens/s: 194029.9 | physical tokens/s: 194410.5 | loss tokens/s: 143310.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.899629E-01 | lm-global-supervised-tokens: 1.239166E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.202432E-02 | lm-token-buffer-mib: 5.801727E+01 | global_load_balancing_loss: 9.997436E-01 | loss scale: 1.0 | grad norm: 0.101 | params norm: 11630.496 | params rms: 0.062975 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 7:49:17.919105 | finish at 2026-10-10 12:59:58 + [2026-10-04 05:12:09] iteration 535/ 6855 | consumed samples: 68480 | elapsed time per iteration (ms): 89206.9 | throughput per GPU (TFLOP/s/GPU): 41.2 | MFU 13.19% | FLOPs estimator: workload-aware | logical tokens/s: 188070.5 | valid tokens/s: 188070.5 | physical tokens/s: 188573.5 | loss tokens/s: 138399.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.722358E-01 | lm-global-supervised-tokens: 1.234620E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.829395E-02 | lm-token-buffer-mib: 5.818143E+01 | global_load_balancing_loss: 1.000285E+00 | loss scale: 1.0 | grad norm: 0.040 | params norm: 11630.405 | params rms: 0.062975 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 12:36:27.904091 | finish at 2026-10-10 17:48:37 + [2026-10-04 05:13:38] iteration 536/ 6855 | consumed samples: 68608 | elapsed time per iteration (ms): 88441.2 | throughput per GPU (TFLOP/s/GPU): 41.6 | MFU 13.32% | FLOPs estimator: workload-aware | logical tokens/s: 189698.6 | valid tokens/s: 189698.6 | physical tokens/s: 190168.1 | loss tokens/s: 136515.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.789330E-01 | lm-global-supervised-tokens: 1.207361E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.165816E-02 | lm-token-buffer-mib: 5.805840E+01 | global_load_balancing_loss: 1.001279E+00 | loss scale: 1.0 | grad norm: 0.039 | params norm: 11630.315 | params rms: 0.062974 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 11:14:20.144793 | finish at 2026-10-10 16:27:58 + [2026-10-04 05:15:07] iteration 537/ 6855 | consumed samples: 68736 | elapsed time per iteration (ms): 89305.1 | throughput per GPU (TFLOP/s/GPU): 40.6 | MFU 13.00% | FLOPs estimator: workload-aware | logical tokens/s: 187863.9 | valid tokens/s: 187863.9 | physical tokens/s: 188500.1 | loss tokens/s: 145136.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.464634E-01 | lm-global-supervised-tokens: 1.296143E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.580554E-02 | lm-token-buffer-mib: 5.826390E+01 | global_load_balancing_loss: 1.000493E+00 | loss scale: 1.0 | grad norm: 0.107 | params norm: 11630.224 | params rms: 0.062974 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 12:43:49.871624 | finish at 2026-10-10 17:58:57 + [2026-10-04 05:16:34] iteration 538/ 6855 | consumed samples: 68864 | elapsed time per iteration (ms): 87583.9 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.66% | FLOPs estimator: workload-aware | logical tokens/s: 191555.9 | valid tokens/s: 191555.8 | physical tokens/s: 192040.9 | loss tokens/s: 139299.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.319123E-01 | lm-global-supervised-tokens: 1.220038E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.549415E-02 | lm-token-buffer-mib: 5.812921E+01 | global_load_balancing_loss: 1.000498E+00 | loss scale: 1.0 | grad norm: 0.035 | params norm: 11630.132 | params rms: 0.062973 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 9:41:07.578976 | finish at 2026-10-10 14:57:42 + [2026-10-04 05:18:04] iteration 539/ 6855 | consumed samples: 68992 | elapsed time per iteration (ms): 89261.6 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.40% | FLOPs estimator: workload-aware | logical tokens/s: 187954.9 | valid tokens/s: 187954.9 | physical tokens/s: 188503.0 | loss tokens/s: 142276.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.163706E-01 | lm-global-supervised-tokens: 1.269981E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.165459E-02 | lm-token-buffer-mib: 5.830999E+01 | global_load_balancing_loss: 1.000328E+00 | loss scale: 1.0 | grad norm: 0.049 | params norm: 11630.040 | params rms: 0.062973 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 12:36:16.383167 | finish at 2026-10-10 17:54:20 + [2026-10-04 05:19:50] iteration 540/ 6855 | consumed samples: 69120 | elapsed time per iteration (ms): 106385.3 | throughput per GPU (TFLOP/s/GPU): 34.4 | MFU 11.03% | FLOPs estimator: workload-aware | logical tokens/s: 157701.7 | valid tokens/s: 157701.7 | physical tokens/s: 158213.3 | loss tokens/s: 122589.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.983231E-01 | lm-global-supervised-tokens: 1.304175E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.556547E-02 | lm-token-buffer-mib: 5.825243E+01 | global_load_balancing_loss: 9.992125E-01 | loss scale: 1.0 | grad norm: 0.044 | params norm: 11629.949 | params rms: 0.062972 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 7 days, 18:37:03.000373 | finish at 2026-10-11 23:56:53 + [2026-10-04 05:21:15] iteration 541/ 6855 | consumed samples: 69248 | elapsed time per iteration (ms): 84606.7 | throughput per GPU (TFLOP/s/GPU): 43.2 | MFU 13.85% | FLOPs estimator: workload-aware | logical tokens/s: 198295.9 | valid tokens/s: 198295.8 | physical tokens/s: 198848.3 | loss tokens/s: 151975.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.705003E-01 | lm-global-supervised-tokens: 1.285812E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.560959E-02 | lm-token-buffer-mib: 5.822653E+01 | global_load_balancing_loss: 1.000401E+00 | loss scale: 1.0 | grad norm: 0.052 | params norm: 11629.857 | params rms: 0.062972 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 4:23:26.996992 | finish at 2026-10-10 09:44:42 + [2026-10-04 05:22:40] iteration 542/ 6855 | consumed samples: 69376 | elapsed time per iteration (ms): 84892.9 | throughput per GPU (TFLOP/s/GPU): 43.7 | MFU 14.00% | FLOPs estimator: workload-aware | logical tokens/s: 197627.7 | valid tokens/s: 197627.7 | physical tokens/s: 198025.8 | loss tokens/s: 147318.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.782724E-01 | lm-global-supervised-tokens: 1.250632E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.479500E-02 | lm-token-buffer-mib: 5.804831E+01 | global_load_balancing_loss: 1.000848E+00 | loss scale: 1.0 | grad norm: 0.244 | params norm: 11629.765 | params rms: 0.062971 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 4:52:08.712072 | finish at 2026-10-10 10:14:48 + [2026-10-04 05:24:08] iteration 543/ 6855 | consumed samples: 69504 | elapsed time per iteration (ms): 88128.6 | throughput per GPU (TFLOP/s/GPU): 42.8 | MFU 13.73% | FLOPs estimator: workload-aware | logical tokens/s: 190371.8 | valid tokens/s: 190371.7 | physical tokens/s: 190744.7 | loss tokens/s: 138421.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.766405E-01 | lm-global-supervised-tokens: 1.219889E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.150792E-02 | lm-token-buffer-mib: 5.808100E+01 | global_load_balancing_loss: 1.001895E+00 | loss scale: 1.0 | grad norm: 0.057 | params norm: 11629.662 | params rms: 0.062971 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 10:31:07.784157 | finish at 2026-10-10 15:55:16 + [2026-10-04 05:25:37] iteration 544/ 6855 | consumed samples: 69632 | elapsed time per iteration (ms): 89267.0 | throughput per GPU (TFLOP/s/GPU): 41.2 | MFU 13.20% | FLOPs estimator: workload-aware | logical tokens/s: 187943.7 | valid tokens/s: 187943.6 | physical tokens/s: 188526.8 | loss tokens/s: 140751.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.195144E-01 | lm-global-supervised-tokens: 1.256443E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.178283E-02 | lm-token-buffer-mib: 5.812044E+01 | global_load_balancing_loss: 1.000593E+00 | loss scale: 1.0 | grad norm: 0.061 | params norm: 11629.569 | params rms: 0.062970 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 12:29:24.251914 | finish at 2026-10-10 17:55:01 + [2026-10-04 05:27:06] iteration 545/ 6855 | consumed samples: 69760 | elapsed time per iteration (ms): 88654.3 | throughput per GPU (TFLOP/s/GPU): 42.8 | MFU 13.71% | FLOPs estimator: workload-aware | logical tokens/s: 189242.8 | valid tokens/s: 189242.8 | physical tokens/s: 189645.1 | loss tokens/s: 136792.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.969240E-01 | lm-global-supervised-tokens: 1.212720E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.836590E-02 | lm-token-buffer-mib: 5.807677E+01 | global_load_balancing_loss: 1.000864E+00 | loss scale: 1.0 | grad norm: 0.041 | params norm: 11629.484 | params rms: 0.062970 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 11:23:28.360538 | finish at 2026-10-10 16:50:34 + [2026-10-04 05:28:32] iteration 546/ 6855 | consumed samples: 69888 | elapsed time per iteration (ms): 86237.4 | throughput per GPU (TFLOP/s/GPU): 42.8 | MFU 13.70% | FLOPs estimator: workload-aware | logical tokens/s: 194546.8 | valid tokens/s: 194546.8 | physical tokens/s: 194967.3 | loss tokens/s: 145152.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.576971E-01 | lm-global-supervised-tokens: 1.251759E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.711096E-02 | lm-token-buffer-mib: 5.813135E+01 | global_load_balancing_loss: 9.985192E-01 | loss scale: 1.0 | grad norm: 0.153 | params norm: 11629.407 | params rms: 0.062969 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 7:07:51.856223 | finish at 2026-10-10 12:36:24 + [2026-10-04 05:30:01] iteration 547/ 6855 | consumed samples: 70016 | elapsed time per iteration (ms): 89234.0 | throughput per GPU (TFLOP/s/GPU): 41.6 | MFU 13.34% | FLOPs estimator: workload-aware | logical tokens/s: 188012.9 | valid tokens/s: 188012.9 | physical tokens/s: 188491.1 | loss tokens/s: 130770.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.697059E-01 | lm-global-supervised-tokens: 1.166919E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.161184E-02 | lm-token-buffer-mib: 5.804034E+01 | global_load_balancing_loss: 9.999399E-01 | loss scale: 1.0 | grad norm: 0.141 | params norm: 11629.336 | params rms: 0.062969 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 12:21:27.885306 | finish at 2026-10-10 17:51:29 + [2026-10-04 05:31:29] iteration 548/ 6855 | consumed samples: 70144 | elapsed time per iteration (ms): 87474.8 | throughput per GPU (TFLOP/s/GPU): 42.8 | MFU 13.73% | FLOPs estimator: workload-aware | logical tokens/s: 191794.7 | valid tokens/s: 191794.7 | physical tokens/s: 192197.9 | loss tokens/s: 138099.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.969979E-01 | lm-global-supervised-tokens: 1.208027E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.874215E-02 | lm-token-buffer-mib: 5.813712E+01 | global_load_balancing_loss: 9.992324E-01 | loss scale: 1.0 | grad norm: 0.041 | params norm: 11629.267 | params rms: 0.062969 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 9:15:03.830449 | finish at 2026-10-10 14:46:32 + [2026-10-04 05:32:55] iteration 549/ 6855 | consumed samples: 70272 | elapsed time per iteration (ms): 86718.3 | throughput per GPU (TFLOP/s/GPU): 43.3 | MFU 13.89% | FLOPs estimator: workload-aware | logical tokens/s: 193467.9 | valid tokens/s: 193467.9 | physical tokens/s: 193937.3 | loss tokens/s: 136967.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.339153E-01 | lm-global-supervised-tokens: 1.187758E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.425813E-02 | lm-token-buffer-mib: 5.825319E+01 | global_load_balancing_loss: 9.995533E-01 | loss scale: 1.0 | grad norm: 0.064 | params norm: 11629.197 | params rms: 0.062968 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 7:54:05.290742 | finish at 2026-10-10 13:27:01 + [2026-10-04 05:34:44] iteration 550/ 6855 | consumed samples: 70400 | elapsed time per iteration (ms): 108563.7 | throughput per GPU (TFLOP/s/GPU): 34.2 | MFU 10.95% | FLOPs estimator: workload-aware | logical tokens/s: 154537.8 | valid tokens/s: 154537.8 | physical tokens/s: 154864.8 | loss tokens/s: 112559.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.823443E-01 | lm-global-supervised-tokens: 1.221982E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.152036E-02 | lm-token-buffer-mib: 5.824684E+01 | global_load_balancing_loss: 1.000391E+00 | loss scale: 1.0 | grad norm: 0.063 | params norm: 11629.127 | params rms: 0.062968 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 7 days, 22:08:13.823097 | finish at 2026-10-12 03:42:58 + [2026-10-04 05:36:12] iteration 551/ 6855 | consumed samples: 70528 | elapsed time per iteration (ms): 88543.3 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.39% | FLOPs estimator: workload-aware | logical tokens/s: 189480.0 | valid tokens/s: 189480.0 | physical tokens/s: 189910.3 | loss tokens/s: 138646.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.930411E-01 | lm-global-supervised-tokens: 1.227623E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.377886E-02 | lm-token-buffer-mib: 5.819577E+01 | global_load_balancing_loss: 1.001148E+00 | loss scale: 1.0 | grad norm: 0.184 | params norm: 11629.055 | params rms: 0.062968 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 11:02:56.843918 | finish at 2026-10-10 16:39:09 + [2026-10-04 05:37:39] iteration 552/ 6855 | consumed samples: 70656 | elapsed time per iteration (ms): 87001.8 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.66% | FLOPs estimator: workload-aware | logical tokens/s: 192837.0 | valid tokens/s: 192837.0 | physical tokens/s: 193222.2 | loss tokens/s: 145334.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.865013E-01 | lm-global-supervised-tokens: 1.264431E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.142324E-02 | lm-token-buffer-mib: 5.809084E+01 | global_load_balancing_loss: 9.997066E-01 | loss scale: 1.0 | grad norm: 0.098 | params norm: 11628.977 | params rms: 0.062967 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 8:19:32.069273 | finish at 2026-10-10 13:57:11 + [2026-10-04 05:39:05] iteration 553/ 6855 | consumed samples: 70784 | elapsed time per iteration (ms): 85198.6 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.75% | FLOPs estimator: workload-aware | logical tokens/s: 196918.0 | valid tokens/s: 196918.0 | physical tokens/s: 197445.9 | loss tokens/s: 150255.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.797142E-01 | lm-global-supervised-tokens: 1.280155E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.839523E-02 | lm-token-buffer-mib: 5.816879E+01 | global_load_balancing_loss: 1.000886E+00 | loss scale: 1.0 | grad norm: 0.160 | params norm: 11628.902 | params rms: 0.062967 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 5:08:41.449825 | finish at 2026-10-10 10:47:46 + [2026-10-04 05:40:35] iteration 554/ 6855 | consumed samples: 70912 | elapsed time per iteration (ms): 90347.5 | throughput per GPU (TFLOP/s/GPU): 40.8 | MFU 13.08% | FLOPs estimator: workload-aware | logical tokens/s: 185695.9 | valid tokens/s: 185695.9 | physical tokens/s: 186189.3 | loss tokens/s: 137278.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.761593E-01 | lm-global-supervised-tokens: 1.240278E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.235699E-02 | lm-token-buffer-mib: 5.817001E+01 | global_load_balancing_loss: 1.000753E+00 | loss scale: 1.0 | grad norm: 0.093 | params norm: 11628.831 | params rms: 0.062966 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 14:07:59.658132 | finish at 2026-10-10 19:48:35 + [2026-10-04 05:42:08] iteration 555/ 6855 | consumed samples: 71040 | elapsed time per iteration (ms): 92609.3 | throughput per GPU (TFLOP/s/GPU): 41.5 | MFU 13.30% | FLOPs estimator: workload-aware | logical tokens/s: 181160.5 | valid tokens/s: 181160.5 | physical tokens/s: 181540.6 | loss tokens/s: 130971.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.191372E-01 | lm-global-supervised-tokens: 1.212919E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.259575E-02 | lm-token-buffer-mib: 5.812492E+01 | global_load_balancing_loss: 1.002022E+00 | loss scale: 1.0 | grad norm: 0.085 | params norm: 11628.760 | params rms: 0.062966 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 18:03:58.754582 | finish at 2026-10-10 23:46:06 + [2026-10-04 05:43:35] iteration 556/ 6855 | consumed samples: 71168 | elapsed time per iteration (ms): 87481.5 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.53% | FLOPs estimator: workload-aware | logical tokens/s: 191780.0 | valid tokens/s: 191780.0 | physical tokens/s: 192264.7 | loss tokens/s: 141845.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.399000E-01 | lm-global-supervised-tokens: 1.240886E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.285168E-02 | lm-token-buffer-mib: 5.815807E+01 | global_load_balancing_loss: 1.003074E+00 | loss scale: 1.0 | grad norm: 0.183 | params norm: 11628.689 | params rms: 0.062966 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 9:04:06.230750 | finish at 2026-10-10 14:47:41 + [2026-10-04 05:45:00] iteration 557/ 6855 | consumed samples: 71296 | elapsed time per iteration (ms): 84818.1 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.66% | FLOPs estimator: workload-aware | logical tokens/s: 197802.0 | valid tokens/s: 197802.0 | physical tokens/s: 198324.2 | loss tokens/s: 147504.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.503915E-01 | lm-global-supervised-tokens: 1.251105E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.151755E-02 | lm-token-buffer-mib: 5.833748E+01 | global_load_balancing_loss: 1.000570E+00 | loss scale: 1.0 | grad norm: 0.105 | params norm: 11628.619 | params rms: 0.062965 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 4:23:04.112854 | finish at 2026-10-10 10:08:04 + [2026-10-04 05:46:27] iteration 558/ 6855 | consumed samples: 71424 | elapsed time per iteration (ms): 86842.7 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.51% | FLOPs estimator: workload-aware | logical tokens/s: 193190.1 | valid tokens/s: 193190.1 | physical tokens/s: 193717.9 | loss tokens/s: 144420.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.741868E-01 | lm-global-supervised-tokens: 1.254186E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.694538E-02 | lm-token-buffer-mib: 5.816261E+01 | global_load_balancing_loss: 9.986932E-01 | loss scale: 1.0 | grad norm: 0.141 | params norm: 11628.549 | params rms: 0.062965 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 7:54:08.657583 | finish at 2026-10-10 13:40:35 + [2026-10-04 05:47:55] iteration 559/ 6855 | consumed samples: 71552 | elapsed time per iteration (ms): 88058.2 | throughput per GPU (TFLOP/s/GPU): 41.4 | MFU 13.26% | FLOPs estimator: workload-aware | logical tokens/s: 190523.8 | valid tokens/s: 190523.8 | physical tokens/s: 190873.9 | loss tokens/s: 144355.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.804892E-01 | lm-global-supervised-tokens: 1.271170E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.236399E-02 | lm-token-buffer-mib: 5.804435E+01 | global_load_balancing_loss: 9.991290E-01 | loss scale: 1.0 | grad norm: 0.069 | params norm: 11628.478 | params rms: 0.062964 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 10:00:14.546547 | finish at 2026-10-10 15:48:09 + [2026-10-04 05:49:39] iteration 560/ 6855 | consumed samples: 71680 | elapsed time per iteration (ms): 104491.6 | throughput per GPU (TFLOP/s/GPU): 36.2 | MFU 11.60% | FLOPs estimator: workload-aware | logical tokens/s: 160560.4 | valid tokens/s: 160560.4 | physical tokens/s: 160834.7 | loss tokens/s: 116136.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.463820E-01 | lm-global-supervised-tokens: 1.213530E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.318692E-02 | lm-token-buffer-mib: 5.825036E+01 | global_load_balancing_loss: 9.997176E-01 | loss scale: 1.0 | grad norm: 0.067 | params norm: 11628.404 | params rms: 0.062964 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 7 days, 14:42:54.661252 | finish at 2026-10-11 20:32:34 +times across ranks (ms): + save-checkpoint: + rank 0: 23859.45 + rank 1: 23859.46 + rank 2: 23859.46 + rank 3: 23859.50 + rank 4: 23859.43 + rank 5: 23859.44 + rank 6: 23859.42 + rank 7: 23859.46 + rank 8: 23859.73 + rank 9: 23859.67 + rank 10: 23859.65 + rank 11: 23859.68 + rank 12: 23859.67 + rank 13: 23859.68 + rank 14: 23859.67 + rank 15: 23859.65 + rank 16: 23859.58 + rank 17: 23859.65 + rank 18: 23859.55 + rank 19: 23859.61 + rank 20: 23859.55 + rank 21: 23859.61 + rank 22: 23859.55 + rank 23: 23859.57 + rank 24: 23859.77 + rank 25: 23859.74 + rank 26: 23859.85 + rank 27: 23859.75 + rank 28: 23859.71 + rank 29: 23859.74 + rank 30: 23859.72 + rank 31: 23859.70 + rank 32: 23859.83 + rank 33: 23859.84 + rank 34: 23859.83 + rank 35: 23859.79 + rank 36: 23859.75 + rank 37: 23859.77 + rank 38: 23859.81 + rank 39: 23859.77 + rank 40: 23861.43 + rank 41: 23861.37 + rank 42: 23861.39 + rank 43: 23861.42 + rank 44: 23861.36 + rank 45: 23861.38 + rank 46: 23861.39 + rank 47: 23861.37 + rank 48: 23859.52 + rank 49: 23859.52 + rank 50: 23859.46 + rank 51: 23859.48 + rank 52: 23859.47 + rank 53: 23859.47 + rank 54: 23859.48 + rank 55: 23859.47 + rank 56: 23859.02 + rank 57: 23858.96 + rank 58: 23858.96 + rank 59: 23858.95 + rank 60: 23858.96 + rank 61: 23858.95 + rank 62: 23858.95 + rank 63: 23858.97 + [2026-10-04 05:51:53] iteration 561/ 6855 | consumed samples: 71808 | elapsed time per iteration (ms): 104137.4 | throughput per GPU (TFLOP/s/GPU): 34.5 | MFU 11.05% | FLOPs estimator: workload-aware | logical tokens/s: 161106.5 | valid tokens/s: 161106.5 | physical tokens/s: 161545.7 | loss tokens/s: 125350.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.805717E-01 | lm-global-supervised-tokens: 1.305364E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.806547E+00 | lm-token-buffer-mib: 5.822410E+01 | global_load_balancing_loss: 9.992918E-01 | loss scale: 1.0 | grad norm: 0.074 | params norm: 11628.330 | params rms: 0.062964 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 7 days, 14:04:00.565453 | finish at 2026-10-11 19:55:54 + [2026-10-04 05:53:24] iteration 562/ 6855 | consumed samples: 71936 | elapsed time per iteration (ms): 90948.7 | throughput per GPU (TFLOP/s/GPU): 39.9 | MFU 12.78% | FLOPs estimator: workload-aware | logical tokens/s: 184469.1 | valid tokens/s: 184469.1 | physical tokens/s: 184898.5 | loss tokens/s: 137627.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.694428E-01 | lm-global-supervised-tokens: 1.251699E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.484957E-02 | lm-token-buffer-mib: 5.813127E+01 | global_load_balancing_loss: 1.000463E+00 | loss scale: 1.0 | grad norm: 0.059 | params norm: 11628.253 | params rms: 0.062963 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 14:58:59.880722 | finish at 2026-10-10 20:52:24 + [2026-10-04 05:54:56] iteration 563/ 6855 | consumed samples: 72064 | elapsed time per iteration (ms): 92084.5 | throughput per GPU (TFLOP/s/GPU): 41.4 | MFU 13.28% | FLOPs estimator: workload-aware | logical tokens/s: 182192.5 | valid tokens/s: 182192.5 | physical tokens/s: 182706.4 | loss tokens/s: 129396.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.173488E-01 | lm-global-supervised-tokens: 1.191544E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.168419E-02 | lm-token-buffer-mib: 5.817004E+01 | global_load_balancing_loss: 1.001215E+00 | loss scale: 1.0 | grad norm: 0.064 | params norm: 11628.174 | params rms: 0.062963 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 16:56:35.852983 | finish at 2026-10-10 22:51:32 + [2026-10-04 05:56:23] iteration 564/ 6855 | consumed samples: 72192 | elapsed time per iteration (ms): 87005.6 | throughput per GPU (TFLOP/s/GPU): 43.4 | MFU 13.90% | FLOPs estimator: workload-aware | logical tokens/s: 192828.9 | valid tokens/s: 192828.9 | physical tokens/s: 193247.1 | loss tokens/s: 145299.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.110207E-01 | lm-global-supervised-tokens: 1.264190E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.282609E-02 | lm-token-buffer-mib: 5.825037E+01 | global_load_balancing_loss: 1.001130E+00 | loss scale: 1.0 | grad norm: 0.041 | params norm: 11628.095 | params rms: 0.062962 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 8:02:32.427432 | finish at 2026-10-10 13:58:56 + [2026-10-04 05:57:49] iteration 565/ 6855 | consumed samples: 72320 | elapsed time per iteration (ms): 86141.6 | throughput per GPU (TFLOP/s/GPU): 43.2 | MFU 13.84% | FLOPs estimator: workload-aware | logical tokens/s: 194762.6 | valid tokens/s: 194762.5 | physical tokens/s: 195204.8 | loss tokens/s: 144326.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.879237E-01 | lm-global-supervised-tokens: 1.243249E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.478429E-02 | lm-token-buffer-mib: 5.815662E+01 | global_load_balancing_loss: 1.001498E+00 | loss scale: 1.0 | grad norm: 0.051 | params norm: 11628.013 | params rms: 0.062962 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 6:30:30.610843 | finish at 2026-10-10 12:28:20 + [2026-10-04 05:59:17] iteration 566/ 6855 | consumed samples: 72448 | elapsed time per iteration (ms): 87180.3 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.53% | FLOPs estimator: workload-aware | logical tokens/s: 192442.8 | valid tokens/s: 192442.8 | physical tokens/s: 192837.8 | loss tokens/s: 143735.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.903045E-01 | lm-global-supervised-tokens: 1.253090E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.710170E-02 | lm-token-buffer-mib: 5.835995E+01 | global_load_balancing_loss: 9.991645E-01 | loss scale: 1.0 | grad norm: 0.049 | params norm: 11627.929 | params rms: 0.062961 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 8:17:56.627792 | finish at 2026-10-10 14:17:13 + [2026-10-04 06:00:46] iteration 567/ 6855 | consumed samples: 72576 | elapsed time per iteration (ms): 89101.7 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.64% | FLOPs estimator: workload-aware | logical tokens/s: 188292.6 | valid tokens/s: 188292.6 | physical tokens/s: 188761.6 | loss tokens/s: 132482.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.792721E-01 | lm-global-supervised-tokens: 1.180442E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 4.104806E-02 | lm-token-buffer-mib: 5.822776E+01 | global_load_balancing_loss: 1.000221E+00 | loss scale: 1.0 | grad norm: 0.042 | params norm: 11627.844 | params rms: 0.062961 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 11:37:51.352100 | finish at 2026-10-10 17:38:37 + [2026-10-04 06:02:12] iteration 568/ 6855 | consumed samples: 72704 | elapsed time per iteration (ms): 85990.1 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.68% | FLOPs estimator: workload-aware | logical tokens/s: 195105.6 | valid tokens/s: 195105.6 | physical tokens/s: 195728.0 | loss tokens/s: 141586.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.002485E-01 | lm-global-supervised-tokens: 1.217503E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.767851E-02 | lm-token-buffer-mib: 5.834406E+01 | global_load_balancing_loss: 1.000769E+00 | loss scale: 1.0 | grad norm: 0.071 | params norm: 11627.758 | params rms: 0.062961 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 6:10:19.876531 | finish at 2026-10-10 12:12:32 + [2026-10-04 06:03:42] iteration 569/ 6855 | consumed samples: 72832 | elapsed time per iteration (ms): 89891.2 | throughput per GPU (TFLOP/s/GPU): 41.3 | MFU 13.23% | FLOPs estimator: workload-aware | logical tokens/s: 186638.1 | valid tokens/s: 186638.1 | physical tokens/s: 187220.7 | loss tokens/s: 137272.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.911137E-01 | lm-global-supervised-tokens: 1.233956E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.111363E-02 | lm-token-buffer-mib: 5.814828E+01 | global_load_balancing_loss: 1.000138E+00 | loss scale: 1.0 | grad norm: 0.046 | params norm: 11627.670 | params rms: 0.062960 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 12:57:36.026662 | finish at 2026-10-10 19:01:18 + [2026-10-04 06:05:29] iteration 570/ 6855 | consumed samples: 72960 | elapsed time per iteration (ms): 107217.2 | throughput per GPU (TFLOP/s/GPU): 34.1 | MFU 10.93% | FLOPs estimator: workload-aware | logical tokens/s: 156478.8 | valid tokens/s: 156478.8 | physical tokens/s: 156813.7 | loss tokens/s: 115885.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.797501E-01 | lm-global-supervised-tokens: 1.242492E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.149084E-02 | lm-token-buffer-mib: 5.805762E+01 | global_load_balancing_loss: 9.998174E-01 | loss scale: 1.0 | grad norm: 0.055 | params norm: 11627.579 | params rms: 0.062960 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 7 days, 19:10:59.944918 | finish at 2026-10-12 01:16:29 + [2026-10-04 06:06:57] iteration 571/ 6855 | consumed samples: 73088 | elapsed time per iteration (ms): 87968.7 | throughput per GPU (TFLOP/s/GPU): 41.4 | MFU 13.26% | FLOPs estimator: workload-aware | logical tokens/s: 190717.9 | valid tokens/s: 190717.9 | physical tokens/s: 191527.5 | loss tokens/s: 142208.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.728792E-01 | lm-global-supervised-tokens: 1.250990E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.266752E-02 | lm-token-buffer-mib: 5.825271E+01 | global_load_balancing_loss: 1.000073E+00 | loss scale: 1.0 | grad norm: 0.038 | params norm: 11627.487 | params rms: 0.062959 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 9:33:15.042192 | finish at 2026-10-10 15:40:12 + [2026-10-04 06:08:24] iteration 572/ 6855 | consumed samples: 73216 | elapsed time per iteration (ms): 87228.3 | throughput per GPU (TFLOP/s/GPU): 41.5 | MFU 13.29% | FLOPs estimator: workload-aware | logical tokens/s: 192336.6 | valid tokens/s: 192336.6 | physical tokens/s: 192800.2 | loss tokens/s: 142213.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.793604E-01 | lm-global-supervised-tokens: 1.240504E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.356586E-02 | lm-token-buffer-mib: 5.814125E+01 | global_load_balancing_loss: 1.001620E+00 | loss scale: 1.0 | grad norm: 0.051 | params norm: 11627.393 | params rms: 0.062959 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 8:14:15.482896 | finish at 2026-10-10 14:22:40 + [2026-10-04 06:09:54] iteration 573/ 6855 | consumed samples: 73344 | elapsed time per iteration (ms): 90368.0 | throughput per GPU (TFLOP/s/GPU): 41.2 | MFU 13.21% | FLOPs estimator: workload-aware | logical tokens/s: 185653.1 | valid tokens/s: 185653.1 | physical tokens/s: 186143.2 | loss tokens/s: 130931.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.966503E-01 | lm-global-supervised-tokens: 1.183202E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.154999E-02 | lm-token-buffer-mib: 5.818183E+01 | global_load_balancing_loss: 1.000500E+00 | loss scale: 1.0 | grad norm: 0.043 | params norm: 11627.298 | params rms: 0.062958 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 13:41:31.996360 | finish at 2026-10-10 19:51:26 + [2026-10-04 06:11:25] iteration 574/ 6855 | consumed samples: 73472 | elapsed time per iteration (ms): 90974.0 | throughput per GPU (TFLOP/s/GPU): 41.7 | MFU 13.36% | FLOPs estimator: workload-aware | logical tokens/s: 184417.7 | valid tokens/s: 184417.7 | physical tokens/s: 184794.3 | loss tokens/s: 131874.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.046933E-01 | lm-global-supervised-tokens: 1.199714E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.598317E-02 | lm-token-buffer-mib: 5.812845E+01 | global_load_balancing_loss: 1.001209E+00 | loss scale: 1.0 | grad norm: 0.039 | params norm: 11627.203 | params rms: 0.062958 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 14:43:27.518648 | finish at 2026-10-10 20:54:53 + [2026-10-04 06:12:54] iteration 575/ 6855 | consumed samples: 73600 | elapsed time per iteration (ms): 88107.2 | throughput per GPU (TFLOP/s/GPU): 41.5 | MFU 13.31% | FLOPs estimator: workload-aware | logical tokens/s: 190417.3 | valid tokens/s: 190417.3 | physical tokens/s: 190861.3 | loss tokens/s: 137715.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.381037E-01 | lm-global-supervised-tokens: 1.213376E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.544278E-02 | lm-token-buffer-mib: 5.807169E+01 | global_load_balancing_loss: 9.985049E-01 | loss scale: 1.0 | grad norm: 0.054 | params norm: 11627.106 | params rms: 0.062957 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 9:41:53.218412 | finish at 2026-10-10 15:54:47 + [2026-10-04 06:14:21] iteration 576/ 6855 | consumed samples: 73728 | elapsed time per iteration (ms): 87645.0 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.64% | FLOPs estimator: workload-aware | logical tokens/s: 191421.5 | valid tokens/s: 191421.5 | physical tokens/s: 191721.7 | loss tokens/s: 136657.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.774379E-01 | lm-global-supervised-tokens: 1.197734E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.583979E-02 | lm-token-buffer-mib: 5.809066E+01 | global_load_balancing_loss: 9.992144E-01 | loss scale: 1.0 | grad norm: 0.038 | params norm: 11627.008 | params rms: 0.062956 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 8:52:03.068655 | finish at 2026-10-10 15:06:24 + [2026-10-04 06:15:49] iteration 577/ 6855 | consumed samples: 73856 | elapsed time per iteration (ms): 88141.1 | throughput per GPU (TFLOP/s/GPU): 41.4 | MFU 13.27% | FLOPs estimator: workload-aware | logical tokens/s: 190344.2 | valid tokens/s: 190344.2 | physical tokens/s: 190620.0 | loss tokens/s: 145382.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.440853E-01 | lm-global-supervised-tokens: 1.281418E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.455304E-02 | lm-token-buffer-mib: 5.806194E+01 | global_load_balancing_loss: 9.996989E-01 | loss scale: 1.0 | grad norm: 0.055 | params norm: 11626.910 | params rms: 0.062956 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 9:42:29.847811 | finish at 2026-10-10 15:58:19 + [2026-10-04 06:17:16] iteration 578/ 6855 | consumed samples: 73984 | elapsed time per iteration (ms): 86767.5 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.58% | FLOPs estimator: workload-aware | logical tokens/s: 193358.2 | valid tokens/s: 193358.2 | physical tokens/s: 193782.1 | loss tokens/s: 141110.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.527841E-01 | lm-global-supervised-tokens: 1.224383E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.262003E-02 | lm-token-buffer-mib: 5.807439E+01 | global_load_balancing_loss: 1.000220E+00 | loss scale: 1.0 | grad norm: 0.043 | params norm: 11626.811 | params rms: 0.062955 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 7:17:19.381517 | finish at 2026-10-10 13:34:35 + [2026-10-04 06:18:43] iteration 579/ 6855 | consumed samples: 74112 | elapsed time per iteration (ms): 86580.7 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.60% | FLOPs estimator: workload-aware | logical tokens/s: 193775.2 | valid tokens/s: 193775.2 | physical tokens/s: 194175.2 | loss tokens/s: 142551.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.831177E-01 | lm-global-supervised-tokens: 1.234219E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.316932E-02 | lm-token-buffer-mib: 5.828561E+01 | global_load_balancing_loss: 1.001205E+00 | loss scale: 1.0 | grad norm: 0.038 | params norm: 11626.710 | params rms: 0.062955 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 6:56:20.496643 | finish at 2026-10-10 13:15:03 + [2026-10-04 06:20:27] iteration 580/ 6855 | consumed samples: 74240 | elapsed time per iteration (ms): 104398.7 | throughput per GPU (TFLOP/s/GPU): 34.5 | MFU 11.06% | FLOPs estimator: workload-aware | logical tokens/s: 160703.3 | valid tokens/s: 160703.3 | physical tokens/s: 161080.6 | loss tokens/s: 121380.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.260146E-01 | lm-global-supervised-tokens: 1.267196E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.248095E-02 | lm-token-buffer-mib: 5.806464E+01 | global_load_balancing_loss: 1.000812E+00 | loss scale: 1.0 | grad norm: 0.044 | params norm: 11626.610 | params rms: 0.062954 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 7 days, 13:58:21.897848 | finish at 2026-10-11 20:18:49 + [2026-10-04 06:21:56] iteration 581/ 6855 | consumed samples: 74368 | elapsed time per iteration (ms): 88486.1 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.50% | FLOPs estimator: workload-aware | logical tokens/s: 189602.7 | valid tokens/s: 189602.7 | physical tokens/s: 190014.5 | loss tokens/s: 135336.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.734777E-01 | lm-global-supervised-tokens: 1.197539E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.292102E-02 | lm-token-buffer-mib: 5.802232E+01 | global_load_balancing_loss: 1.000739E+00 | loss scale: 1.0 | grad norm: 0.031 | params norm: 11626.510 | params rms: 0.062954 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 10:12:41.521888 | finish at 2026-10-10 16:34:37 + [2026-10-04 06:23:19] iteration 582/ 6855 | consumed samples: 74496 | elapsed time per iteration (ms): 83670.8 | throughput per GPU (TFLOP/s/GPU): 44.1 | MFU 14.14% | FLOPs estimator: workload-aware | logical tokens/s: 200513.7 | valid tokens/s: 200513.7 | physical tokens/s: 201045.8 | loss tokens/s: 148867.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.806553E-01 | lm-global-supervised-tokens: 1.245590E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.400350E-02 | lm-token-buffer-mib: 5.813866E+01 | global_load_balancing_loss: 1.000657E+00 | loss scale: 1.0 | grad norm: 0.092 | params norm: 11626.411 | params rms: 0.062953 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 1:47:47.070298 | finish at 2026-10-10 08:11:06 + [2026-10-04 06:24:47] iteration 583/ 6855 | consumed samples: 74624 | elapsed time per iteration (ms): 87838.0 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.39% | FLOPs estimator: workload-aware | logical tokens/s: 191001.5 | valid tokens/s: 191001.5 | physical tokens/s: 191587.4 | loss tokens/s: 138699.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.450999E-01 | lm-global-supervised-tokens: 1.218305E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.536548E-02 | lm-token-buffer-mib: 5.822236E+01 | global_load_balancing_loss: 1.001543E+00 | loss scale: 1.0 | grad norm: 0.111 | params norm: 11626.310 | params rms: 0.062953 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 9:01:59.987213 | finish at 2026-10-10 15:26:47 + [2026-10-04 06:26:13] iteration 584/ 6855 | consumed samples: 74752 | elapsed time per iteration (ms): 86137.7 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.74% | FLOPs estimator: workload-aware | logical tokens/s: 194771.5 | valid tokens/s: 194771.5 | physical tokens/s: 195226.7 | loss tokens/s: 145301.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.284944E-01 | lm-global-supervised-tokens: 1.251591E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.389866E-02 | lm-token-buffer-mib: 5.807878E+01 | global_load_balancing_loss: 9.993861E-01 | loss scale: 1.0 | grad norm: 0.086 | params norm: 11626.210 | params rms: 0.062952 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 6:02:49.760912 | finish at 2026-10-10 12:29:03 + [2026-10-04 06:27:44] iteration 585/ 6855 | consumed samples: 74880 | elapsed time per iteration (ms): 91098.2 | throughput per GPU (TFLOP/s/GPU): 41.3 | MFU 13.23% | FLOPs estimator: workload-aware | logical tokens/s: 184166.1 | valid tokens/s: 184166.1 | physical tokens/s: 184769.1 | loss tokens/s: 144283.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.921028E-01 | lm-global-supervised-tokens: 1.314392E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.296990E-02 | lm-token-buffer-mib: 5.811868E+01 | global_load_balancing_loss: 9.987231E-01 | loss scale: 1.0 | grad norm: 0.081 | params norm: 11626.109 | params rms: 0.062952 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 14:39:45.541112 | finish at 2026-10-10 21:07:30 + [2026-10-04 06:29:14] iteration 586/ 6855 | consumed samples: 75008 | elapsed time per iteration (ms): 90074.7 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.50% | FLOPs estimator: workload-aware | logical tokens/s: 186258.9 | valid tokens/s: 186258.9 | physical tokens/s: 186638.5 | loss tokens/s: 137099.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.435255E-01 | lm-global-supervised-tokens: 1.234922E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.141527E-02 | lm-token-buffer-mib: 5.810812E+01 | global_load_balancing_loss: 1.000580E+00 | loss scale: 1.0 | grad norm: 0.061 | params norm: 11626.011 | params rms: 0.062951 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 12:51:18.299518 | finish at 2026-10-10 19:20:33 + [2026-10-04 06:30:43] iteration 587/ 6855 | consumed samples: 75136 | elapsed time per iteration (ms): 89146.0 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.40% | FLOPs estimator: workload-aware | logical tokens/s: 188199.0 | valid tokens/s: 188199.0 | physical tokens/s: 188574.5 | loss tokens/s: 136092.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.655878E-01 | lm-global-supervised-tokens: 1.213212E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.169994E-02 | lm-token-buffer-mib: 5.824915E+01 | global_load_balancing_loss: 9.997050E-01 | loss scale: 1.0 | grad norm: 0.059 | params norm: 11625.912 | params rms: 0.062951 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 11:12:46.967525 | finish at 2026-10-10 17:43:30 + [2026-10-04 06:32:14] iteration 588/ 6855 | consumed samples: 75264 | elapsed time per iteration (ms): 90420.2 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.44% | FLOPs estimator: workload-aware | logical tokens/s: 185547.0 | valid tokens/s: 185547.0 | physical tokens/s: 186057.6 | loss tokens/s: 135382.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.841660E-01 | lm-global-supervised-tokens: 1.224132E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.071846E-02 | lm-token-buffer-mib: 5.811875E+01 | global_load_balancing_loss: 1.000079E+00 | loss scale: 1.0 | grad norm: 0.064 | params norm: 11625.813 | params rms: 0.062950 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 13:24:23.277541 | finish at 2026-10-10 19:56:37 + [2026-10-04 06:33:43] iteration 589/ 6855 | consumed samples: 75392 | elapsed time per iteration (ms): 89422.6 | throughput per GPU (TFLOP/s/GPU): 41.5 | MFU 13.30% | FLOPs estimator: workload-aware | logical tokens/s: 187617.1 | valid tokens/s: 187617.1 | physical tokens/s: 188122.4 | loss tokens/s: 134964.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.097169E-01 | lm-global-supervised-tokens: 1.206892E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.452203E-02 | lm-token-buffer-mib: 5.829131E+01 | global_load_balancing_loss: 1.000548E+00 | loss scale: 1.0 | grad norm: 0.036 | params norm: 11625.713 | params rms: 0.062949 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 11:38:42.170150 | finish at 2026-10-10 18:12:26 + [2026-10-04 06:35:29] iteration 590/ 6855 | consumed samples: 75520 | elapsed time per iteration (ms): 106026.8 | throughput per GPU (TFLOP/s/GPU): 35.3 | MFU 11.32% | FLOPs estimator: workload-aware | logical tokens/s: 158235.3 | valid tokens/s: 158235.3 | physical tokens/s: 158444.8 | loss tokens/s: 113944.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.611761E-01 | lm-global-supervised-tokens: 1.208118E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.816773E-02 | lm-token-buffer-mib: 5.809782E+01 | global_load_balancing_loss: 1.000751E+00 | loss scale: 1.0 | grad norm: 0.047 | params norm: 11625.615 | params rms: 0.062949 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 7 days, 16:30:57.943305 | finish at 2026-10-11 23:06:27 + [2026-10-04 06:36:59] iteration 591/ 6855 | consumed samples: 75648 | elapsed time per iteration (ms): 89691.7 | throughput per GPU (TFLOP/s/GPU): 41.4 | MFU 13.26% | FLOPs estimator: workload-aware | logical tokens/s: 187053.6 | valid tokens/s: 187053.6 | physical tokens/s: 187604.1 | loss tokens/s: 132484.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.954807E-01 | lm-global-supervised-tokens: 1.188278E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.242356E-02 | lm-token-buffer-mib: 5.816172E+01 | global_load_balancing_loss: 1.000567E+00 | loss scale: 1.0 | grad norm: 0.058 | params norm: 11625.519 | params rms: 0.062948 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 12:03:48.689209 | finish at 2026-10-10 18:40:48 + [2026-10-04 06:38:26] iteration 592/ 6855 | consumed samples: 75776 | elapsed time per iteration (ms): 86860.4 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.51% | FLOPs estimator: workload-aware | logical tokens/s: 193151.2 | valid tokens/s: 193151.1 | physical tokens/s: 193725.0 | loss tokens/s: 142172.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.830813E-01 | lm-global-supervised-tokens: 1.234915E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.307198E-02 | lm-token-buffer-mib: 5.819228E+01 | global_load_balancing_loss: 1.000962E+00 | loss scale: 1.0 | grad norm: 0.097 | params norm: 11625.421 | params rms: 0.062948 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 7:06:46.968670 | finish at 2026-10-10 13:45:13 + [2026-10-04 06:39:52] iteration 593/ 6855 | consumed samples: 75904 | elapsed time per iteration (ms): 86140.6 | throughput per GPU (TFLOP/s/GPU): 43.7 | MFU 14.02% | FLOPs estimator: workload-aware | logical tokens/s: 194764.7 | valid tokens/s: 194764.7 | physical tokens/s: 195092.2 | loss tokens/s: 137946.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.715440E-01 | lm-global-supervised-tokens: 1.188277E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.160785E-02 | lm-token-buffer-mib: 5.812950E+01 | global_load_balancing_loss: 1.000109E+00 | loss scale: 1.0 | grad norm: 0.121 | params norm: 11625.324 | params rms: 0.062947 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 5:50:12.448931 | finish at 2026-10-10 12:30:05 + [2026-10-04 06:41:17] iteration 594/ 6855 | consumed samples: 76032 | elapsed time per iteration (ms): 84562.5 | throughput per GPU (TFLOP/s/GPU): 43.4 | MFU 13.90% | FLOPs estimator: workload-aware | logical tokens/s: 198399.5 | valid tokens/s: 198399.5 | physical tokens/s: 198975.2 | loss tokens/s: 148175.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.338880E-01 | lm-global-supervised-tokens: 1.253010E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.347373E-02 | lm-token-buffer-mib: 5.815370E+01 | global_load_balancing_loss: 1.000946E+00 | loss scale: 1.0 | grad norm: 0.107 | params norm: 11625.226 | params rms: 0.062947 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 3:04:05.708008 | finish at 2026-10-10 09:45:22 + [2026-10-04 06:42:48] iteration 595/ 6855 | consumed samples: 76160 | elapsed time per iteration (ms): 91174.2 | throughput per GPU (TFLOP/s/GPU): 40.6 | MFU 13.00% | FLOPs estimator: workload-aware | logical tokens/s: 184012.7 | valid tokens/s: 184012.7 | physical tokens/s: 184437.1 | loss tokens/s: 133924.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.694814E-01 | lm-global-supervised-tokens: 1.221050E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.215376E-02 | lm-token-buffer-mib: 5.816837E+01 | global_load_balancing_loss: 9.995620E-01 | loss scale: 1.0 | grad norm: 0.053 | params norm: 11625.130 | params rms: 0.062946 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 14:32:30.575943 | finish at 2026-10-10 21:15:18 + [2026-10-04 06:44:11] iteration 596/ 6855 | consumed samples: 76288 | elapsed time per iteration (ms): 83620.9 | throughput per GPU (TFLOP/s/GPU): 43.4 | MFU 13.93% | FLOPs estimator: workload-aware | logical tokens/s: 200634.0 | valid tokens/s: 200634.0 | physical tokens/s: 201169.4 | loss tokens/s: 148253.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.649137E-01 | lm-global-supervised-tokens: 1.239711E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.414953E-02 | lm-token-buffer-mib: 5.811912E+01 | global_load_balancing_loss: 9.984984E-01 | loss scale: 1.0 | grad norm: 0.059 | params norm: 11625.033 | params rms: 0.062946 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 1:23:03.233464 | finish at 2026-10-10 08:07:15 + [2026-10-04 06:45:38] iteration 597/ 6855 | consumed samples: 76416 | elapsed time per iteration (ms): 86661.7 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.60% | FLOPs estimator: workload-aware | logical tokens/s: 193593.3 | valid tokens/s: 193593.3 | physical tokens/s: 194265.5 | loss tokens/s: 136990.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.983031E-01 | lm-global-supervised-tokens: 1.187184E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.487052E-02 | lm-token-buffer-mib: 5.817248E+01 | global_load_balancing_loss: 1.000458E+00 | loss scale: 1.0 | grad norm: 0.095 | params norm: 11624.935 | params rms: 0.062945 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 6:38:48.815715 | finish at 2026-10-10 13:24:27 + [2026-10-04 06:47:04] iteration 598/ 6855 | consumed samples: 76544 | elapsed time per iteration (ms): 85731.5 | throughput per GPU (TFLOP/s/GPU): 44.3 | MFU 14.21% | FLOPs estimator: workload-aware | logical tokens/s: 195694.5 | valid tokens/s: 195694.5 | physical tokens/s: 196143.7 | loss tokens/s: 149689.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.345493E-01 | lm-global-supervised-tokens: 1.283310E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.662844E-02 | lm-token-buffer-mib: 5.814450E+01 | global_load_balancing_loss: 1.000657E+00 | loss scale: 1.0 | grad norm: 0.046 | params norm: 11624.837 | params rms: 0.062945 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 5:00:22.197822 | finish at 2026-10-10 11:47:26 + [2026-10-04 06:48:31] iteration 599/ 6855 | consumed samples: 76672 | elapsed time per iteration (ms): 86831.0 | throughput per GPU (TFLOP/s/GPU): 42.8 | MFU 13.71% | FLOPs estimator: workload-aware | logical tokens/s: 193216.2 | valid tokens/s: 193216.2 | physical tokens/s: 193837.7 | loss tokens/s: 142078.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.587772E-01 | lm-global-supervised-tokens: 1.233683E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.381784E-02 | lm-token-buffer-mib: 5.826493E+01 | global_load_balancing_loss: 1.000516E+00 | loss scale: 1.0 | grad norm: 0.059 | params norm: 11624.741 | params rms: 0.062944 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 6:53:34.727612 | finish at 2026-10-10 13:42:05 + [2026-10-04 06:50:20] iteration 600/ 6855 | consumed samples: 76800 | elapsed time per iteration (ms): 109226.2 | throughput per GPU (TFLOP/s/GPU): 33.7 | MFU 10.80% | FLOPs estimator: workload-aware | logical tokens/s: 153600.2 | valid tokens/s: 153600.2 | physical tokens/s: 154042.9 | loss tokens/s: 115686.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.061980E-01 | lm-global-supervised-tokens: 1.263595E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.276436E-02 | lm-token-buffer-mib: 5.820156E+01 | global_load_balancing_loss: 1.000008E+00 | loss scale: 1.0 | grad norm: 0.047 | params norm: 11624.643 | params rms: 0.062944 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 7 days, 21:46:49.778749 | finish at 2026-10-12 04:37:10 +times across ranks (ms): + save-checkpoint: + rank 0: 25583.51 + rank 1: 25583.48 + rank 2: 25583.54 + rank 3: 25583.52 + rank 4: 25583.51 + rank 5: 25583.48 + rank 6: 25583.47 + rank 7: 25583.52 + rank 8: 25583.32 + rank 9: 25583.28 + rank 10: 25583.29 + rank 11: 25583.28 + rank 12: 25583.29 + rank 13: 25583.28 + rank 14: 25583.29 + rank 15: 25583.28 + rank 16: 25583.23 + rank 17: 25583.20 + rank 18: 25583.19 + rank 19: 25583.23 + rank 20: 25583.20 + rank 21: 25583.23 + rank 22: 25583.20 + rank 23: 25583.18 + rank 24: 25583.63 + rank 25: 25583.84 + rank 26: 25583.61 + rank 27: 25583.66 + rank 28: 25584.07 + rank 29: 25583.56 + rank 30: 25583.57 + rank 31: 25583.62 + rank 32: 25583.81 + rank 33: 25583.74 + rank 34: 25583.70 + rank 35: 25583.76 + rank 36: 25583.67 + rank 37: 25583.68 + rank 38: 25583.75 + rank 39: 25583.69 + rank 40: 25583.22 + rank 41: 25583.22 + rank 42: 25583.25 + rank 43: 25583.23 + rank 44: 25583.20 + rank 45: 25583.20 + rank 46: 25583.21 + rank 47: 25583.25 + rank 48: 25583.46 + rank 49: 25583.39 + rank 50: 25583.38 + rank 51: 25583.45 + rank 52: 25583.46 + rank 53: 25583.46 + rank 54: 25583.50 + rank 55: 25583.41 + rank 56: 25583.29 + rank 57: 25583.24 + rank 58: 25583.21 + rank 59: 25583.21 + rank 60: 25583.21 + rank 61: 25583.06 + rank 62: 25583.24 + rank 63: 25583.27 + [2026-10-04 06:52:12] iteration 601/ 6855 | consumed samples: 76928 | elapsed time per iteration (ms): 85978.9 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.80% | FLOPs estimator: workload-aware | logical tokens/s: 195131.3 | valid tokens/s: 195131.3 | physical tokens/s: 195484.3 | loss tokens/s: 139291.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.641868E-01 | lm-global-supervised-tokens: 1.197614E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.841483E-01 | lm-token-buffer-mib: 5.814209E+01 | global_load_balancing_loss: 1.000518E+00 | loss scale: 1.0 | grad norm: 0.052 | params norm: 11624.545 | params rms: 0.062943 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 5:21:52.308716 | finish at 2026-10-10 12:14:04 + [2026-10-04 06:53:42] iteration 602/ 6855 | consumed samples: 77056 | elapsed time per iteration (ms): 89795.5 | throughput per GPU (TFLOP/s/GPU): 40.2 | MFU 12.89% | FLOPs estimator: workload-aware | logical tokens/s: 186837.6 | valid tokens/s: 186837.6 | physical tokens/s: 187176.6 | loss tokens/s: 139743.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.283265E-01 | lm-global-supervised-tokens: 1.254830E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.439307E-02 | lm-token-buffer-mib: 5.805040E+01 | global_load_balancing_loss: 1.000785E+00 | loss scale: 1.0 | grad norm: 0.037 | params norm: 11624.448 | params rms: 0.062943 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 11:58:11.214044 | finish at 2026-10-10 18:51:53 + [2026-10-04 06:55:09] iteration 603/ 6855 | consumed samples: 77184 | elapsed time per iteration (ms): 87844.8 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.70% | FLOPs estimator: workload-aware | logical tokens/s: 190986.8 | valid tokens/s: 190986.8 | physical tokens/s: 191483.7 | loss tokens/s: 136953.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.384393E-01 | lm-global-supervised-tokens: 1.203067E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.769250E-02 | lm-token-buffer-mib: 5.812852E+01 | global_load_balancing_loss: 1.000745E+00 | loss scale: 1.0 | grad norm: 0.049 | params norm: 11624.350 | params rms: 0.062942 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 8:33:25.591192 | finish at 2026-10-10 15:28:35 + [2026-10-04 06:56:33] iteration 604/ 6855 | consumed samples: 77312 | elapsed time per iteration (ms): 83641.5 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.76% | FLOPs estimator: workload-aware | logical tokens/s: 200584.5 | valid tokens/s: 200584.5 | physical tokens/s: 201005.3 | loss tokens/s: 149106.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.666750E-01 | lm-global-supervised-tokens: 1.247152E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.474643E-02 | lm-token-buffer-mib: 5.808175E+01 | global_load_balancing_loss: 9.999314E-01 | loss scale: 1.0 | grad norm: 0.051 | params norm: 11624.253 | params rms: 0.062942 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 1:14:03.028399 | finish at 2026-10-10 08:10:36 + [2026-10-04 06:57:59] iteration 605/ 6855 | consumed samples: 77440 | elapsed time per iteration (ms): 86317.1 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.61% | FLOPs estimator: workload-aware | logical tokens/s: 194367.0 | valid tokens/s: 194367.0 | physical tokens/s: 194890.2 | loss tokens/s: 142058.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.654527E-01 | lm-global-supervised-tokens: 1.226203E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.581771E-02 | lm-token-buffer-mib: 5.824049E+01 | global_load_balancing_loss: 9.998336E-01 | loss scale: 1.0 | grad norm: 0.067 | params norm: 11624.157 | params rms: 0.062941 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 5:51:21.604099 | finish at 2026-10-10 12:49:21 + [2026-10-04 06:59:28] iteration 606/ 6855 | consumed samples: 77568 | elapsed time per iteration (ms): 88490.0 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.49% | FLOPs estimator: workload-aware | logical tokens/s: 189593.7 | valid tokens/s: 189593.7 | physical tokens/s: 190117.5 | loss tokens/s: 134917.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.404946E-01 | lm-global-supervised-tokens: 1.193884E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.158030E-02 | lm-token-buffer-mib: 5.808627E+01 | global_load_balancing_loss: 9.996748E-01 | loss scale: 1.0 | grad norm: 0.161 | params norm: 11624.060 | params rms: 0.062941 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 9:36:13.764230 | finish at 2026-10-10 16:35:42 + [2026-10-04 07:00:53] iteration 607/ 6855 | consumed samples: 77696 | elapsed time per iteration (ms): 84765.6 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.54% | FLOPs estimator: workload-aware | logical tokens/s: 197923.1 | valid tokens/s: 197923.1 | physical tokens/s: 198470.4 | loss tokens/s: 149387.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.610364E-01 | lm-global-supervised-tokens: 1.266293E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.311233E-02 | lm-token-buffer-mib: 5.804879E+01 | global_load_balancing_loss: 9.988772E-01 | loss scale: 1.0 | grad norm: 0.341 | params norm: 11623.964 | params rms: 0.062940 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 3:06:55.650324 | finish at 2026-10-10 10:07:48 + [2026-10-04 07:02:20] iteration 608/ 6855 | consumed samples: 77824 | elapsed time per iteration (ms): 87164.6 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.59% | FLOPs estimator: workload-aware | logical tokens/s: 192476.4 | valid tokens/s: 192476.4 | physical tokens/s: 193303.3 | loss tokens/s: 144557.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.844691E-01 | lm-global-supervised-tokens: 1.260033E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.284896E-02 | lm-token-buffer-mib: 5.808289E+01 | global_load_balancing_loss: 1.000506E+00 | loss scale: 1.0 | grad norm: 0.212 | params norm: 11623.866 | params rms: 0.062939 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 7:15:17.172141 | finish at 2026-10-10 14:17:37 + [2026-10-04 07:03:49] iteration 609/ 6855 | consumed samples: 77952 | elapsed time per iteration (ms): 89093.3 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.47% | FLOPs estimator: workload-aware | logical tokens/s: 188310.3 | valid tokens/s: 188310.3 | physical tokens/s: 188767.4 | loss tokens/s: 136659.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.793706E-01 | lm-global-supervised-tokens: 1.217541E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.153172E-02 | lm-token-buffer-mib: 5.833028E+01 | global_load_balancing_loss: 1.000648E+00 | loss scale: 1.0 | grad norm: 0.126 | params norm: 11623.770 | params rms: 0.062939 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 10:34:36.591621 | finish at 2026-10-10 17:38:25 + [2026-10-04 07:05:36] iteration 610/ 6855 | consumed samples: 78080 | elapsed time per iteration (ms): 106807.9 | throughput per GPU (TFLOP/s/GPU): 33.6 | MFU 10.78% | FLOPs estimator: workload-aware | logical tokens/s: 157078.0 | valid tokens/s: 157078.0 | physical tokens/s: 157517.4 | loss tokens/s: 120521.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.145768E-01 | lm-global-supervised-tokens: 1.287263E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.177318E-02 | lm-token-buffer-mib: 5.816269E+01 | global_load_balancing_loss: 1.001391E+00 | loss scale: 1.0 | grad norm: 0.071 | params norm: 11623.673 | params rms: 0.062938 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 7 days, 17:16:55.144618 | finish at 2026-10-12 00:22:31 + [2026-10-04 07:07:02] iteration 611/ 6855 | consumed samples: 78208 | elapsed time per iteration (ms): 86345.4 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.51% | FLOPs estimator: workload-aware | logical tokens/s: 194303.4 | valid tokens/s: 194303.4 | physical tokens/s: 194685.2 | loss tokens/s: 148304.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.147263E-01 | lm-global-supervised-tokens: 1.280538E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.547890E-02 | lm-token-buffer-mib: 5.822827E+01 | global_load_balancing_loss: 1.000506E+00 | loss scale: 1.0 | grad norm: 0.156 | params norm: 11623.576 | params rms: 0.062938 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 5:45:40.928292 | finish at 2026-10-10 12:52:43 + [2026-10-04 07:08:29] iteration 612/ 6855 | consumed samples: 78336 | elapsed time per iteration (ms): 87289.6 | throughput per GPU (TFLOP/s/GPU): 41.5 | MFU 13.31% | FLOPs estimator: workload-aware | logical tokens/s: 192201.7 | valid tokens/s: 192201.7 | physical tokens/s: 192590.5 | loss tokens/s: 139933.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.921495E-01 | lm-global-supervised-tokens: 1.221474E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.536246E-02 | lm-token-buffer-mib: 5.814172E+01 | global_load_balancing_loss: 9.997589E-01 | loss scale: 1.0 | grad norm: 0.036 | params norm: 11623.479 | params rms: 0.062937 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 7:22:29.061454 | finish at 2026-10-10 14:30:58 + [2026-10-04 07:09:59] iteration 613/ 6855 | consumed samples: 78464 | elapsed time per iteration (ms): 89232.7 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.40% | FLOPs estimator: workload-aware | logical tokens/s: 188016.2 | valid tokens/s: 188016.2 | physical tokens/s: 188495.0 | loss tokens/s: 137791.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.494799E-01 | lm-global-supervised-tokens: 1.229547E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.186338E-02 | lm-token-buffer-mib: 5.819684E+01 | global_load_balancing_loss: 9.996033E-01 | loss scale: 1.0 | grad norm: 0.185 | params norm: 11623.381 | params rms: 0.062937 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 10:43:10.576588 | finish at 2026-10-10 17:53:09 + [2026-10-04 07:11:25] iteration 614/ 6855 | consumed samples: 78592 | elapsed time per iteration (ms): 86746.2 | throughput per GPU (TFLOP/s/GPU): 41.4 | MFU 13.28% | FLOPs estimator: workload-aware | logical tokens/s: 193405.7 | valid tokens/s: 193405.7 | physical tokens/s: 193820.9 | loss tokens/s: 143169.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.750672E-01 | lm-global-supervised-tokens: 1.241944E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.796692E-02 | lm-token-buffer-mib: 5.811501E+01 | global_load_balancing_loss: 9.995926E-01 | loss scale: 1.0 | grad norm: 0.031 | params norm: 11623.284 | params rms: 0.062936 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 6:23:03.269828 | finish at 2026-10-10 13:34:29 + [2026-10-04 07:12:51] iteration 615/ 6855 | consumed samples: 78720 | elapsed time per iteration (ms): 85661.8 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.46% | FLOPs estimator: workload-aware | logical tokens/s: 195853.7 | valid tokens/s: 195853.7 | physical tokens/s: 196332.4 | loss tokens/s: 148825.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.108674E-01 | lm-global-supervised-tokens: 1.274864E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.342621E-02 | lm-token-buffer-mib: 5.807597E+01 | global_load_balancing_loss: 9.992098E-01 | loss scale: 1.0 | grad norm: 0.139 | params norm: 11623.187 | params rms: 0.062936 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 4:28:49.743004 | finish at 2026-10-10 11:41:41 + [2026-10-04 07:14:21] iteration 616/ 6855 | consumed samples: 78848 | elapsed time per iteration (ms): 89783.8 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.39% | FLOPs estimator: workload-aware | logical tokens/s: 186862.0 | valid tokens/s: 186862.0 | physical tokens/s: 187354.4 | loss tokens/s: 135824.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.842402E-01 | lm-global-supervised-tokens: 1.219480E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.164962E-02 | lm-token-buffer-mib: 5.811261E+01 | global_load_balancing_loss: 1.001050E+00 | loss scale: 1.0 | grad norm: 0.064 | params norm: 11623.090 | params rms: 0.062935 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 11:36:01.127493 | finish at 2026-10-10 18:50:22 + [2026-10-04 07:15:47] iteration 617/ 6855 | consumed samples: 78976 | elapsed time per iteration (ms): 86022.7 | throughput per GPU (TFLOP/s/GPU): 43.8 | MFU 14.03% | FLOPs estimator: workload-aware | logical tokens/s: 195031.7 | valid tokens/s: 195031.7 | physical tokens/s: 195360.4 | loss tokens/s: 150067.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.262774E-01 | lm-global-supervised-tokens: 1.290923E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.508112E-02 | lm-token-buffer-mib: 5.808752E+01 | global_load_balancing_loss: 9.990295E-01 | loss scale: 1.0 | grad norm: 0.055 | params norm: 11622.991 | params rms: 0.062935 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 5:03:29.432011 | finish at 2026-10-10 12:19:16 + [2026-10-04 07:17:13] iteration 618/ 6855 | consumed samples: 79104 | elapsed time per iteration (ms): 86553.9 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.52% | FLOPs estimator: workload-aware | logical tokens/s: 193834.7 | valid tokens/s: 193834.7 | physical tokens/s: 194258.8 | loss tokens/s: 144383.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.721137E-01 | lm-global-supervised-tokens: 1.249699E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.327701E-02 | lm-token-buffer-mib: 5.807306E+01 | global_load_balancing_loss: 1.002946E+00 | loss scale: 1.0 | grad norm: 0.166 | params norm: 11622.893 | params rms: 0.062934 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 5:57:16.599971 | finish at 2026-10-10 13:14:30 + [2026-10-04 07:18:42] iteration 619/ 6855 | consumed samples: 79232 | elapsed time per iteration (ms): 88665.3 | throughput per GPU (TFLOP/s/GPU): 41.6 | MFU 13.33% | FLOPs estimator: workload-aware | logical tokens/s: 189219.4 | valid tokens/s: 189219.4 | physical tokens/s: 189732.1 | loss tokens/s: 139030.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.297010E-01 | lm-global-supervised-tokens: 1.232715E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.544840E-02 | lm-token-buffer-mib: 5.818412E+01 | global_load_balancing_loss: 9.994529E-01 | loss scale: 1.0 | grad norm: 0.067 | params norm: 11622.797 | params rms: 0.062934 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 9:35:16.920151 | finish at 2026-10-10 16:53:59 + [2026-10-04 07:20:30] iteration 620/ 6855 | consumed samples: 79360 | elapsed time per iteration (ms): 107728.8 | throughput per GPU (TFLOP/s/GPU): 35.1 | MFU 11.23% | FLOPs estimator: workload-aware | logical tokens/s: 155735.4 | valid tokens/s: 155735.4 | physical tokens/s: 156006.1 | loss tokens/s: 108085.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.064906E-01 | lm-global-supervised-tokens: 1.164390E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.511293E-02 | lm-token-buffer-mib: 5.804519E+01 | global_load_balancing_loss: 9.996631E-01 | loss scale: 1.0 | grad norm: 0.100 | params norm: 11622.701 | params rms: 0.062933 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 7 days, 18:34:49.025254 | finish at 2026-10-12 01:55:19 + [2026-10-04 07:21:57] iteration 621/ 6855 | consumed samples: 79488 | elapsed time per iteration (ms): 86984.1 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.44% | FLOPs estimator: workload-aware | logical tokens/s: 192876.2 | valid tokens/s: 192876.2 | physical tokens/s: 193351.5 | loss tokens/s: 140478.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.736990E-01 | lm-global-supervised-tokens: 1.221936E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.619275E-02 | lm-token-buffer-mib: 5.822057E+01 | global_load_balancing_loss: 9.987807E-01 | loss scale: 1.0 | grad norm: 0.078 | params norm: 11622.603 | params rms: 0.062933 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 6:37:38.668990 | finish at 2026-10-10 13:59:35 + [2026-10-04 07:23:22] iteration 622/ 6855 | consumed samples: 79616 | elapsed time per iteration (ms): 85471.7 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.55% | FLOPs estimator: workload-aware | logical tokens/s: 196289.4 | valid tokens/s: 196289.4 | physical tokens/s: 196829.5 | loss tokens/s: 147486.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.590226E-01 | lm-global-supervised-tokens: 1.260595E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.407086E-02 | lm-token-buffer-mib: 5.810788E+01 | global_load_balancing_loss: 9.995072E-01 | loss scale: 1.0 | grad norm: 0.099 | params norm: 11622.505 | params rms: 0.062932 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 3:59:04.934910 | finish at 2026-10-10 11:22:27 + [2026-10-04 07:24:50] iteration 623/ 6855 | consumed samples: 79744 | elapsed time per iteration (ms): 87427.8 | throughput per GPU (TFLOP/s/GPU): 41.7 | MFU 13.38% | FLOPs estimator: workload-aware | logical tokens/s: 191897.7 | valid tokens/s: 191897.7 | physical tokens/s: 192199.3 | loss tokens/s: 143352.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.924486E-01 | lm-global-supervised-tokens: 1.253296E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.192736E-02 | lm-token-buffer-mib: 5.810563E+01 | global_load_balancing_loss: 1.000481E+00 | loss scale: 1.0 | grad norm: 0.057 | params norm: 11622.409 | params rms: 0.062932 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 7:20:49.851612 | finish at 2026-10-10 14:45:39 + [2026-10-04 07:26:15] iteration 624/ 6855 | consumed samples: 79872 | elapsed time per iteration (ms): 85045.7 | throughput per GPU (TFLOP/s/GPU): 42.8 | MFU 13.72% | FLOPs estimator: workload-aware | logical tokens/s: 197272.4 | valid tokens/s: 197272.4 | physical tokens/s: 197817.6 | loss tokens/s: 147396.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.684828E-01 | lm-global-supervised-tokens: 1.253545E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.512053E-02 | lm-token-buffer-mib: 5.807623E+01 | global_load_balancing_loss: 1.001245E+00 | loss scale: 1.0 | grad norm: 0.081 | params norm: 11622.315 | params rms: 0.062931 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 3:11:59.803210 | finish at 2026-10-10 10:38:14 + [2026-10-04 07:27:41] iteration 625/ 6855 | consumed samples: 80000 | elapsed time per iteration (ms): 85983.9 | throughput per GPU (TFLOP/s/GPU): 43.7 | MFU 14.01% | FLOPs estimator: workload-aware | logical tokens/s: 195119.9 | valid tokens/s: 195119.9 | physical tokens/s: 195555.8 | loss tokens/s: 134551.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.385710E-01 | lm-global-supervised-tokens: 1.156930E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.139662E-02 | lm-token-buffer-mib: 5.804804E+01 | global_load_balancing_loss: 1.001919E+00 | loss scale: 1.0 | grad norm: 0.067 | params norm: 11622.219 | params rms: 0.062931 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 4:47:59.972227 | finish at 2026-10-10 12:15:41 + [2026-10-04 07:29:07] iteration 626/ 6855 | consumed samples: 80128 | elapsed time per iteration (ms): 86605.5 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.47% | FLOPs estimator: workload-aware | logical tokens/s: 193720.0 | valid tokens/s: 193720.0 | physical tokens/s: 194230.0 | loss tokens/s: 151188.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.684555E-01 | lm-global-supervised-tokens: 1.309379E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.183669E-02 | lm-token-buffer-mib: 5.815558E+01 | global_load_balancing_loss: 1.001109E+00 | loss scale: 1.0 | grad norm: 0.042 | params norm: 11622.123 | params rms: 0.062930 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 5:51:05.723253 | finish at 2026-10-10 13:20:13 + [2026-10-04 07:30:36] iteration 627/ 6855 | consumed samples: 80256 | elapsed time per iteration (ms): 88663.1 | throughput per GPU (TFLOP/s/GPU): 41.0 | MFU 13.14% | FLOPs estimator: workload-aware | logical tokens/s: 189223.7 | valid tokens/s: 189223.7 | physical tokens/s: 189821.1 | loss tokens/s: 138877.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.821325E-01 | lm-global-supervised-tokens: 1.231335E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.182230E-02 | lm-token-buffer-mib: 5.818413E+01 | global_load_balancing_loss: 9.996465E-01 | loss scale: 1.0 | grad norm: 0.050 | params norm: 11622.029 | params rms: 0.062930 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 9:23:13.688825 | finish at 2026-10-10 16:53:50 + [2026-10-04 07:32:07] iteration 628/ 6855 | consumed samples: 80384 | elapsed time per iteration (ms): 91364.3 | throughput per GPU (TFLOP/s/GPU): 41.6 | MFU 13.35% | FLOPs estimator: workload-aware | logical tokens/s: 183629.9 | valid tokens/s: 183629.9 | physical tokens/s: 184056.9 | loss tokens/s: 137298.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.664711E-01 | lm-global-supervised-tokens: 1.254414E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.605416E-02 | lm-token-buffer-mib: 5.815020E+01 | global_load_balancing_loss: 9.990227E-01 | loss scale: 1.0 | grad norm: 0.049 | params norm: 11621.937 | params rms: 0.062929 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 14:02:05.318022 | finish at 2026-10-10 21:34:13 + [2026-10-04 07:33:36] iteration 629/ 6855 | consumed samples: 80512 | elapsed time per iteration (ms): 88426.9 | throughput per GPU (TFLOP/s/GPU): 41.3 | MFU 13.22% | FLOPs estimator: workload-aware | logical tokens/s: 189729.3 | valid tokens/s: 189729.3 | physical tokens/s: 190292.3 | loss tokens/s: 136517.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.298194E-01 | lm-global-supervised-tokens: 1.207178E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.130859E-02 | lm-token-buffer-mib: 5.834351E+01 | global_load_balancing_loss: 9.995681E-01 | loss scale: 1.0 | grad norm: 0.044 | params norm: 11621.843 | params rms: 0.062928 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 8:55:45.975325 | finish at 2026-10-10 16:29:22 + [2026-10-04 07:35:25] iteration 630/ 6855 | consumed samples: 80640 | elapsed time per iteration (ms): 108868.9 | throughput per GPU (TFLOP/s/GPU): 34.3 | MFU 10.98% | FLOPs estimator: workload-aware | logical tokens/s: 154103.2 | valid tokens/s: 154103.2 | physical tokens/s: 154492.6 | loss tokens/s: 110197.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.685285E-01 | lm-global-supervised-tokens: 1.199711E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.255413E-02 | lm-token-buffer-mib: 5.810611E+01 | global_load_balancing_loss: 9.989785E-01 | loss scale: 1.0 | grad norm: 0.054 | params norm: 11621.749 | params rms: 0.062928 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 7 days, 20:15:08.931077 | finish at 2026-10-12 03:50:33 + [2026-10-04 07:36:53] iteration 631/ 6855 | consumed samples: 80768 | elapsed time per iteration (ms): 88732.7 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.44% | FLOPs estimator: workload-aware | logical tokens/s: 189075.7 | valid tokens/s: 189075.7 | physical tokens/s: 189656.6 | loss tokens/s: 139997.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.891226E-01 | lm-global-supervised-tokens: 1.242239E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.450962E-02 | lm-token-buffer-mib: 5.833486E+01 | global_load_balancing_loss: 1.001045E+00 | loss scale: 1.0 | grad norm: 0.051 | params norm: 11621.656 | params rms: 0.062927 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 9:24:32.266125 | finish at 2026-10-10 17:01:26 + [2026-10-04 07:38:18] iteration 632/ 6855 | consumed samples: 80896 | elapsed time per iteration (ms): 84681.4 | throughput per GPU (TFLOP/s/GPU): 43.4 | MFU 13.91% | FLOPs estimator: workload-aware | logical tokens/s: 198121.5 | valid tokens/s: 198121.5 | physical tokens/s: 198763.0 | loss tokens/s: 143178.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.746182E-01 | lm-global-supervised-tokens: 1.212455E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.145743E-02 | lm-token-buffer-mib: 5.810143E+01 | global_load_balancing_loss: 9.993068E-01 | loss scale: 1.0 | grad norm: 0.075 | params norm: 11621.563 | params rms: 0.062927 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 2:22:52.641895 | finish at 2026-10-10 10:01:11 + [2026-10-04 07:39:47] iteration 633/ 6855 | consumed samples: 81024 | elapsed time per iteration (ms): 89125.0 | throughput per GPU (TFLOP/s/GPU): 41.5 | MFU 13.29% | FLOPs estimator: workload-aware | logical tokens/s: 188243.4 | valid tokens/s: 188243.4 | physical tokens/s: 188685.4 | loss tokens/s: 140210.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.276778E-01 | lm-global-supervised-tokens: 1.249629E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.388356E-02 | lm-token-buffer-mib: 5.808367E+01 | global_load_balancing_loss: 9.985974E-01 | loss scale: 1.0 | grad norm: 0.036 | params norm: 11621.470 | params rms: 0.062926 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 10:02:15.935430 | finish at 2026-10-10 17:42:03 + [2026-10-04 07:41:15] iteration 634/ 6855 | consumed samples: 81152 | elapsed time per iteration (ms): 87828.0 | throughput per GPU (TFLOP/s/GPU): 42.8 | MFU 13.73% | FLOPs estimator: workload-aware | logical tokens/s: 191022.9 | valid tokens/s: 191022.9 | physical tokens/s: 191543.2 | loss tokens/s: 136464.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.060826E-01 | lm-global-supervised-tokens: 1.198537E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.274957E-02 | lm-token-buffer-mib: 5.838213E+01 | global_load_balancing_loss: 1.003565E+00 | loss scale: 1.0 | grad norm: 0.077 | params norm: 11621.376 | params rms: 0.062926 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 7:46:17.740732 | finish at 2026-10-10 15:27:33 + [2026-10-04 07:42:42] iteration 635/ 6855 | consumed samples: 81280 | elapsed time per iteration (ms): 87316.4 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.76% | FLOPs estimator: workload-aware | logical tokens/s: 192142.1 | valid tokens/s: 192142.1 | physical tokens/s: 192492.5 | loss tokens/s: 137748.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.324639E-01 | lm-global-supervised-tokens: 1.202773E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.585841E-02 | lm-token-buffer-mib: 5.810645E+01 | global_load_balancing_loss: 1.001327E+00 | loss scale: 1.0 | grad norm: 0.039 | params norm: 11621.285 | params rms: 0.062925 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 6:51:48.293047 | finish at 2026-10-10 14:34:31 + [2026-10-04 07:44:08] iteration 636/ 6855 | consumed samples: 81408 | elapsed time per iteration (ms): 85779.0 | throughput per GPU (TFLOP/s/GPU): 44.1 | MFU 14.13% | FLOPs estimator: workload-aware | logical tokens/s: 195586.6 | valid tokens/s: 195586.6 | physical tokens/s: 196045.2 | loss tokens/s: 143458.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.041767E-01 | lm-global-supervised-tokens: 1.230572E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.497618E-02 | lm-token-buffer-mib: 5.830354E+01 | global_load_balancing_loss: 1.000656E+00 | loss scale: 1.0 | grad norm: 0.058 | params norm: 11621.195 | params rms: 0.062925 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 4:10:59.454483 | finish at 2026-10-10 11:55:07 + [2026-10-04 07:45:33] iteration 637/ 6855 | consumed samples: 81536 | elapsed time per iteration (ms): 85196.6 | throughput per GPU (TFLOP/s/GPU): 43.5 | MFU 13.94% | FLOPs estimator: workload-aware | logical tokens/s: 196923.2 | valid tokens/s: 196923.2 | physical tokens/s: 197513.5 | loss tokens/s: 149720.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.036409E-01 | lm-global-supervised-tokens: 1.275569E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.807707E-02 | lm-token-buffer-mib: 5.825156E+01 | global_load_balancing_loss: 9.998466E-01 | loss scale: 1.0 | grad norm: 0.080 | params norm: 11621.104 | params rms: 0.062924 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 3:09:12.677961 | finish at 2026-10-10 10:54:46 + [2026-10-04 07:47:02] iteration 638/ 6855 | consumed samples: 81664 | elapsed time per iteration (ms): 89238.8 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.39% | FLOPs estimator: workload-aware | logical tokens/s: 188003.6 | valid tokens/s: 188003.5 | physical tokens/s: 188476.1 | loss tokens/s: 137821.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.850247E-01 | lm-global-supervised-tokens: 1.229902E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.245063E-02 | lm-token-buffer-mib: 5.820209E+01 | global_load_balancing_loss: 9.986969E-01 | loss scale: 1.0 | grad norm: 0.051 | params norm: 11621.012 | params rms: 0.062924 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 10:06:37.485019 | finish at 2026-10-10 17:53:40 + [2026-10-04 07:48:30] iteration 639/ 6855 | consumed samples: 81792 | elapsed time per iteration (ms): 87142.4 | throughput per GPU (TFLOP/s/GPU): 43.5 | MFU 13.95% | FLOPs estimator: workload-aware | logical tokens/s: 192526.0 | valid tokens/s: 192526.0 | physical tokens/s: 192990.7 | loss tokens/s: 139084.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.714363E-01 | lm-global-supervised-tokens: 1.212012E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.157722E-02 | lm-token-buffer-mib: 5.814018E+01 | global_load_balancing_loss: 9.994210E-01 | loss scale: 1.0 | grad norm: 0.079 | params norm: 11620.921 | params rms: 0.062924 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 6:27:56.881428 | finish at 2026-10-10 14:16:26 + [2026-10-04 07:50:17] iteration 640/ 6855 | consumed samples: 81920 | elapsed time per iteration (ms): 107436.7 | throughput per GPU (TFLOP/s/GPU): 34.0 | MFU 10.91% | FLOPs estimator: workload-aware | logical tokens/s: 156158.5 | valid tokens/s: 156158.5 | physical tokens/s: 156494.2 | loss tokens/s: 114588.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.385173E-01 | lm-global-supervised-tokens: 1.231098E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.127655E-02 | lm-token-buffer-mib: 5.811857E+01 | global_load_balancing_loss: 1.000445E+00 | loss scale: 1.0 | grad norm: 0.110 | params norm: 11620.831 | params rms: 0.062923 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 7 days, 17:28:38.925198 | finish at 2026-10-12 01:18:56 +times across ranks (ms): + save-checkpoint: + rank 0: 25301.71 + rank 1: 25301.70 + rank 2: 25301.74 + rank 3: 25301.73 + rank 4: 25301.72 + rank 5: 25301.69 + rank 6: 25301.67 + rank 7: 25301.72 + rank 8: 25302.28 + rank 9: 25302.25 + rank 10: 25302.25 + rank 11: 25302.24 + rank 12: 25302.24 + rank 13: 25302.25 + rank 14: 25302.26 + rank 15: 25302.24 + rank 16: 25301.90 + rank 17: 25301.85 + rank 18: 25301.87 + rank 19: 25301.89 + rank 20: 25301.87 + rank 21: 25301.89 + rank 22: 25301.87 + rank 23: 25301.86 + rank 24: 25302.45 + rank 25: 25302.38 + rank 26: 25302.47 + rank 27: 25302.36 + rank 28: 25302.38 + rank 29: 25302.42 + rank 30: 25302.39 + rank 31: 25302.45 + rank 32: 25301.87 + rank 33: 25301.86 + rank 34: 25301.90 + rank 35: 25301.84 + rank 36: 25301.79 + rank 37: 25301.79 + rank 38: 25301.86 + rank 39: 25301.80 + rank 40: 25302.60 + rank 41: 25302.58 + rank 42: 25302.59 + rank 43: 25302.59 + rank 44: 25302.59 + rank 45: 25302.61 + rank 46: 25302.59 + rank 47: 25302.56 + rank 48: 25302.50 + rank 49: 25302.10 + rank 50: 25302.09 + rank 51: 25302.14 + rank 52: 25302.14 + rank 53: 25302.12 + rank 54: 25302.14 + rank 55: 25302.12 + rank 56: 25302.12 + rank 57: 25302.11 + rank 58: 25301.93 + rank 59: 25302.06 + rank 60: 25302.07 + rank 61: 25302.14 + rank 62: 25302.09 + rank 63: 25302.15 + [2026-10-04 07:52:09] iteration 641/ 6855 | consumed samples: 82048 | elapsed time per iteration (ms): 86488.4 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.61% | FLOPs estimator: workload-aware | logical tokens/s: 193982.1 | valid tokens/s: 193982.1 | physical tokens/s: 194538.4 | loss tokens/s: 142299.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.871833E-01 | lm-global-supervised-tokens: 1.230722E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.005458E-01 | lm-token-buffer-mib: 5.810026E+01 | global_load_balancing_loss: 1.000772E+00 | loss scale: 1.0 | grad norm: 0.060 | params norm: 11620.740 | params rms: 0.062923 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 5:17:19.213798 | finish at 2026-10-10 13:09:28 + [2026-10-04 07:53:44] iteration 642/ 6855 | consumed samples: 82176 | elapsed time per iteration (ms): 94555.6 | throughput per GPU (TFLOP/s/GPU): 39.1 | MFU 12.55% | FLOPs estimator: workload-aware | logical tokens/s: 177431.8 | valid tokens/s: 177431.8 | physical tokens/s: 178008.5 | loss tokens/s: 128474.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.583769E-01 | lm-global-supervised-tokens: 1.214797E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.405864E-02 | lm-token-buffer-mib: 5.807229E+01 | global_load_balancing_loss: 9.998069E-01 | loss scale: 1.0 | grad norm: 0.112 | params norm: 11620.648 | params rms: 0.062922 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 19:11:13.849030 | finish at 2026-10-11 03:04:57 + [2026-10-04 07:55:10] iteration 643/ 6855 | consumed samples: 82304 | elapsed time per iteration (ms): 86524.7 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.52% | FLOPs estimator: workload-aware | logical tokens/s: 193899.9 | valid tokens/s: 193899.9 | physical tokens/s: 194307.3 | loss tokens/s: 145992.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.723388E-01 | lm-global-supervised-tokens: 1.263191E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.700458E-02 | lm-token-buffer-mib: 5.823081E+01 | global_load_balancing_loss: 1.000496E+00 | loss scale: 1.0 | grad norm: 0.078 | params norm: 11620.558 | params rms: 0.062922 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 5:18:11.130845 | finish at 2026-10-10 13:13:21 + [2026-10-04 07:56:39] iteration 644/ 6855 | consumed samples: 82432 | elapsed time per iteration (ms): 88638.0 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.54% | FLOPs estimator: workload-aware | logical tokens/s: 189277.5 | valid tokens/s: 189277.5 | physical tokens/s: 189720.6 | loss tokens/s: 139247.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.127652E-01 | lm-global-supervised-tokens: 1.234259E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.745744E-02 | lm-token-buffer-mib: 5.819956E+01 | global_load_balancing_loss: 9.991461E-01 | loss scale: 1.0 | grad norm: 0.094 | params norm: 11620.469 | params rms: 0.062921 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 8:55:30.826866 | finish at 2026-10-10 16:52:10 + [2026-10-04 07:58:04] iteration 645/ 6855 | consumed samples: 82560 | elapsed time per iteration (ms): 85152.4 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.61% | FLOPs estimator: workload-aware | logical tokens/s: 197025.3 | valid tokens/s: 197025.3 | physical tokens/s: 197632.8 | loss tokens/s: 146049.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.258351E-01 | lm-global-supervised-tokens: 1.243648E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.164827E-02 | lm-token-buffer-mib: 5.814317E+01 | global_load_balancing_loss: 1.001133E+00 | loss scale: 1.0 | grad norm: 0.117 | params norm: 11620.380 | params rms: 0.062921 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 2:53:16.424832 | finish at 2026-10-10 10:51:20 + [2026-10-04 07:59:30] iteration 646/ 6855 | consumed samples: 82688 | elapsed time per iteration (ms): 85583.7 | throughput per GPU (TFLOP/s/GPU): 43.4 | MFU 13.90% | FLOPs estimator: workload-aware | logical tokens/s: 196032.4 | valid tokens/s: 196032.4 | physical tokens/s: 196536.6 | loss tokens/s: 140779.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.593484E-01 | lm-global-supervised-tokens: 1.204840E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.205094E-02 | lm-token-buffer-mib: 5.827265E+01 | global_load_balancing_loss: 1.000002E+00 | loss scale: 1.0 | grad norm: 0.073 | params norm: 11620.291 | params rms: 0.062920 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 3:36:29.272876 | finish at 2026-10-10 11:35:59 + [2026-10-04 08:00:56] iteration 647/ 6855 | consumed samples: 82816 | elapsed time per iteration (ms): 86524.9 | throughput per GPU (TFLOP/s/GPU): 43.4 | MFU 13.92% | FLOPs estimator: workload-aware | logical tokens/s: 193899.1 | valid tokens/s: 193899.1 | physical tokens/s: 194245.7 | loss tokens/s: 147186.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.967469E-01 | lm-global-supervised-tokens: 1.273528E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.212740E-02 | lm-token-buffer-mib: 5.821387E+01 | global_load_balancing_loss: 1.000589E+00 | loss scale: 1.0 | grad norm: 0.174 | params norm: 11620.201 | params rms: 0.062920 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 5:12:26.407257 | finish at 2026-10-10 13:13:22 + [2026-10-04 08:02:19] iteration 648/ 6855 | consumed samples: 82944 | elapsed time per iteration (ms): 82974.8 | throughput per GPU (TFLOP/s/GPU): 43.9 | MFU 14.06% | FLOPs estimator: workload-aware | logical tokens/s: 202196.4 | valid tokens/s: 202196.4 | physical tokens/s: 202772.7 | loss tokens/s: 148740.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.884022E-01 | lm-global-supervised-tokens: 1.234173E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.171876E-02 | lm-token-buffer-mib: 5.818787E+01 | global_load_balancing_loss: 1.000560E+00 | loss scale: 1.0 | grad norm: 0.059 | params norm: 11620.112 | params rms: 0.062919 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 23:03:44.482171 | finish at 2026-10-10 07:06:03 + [2026-10-04 08:03:50] iteration 649/ 6855 | consumed samples: 83072 | elapsed time per iteration (ms): 90516.3 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.41% | FLOPs estimator: workload-aware | logical tokens/s: 185350.0 | valid tokens/s: 185350.0 | physical tokens/s: 185693.4 | loss tokens/s: 128445.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.814153E-01 | lm-global-supervised-tokens: 1.162639E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.485837E-02 | lm-token-buffer-mib: 5.816199E+01 | global_load_balancing_loss: 1.001658E+00 | loss scale: 1.0 | grad norm: 0.177 | params norm: 11620.023 | params rms: 0.062919 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 12:02:24.311451 | finish at 2026-10-10 20:06:14 + [2026-10-04 08:05:39] iteration 650/ 6855 | consumed samples: 83200 | elapsed time per iteration (ms): 109254.1 | throughput per GPU (TFLOP/s/GPU): 33.2 | MFU 10.65% | FLOPs estimator: workload-aware | logical tokens/s: 153561.0 | valid tokens/s: 153561.0 | physical tokens/s: 153971.2 | loss tokens/s: 116434.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.210849E-01 | lm-global-supervised-tokens: 1.272099E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.535361E-02 | lm-token-buffer-mib: 5.817356E+01 | global_load_balancing_loss: 1.000478E+00 | loss scale: 1.0 | grad norm: 0.074 | params norm: 11619.936 | params rms: 0.062918 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 7 days, 20:18:41.538646 | finish at 2026-10-12 04:24:20 + [2026-10-04 08:07:06] iteration 651/ 6855 | consumed samples: 83328 | elapsed time per iteration (ms): 86724.3 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.43% | FLOPs estimator: workload-aware | logical tokens/s: 193454.4 | valid tokens/s: 193454.3 | physical tokens/s: 193912.3 | loss tokens/s: 147245.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.627431E-01 | lm-global-supervised-tokens: 1.276980E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.244922E-02 | lm-token-buffer-mib: 5.814601E+01 | global_load_balancing_loss: 1.000052E+00 | loss scale: 1.0 | grad norm: 0.116 | params norm: 11619.849 | params rms: 0.062918 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 5:27:17.741521 | finish at 2026-10-10 13:34:23 + [2026-10-04 08:08:34] iteration 652/ 6855 | consumed samples: 83456 | elapsed time per iteration (ms): 88143.1 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.58% | FLOPs estimator: workload-aware | logical tokens/s: 190339.9 | valid tokens/s: 190339.9 | physical tokens/s: 190978.3 | loss tokens/s: 139971.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.094001E-01 | lm-global-supervised-tokens: 1.233752E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.202473E-02 | lm-token-buffer-mib: 5.815106E+01 | global_load_balancing_loss: 9.992744E-01 | loss scale: 1.0 | grad norm: 0.070 | params norm: 11619.760 | params rms: 0.062917 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 7:52:31.735221 | finish at 2026-10-10 16:01:05 + [2026-10-04 08:10:06] iteration 653/ 6855 | consumed samples: 83584 | elapsed time per iteration (ms): 92131.3 | throughput per GPU (TFLOP/s/GPU): 41.5 | MFU 13.29% | FLOPs estimator: workload-aware | logical tokens/s: 182101.2 | valid tokens/s: 182101.2 | physical tokens/s: 182529.1 | loss tokens/s: 133239.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.464604E-01 | lm-global-supervised-tokens: 1.227549E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.192301E-02 | lm-token-buffer-mib: 5.810307E+01 | global_load_balancing_loss: 9.985015E-01 | loss scale: 1.0 | grad norm: 0.082 | params norm: 11619.674 | params rms: 0.062917 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 14:43:18.068098 | finish at 2026-10-10 22:53:24 + [2026-10-04 08:11:33] iteration 654/ 6855 | consumed samples: 83712 | elapsed time per iteration (ms): 87127.5 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.51% | FLOPs estimator: workload-aware | logical tokens/s: 192558.5 | valid tokens/s: 192558.5 | physical tokens/s: 193210.6 | loss tokens/s: 138913.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.793255E-01 | lm-global-supervised-tokens: 1.210320E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.453264E-02 | lm-token-buffer-mib: 5.811061E+01 | global_load_balancing_loss: 9.999728E-01 | loss scale: 1.0 | grad norm: 0.129 | params norm: 11619.589 | params rms: 0.062916 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 6:04:37.422353 | finish at 2026-10-10 14:16:10 + [2026-10-04 08:12:57] iteration 655/ 6855 | consumed samples: 83840 | elapsed time per iteration (ms): 83648.6 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.61% | FLOPs estimator: workload-aware | logical tokens/s: 200567.8 | valid tokens/s: 200567.8 | physical tokens/s: 201020.4 | loss tokens/s: 145990.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.805541E-01 | lm-global-supervised-tokens: 1.221192E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.528497E-02 | lm-token-buffer-mib: 5.805674E+01 | global_load_balancing_loss: 9.998143E-01 | loss scale: 1.0 | grad norm: 0.117 | params norm: 11619.502 | params rms: 0.062916 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 0:03:41.144724 | finish at 2026-10-10 08:16:38 + [2026-10-04 08:14:24] iteration 656/ 6855 | consumed samples: 83968 | elapsed time per iteration (ms): 87833.3 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.39% | FLOPs estimator: workload-aware | logical tokens/s: 191011.5 | valid tokens/s: 191011.5 | physical tokens/s: 191468.9 | loss tokens/s: 138061.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.220319E-01 | lm-global-supervised-tokens: 1.212642E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.632399E-02 | lm-token-buffer-mib: 5.821577E+01 | global_load_balancing_loss: 1.000640E+00 | loss scale: 1.0 | grad norm: 0.144 | params norm: 11619.414 | params rms: 0.062915 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 7:14:38.674699 | finish at 2026-10-10 15:29:03 + [2026-10-04 08:15:52] iteration 657/ 6855 | consumed samples: 84096 | elapsed time per iteration (ms): 87333.5 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.55% | FLOPs estimator: workload-aware | logical tokens/s: 192104.5 | valid tokens/s: 192104.5 | physical tokens/s: 192558.0 | loss tokens/s: 142407.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.608506E-01 | lm-global-supervised-tokens: 1.243692E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.223531E-02 | lm-token-buffer-mib: 5.810317E+01 | global_load_balancing_loss: 1.000800E+00 | loss scale: 1.0 | grad norm: 0.079 | params norm: 11619.327 | params rms: 0.062915 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 6:21:33.192026 | finish at 2026-10-10 14:37:25 + [2026-10-04 08:17:19] iteration 658/ 6855 | consumed samples: 84224 | elapsed time per iteration (ms): 87142.4 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.49% | FLOPs estimator: workload-aware | logical tokens/s: 192526.5 | valid tokens/s: 192526.5 | physical tokens/s: 193034.9 | loss tokens/s: 146967.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.307519E-01 | lm-global-supervised-tokens: 1.280709E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.661881E-02 | lm-token-buffer-mib: 5.813987E+01 | global_load_balancing_loss: 9.999206E-01 | loss scale: 1.0 | grad norm: 0.164 | params norm: 11619.241 | params rms: 0.062914 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 6:00:21.266801 | finish at 2026-10-10 14:17:40 + [2026-10-04 08:18:44] iteration 659/ 6855 | consumed samples: 84352 | elapsed time per iteration (ms): 85619.9 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.59% | FLOPs estimator: workload-aware | logical tokens/s: 195949.9 | valid tokens/s: 195949.9 | physical tokens/s: 196543.8 | loss tokens/s: 149800.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.247807E-01 | lm-global-supervised-tokens: 1.282590E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.175115E-02 | lm-token-buffer-mib: 5.809283E+01 | global_load_balancing_loss: 1.000936E+00 | loss scale: 1.0 | grad norm: 0.084 | params norm: 11619.155 | params rms: 0.062914 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 3:21:40.956462 | finish at 2026-10-10 11:40:25 + [2026-10-04 08:20:28] iteration 660/ 6855 | consumed samples: 84480 | elapsed time per iteration (ms): 103793.4 | throughput per GPU (TFLOP/s/GPU): 34.7 | MFU 11.11% | FLOPs estimator: workload-aware | logical tokens/s: 161639.2 | valid tokens/s: 161639.2 | physical tokens/s: 162027.9 | loss tokens/s: 123357.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.773521E-01 | lm-global-supervised-tokens: 1.280366E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.453264E-02 | lm-token-buffer-mib: 5.835204E+01 | global_load_balancing_loss: 1.000268E+00 | loss scale: 1.0 | grad norm: 0.175 | params norm: 11619.068 | params rms: 0.062913 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 7 days, 10:36:39.943449 | finish at 2026-10-11 18:57:08 + [2026-10-04 08:21:59] iteration 661/ 6855 | consumed samples: 84608 | elapsed time per iteration (ms): 90559.0 | throughput per GPU (TFLOP/s/GPU): 41.3 | MFU 13.24% | FLOPs estimator: workload-aware | logical tokens/s: 185262.7 | valid tokens/s: 185262.7 | physical tokens/s: 185634.6 | loss tokens/s: 132864.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.094223E-01 | lm-global-supervised-tokens: 1.203212E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.282106E-02 | lm-token-buffer-mib: 5.807377E+01 | global_load_balancing_loss: 1.000969E+00 | loss scale: 1.0 | grad norm: 0.049 | params norm: 11618.982 | params rms: 0.062913 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 11:48:42.456432 | finish at 2026-10-10 20:10:41 + [2026-10-04 08:23:24] iteration 662/ 6855 | consumed samples: 84736 | elapsed time per iteration (ms): 85357.9 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.78% | FLOPs estimator: workload-aware | logical tokens/s: 196551.1 | valid tokens/s: 196550.9 | physical tokens/s: 197034.9 | loss tokens/s: 137173.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.345767E-01 | lm-global-supervised-tokens: 1.170883E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.164621E-02 | lm-token-buffer-mib: 5.808013E+01 | global_load_balancing_loss: 9.995666E-01 | loss scale: 1.0 | grad norm: 0.117 | params norm: 11618.895 | params rms: 0.062913 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 2:50:21.714781 | finish at 2026-10-10 11:13:46 + [2026-10-04 08:24:52] iteration 663/ 6855 | consumed samples: 84864 | elapsed time per iteration (ms): 87727.4 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.55% | FLOPs estimator: workload-aware | logical tokens/s: 191242.6 | valid tokens/s: 191242.6 | physical tokens/s: 191665.6 | loss tokens/s: 137910.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.200101E-01 | lm-global-supervised-tokens: 1.209850E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.368988E-02 | lm-token-buffer-mib: 5.806498E+01 | global_load_balancing_loss: 1.000511E+00 | loss scale: 1.0 | grad norm: 0.059 | params norm: 11618.807 | params rms: 0.062912 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 6:53:27.968193 | finish at 2026-10-10 15:18:20 + [2026-10-04 08:26:20] iteration 664/ 6855 | consumed samples: 84992 | elapsed time per iteration (ms): 88070.8 | throughput per GPU (TFLOP/s/GPU): 42.8 | MFU 13.72% | FLOPs estimator: workload-aware | logical tokens/s: 190495.8 | valid tokens/s: 190495.8 | physical tokens/s: 190947.7 | loss tokens/s: 137829.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.940799E-01 | lm-global-supervised-tokens: 1.213875E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.522909E-02 | lm-token-buffer-mib: 5.806450E+01 | global_load_balancing_loss: 9.998661E-01 | loss scale: 1.0 | grad norm: 0.068 | params norm: 11618.719 | params rms: 0.062912 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 7:27:26.243502 | finish at 2026-10-10 15:53:46 + [2026-10-04 08:27:48] iteration 665/ 6855 | consumed samples: 85120 | elapsed time per iteration (ms): 87639.8 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.45% | FLOPs estimator: workload-aware | logical tokens/s: 191433.3 | valid tokens/s: 191433.3 | physical tokens/s: 191961.3 | loss tokens/s: 142617.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.624365E-01 | lm-global-supervised-tokens: 1.249901E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.408719E-02 | lm-token-buffer-mib: 5.826139E+01 | global_load_balancing_loss: 9.988545E-01 | loss scale: 1.0 | grad norm: 0.052 | params norm: 11618.631 | params rms: 0.062911 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 6:41:30.427380 | finish at 2026-10-10 15:09:18 + [2026-10-04 08:29:16] iteration 666/ 6855 | consumed samples: 85248 | elapsed time per iteration (ms): 88017.8 | throughput per GPU (TFLOP/s/GPU): 41.1 | MFU 13.19% | FLOPs estimator: workload-aware | logical tokens/s: 190611.3 | valid tokens/s: 190611.3 | physical tokens/s: 191265.5 | loss tokens/s: 141914.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.060014E-01 | lm-global-supervised-tokens: 1.249103E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.165121E-02 | lm-token-buffer-mib: 5.828326E+01 | global_load_balancing_loss: 1.000181E+00 | loss scale: 1.0 | grad norm: 0.067 | params norm: 11618.544 | params rms: 0.062911 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 7:19:02.226748 | finish at 2026-10-10 15:48:18 + [2026-10-04 08:30:46] iteration 667/ 6855 | consumed samples: 85376 | elapsed time per iteration (ms): 90103.2 | throughput per GPU (TFLOP/s/GPU): 40.2 | MFU 12.89% | FLOPs estimator: workload-aware | logical tokens/s: 186199.8 | valid tokens/s: 186199.8 | physical tokens/s: 186910.2 | loss tokens/s: 144221.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.651829E-01 | lm-global-supervised-tokens: 1.299484E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.157962E-02 | lm-token-buffer-mib: 5.816991E+01 | global_load_balancing_loss: 1.000603E+00 | loss scale: 1.0 | grad norm: 0.114 | params norm: 11618.457 | params rms: 0.062910 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 10:52:38.486269 | finish at 2026-10-10 19:23:24 + [2026-10-04 08:32:11] iteration 668/ 6855 | consumed samples: 85504 | elapsed time per iteration (ms): 85490.6 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.57% | FLOPs estimator: workload-aware | logical tokens/s: 196245.7 | valid tokens/s: 196245.7 | physical tokens/s: 196740.7 | loss tokens/s: 147969.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.001898E-01 | lm-global-supervised-tokens: 1.264996E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.430979E-02 | lm-token-buffer-mib: 5.826134E+01 | global_load_balancing_loss: 1.000156E+00 | loss scale: 1.0 | grad norm: 0.130 | params norm: 11618.371 | params rms: 0.062910 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 2:55:30.487434 | finish at 2026-10-10 11:27:42 + [2026-10-04 08:33:38] iteration 669/ 6855 | consumed samples: 85632 | elapsed time per iteration (ms): 86571.2 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.54% | FLOPs estimator: workload-aware | logical tokens/s: 193796.2 | valid tokens/s: 193796.2 | physical tokens/s: 194115.3 | loss tokens/s: 139310.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.662510E-01 | lm-global-supervised-tokens: 1.206029E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.604955E-02 | lm-token-buffer-mib: 5.815118E+01 | global_load_balancing_loss: 1.001565E+00 | loss scale: 1.0 | grad norm: 0.034 | params norm: 11618.285 | params rms: 0.062909 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 4:45:29.678521 | finish at 2026-10-10 13:19:08 + [2026-10-04 08:35:24] iteration 670/ 6855 | consumed samples: 85760 | elapsed time per iteration (ms): 105984.3 | throughput per GPU (TFLOP/s/GPU): 33.9 | MFU 10.86% | FLOPs estimator: workload-aware | logical tokens/s: 158298.2 | valid tokens/s: 158298.2 | physical tokens/s: 158691.8 | loss tokens/s: 122873.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.815449E-01 | lm-global-supervised-tokens: 1.302267E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.349460E-02 | lm-token-buffer-mib: 5.808723E+01 | global_load_balancing_loss: 1.001267E+00 | loss scale: 1.0 | grad norm: 0.077 | params norm: 11618.199 | params rms: 0.062909 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 7 days, 14:05:12.803444 | finish at 2026-10-11 22:40:37 + [2026-10-04 08:36:53] iteration 671/ 6855 | consumed samples: 85888 | elapsed time per iteration (ms): 89002.7 | throughput per GPU (TFLOP/s/GPU): 41.7 | MFU 13.37% | FLOPs estimator: workload-aware | logical tokens/s: 188502.2 | valid tokens/s: 188502.2 | physical tokens/s: 188878.3 | loss tokens/s: 140085.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.977007E-01 | lm-global-supervised-tokens: 1.246797E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.545674E-02 | lm-token-buffer-mib: 5.827163E+01 | global_load_balancing_loss: 1.000455E+00 | loss scale: 1.0 | grad norm: 0.044 | params norm: 11618.113 | params rms: 0.062908 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 8:53:12.992235 | finish at 2026-10-10 17:30:06 + [2026-10-04 08:38:18] iteration 672/ 6855 | consumed samples: 86016 | elapsed time per iteration (ms): 84945.5 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.65% | FLOPs estimator: workload-aware | logical tokens/s: 197504.6 | valid tokens/s: 197504.6 | physical tokens/s: 198061.7 | loss tokens/s: 148745.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.748679E-01 | lm-global-supervised-tokens: 1.263525E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.784896E-02 | lm-token-buffer-mib: 5.820721E+01 | global_load_balancing_loss: 9.993958E-01 | loss scale: 1.0 | grad norm: 0.061 | params norm: 11618.027 | params rms: 0.062908 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 1:53:37.882871 | finish at 2026-10-10 10:31:56 + [2026-10-04 08:39:41] iteration 673/ 6855 | consumed samples: 86144 | elapsed time per iteration (ms): 83369.7 | throughput per GPU (TFLOP/s/GPU): 43.8 | MFU 14.03% | FLOPs estimator: workload-aware | logical tokens/s: 201238.2 | valid tokens/s: 201238.1 | physical tokens/s: 201700.4 | loss tokens/s: 149084.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.028534E-01 | lm-global-supervised-tokens: 1.242915E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.233326E-02 | lm-token-buffer-mib: 5.810783E+01 | global_load_balancing_loss: 9.998915E-01 | loss scale: 1.0 | grad norm: 0.096 | params norm: 11617.942 | params rms: 0.062907 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 23:09:51.424692 | finish at 2026-10-10 07:49:33 + [2026-10-04 08:41:08] iteration 674/ 6855 | consumed samples: 86272 | elapsed time per iteration (ms): 87096.0 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.61% | FLOPs estimator: workload-aware | logical tokens/s: 192629.0 | valid tokens/s: 192629.0 | physical tokens/s: 192992.2 | loss tokens/s: 136530.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.730176E-01 | lm-global-supervised-tokens: 1.189122E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.003936E-02 | lm-token-buffer-mib: 5.820107E+01 | global_load_balancing_loss: 9.991378E-01 | loss scale: 1.0 | grad norm: 0.036 | params norm: 11617.858 | params rms: 0.062907 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 5:32:20.294677 | finish at 2026-10-10 14:13:29 + [2026-10-04 08:42:38] iteration 675/ 6855 | consumed samples: 86400 | elapsed time per iteration (ms): 90000.7 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.48% | FLOPs estimator: workload-aware | logical tokens/s: 186412.0 | valid tokens/s: 186412.0 | physical tokens/s: 186830.2 | loss tokens/s: 136420.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.525623E-01 | lm-global-supervised-tokens: 1.227796E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.484456E-02 | lm-token-buffer-mib: 5.816599E+01 | global_load_balancing_loss: 9.999583E-01 | loss scale: 1.0 | grad norm: 0.090 | params norm: 11617.772 | params rms: 0.062906 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 10:30:04.330401 | finish at 2026-10-10 19:12:43 + [2026-10-04 08:44:03] iteration 676/ 6855 | consumed samples: 86528 | elapsed time per iteration (ms): 85245.4 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.74% | FLOPs estimator: workload-aware | logical tokens/s: 196810.7 | valid tokens/s: 196810.7 | physical tokens/s: 197223.1 | loss tokens/s: 149857.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.112074E-01 | lm-global-supervised-tokens: 1.277470E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.152530E-02 | lm-token-buffer-mib: 5.805273E+01 | global_load_balancing_loss: 1.000023E+00 | loss scale: 1.0 | grad norm: 0.048 | params norm: 11617.686 | params rms: 0.062906 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 2:18:51.597370 | finish at 2026-10-10 11:02:55 + [2026-10-04 08:45:32] iteration 677/ 6855 | consumed samples: 86656 | elapsed time per iteration (ms): 88703.3 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.62% | FLOPs estimator: workload-aware | logical tokens/s: 189138.0 | valid tokens/s: 189138.0 | physical tokens/s: 189576.9 | loss tokens/s: 138873.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.024463E-01 | lm-global-supervised-tokens: 1.231853E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.326703E-02 | lm-token-buffer-mib: 5.817344E+01 | global_load_balancing_loss: 1.000874E+00 | loss scale: 1.0 | grad norm: 0.133 | params norm: 11617.602 | params rms: 0.062906 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 8:13:29.056624 | finish at 2026-10-10 16:59:01 + [2026-10-04 08:47:02] iteration 678/ 6855 | consumed samples: 86784 | elapsed time per iteration (ms): 89470.3 | throughput per GPU (TFLOP/s/GPU): 41.6 | MFU 13.34% | FLOPs estimator: workload-aware | logical tokens/s: 187517.1 | valid tokens/s: 187517.1 | physical tokens/s: 187954.9 | loss tokens/s: 131260.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.396906E-01 | lm-global-supervised-tokens: 1.174394E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.566802E-02 | lm-token-buffer-mib: 5.813344E+01 | global_load_balancing_loss: 1.001426E+00 | loss scale: 1.0 | grad norm: 0.042 | params norm: 11617.516 | params rms: 0.062905 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 9:30:58.048739 | finish at 2026-10-10 18:18:00 + [2026-10-04 08:48:30] iteration 679/ 6855 | consumed samples: 86912 | elapsed time per iteration (ms): 88522.2 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.52% | FLOPs estimator: workload-aware | logical tokens/s: 189525.1 | valid tokens/s: 189525.1 | physical tokens/s: 189927.9 | loss tokens/s: 138197.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.704478E-01 | lm-global-supervised-tokens: 1.223354E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.482824E-02 | lm-token-buffer-mib: 5.816927E+01 | global_load_balancing_loss: 1.001007E+00 | loss scale: 1.0 | grad norm: 0.085 | params norm: 11617.430 | params rms: 0.062905 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 7:51:53.103447 | finish at 2026-10-10 16:40:23 + [2026-10-04 08:50:17] iteration 680/ 6855 | consumed samples: 87040 | elapsed time per iteration (ms): 107036.5 | throughput per GPU (TFLOP/s/GPU): 34.5 | MFU 11.06% | FLOPs estimator: workload-aware | logical tokens/s: 156742.7 | valid tokens/s: 156742.7 | physical tokens/s: 157103.1 | loss tokens/s: 114010.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.939241E-01 | lm-global-supervised-tokens: 1.220330E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.490805E-02 | lm-token-buffer-mib: 5.823885E+01 | global_load_balancing_loss: 9.994292E-01 | loss scale: 1.0 | grad norm: 0.084 | params norm: 11617.343 | params rms: 0.062904 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 7 days, 15:35:50.593472 | finish at 2026-10-12 00:26:08 +times across ranks (ms): + save-checkpoint: + rank 0: 37264.98 + rank 1: 37264.92 + rank 2: 37265.00 + rank 3: 37264.93 + rank 4: 37264.88 + rank 5: 37264.91 + rank 6: 37264.95 + rank 7: 37264.97 + rank 8: 37261.73 + rank 9: 37261.68 + rank 10: 37261.66 + rank 11: 37261.66 + rank 12: 37261.66 + rank 13: 37261.69 + rank 14: 37261.66 + rank 15: 37261.64 + rank 16: 37263.79 + rank 17: 37263.79 + rank 18: 37263.74 + rank 19: 37263.80 + rank 20: 37263.73 + rank 21: 37263.79 + rank 22: 37263.75 + rank 23: 37263.74 + rank 24: 37961.02 + rank 25: 37961.04 + rank 26: 37961.08 + rank 27: 37961.53 + rank 28: 37961.06 + rank 29: 37960.93 + rank 30: 37961.07 + rank 31: 37961.14 + rank 32: 37263.34 + rank 33: 37263.38 + rank 34: 37263.27 + rank 35: 37263.33 + rank 36: 37263.25 + rank 37: 37263.29 + rank 38: 37263.29 + rank 39: 37263.30 + rank 40: 37263.14 + rank 41: 37263.10 + rank 42: 37263.12 + rank 43: 37263.11 + rank 44: 37263.11 + rank 45: 37263.10 + rank 46: 37263.13 + rank 47: 37263.09 + rank 48: 37263.50 + rank 49: 37263.54 + rank 50: 37263.55 + rank 51: 37263.55 + rank 52: 37263.60 + rank 53: 37263.48 + rank 54: 37263.59 + rank 55: 37263.54 + rank 56: 37265.16 + rank 57: 37265.16 + rank 58: 37265.12 + rank 59: 37265.12 + rank 60: 37265.11 + rank 61: 37265.13 + rank 62: 37265.09 + rank 63: 37265.14 + [2026-10-04 08:52:45] iteration 681/ 6855 | consumed samples: 87168 | elapsed time per iteration (ms): 103442.5 | throughput per GPU (TFLOP/s/GPU): 35.7 | MFU 11.45% | FLOPs estimator: workload-aware | logical tokens/s: 162188.1 | valid tokens/s: 162188.1 | physical tokens/s: 162645.1 | loss tokens/s: 115179.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.637906E-01 | lm-global-supervised-tokens: 1.191447E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 5.484615E+00 | lm-token-buffer-mib: 5.834940E+01 | global_load_balancing_loss: 9.989502E-01 | loss scale: 1.0 | grad norm: 0.082 | params norm: 11617.257 | params rms: 0.062904 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 7 days, 9:24:14.061946 | finish at 2026-10-11 18:16:59 + [2026-10-04 08:54:16] iteration 682/ 6855 | consumed samples: 87296 | elapsed time per iteration (ms): 90946.6 | throughput per GPU (TFLOP/s/GPU): 41.1 | MFU 13.17% | FLOPs estimator: workload-aware | logical tokens/s: 184472.9 | valid tokens/s: 184472.9 | physical tokens/s: 184919.8 | loss tokens/s: 140720.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.906803E-01 | lm-global-supervised-tokens: 1.279807E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.675365E-02 | lm-token-buffer-mib: 5.808921E+01 | global_load_balancing_loss: 9.994911E-01 | loss scale: 1.0 | grad norm: 0.057 | params norm: 11617.171 | params rms: 0.062903 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 11:56:53.176114 | finish at 2026-10-10 20:51:09 + [2026-10-04 08:55:42] iteration 683/ 6855 | consumed samples: 87424 | elapsed time per iteration (ms): 86266.1 | throughput per GPU (TFLOP/s/GPU): 42.8 | MFU 13.71% | FLOPs estimator: workload-aware | logical tokens/s: 194481.8 | valid tokens/s: 194481.8 | physical tokens/s: 194965.7 | loss tokens/s: 150369.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.177800E-01 | lm-global-supervised-tokens: 1.297182E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.176403E-02 | lm-token-buffer-mib: 5.821370E+01 | global_load_balancing_loss: 1.000742E+00 | loss scale: 1.0 | grad norm: 0.057 | params norm: 11617.086 | params rms: 0.062903 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 3:53:54.399669 | finish at 2026-10-10 12:49:36 + [2026-10-04 08:57:08] iteration 684/ 6855 | consumed samples: 87552 | elapsed time per iteration (ms): 86079.3 | throughput per GPU (TFLOP/s/GPU): 43.8 | MFU 14.03% | FLOPs estimator: workload-aware | logical tokens/s: 194903.9 | valid tokens/s: 194903.9 | physical tokens/s: 195347.4 | loss tokens/s: 144499.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.153699E-01 | lm-global-supervised-tokens: 1.243843E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.482638E-02 | lm-token-buffer-mib: 5.818098E+01 | global_load_balancing_loss: 1.000341E+00 | loss scale: 1.0 | grad norm: 0.075 | params norm: 11617.002 | params rms: 0.062902 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 3:33:15.557000 | finish at 2026-10-10 12:30:23 + [2026-10-04 08:58:33] iteration 685/ 6855 | consumed samples: 87680 | elapsed time per iteration (ms): 84697.4 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.79% | FLOPs estimator: workload-aware | logical tokens/s: 198084.1 | valid tokens/s: 198084.1 | physical tokens/s: 198865.0 | loss tokens/s: 148155.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.702532E-01 | lm-global-supervised-tokens: 1.254837E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.547687E-02 | lm-token-buffer-mib: 5.816082E+01 | global_load_balancing_loss: 1.001785E+00 | loss scale: 1.0 | grad norm: 0.044 | params norm: 11616.917 | params rms: 0.062902 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 1:09:43.220420 | finish at 2026-10-10 10:08:16 + [2026-10-04 08:59:58] iteration 686/ 6855 | consumed samples: 87808 | elapsed time per iteration (ms): 84896.7 | throughput per GPU (TFLOP/s/GPU): 43.2 | MFU 13.84% | FLOPs estimator: workload-aware | logical tokens/s: 197619.2 | valid tokens/s: 197619.2 | physical tokens/s: 198235.0 | loss tokens/s: 148783.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.889741E-01 | lm-global-supervised-tokens: 1.263122E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.348636E-02 | lm-token-buffer-mib: 5.816747E+01 | global_load_balancing_loss: 1.000744E+00 | loss scale: 1.0 | grad norm: 0.057 | params norm: 11616.831 | params rms: 0.062901 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 1:28:47.541687 | finish at 2026-10-10 10:28:45 + [2026-10-04 09:01:25] iteration 687/ 6855 | consumed samples: 87936 | elapsed time per iteration (ms): 87723.4 | throughput per GPU (TFLOP/s/GPU): 43.3 | MFU 13.89% | FLOPs estimator: workload-aware | logical tokens/s: 191251.3 | valid tokens/s: 191251.3 | physical tokens/s: 191549.5 | loss tokens/s: 138742.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.150676E-01 | lm-global-supervised-tokens: 1.217098E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.457894E-02 | lm-token-buffer-mib: 5.805658E+01 | global_load_balancing_loss: 9.999331E-01 | loss scale: 1.0 | grad norm: 0.051 | params norm: 11616.745 | params rms: 0.062901 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 6:17:57.877504 | finish at 2026-10-10 15:19:23 + [2026-10-04 09:02:56] iteration 688/ 6855 | consumed samples: 88064 | elapsed time per iteration (ms): 90349.4 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.46% | FLOPs estimator: workload-aware | logical tokens/s: 185691.7 | valid tokens/s: 185691.7 | physical tokens/s: 186128.8 | loss tokens/s: 135579.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.149450E-01 | lm-global-supervised-tokens: 1.224956E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.809663E-02 | lm-token-buffer-mib: 5.808739E+01 | global_load_balancing_loss: 1.001150E+00 | loss scale: 1.0 | grad norm: 0.036 | params norm: 11616.659 | params rms: 0.062900 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 10:46:24.586862 | finish at 2026-10-10 19:49:20 + [2026-10-04 09:04:26] iteration 689/ 6855 | consumed samples: 88192 | elapsed time per iteration (ms): 90058.5 | throughput per GPU (TFLOP/s/GPU): 40.6 | MFU 13.00% | FLOPs estimator: workload-aware | logical tokens/s: 186292.1 | valid tokens/s: 186292.1 | physical tokens/s: 186759.7 | loss tokens/s: 138789.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.546771E-01 | lm-global-supervised-tokens: 1.249917E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.158567E-02 | lm-token-buffer-mib: 5.810341E+01 | global_load_balancing_loss: 9.996150E-01 | loss scale: 1.0 | grad norm: 0.037 | params norm: 11616.573 | params rms: 0.062900 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 10:15:00.749540 | finish at 2026-10-10 19:19:26 + [2026-10-04 09:06:15] iteration 690/ 6855 | consumed samples: 88320 | elapsed time per iteration (ms): 109157.6 | throughput per GPU (TFLOP/s/GPU): 33.9 | MFU 10.86% | FLOPs estimator: workload-aware | logical tokens/s: 153696.4 | valid tokens/s: 153696.4 | physical tokens/s: 154010.0 | loss tokens/s: 113799.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.518387E-01 | lm-global-supervised-tokens: 1.242210E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.231082E-02 | lm-token-buffer-mib: 5.819187E+01 | global_load_balancing_loss: 9.988608E-01 | loss scale: 1.0 | grad norm: 0.054 | params norm: 11616.489 | params rms: 0.062900 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 7 days, 18:55:56.469787 | finish at 2026-10-12 04:02:11 + [2026-10-04 09:07:40] iteration 691/ 6855 | consumed samples: 88448 | elapsed time per iteration (ms): 84732.4 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.59% | FLOPs estimator: workload-aware | logical tokens/s: 198002.0 | valid tokens/s: 198002.0 | physical tokens/s: 198466.4 | loss tokens/s: 156499.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.850325E-01 | lm-global-supervised-tokens: 1.326057E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.475375E-02 | lm-token-buffer-mib: 5.815842E+01 | global_load_balancing_loss: 9.984844E-01 | loss scale: 1.0 | grad norm: 0.048 | params norm: 11616.403 | params rms: 0.062899 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 1:04:50.226659 | finish at 2026-10-10 10:12:30 + [2026-10-04 09:09:11] iteration 692/ 6855 | consumed samples: 88576 | elapsed time per iteration (ms): 91231.9 | throughput per GPU (TFLOP/s/GPU): 41.2 | MFU 13.20% | FLOPs estimator: workload-aware | logical tokens/s: 183895.9 | valid tokens/s: 183895.9 | physical tokens/s: 184333.9 | loss tokens/s: 130932.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.480997E-01 | lm-global-supervised-tokens: 1.194526E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.535274E-02 | lm-token-buffer-mib: 5.808477E+01 | global_load_balancing_loss: 9.996295E-01 | loss scale: 1.0 | grad norm: 0.067 | params norm: 11616.317 | params rms: 0.062899 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 12:11:02.381759 | finish at 2026-10-10 21:20:13 + [2026-10-04 09:10:38] iteration 693/ 6855 | consumed samples: 88704 | elapsed time per iteration (ms): 87710.0 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.56% | FLOPs estimator: workload-aware | logical tokens/s: 191280.5 | valid tokens/s: 191280.5 | physical tokens/s: 191691.2 | loss tokens/s: 141463.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.358121E-01 | lm-global-supervised-tokens: 1.240772E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.345972E-02 | lm-token-buffer-mib: 5.818484E+01 | global_load_balancing_loss: 1.000564E+00 | loss scale: 1.0 | grad norm: 0.080 | params norm: 11616.229 | params rms: 0.062898 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 6:07:48.970284 | finish at 2026-10-10 15:18:27 + [2026-10-04 09:12:05] iteration 694/ 6855 | consumed samples: 88832 | elapsed time per iteration (ms): 86224.0 | throughput per GPU (TFLOP/s/GPU): 41.7 | MFU 13.37% | FLOPs estimator: workload-aware | logical tokens/s: 194576.6 | valid tokens/s: 194576.6 | physical tokens/s: 195118.8 | loss tokens/s: 143645.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.382018E-01 | lm-global-supervised-tokens: 1.238568E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.448981E-02 | lm-token-buffer-mib: 5.823598E+01 | global_load_balancing_loss: 1.001803E+00 | loss scale: 1.0 | grad norm: 0.096 | params norm: 11616.141 | params rms: 0.062898 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 3:33:45.941941 | finish at 2026-10-10 12:45:51 + [2026-10-04 09:13:31] iteration 695/ 6855 | consumed samples: 88960 | elapsed time per iteration (ms): 86453.8 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.64% | FLOPs estimator: workload-aware | logical tokens/s: 194059.6 | valid tokens/s: 194059.6 | physical tokens/s: 194497.5 | loss tokens/s: 149589.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.466857E-01 | lm-global-supervised-tokens: 1.293259E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.510913E-02 | lm-token-buffer-mib: 5.815276E+01 | global_load_balancing_loss: 1.001776E+00 | loss scale: 1.0 | grad norm: 0.090 | params norm: 11616.052 | params rms: 0.062897 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 3:55:55.244751 | finish at 2026-10-10 13:09:26 + [2026-10-04 09:15:01] iteration 696/ 6855 | consumed samples: 89088 | elapsed time per iteration (ms): 89498.6 | throughput per GPU (TFLOP/s/GPU): 41.5 | MFU 13.31% | FLOPs estimator: workload-aware | logical tokens/s: 187457.6 | valid tokens/s: 187457.6 | physical tokens/s: 187885.8 | loss tokens/s: 137168.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.569335E-01 | lm-global-supervised-tokens: 1.227635E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.507846E-02 | lm-token-buffer-mib: 5.813270E+01 | global_load_balancing_loss: 1.000202E+00 | loss scale: 1.0 | grad norm: 0.090 | params norm: 11615.965 | params rms: 0.062897 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 9:07:01.984632 | finish at 2026-10-10 18:22:03 + [2026-10-04 09:16:30] iteration 697/ 6855 | consumed samples: 89216 | elapsed time per iteration (ms): 88867.1 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.41% | FLOPs estimator: workload-aware | logical tokens/s: 188789.1 | valid tokens/s: 188789.1 | physical tokens/s: 189114.1 | loss tokens/s: 139763.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.755351E-01 | lm-global-supervised-tokens: 1.242035E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.146475E-02 | lm-token-buffer-mib: 5.818752E+01 | global_load_balancing_loss: 1.000156E+00 | loss scale: 1.0 | grad norm: 0.087 | params norm: 11615.880 | params rms: 0.062896 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 8:00:43.399193 | finish at 2026-10-10 17:17:13 + [2026-10-04 09:17:57] iteration 698/ 6855 | consumed samples: 89344 | elapsed time per iteration (ms): 87693.2 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.42% | FLOPs estimator: workload-aware | logical tokens/s: 191316.7 | valid tokens/s: 191316.7 | physical tokens/s: 191886.2 | loss tokens/s: 146555.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.772055E-01 | lm-global-supervised-tokens: 1.285195E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.710108E-02 | lm-token-buffer-mib: 5.830804E+01 | global_load_balancing_loss: 1.001002E+00 | loss scale: 1.0 | grad norm: 0.041 | params norm: 11615.796 | params rms: 0.062896 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 5:58:47.112355 | finish at 2026-10-10 15:16:44 + [2026-10-04 09:19:24] iteration 699/ 6855 | consumed samples: 89472 | elapsed time per iteration (ms): 87192.3 | throughput per GPU (TFLOP/s/GPU): 44.0 | MFU 14.09% | FLOPs estimator: workload-aware | logical tokens/s: 192416.1 | valid tokens/s: 192416.1 | physical tokens/s: 192992.9 | loss tokens/s: 135122.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.526738E-01 | lm-global-supervised-tokens: 1.178161E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.196324E-02 | lm-token-buffer-mib: 5.802245E+01 | global_load_balancing_loss: 9.993128E-01 | loss scale: 1.0 | grad norm: 0.090 | params norm: 11615.712 | params rms: 0.062895 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 5:05:55.661336 | finish at 2026-10-10 14:25:20 + [2026-10-04 09:21:09] iteration 700/ 6855 | consumed samples: 89600 | elapsed time per iteration (ms): 104478.1 | throughput per GPU (TFLOP/s/GPU): 34.4 | MFU 11.04% | FLOPs estimator: workload-aware | logical tokens/s: 160581.1 | valid tokens/s: 160581.1 | physical tokens/s: 161100.3 | loss tokens/s: 120222.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.410931E-01 | lm-global-supervised-tokens: 1.256065E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.187248E-02 | lm-token-buffer-mib: 5.817055E+01 | global_load_balancing_loss: 9.988189E-01 | loss scale: 1.0 | grad norm: 0.090 | params norm: 11615.628 | params rms: 0.062895 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 7 days, 10:37:42.711748 | finish at 2026-10-11 19:58:52 + [2026-10-04 09:22:32] iteration 701/ 6855 | consumed samples: 89728 | elapsed time per iteration (ms): 82872.5 | throughput per GPU (TFLOP/s/GPU): 43.2 | MFU 13.83% | FLOPs estimator: workload-aware | logical tokens/s: 202444.9 | valid tokens/s: 202444.9 | physical tokens/s: 202860.5 | loss tokens/s: 154025.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.726989E-01 | lm-global-supervised-tokens: 1.276450E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.364433E-02 | lm-token-buffer-mib: 5.811192E+01 | global_load_balancing_loss: 9.984523E-01 | loss scale: 1.0 | grad norm: 0.045 | params norm: 11615.548 | params rms: 0.062894 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 21:39:57.264876 | finish at 2026-10-10 07:02:29 + [2026-10-04 09:24:01] iteration 702/ 6855 | consumed samples: 89856 | elapsed time per iteration (ms): 89080.7 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.44% | FLOPs estimator: workload-aware | logical tokens/s: 188337.1 | valid tokens/s: 188337.1 | physical tokens/s: 188715.3 | loss tokens/s: 137673.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.601176E-01 | lm-global-supervised-tokens: 1.226402E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.310484E-02 | lm-token-buffer-mib: 5.822297E+01 | global_load_balancing_loss: 1.000046E+00 | loss scale: 1.0 | grad norm: 0.071 | params norm: 11615.467 | params rms: 0.062894 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 8:15:13.658103 | finish at 2026-10-10 17:39:15 + [2026-10-04 09:25:30] iteration 703/ 6855 | consumed samples: 89984 | elapsed time per iteration (ms): 89413.0 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.44% | FLOPs estimator: workload-aware | logical tokens/s: 187636.6 | valid tokens/s: 187636.6 | physical tokens/s: 187974.1 | loss tokens/s: 141303.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.970843E-01 | lm-global-supervised-tokens: 1.263438E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.139326E-02 | lm-token-buffer-mib: 5.815143E+01 | global_load_balancing_loss: 1.000862E+00 | loss scale: 1.0 | grad norm: 0.095 | params norm: 11615.385 | params rms: 0.062894 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 8:47:48.908665 | finish at 2026-10-10 18:13:19 + [2026-10-04 09:27:01] iteration 704/ 6855 | consumed samples: 90112 | elapsed time per iteration (ms): 90564.0 | throughput per GPU (TFLOP/s/GPU): 41.4 | MFU 13.27% | FLOPs estimator: workload-aware | logical tokens/s: 185251.8 | valid tokens/s: 185251.8 | physical tokens/s: 185842.2 | loss tokens/s: 133255.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.848063E-01 | lm-global-supervised-tokens: 1.206814E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.376268E-02 | lm-token-buffer-mib: 5.819302E+01 | global_load_balancing_loss: 1.001940E+00 | loss scale: 1.0 | grad norm: 0.051 | params norm: 11615.303 | params rms: 0.062893 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 10:44:19.411172 | finish at 2026-10-10 20:11:20 + [2026-10-04 09:28:29] iteration 705/ 6855 | consumed samples: 90240 | elapsed time per iteration (ms): 87764.6 | throughput per GPU (TFLOP/s/GPU): 41.6 | MFU 13.33% | FLOPs estimator: workload-aware | logical tokens/s: 191161.2 | valid tokens/s: 191161.2 | physical tokens/s: 191560.0 | loss tokens/s: 147761.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.842841E-01 | lm-global-supervised-tokens: 1.296825E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.195432E-02 | lm-token-buffer-mib: 5.819716E+01 | global_load_balancing_loss: 1.001416E+00 | loss scale: 1.0 | grad norm: 0.137 | params norm: 11615.222 | params rms: 0.062893 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 5:55:52.552700 | finish at 2026-10-10 15:24:21 + [2026-10-04 09:29:54] iteration 706/ 6855 | consumed samples: 90368 | elapsed time per iteration (ms): 85090.1 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.66% | FLOPs estimator: workload-aware | logical tokens/s: 197168.6 | valid tokens/s: 197168.6 | physical tokens/s: 197711.4 | loss tokens/s: 146893.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.603325E-01 | lm-global-supervised-tokens: 1.249920E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.202258E-02 | lm-token-buffer-mib: 5.812560E+01 | global_load_balancing_loss: 1.000023E+00 | loss scale: 1.0 | grad norm: 0.135 | params norm: 11615.141 | params rms: 0.062892 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 1:20:18.817030 | finish at 2026-10-10 10:50:12 + [2026-10-04 09:31:19] iteration 707/ 6855 | consumed samples: 90496 | elapsed time per iteration (ms): 85297.9 | throughput per GPU (TFLOP/s/GPU): 43.3 | MFU 13.89% | FLOPs estimator: workload-aware | logical tokens/s: 196689.6 | valid tokens/s: 196689.6 | physical tokens/s: 197033.0 | loss tokens/s: 148465.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.145277E-01 | lm-global-supervised-tokens: 1.266382E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.374780E-02 | lm-token-buffer-mib: 5.812553E+01 | global_load_balancing_loss: 9.994148E-01 | loss scale: 1.0 | grad norm: 0.047 | params norm: 11615.058 | params rms: 0.062892 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 1:40:11.701504 | finish at 2026-10-10 11:11:31 + [2026-10-04 09:32:48] iteration 708/ 6855 | consumed samples: 90624 | elapsed time per iteration (ms): 88970.6 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.40% | FLOPs estimator: workload-aware | logical tokens/s: 188569.6 | valid tokens/s: 188569.6 | physical tokens/s: 189236.8 | loss tokens/s: 143370.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.871962E-01 | lm-global-supervised-tokens: 1.275574E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.423521E-02 | lm-token-buffer-mib: 5.862150E+01 | global_load_balancing_loss: 9.991890E-01 | loss scale: 1.0 | grad norm: 0.091 | params norm: 11614.975 | params rms: 0.062891 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 7:55:02.410888 | finish at 2026-10-10 17:27:51 + [2026-10-04 09:34:16] iteration 709/ 6855 | consumed samples: 90752 | elapsed time per iteration (ms): 87763.5 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.63% | FLOPs estimator: workload-aware | logical tokens/s: 191163.3 | valid tokens/s: 191163.3 | physical tokens/s: 191723.4 | loss tokens/s: 138685.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.883587E-01 | lm-global-supervised-tokens: 1.217153E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.494142E-02 | lm-token-buffer-mib: 5.825938E+01 | global_load_balancing_loss: 1.000675E+00 | loss scale: 1.0 | grad norm: 0.075 | params norm: 11614.890 | params rms: 0.062891 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 5:49:54.510411 | finish at 2026-10-10 15:24:10 + [2026-10-04 09:36:02] iteration 710/ 6855 | consumed samples: 90880 | elapsed time per iteration (ms): 105549.0 | throughput per GPU (TFLOP/s/GPU): 35.2 | MFU 11.29% | FLOPs estimator: workload-aware | logical tokens/s: 158951.3 | valid tokens/s: 158951.3 | physical tokens/s: 159211.4 | loss tokens/s: 118415.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.023998E-01 | lm-global-supervised-tokens: 1.249867E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.155314E-02 | lm-token-buffer-mib: 5.815362E+01 | global_load_balancing_loss: 1.000654E+00 | loss scale: 1.0 | grad norm: 0.048 | params norm: 11614.805 | params rms: 0.062890 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 7 days, 12:09:58.605152 | finish at 2026-10-11 21:46:00 + [2026-10-04 09:37:29] iteration 711/ 6855 | consumed samples: 91008 | elapsed time per iteration (ms): 87475.3 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.44% | FLOPs estimator: workload-aware | logical tokens/s: 191793.2 | valid tokens/s: 191793.2 | physical tokens/s: 192423.7 | loss tokens/s: 147981.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.092288E-01 | lm-global-supervised-tokens: 1.294471E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.166868E-02 | lm-token-buffer-mib: 5.807796E+01 | global_load_balancing_loss: 1.000385E+00 | loss scale: 1.0 | grad norm: 0.039 | params norm: 11614.723 | params rms: 0.062890 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 5:17:28.130859 | finish at 2026-10-10 14:54:57 + [2026-10-04 09:38:55] iteration 712/ 6855 | consumed samples: 91136 | elapsed time per iteration (ms): 86453.8 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.56% | FLOPs estimator: workload-aware | logical tokens/s: 194059.8 | valid tokens/s: 194059.8 | physical tokens/s: 194566.9 | loss tokens/s: 144247.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.532695E-01 | lm-global-supervised-tokens: 1.247071E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.617738E-02 | lm-token-buffer-mib: 5.804546E+01 | global_load_balancing_loss: 9.999353E-01 | loss scale: 1.0 | grad norm: 0.059 | params norm: 11614.639 | params rms: 0.062889 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 3:31:25.398787 | finish at 2026-10-10 13:10:21 + [2026-10-04 09:40:21] iteration 713/ 6855 | consumed samples: 91264 | elapsed time per iteration (ms): 85893.3 | throughput per GPU (TFLOP/s/GPU): 43.1 | MFU 13.80% | FLOPs estimator: workload-aware | logical tokens/s: 195326.3 | valid tokens/s: 195326.3 | physical tokens/s: 195829.3 | loss tokens/s: 141016.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.047582E-01 | lm-global-supervised-tokens: 1.211240E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.377075E-02 | lm-token-buffer-mib: 5.818362E+01 | global_load_balancing_loss: 1.000338E+00 | loss scale: 1.0 | grad norm: 0.063 | params norm: 11614.554 | params rms: 0.062889 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 2:32:36.460043 | finish at 2026-10-10 12:12:58 + [2026-10-04 09:41:48] iteration 714/ 6855 | consumed samples: 91392 | elapsed time per iteration (ms): 86798.3 | throughput per GPU (TFLOP/s/GPU): 42.8 | MFU 13.71% | FLOPs estimator: workload-aware | logical tokens/s: 193289.4 | valid tokens/s: 193289.3 | physical tokens/s: 193697.5 | loss tokens/s: 146018.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.055162E-01 | lm-global-supervised-tokens: 1.267412E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.183482E-02 | lm-token-buffer-mib: 5.809068E+01 | global_load_balancing_loss: 9.999415E-01 | loss scale: 1.0 | grad norm: 0.059 | params norm: 11614.471 | params rms: 0.062889 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 4:03:48.312155 | finish at 2026-10-10 13:45:36 + [2026-10-04 09:43:15] iteration 715/ 6855 | consumed samples: 91520 | elapsed time per iteration (ms): 86670.4 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.74% | FLOPs estimator: workload-aware | logical tokens/s: 193573.6 | valid tokens/s: 193573.6 | physical tokens/s: 194224.7 | loss tokens/s: 141786.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.474448E-01 | lm-global-supervised-tokens: 1.228872E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.535799E-02 | lm-token-buffer-mib: 5.845296E+01 | global_load_balancing_loss: 1.000141E+00 | loss scale: 1.0 | grad norm: 0.046 | params norm: 11614.387 | params rms: 0.062888 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 3:49:16.265659 | finish at 2026-10-10 13:32:31 + [2026-10-04 09:44:41] iteration 716/ 6855 | consumed samples: 91648 | elapsed time per iteration (ms): 85995.3 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.59% | FLOPs estimator: workload-aware | logical tokens/s: 195093.4 | valid tokens/s: 195093.3 | physical tokens/s: 195546.6 | loss tokens/s: 145741.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.389081E-01 | lm-global-supervised-tokens: 1.253313E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.413364E-02 | lm-token-buffer-mib: 5.811002E+01 | global_load_balancing_loss: 9.997502E-01 | loss scale: 1.0 | grad norm: 0.104 | params norm: 11614.303 | params rms: 0.062888 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 2:38:45.438302 | finish at 2026-10-10 12:23:26 + [2026-10-04 09:46:12] iteration 717/ 6855 | consumed samples: 91776 | elapsed time per iteration (ms): 91318.1 | throughput per GPU (TFLOP/s/GPU): 41.7 | MFU 13.36% | FLOPs estimator: workload-aware | logical tokens/s: 183722.7 | valid tokens/s: 183722.7 | physical tokens/s: 184071.4 | loss tokens/s: 129229.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.215750E-01 | lm-global-supervised-tokens: 1.180101E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.112516E-02 | lm-token-buffer-mib: 5.814513E+01 | global_load_balancing_loss: 1.000644E+00 | loss scale: 1.0 | grad norm: 0.236 | params norm: 11614.220 | params rms: 0.062887 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 11:41:50.756674 | finish at 2026-10-10 21:28:03 + [2026-10-04 09:47:40] iteration 718/ 6855 | consumed samples: 91904 | elapsed time per iteration (ms): 88290.4 | throughput per GPU (TFLOP/s/GPU): 42.8 | MFU 13.72% | FLOPs estimator: workload-aware | logical tokens/s: 190023.2 | valid tokens/s: 190023.2 | physical tokens/s: 190307.0 | loss tokens/s: 136568.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.323912E-01 | lm-global-supervised-tokens: 1.205769E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.770961E-02 | lm-token-buffer-mib: 5.805613E+01 | global_load_balancing_loss: 1.001481E+00 | loss scale: 1.0 | grad norm: 0.181 | params norm: 11614.136 | params rms: 0.062887 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 6:30:37.983055 | finish at 2026-10-10 16:18:18 + [2026-10-04 09:49:06] iteration 719/ 6855 | consumed samples: 92032 | elapsed time per iteration (ms): 85535.5 | throughput per GPU (TFLOP/s/GPU): 43.1 | MFU 13.82% | FLOPs estimator: workload-aware | logical tokens/s: 196143.1 | valid tokens/s: 196143.1 | physical tokens/s: 196525.6 | loss tokens/s: 144405.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.446495E-01 | lm-global-supervised-tokens: 1.235175E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.162360E-02 | lm-token-buffer-mib: 5.816093E+01 | global_load_balancing_loss: 1.000954E+00 | loss scale: 1.0 | grad norm: 0.123 | params norm: 11614.054 | params rms: 0.062886 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 1:47:25.986301 | finish at 2026-10-10 11:36:32 + [2026-10-04 09:50:55] iteration 720/ 6855 | consumed samples: 92160 | elapsed time per iteration (ms): 108694.5 | throughput per GPU (TFLOP/s/GPU): 35.0 | MFU 11.21% | FLOPs estimator: workload-aware | logical tokens/s: 154351.9 | valid tokens/s: 154351.9 | physical tokens/s: 154594.9 | loss tokens/s: 114262.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.201050E-01 | lm-global-supervised-tokens: 1.241972E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.402416E-02 | lm-token-buffer-mib: 5.804840E+01 | global_load_balancing_loss: 9.988327E-01 | loss scale: 1.0 | grad norm: 0.146 | params norm: 11613.972 | params rms: 0.062886 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 7 days, 17:14:00.735372 | finish at 2026-10-12 03:04:55 +times across ranks (ms): + save-checkpoint: + rank 0: 24751.68 + rank 1: 24751.68 + rank 2: 24751.72 + rank 3: 24751.68 + rank 4: 24751.70 + rank 5: 24751.71 + rank 6: 24751.69 + rank 7: 24751.67 + rank 8: 24753.58 + rank 9: 24753.56 + rank 10: 24753.57 + rank 11: 24753.57 + rank 12: 24753.57 + rank 13: 24753.55 + rank 14: 24753.57 + rank 15: 24753.55 + rank 16: 24753.85 + rank 17: 24753.85 + rank 18: 24753.82 + rank 19: 24753.83 + rank 20: 24753.83 + rank 21: 24753.84 + rank 22: 24753.82 + rank 23: 24753.82 + rank 24: 24753.79 + rank 25: 24753.78 + rank 26: 24753.81 + rank 27: 24753.88 + rank 28: 24753.76 + rank 29: 24753.81 + rank 30: 24753.78 + rank 31: 24753.78 + rank 32: 24753.32 + rank 33: 24753.33 + rank 34: 24753.30 + rank 35: 24753.28 + rank 36: 24753.26 + rank 37: 24753.26 + rank 38: 24753.31 + rank 39: 24753.28 + rank 40: 24753.53 + rank 41: 24753.80 + rank 42: 24753.75 + rank 43: 24753.72 + rank 44: 24753.77 + rank 45: 24753.82 + rank 46: 24753.53 + rank 47: 24753.80 + rank 48: 24754.08 + rank 49: 24754.15 + rank 50: 24754.06 + rank 51: 24754.24 + rank 52: 24754.08 + rank 53: 24754.04 + rank 54: 24754.08 + rank 55: 24754.05 + rank 56: 24753.94 + rank 57: 24753.92 + rank 58: 24753.90 + rank 59: 24753.90 + rank 60: 24753.92 + rank 61: 24753.90 + rank 62: 24753.92 + rank 63: 24753.93 + [2026-10-04 09:53:13] iteration 721/ 6855 | consumed samples: 92288 | elapsed time per iteration (ms): 113943.4 | throughput per GPU (TFLOP/s/GPU): 33.4 | MFU 10.69% | FLOPs estimator: workload-aware | logical tokens/s: 147240.9 | valid tokens/s: 147240.9 | physical tokens/s: 147575.9 | loss tokens/s: 106720.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.812762E-01 | lm-global-supervised-tokens: 1.216007E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 8.483086E-01 | lm-token-buffer-mib: 5.811327E+01 | global_load_balancing_loss: 1.000386E+00 | loss scale: 1.0 | grad norm: 0.210 | params norm: 11613.886 | params rms: 0.062885 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 8 days, 2:08:48.581031 | finish at 2026-10-12 12:02:02 + [2026-10-04 09:54:45] iteration 722/ 6855 | consumed samples: 92416 | elapsed time per iteration (ms): 91544.6 | throughput per GPU (TFLOP/s/GPU): 40.5 | MFU 12.99% | FLOPs estimator: workload-aware | logical tokens/s: 183267.3 | valid tokens/s: 183267.3 | physical tokens/s: 183634.4 | loss tokens/s: 131332.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.275387E-01 | lm-global-supervised-tokens: 1.202280E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.984004E-02 | lm-token-buffer-mib: 5.813771E+01 | global_load_balancing_loss: 1.000808E+00 | loss scale: 1.0 | grad norm: 0.216 | params norm: 11613.798 | params rms: 0.062885 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 11:57:22.897336 | finish at 2026-10-10 21:52:08 + [2026-10-04 09:56:11] iteration 723/ 6855 | consumed samples: 92544 | elapsed time per iteration (ms): 85756.8 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.64% | FLOPs estimator: workload-aware | logical tokens/s: 195636.4 | valid tokens/s: 195636.4 | physical tokens/s: 196413.2 | loss tokens/s: 146901.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.564956E-01 | lm-global-supervised-tokens: 1.259780E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.165524E-02 | lm-token-buffer-mib: 5.822935E+01 | global_load_balancing_loss: 9.991345E-01 | loss scale: 1.0 | grad norm: 0.251 | params norm: 11613.714 | params rms: 0.062884 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 2:04:20.502766 | finish at 2026-10-10 12:00:31 + [2026-10-04 09:57:37] iteration 724/ 6855 | consumed samples: 92672 | elapsed time per iteration (ms): 85840.2 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.57% | FLOPs estimator: workload-aware | logical tokens/s: 195446.3 | valid tokens/s: 195446.3 | physical tokens/s: 195968.3 | loss tokens/s: 146851.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.755037E-01 | lm-global-supervised-tokens: 1.260572E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.179592E-02 | lm-token-buffer-mib: 5.810306E+01 | global_load_balancing_loss: 9.993628E-01 | loss scale: 1.0 | grad norm: 0.267 | params norm: 11613.633 | params rms: 0.062884 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 2:11:26.201560 | finish at 2026-10-10 12:09:03 + [2026-10-04 09:59:06] iteration 725/ 6855 | consumed samples: 92800 | elapsed time per iteration (ms): 89491.4 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.48% | FLOPs estimator: workload-aware | logical tokens/s: 187472.1 | valid tokens/s: 187472.1 | physical tokens/s: 187891.7 | loss tokens/s: 138039.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.649124E-01 | lm-global-supervised-tokens: 1.235336E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.744000E-02 | lm-token-buffer-mib: 5.805023E+01 | global_load_balancing_loss: 1.000669E+00 | loss scale: 1.0 | grad norm: 0.163 | params norm: 11613.549 | params rms: 0.062884 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 8:23:02.558246 | finish at 2026-10-10 18:22:09 + [2026-10-04 10:00:32] iteration 726/ 6855 | consumed samples: 92928 | elapsed time per iteration (ms): 86094.2 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.47% | FLOPs estimator: workload-aware | logical tokens/s: 194870.1 | valid tokens/s: 194870.1 | physical tokens/s: 195396.6 | loss tokens/s: 148280.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.432699E-01 | lm-global-supervised-tokens: 1.276606E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.401894E-02 | lm-token-buffer-mib: 5.815289E+01 | global_load_balancing_loss: 9.999741E-01 | loss scale: 1.0 | grad norm: 0.150 | params norm: 11613.465 | params rms: 0.062883 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 2:34:31.380387 | finish at 2026-10-10 12:35:04 + [2026-10-04 10:01:59] iteration 727/ 6855 | consumed samples: 93056 | elapsed time per iteration (ms): 86799.4 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.56% | FLOPs estimator: workload-aware | logical tokens/s: 193286.1 | valid tokens/s: 193286.1 | physical tokens/s: 193648.2 | loss tokens/s: 136432.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.054905E-01 | lm-global-supervised-tokens: 1.184222E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.230610E-02 | lm-token-buffer-mib: 5.807836E+01 | global_load_balancing_loss: 1.001266E+00 | loss scale: 1.0 | grad norm: 0.173 | params norm: 11613.383 | params rms: 0.062883 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 3:45:06.627331 | finish at 2026-10-10 13:47:06 + [2026-10-04 10:03:28] iteration 728/ 6855 | consumed samples: 93184 | elapsed time per iteration (ms): 89032.6 | throughput per GPU (TFLOP/s/GPU): 40.8 | MFU 13.07% | FLOPs estimator: workload-aware | logical tokens/s: 188438.4 | valid tokens/s: 188438.4 | physical tokens/s: 189114.1 | loss tokens/s: 139263.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.022991E-01 | lm-global-supervised-tokens: 1.239903E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.132475E-02 | lm-token-buffer-mib: 5.813113E+01 | global_load_balancing_loss: 1.000145E+00 | loss scale: 1.0 | grad norm: 0.136 | params norm: 11613.302 | params rms: 0.062882 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 7:31:42.979316 | finish at 2026-10-10 17:35:11 + [2026-10-04 10:04:54] iteration 729/ 6855 | consumed samples: 93312 | elapsed time per iteration (ms): 85714.0 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.68% | FLOPs estimator: workload-aware | logical tokens/s: 195734.4 | valid tokens/s: 195734.4 | physical tokens/s: 196218.3 | loss tokens/s: 146253.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.135583E-01 | lm-global-supervised-tokens: 1.253597E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.226271E-02 | lm-token-buffer-mib: 5.816292E+01 | global_load_balancing_loss: 9.994313E-01 | loss scale: 1.0 | grad norm: 0.052 | params norm: 11613.218 | params rms: 0.062882 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 1:51:24.079302 | finish at 2026-10-10 11:56:18 + [2026-10-04 10:06:43] iteration 730/ 6855 | consumed samples: 93440 | elapsed time per iteration (ms): 109718.3 | throughput per GPU (TFLOP/s/GPU): 33.3 | MFU 10.67% | FLOPs estimator: workload-aware | logical tokens/s: 152911.1 | valid tokens/s: 152911.1 | physical tokens/s: 153221.1 | loss tokens/s: 118503.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.827627E-01 | lm-global-supervised-tokens: 1.300197E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.214647E-02 | lm-token-buffer-mib: 5.807607E+01 | global_load_balancing_loss: 1.000386E+00 | loss scale: 1.0 | grad norm: 0.070 | params norm: 11613.134 | params rms: 0.062881 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 7 days, 18:40:24.450868 | finish at 2026-10-12 04:47:08 + [2026-10-04 10:08:13] iteration 731/ 6855 | consumed samples: 93568 | elapsed time per iteration (ms): 89273.1 | throughput per GPU (TFLOP/s/GPU): 41.0 | MFU 13.13% | FLOPs estimator: workload-aware | logical tokens/s: 187931.4 | valid tokens/s: 187931.4 | physical tokens/s: 188361.5 | loss tokens/s: 146225.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.516182E-01 | lm-global-supervised-tokens: 1.305396E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.154621E-02 | lm-token-buffer-mib: 5.813664E+01 | global_load_balancing_loss: 9.999968E-01 | loss scale: 1.0 | grad norm: 0.143 | params norm: 11613.050 | params rms: 0.062881 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 7:51:48.449183 | finish at 2026-10-10 18:00:01 + [2026-10-04 10:09:39] iteration 732/ 6855 | consumed samples: 93696 | elapsed time per iteration (ms): 86286.2 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.40% | FLOPs estimator: workload-aware | logical tokens/s: 194436.6 | valid tokens/s: 194436.6 | physical tokens/s: 195064.3 | loss tokens/s: 146248.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.522605E-01 | lm-global-supervised-tokens: 1.261927E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.422840E-02 | lm-token-buffer-mib: 5.821532E+01 | global_load_balancing_loss: 1.000625E+00 | loss scale: 1.0 | grad norm: 0.044 | params norm: 11612.966 | params rms: 0.062880 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 2:45:30.570766 | finish at 2026-10-10 12:55:10 + [2026-10-04 10:11:11] iteration 733/ 6855 | consumed samples: 93824 | elapsed time per iteration (ms): 92304.8 | throughput per GPU (TFLOP/s/GPU): 40.9 | MFU 13.11% | FLOPs estimator: workload-aware | logical tokens/s: 181758.4 | valid tokens/s: 181758.4 | physical tokens/s: 182362.1 | loss tokens/s: 133746.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.314698E-01 | lm-global-supervised-tokens: 1.234545E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.718868E-02 | lm-token-buffer-mib: 5.843110E+01 | global_load_balancing_loss: 9.998785E-01 | loss scale: 1.0 | grad norm: 0.136 | params norm: 11612.883 | params rms: 0.062880 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 12:58:10.000401 | finish at 2026-10-10 23:09:21 + [2026-10-04 10:12:41] iteration 734/ 6855 | consumed samples: 93952 | elapsed time per iteration (ms): 89216.3 | throughput per GPU (TFLOP/s/GPU): 41.6 | MFU 13.35% | FLOPs estimator: workload-aware | logical tokens/s: 188050.0 | valid tokens/s: 188050.0 | physical tokens/s: 188490.8 | loss tokens/s: 135734.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.116731E-01 | lm-global-supervised-tokens: 1.210969E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.173365E-02 | lm-token-buffer-mib: 5.818017E+01 | global_load_balancing_loss: 9.999849E-01 | loss scale: 1.0 | grad norm: 0.042 | params norm: 11612.798 | params rms: 0.062880 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 7:41:33.252563 | finish at 2026-10-10 17:54:14 + [2026-10-04 10:14:10] iteration 735/ 6855 | consumed samples: 94080 | elapsed time per iteration (ms): 89223.7 | throughput per GPU (TFLOP/s/GPU): 41.3 | MFU 13.25% | FLOPs estimator: workload-aware | logical tokens/s: 188035.2 | valid tokens/s: 188035.2 | physical tokens/s: 188394.0 | loss tokens/s: 137715.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.254489E-01 | lm-global-supervised-tokens: 1.228749E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.233585E-02 | lm-token-buffer-mib: 5.818233E+01 | global_load_balancing_loss: 1.000551E+00 | loss scale: 1.0 | grad norm: 0.060 | params norm: 11612.712 | params rms: 0.062879 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 7:40:49.136209 | finish at 2026-10-10 17:54:59 + [2026-10-04 10:15:36] iteration 736/ 6855 | consumed samples: 94208 | elapsed time per iteration (ms): 86354.6 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.50% | FLOPs estimator: workload-aware | logical tokens/s: 194282.7 | valid tokens/s: 194282.7 | physical tokens/s: 194860.2 | loss tokens/s: 145608.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.376976E-01 | lm-global-supervised-tokens: 1.257398E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.163699E-02 | lm-token-buffer-mib: 5.808441E+01 | global_load_balancing_loss: 1.001083E+00 | loss scale: 1.0 | grad norm: 0.110 | params norm: 11612.627 | params rms: 0.062879 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 2:46:43.719790 | finish at 2026-10-10 13:02:20 + [2026-10-04 10:17:04] iteration 737/ 6855 | consumed samples: 94336 | elapsed time per iteration (ms): 88148.7 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.45% | FLOPs estimator: workload-aware | logical tokens/s: 190328.7 | valid tokens/s: 190328.6 | physical tokens/s: 190835.1 | loss tokens/s: 143945.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.557643E-01 | lm-global-supervised-tokens: 1.268860E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.166603E-02 | lm-token-buffer-mib: 5.808138E+01 | global_load_balancing_loss: 9.997788E-01 | loss scale: 1.0 | grad norm: 0.059 | params norm: 11612.540 | params rms: 0.062878 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 5:48:13.451947 | finish at 2026-10-10 16:05:18 + [2026-10-04 10:18:29] iteration 738/ 6855 | consumed samples: 94464 | elapsed time per iteration (ms): 84464.8 | throughput per GPU (TFLOP/s/GPU): 43.3 | MFU 13.89% | FLOPs estimator: workload-aware | logical tokens/s: 198629.5 | valid tokens/s: 198629.5 | physical tokens/s: 199047.4 | loss tokens/s: 149651.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.641456E-01 | lm-global-supervised-tokens: 1.264024E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.265583E-02 | lm-token-buffer-mib: 5.808146E+01 | global_load_balancing_loss: 1.000617E+00 | loss scale: 1.0 | grad norm: 0.120 | params norm: 11612.454 | params rms: 0.062878 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 23:31:11.183789 | finish at 2026-10-10 09:49:40 + [2026-10-04 10:19:58] iteration 739/ 6855 | consumed samples: 94592 | elapsed time per iteration (ms): 89304.8 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.52% | FLOPs estimator: workload-aware | logical tokens/s: 187864.7 | valid tokens/s: 187864.7 | physical tokens/s: 188134.7 | loss tokens/s: 138413.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.746990E-01 | lm-global-supervised-tokens: 1.236096E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.388564E-02 | lm-token-buffer-mib: 5.806013E+01 | global_load_balancing_loss: 9.999509E-01 | loss scale: 1.0 | grad norm: 0.033 | params norm: 11612.367 | params rms: 0.062877 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 7:43:07.894535 | finish at 2026-10-10 18:03:06 + [2026-10-04 10:21:46] iteration 740/ 6855 | consumed samples: 94720 | elapsed time per iteration (ms): 107668.0 | throughput per GPU (TFLOP/s/GPU): 33.6 | MFU 10.77% | FLOPs estimator: workload-aware | logical tokens/s: 155822.7 | valid tokens/s: 155822.7 | physical tokens/s: 156179.2 | loss tokens/s: 113244.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.706927E-01 | lm-global-supervised-tokens: 1.219279E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.121775E-02 | lm-token-buffer-mib: 5.831268E+01 | global_load_balancing_loss: 9.999777E-01 | loss scale: 1.0 | grad norm: 0.120 | params norm: 11612.282 | params rms: 0.062877 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 7 days, 14:53:09.815521 | finish at 2026-10-12 01:14:56 + [2026-10-04 10:23:16] iteration 741/ 6855 | consumed samples: 94848 | elapsed time per iteration (ms): 90531.7 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.43% | FLOPs estimator: workload-aware | logical tokens/s: 185318.6 | valid tokens/s: 185318.6 | physical tokens/s: 185725.9 | loss tokens/s: 129404.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.109593E-01 | lm-global-supervised-tokens: 1.171520E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.328827E-02 | lm-token-buffer-mib: 5.813934E+01 | global_load_balancing_loss: 9.997413E-01 | loss scale: 1.0 | grad norm: 0.061 | params norm: 11612.196 | params rms: 0.062876 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 9:45:10.687802 | finish at 2026-10-10 20:08:27 + [2026-10-04 10:24:42] iteration 742/ 6855 | consumed samples: 94976 | elapsed time per iteration (ms): 85753.9 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.76% | FLOPs estimator: workload-aware | logical tokens/s: 195643.0 | valid tokens/s: 195643.0 | physical tokens/s: 195925.1 | loss tokens/s: 145925.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.243106E-01 | lm-global-supervised-tokens: 1.251368E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.753196E-02 | lm-token-buffer-mib: 5.808399E+01 | global_load_balancing_loss: 1.000207E+00 | loss scale: 1.0 | grad norm: 0.057 | params norm: 11612.110 | params rms: 0.062876 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 1:36:53.573523 | finish at 2026-10-10 12:01:36 + [2026-10-04 10:26:08] iteration 743/ 6855 | consumed samples: 95104 | elapsed time per iteration (ms): 85946.6 | throughput per GPU (TFLOP/s/GPU): 43.1 | MFU 13.80% | FLOPs estimator: workload-aware | logical tokens/s: 195204.0 | valid tokens/s: 195204.0 | physical tokens/s: 195726.8 | loss tokens/s: 141740.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.432399E-01 | lm-global-supervised-tokens: 1.218208E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.418300E-02 | lm-token-buffer-mib: 5.833721E+01 | global_load_balancing_loss: 9.995418E-01 | loss scale: 1.0 | grad norm: 0.121 | params norm: 11612.024 | params rms: 0.062875 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 1:55:05.388718 | finish at 2026-10-10 12:21:13 + [2026-10-04 10:27:34] iteration 744/ 6855 | consumed samples: 95232 | elapsed time per iteration (ms): 86536.3 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.52% | FLOPs estimator: workload-aware | logical tokens/s: 193874.4 | valid tokens/s: 193874.4 | physical tokens/s: 194409.6 | loss tokens/s: 147747.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.365458E-01 | lm-global-supervised-tokens: 1.278554E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.160316E-02 | lm-token-buffer-mib: 5.829645E+01 | global_load_balancing_loss: 9.998101E-01 | loss scale: 1.0 | grad norm: 0.058 | params norm: 11611.938 | params rms: 0.062875 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 2:53:43.416376 | finish at 2026-10-10 13:21:18 + [2026-10-04 10:28:58] iteration 745/ 6855 | consumed samples: 95360 | elapsed time per iteration (ms): 83586.7 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.67% | FLOPs estimator: workload-aware | logical tokens/s: 200715.8 | valid tokens/s: 200715.8 | physical tokens/s: 201232.0 | loss tokens/s: 151848.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.393103E-01 | lm-global-supervised-tokens: 1.269252E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.201253E-02 | lm-token-buffer-mib: 5.820363E+01 | global_load_balancing_loss: 9.999813E-01 | loss scale: 1.0 | grad norm: 0.077 | params norm: 11611.853 | params rms: 0.062874 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 21:51:54.832916 | finish at 2026-10-10 08:20:53 + [2026-10-04 10:30:28] iteration 746/ 6855 | consumed samples: 95488 | elapsed time per iteration (ms): 89854.3 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.40% | FLOPs estimator: workload-aware | logical tokens/s: 186715.4 | valid tokens/s: 186715.4 | physical tokens/s: 187074.7 | loss tokens/s: 132778.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.069237E-01 | lm-global-supervised-tokens: 1.193068E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.277968E-02 | lm-token-buffer-mib: 5.812621E+01 | global_load_balancing_loss: 1.001743E+00 | loss scale: 1.0 | grad norm: 0.085 | params norm: 11611.768 | params rms: 0.062874 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 8:28:40.137310 | finish at 2026-10-10 18:59:08 + [2026-10-04 10:31:59] iteration 747/ 6855 | consumed samples: 95616 | elapsed time per iteration (ms): 91242.4 | throughput per GPU (TFLOP/s/GPU): 41.3 | MFU 13.23% | FLOPs estimator: workload-aware | logical tokens/s: 183875.1 | valid tokens/s: 183875.1 | physical tokens/s: 184320.8 | loss tokens/s: 129030.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.067848E-01 | lm-global-supervised-tokens: 1.177307E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.787872E-02 | lm-token-buffer-mib: 5.815009E+01 | global_load_balancing_loss: 1.001651E+00 | loss scale: 1.0 | grad norm: 0.069 | params norm: 11611.680 | params rms: 0.062873 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 10:48:28.460821 | finish at 2026-10-10 21:20:28 + [2026-10-04 10:33:22] iteration 748/ 6855 | consumed samples: 95744 | elapsed time per iteration (ms): 83249.6 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.55% | FLOPs estimator: workload-aware | logical tokens/s: 201528.2 | valid tokens/s: 201528.2 | physical tokens/s: 202070.8 | loss tokens/s: 155533.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.375340E-01 | lm-global-supervised-tokens: 1.294808E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.151862E-02 | lm-token-buffer-mib: 5.810301E+01 | global_load_balancing_loss: 1.000482E+00 | loss scale: 1.0 | grad norm: 0.088 | params norm: 11611.594 | params rms: 0.062873 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 21:13:25.070919 | finish at 2026-10-10 07:46:47 + [2026-10-04 10:34:51] iteration 749/ 6855 | consumed samples: 95872 | elapsed time per iteration (ms): 88956.0 | throughput per GPU (TFLOP/s/GPU): 41.5 | MFU 13.31% | FLOPs estimator: workload-aware | logical tokens/s: 188601.3 | valid tokens/s: 188601.3 | physical tokens/s: 188959.4 | loss tokens/s: 143803.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.874356E-01 | lm-global-supervised-tokens: 1.279216E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.299535E-02 | lm-token-buffer-mib: 5.806209E+01 | global_load_balancing_loss: 9.993472E-01 | loss scale: 1.0 | grad norm: 0.034 | params norm: 11611.508 | params rms: 0.062873 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 6:52:45.346738 | finish at 2026-10-10 17:27:37 + [2026-10-04 10:36:37] iteration 750/ 6855 | consumed samples: 96000 | elapsed time per iteration (ms): 105603.8 | throughput per GPU (TFLOP/s/GPU): 34.7 | MFU 11.13% | FLOPs estimator: workload-aware | logical tokens/s: 158869.4 | valid tokens/s: 158869.4 | physical tokens/s: 159251.3 | loss tokens/s: 118854.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.615247E-01 | lm-global-supervised-tokens: 1.255150E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.166720E-02 | lm-token-buffer-mib: 5.818770E+01 | global_load_balancing_loss: 9.986534E-01 | loss scale: 1.0 | grad norm: 0.120 | params norm: 11611.422 | params rms: 0.062872 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 7 days, 11:05:10.927169 | finish at 2026-10-11 21:41:48 + [2026-10-04 10:38:07] iteration 751/ 6855 | consumed samples: 96128 | elapsed time per iteration (ms): 90096.6 | throughput per GPU (TFLOP/s/GPU): 40.9 | MFU 13.12% | FLOPs estimator: workload-aware | logical tokens/s: 186213.0 | valid tokens/s: 186213.0 | physical tokens/s: 186764.2 | loss tokens/s: 135843.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.624672E-01 | lm-global-supervised-tokens: 1.223904E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.184329E-02 | lm-token-buffer-mib: 5.820059E+01 | global_load_balancing_loss: 1.000293E+00 | loss scale: 1.0 | grad norm: 0.073 | params norm: 11611.335 | params rms: 0.062872 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 8:45:49.931980 | finish at 2026-10-10 19:23:57 + [2026-10-04 10:39:34] iteration 752/ 6855 | consumed samples: 96256 | elapsed time per iteration (ms): 86692.0 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.54% | FLOPs estimator: workload-aware | logical tokens/s: 193526.3 | valid tokens/s: 193526.3 | physical tokens/s: 193895.9 | loss tokens/s: 145169.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.705782E-01 | lm-global-supervised-tokens: 1.258499E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.959396E-02 | lm-token-buffer-mib: 5.809279E+01 | global_load_balancing_loss: 9.997513E-01 | loss scale: 1.0 | grad norm: 0.061 | params norm: 11611.247 | params rms: 0.062871 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 2:58:00.977175 | finish at 2026-10-10 13:37:35 + [2026-10-04 10:41:04] iteration 753/ 6855 | consumed samples: 96384 | elapsed time per iteration (ms): 89782.9 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.65% | FLOPs estimator: workload-aware | logical tokens/s: 186864.0 | valid tokens/s: 186864.0 | physical tokens/s: 187355.4 | loss tokens/s: 129473.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.702155E-01 | lm-global-supervised-tokens: 1.162451E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.179231E-02 | lm-token-buffer-mib: 5.804169E+01 | global_load_balancing_loss: 1.001118E+00 | loss scale: 1.0 | grad norm: 0.071 | params norm: 11611.161 | params rms: 0.062871 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 8:10:55.291296 | finish at 2026-10-10 18:51:59 + [2026-10-04 10:42:29] iteration 754/ 6855 | consumed samples: 96512 | elapsed time per iteration (ms): 85727.9 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.60% | FLOPs estimator: workload-aware | logical tokens/s: 195702.7 | valid tokens/s: 195702.7 | physical tokens/s: 196210.2 | loss tokens/s: 145273.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.532385E-01 | lm-global-supervised-tokens: 1.245395E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.576550E-02 | lm-token-buffer-mib: 5.811191E+01 | global_load_balancing_loss: 1.000606E+00 | loss scale: 1.0 | grad norm: 0.052 | params norm: 11611.075 | params rms: 0.062870 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 1:17:05.973347 | finish at 2026-10-10 11:59:35 + [2026-10-04 10:43:56] iteration 755/ 6855 | consumed samples: 96640 | elapsed time per iteration (ms): 86963.3 | throughput per GPU (TFLOP/s/GPU): 43.3 | MFU 13.87% | FLOPs estimator: workload-aware | logical tokens/s: 192922.7 | valid tokens/s: 192922.7 | physical tokens/s: 193335.5 | loss tokens/s: 140996.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.030429E-01 | lm-global-supervised-tokens: 1.226152E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.618274E-02 | lm-token-buffer-mib: 5.818356E+01 | global_load_balancing_loss: 1.001202E+00 | loss scale: 1.0 | grad norm: 0.041 | params norm: 11610.990 | params rms: 0.062870 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 3:21:16.356816 | finish at 2026-10-10 14:05:13 + [2026-10-04 10:45:26] iteration 756/ 6855 | consumed samples: 96768 | elapsed time per iteration (ms): 89798.2 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.52% | FLOPs estimator: workload-aware | logical tokens/s: 186831.6 | valid tokens/s: 186831.6 | physical tokens/s: 187216.8 | loss tokens/s: 138576.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.077677E-01 | lm-global-supervised-tokens: 1.244388E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.386775E-02 | lm-token-buffer-mib: 5.807381E+01 | global_load_balancing_loss: 9.999718E-01 | loss scale: 1.0 | grad norm: 0.048 | params norm: 11610.903 | params rms: 0.062869 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 8:07:59.013203 | finish at 2026-10-10 18:53:25 + [2026-10-04 10:46:54] iteration 757/ 6855 | consumed samples: 96896 | elapsed time per iteration (ms): 87780.7 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.76% | FLOPs estimator: workload-aware | logical tokens/s: 191126.5 | valid tokens/s: 191126.5 | physical tokens/s: 191665.0 | loss tokens/s: 131983.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.054054E-01 | lm-global-supervised-tokens: 1.158558E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.233953E-02 | lm-token-buffer-mib: 5.810773E+01 | global_load_balancing_loss: 1.000048E+00 | loss scale: 1.0 | grad norm: 0.064 | params norm: 11610.817 | params rms: 0.062869 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 4:41:26.464156 | finish at 2026-10-10 15:28:20 + [2026-10-04 10:48:18] iteration 758/ 6855 | consumed samples: 97024 | elapsed time per iteration (ms): 84289.2 | throughput per GPU (TFLOP/s/GPU): 43.5 | MFU 13.94% | FLOPs estimator: workload-aware | logical tokens/s: 199042.1 | valid tokens/s: 199042.1 | physical tokens/s: 199355.4 | loss tokens/s: 141871.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.130226E-01 | lm-global-supervised-tokens: 1.195820E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.531062E-02 | lm-token-buffer-mib: 5.810754E+01 | global_load_balancing_loss: 1.000174E+00 | loss scale: 1.0 | grad norm: 0.060 | params norm: 11610.730 | params rms: 0.062868 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 22:45:11.222285 | finish at 2026-10-10 09:33:29 + [2026-10-04 10:49:47] iteration 759/ 6855 | consumed samples: 97152 | elapsed time per iteration (ms): 88808.8 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.53% | FLOPs estimator: workload-aware | logical tokens/s: 188913.5 | valid tokens/s: 188913.5 | physical tokens/s: 189354.6 | loss tokens/s: 142272.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.304898E-01 | lm-global-supervised-tokens: 1.263503E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.269941E-02 | lm-token-buffer-mib: 5.819056E+01 | global_load_balancing_loss: 9.992709E-01 | loss scale: 1.0 | grad norm: 0.099 | params norm: 11610.646 | params rms: 0.062868 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 6:22:58.306618 | finish at 2026-10-10 17:12:45 + [2026-10-04 10:51:35] iteration 760/ 6855 | consumed samples: 97280 | elapsed time per iteration (ms): 108040.6 | throughput per GPU (TFLOP/s/GPU): 34.8 | MFU 11.15% | FLOPs estimator: workload-aware | logical tokens/s: 155286.1 | valid tokens/s: 155286.1 | physical tokens/s: 155596.5 | loss tokens/s: 114835.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.857550E-01 | lm-global-supervised-tokens: 1.240692E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.176023E-02 | lm-token-buffer-mib: 5.809559E+01 | global_load_balancing_loss: 9.994037E-01 | loss scale: 1.0 | grad norm: 0.051 | params norm: 11610.563 | params rms: 0.062867 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 7 days, 14:55:07.416686 | finish at 2026-10-12 01:46:42 +times across ranks (ms): + save-checkpoint: + rank 0: 25068.77 + rank 1: 25068.76 + rank 2: 25068.85 + rank 3: 25068.77 + rank 4: 25068.76 + rank 5: 25068.76 + rank 6: 25068.81 + rank 7: 25068.78 + rank 8: 25067.60 + rank 9: 25067.55 + rank 10: 25067.57 + rank 11: 25067.55 + rank 12: 25067.58 + rank 13: 25067.56 + rank 14: 25067.56 + rank 15: 25067.49 + rank 16: 25068.10 + rank 17: 25068.09 + rank 18: 25068.09 + rank 19: 25068.08 + rank 20: 25068.06 + rank 21: 25068.11 + rank 22: 25068.08 + rank 23: 25068.06 + rank 24: 25068.87 + rank 25: 25068.83 + rank 26: 25068.60 + rank 27: 25068.93 + rank 28: 25068.81 + rank 29: 25069.27 + rank 30: 25068.82 + rank 31: 25068.96 + rank 32: 25069.32 + rank 33: 25069.27 + rank 34: 25069.29 + rank 35: 25069.27 + rank 36: 25069.21 + rank 37: 25069.19 + rank 38: 25069.27 + rank 39: 25069.23 + rank 40: 25068.62 + rank 41: 25068.55 + rank 42: 25068.54 + rank 43: 25068.56 + rank 44: 25068.55 + rank 45: 25068.56 + rank 46: 25068.55 + rank 47: 25068.56 + rank 48: 25069.38 + rank 49: 25069.38 + rank 50: 25069.33 + rank 51: 25069.22 + rank 52: 25069.41 + rank 53: 25069.33 + rank 54: 25069.35 + rank 55: 25069.37 + rank 56: 25069.05 + rank 57: 25069.09 + rank 58: 25069.03 + rank 59: 25069.05 + rank 60: 25069.02 + rank 61: 25069.04 + rank 62: 25069.03 + rank 63: 25069.04 + [2026-10-04 10:53:49] iteration 761/ 6855 | consumed samples: 97408 | elapsed time per iteration (ms): 108441.3 | throughput per GPU (TFLOP/s/GPU): 33.3 | MFU 10.67% | FLOPs estimator: workload-aware | logical tokens/s: 154711.9 | valid tokens/s: 154711.9 | physical tokens/s: 154969.9 | loss tokens/s: 116571.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.657687E-01 | lm-global-supervised-tokens: 1.264114E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 3.922457E-02 | lm-token-buffer-mib: 5.804699E+01 | global_load_balancing_loss: 9.999694E-01 | loss scale: 1.0 | grad norm: 0.065 | params norm: 11610.480 | params rms: 0.062867 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 7 days, 15:34:01.201773 | finish at 2026-10-12 02:27:50 + [2026-10-04 10:55:22] iteration 762/ 6855 | consumed samples: 97536 | elapsed time per iteration (ms): 93194.8 | throughput per GPU (TFLOP/s/GPU): 39.3 | MFU 12.61% | FLOPs estimator: workload-aware | logical tokens/s: 180022.4 | valid tokens/s: 180022.4 | physical tokens/s: 180429.6 | loss tokens/s: 136690.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.153901E-01 | lm-global-supervised-tokens: 1.273885E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.646726E-02 | lm-token-buffer-mib: 5.811916E+01 | global_load_balancing_loss: 1.000638E+00 | loss scale: 1.0 | grad norm: 0.050 | params norm: 11610.398 | params rms: 0.062867 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 13:43:56.043627 | finish at 2026-10-11 00:39:18 + [2026-10-04 10:56:51] iteration 763/ 6855 | consumed samples: 97664 | elapsed time per iteration (ms): 89487.9 | throughput per GPU (TFLOP/s/GPU): 41.5 | MFU 13.29% | FLOPs estimator: workload-aware | logical tokens/s: 187480.2 | valid tokens/s: 187480.2 | physical tokens/s: 187993.4 | loss tokens/s: 131081.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.692885E-01 | lm-global-supervised-tokens: 1.173016E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.323757E-02 | lm-token-buffer-mib: 5.822388E+01 | global_load_balancing_loss: 1.001211E+00 | loss scale: 1.0 | grad norm: 0.046 | params norm: 11610.317 | params rms: 0.062866 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 7:26:00.191052 | finish at 2026-10-10 18:22:51 + [2026-10-04 10:58:19] iteration 764/ 6855 | consumed samples: 97792 | elapsed time per iteration (ms): 87516.1 | throughput per GPU (TFLOP/s/GPU): 42.8 | MFU 13.71% | FLOPs estimator: workload-aware | logical tokens/s: 191703.8 | valid tokens/s: 191703.8 | physical tokens/s: 192276.3 | loss tokens/s: 147758.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.386302E-01 | lm-global-supervised-tokens: 1.293121E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.443934E-02 | lm-token-buffer-mib: 5.828568E+01 | global_load_balancing_loss: 1.001348E+00 | loss scale: 1.0 | grad norm: 0.047 | params norm: 11610.236 | params rms: 0.062866 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 4:04:20.744746 | finish at 2026-10-10 15:02:40 + [2026-10-04 10:59:41] iteration 765/ 6855 | consumed samples: 97920 | elapsed time per iteration (ms): 82245.7 | throughput per GPU (TFLOP/s/GPU): 43.5 | MFU 13.95% | FLOPs estimator: workload-aware | logical tokens/s: 203988.9 | valid tokens/s: 203988.9 | physical tokens/s: 204474.7 | loss tokens/s: 154122.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.603139E-01 | lm-global-supervised-tokens: 1.267591E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.193209E-02 | lm-token-buffer-mib: 5.816431E+01 | global_load_balancing_loss: 1.001393E+00 | loss scale: 1.0 | grad norm: 0.074 | params norm: 11610.155 | params rms: 0.062865 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 19:07:56.379075 | finish at 2026-10-10 06:07:37 + [2026-10-04 11:01:08] iteration 766/ 6855 | consumed samples: 98048 | elapsed time per iteration (ms): 87309.6 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.42% | FLOPs estimator: workload-aware | logical tokens/s: 192157.4 | valid tokens/s: 192157.4 | physical tokens/s: 192736.8 | loss tokens/s: 143227.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.754845E-01 | lm-global-supervised-tokens: 1.250516E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.242059E-02 | lm-token-buffer-mib: 5.818774E+01 | global_load_balancing_loss: 1.000316E+00 | loss scale: 1.0 | grad norm: 0.067 | params norm: 11610.073 | params rms: 0.062865 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 3:40:28.130420 | finish at 2026-10-10 14:41:36 + [2026-10-04 11:02:34] iteration 767/ 6855 | consumed samples: 98176 | elapsed time per iteration (ms): 86029.7 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.65% | FLOPs estimator: workload-aware | logical tokens/s: 195015.9 | valid tokens/s: 195015.9 | physical tokens/s: 195514.5 | loss tokens/s: 145966.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.824834E-01 | lm-global-supervised-tokens: 1.255744E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.804713E-02 | lm-token-buffer-mib: 5.811749E+01 | global_load_balancing_loss: 9.989498E-01 | loss scale: 1.0 | grad norm: 0.060 | params norm: 11609.994 | params rms: 0.062864 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 1:29:09.050442 | finish at 2026-10-10 12:31:43 + [2026-10-04 11:04:00] iteration 768/ 6855 | consumed samples: 98304 | elapsed time per iteration (ms): 85304.6 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.66% | FLOPs estimator: workload-aware | logical tokens/s: 196673.9 | valid tokens/s: 196673.9 | physical tokens/s: 197162.5 | loss tokens/s: 147810.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.543815E-01 | lm-global-supervised-tokens: 1.260895E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.638936E-02 | lm-token-buffer-mib: 5.811822E+01 | global_load_balancing_loss: 9.986609E-01 | loss scale: 1.0 | grad norm: 0.079 | params norm: 11609.914 | params rms: 0.062864 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 0:14:08.959431 | finish at 2026-10-10 11:18:09 + [2026-10-04 11:05:28] iteration 769/ 6855 | consumed samples: 98432 | elapsed time per iteration (ms): 88263.8 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.66% | FLOPs estimator: workload-aware | logical tokens/s: 190080.4 | valid tokens/s: 190080.4 | physical tokens/s: 190483.4 | loss tokens/s: 130657.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.871808E-01 | lm-global-supervised-tokens: 1.153228E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.257158E-02 | lm-token-buffer-mib: 5.806915E+01 | global_load_balancing_loss: 9.998203E-01 | loss scale: 1.0 | grad norm: 0.034 | params norm: 11609.833 | params rms: 0.062863 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 5:12:53.428186 | finish at 2026-10-10 16:18:21 + [2026-10-04 11:07:15] iteration 770/ 6855 | consumed samples: 98560 | elapsed time per iteration (ms): 107123.6 | throughput per GPU (TFLOP/s/GPU): 33.7 | MFU 10.82% | FLOPs estimator: workload-aware | logical tokens/s: 156615.1 | valid tokens/s: 156615.1 | physical tokens/s: 156936.6 | loss tokens/s: 115263.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.487515E-01 | lm-global-supervised-tokens: 1.234743E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.427112E-02 | lm-token-buffer-mib: 5.805857E+01 | global_load_balancing_loss: 1.000365E+00 | loss scale: 1.0 | grad norm: 0.077 | params norm: 11609.751 | params rms: 0.062863 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 7 days, 13:04:07.393291 | finish at 2026-10-12 00:11:22 + [2026-10-04 11:08:44] iteration 771/ 6855 | consumed samples: 98688 | elapsed time per iteration (ms): 88797.9 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.79% | FLOPs estimator: workload-aware | logical tokens/s: 188936.6 | valid tokens/s: 188936.6 | physical tokens/s: 189390.2 | loss tokens/s: 138330.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.368869E-01 | lm-global-supervised-tokens: 1.228344E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.093441E-02 | lm-token-buffer-mib: 5.813206E+01 | global_load_balancing_loss: 1.000838E+00 | loss scale: 1.0 | grad norm: 0.110 | params norm: 11609.669 | params rms: 0.062863 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 6:04:06.456728 | finish at 2026-10-10 17:12:50 + [2026-10-04 11:10:13] iteration 772/ 6855 | consumed samples: 98816 | elapsed time per iteration (ms): 89608.5 | throughput per GPU (TFLOP/s/GPU): 41.2 | MFU 13.19% | FLOPs estimator: workload-aware | logical tokens/s: 187227.8 | valid tokens/s: 187227.8 | physical tokens/s: 187734.6 | loss tokens/s: 134250.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.560738E-01 | lm-global-supervised-tokens: 1.202995E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.170291E-02 | lm-token-buffer-mib: 5.827321E+01 | global_load_balancing_loss: 1.000661E+00 | loss scale: 1.0 | grad norm: 0.100 | params norm: 11609.584 | params rms: 0.062862 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 7:24:48.678109 | finish at 2026-10-10 18:35:02 + [2026-10-04 11:11:39] iteration 773/ 6855 | consumed samples: 98944 | elapsed time per iteration (ms): 85357.5 | throughput per GPU (TFLOP/s/GPU): 43.2 | MFU 13.85% | FLOPs estimator: workload-aware | logical tokens/s: 196552.0 | valid tokens/s: 196552.0 | physical tokens/s: 196948.5 | loss tokens/s: 148284.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.259361E-01 | lm-global-supervised-tokens: 1.265720E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.139169E-02 | lm-token-buffer-mib: 5.816326E+01 | global_load_balancing_loss: 1.000054E+00 | loss scale: 1.0 | grad norm: 0.055 | params norm: 11609.500 | params rms: 0.062862 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 0:12:24.047203 | finish at 2026-10-10 11:24:03 + [2026-10-04 11:13:11] iteration 774/ 6855 | consumed samples: 99072 | elapsed time per iteration (ms): 92265.2 | throughput per GPU (TFLOP/s/GPU): 40.7 | MFU 13.05% | FLOPs estimator: workload-aware | logical tokens/s: 181836.9 | valid tokens/s: 181836.9 | physical tokens/s: 182491.2 | loss tokens/s: 136924.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.321796E-01 | lm-global-supervised-tokens: 1.263338E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.573443E-02 | lm-token-buffer-mib: 5.804889E+01 | global_load_balancing_loss: 9.989575E-01 | loss scale: 1.0 | grad norm: 0.076 | params norm: 11609.416 | params rms: 0.062861 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 11:51:04.442819 | finish at 2026-10-10 23:04:16 + [2026-10-04 11:14:38] iteration 775/ 6855 | consumed samples: 99200 | elapsed time per iteration (ms): 86706.8 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.54% | FLOPs estimator: workload-aware | logical tokens/s: 193493.5 | valid tokens/s: 193493.5 | physical tokens/s: 194125.1 | loss tokens/s: 150232.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.759913E-01 | lm-global-supervised-tokens: 1.302622E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.604568E-02 | lm-token-buffer-mib: 5.817400E+01 | global_load_balancing_loss: 1.000380E+00 | loss scale: 1.0 | grad norm: 0.111 | params norm: 11609.331 | params rms: 0.062861 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 2:26:17.472916 | finish at 2026-10-10 13:40:55 + [2026-10-04 11:16:03] iteration 776/ 6855 | consumed samples: 99328 | elapsed time per iteration (ms): 84913.7 | throughput per GPU (TFLOP/s/GPU): 41.5 | MFU 13.31% | FLOPs estimator: workload-aware | logical tokens/s: 197578.4 | valid tokens/s: 197578.4 | physical tokens/s: 198260.3 | loss tokens/s: 145045.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 5.938112E-01 | lm-global-supervised-tokens: 1.231631E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.611985E-02 | lm-token-buffer-mib: 5.843201E+01 | global_load_balancing_loss: 1.000987E+00 | loss scale: 1.0 | grad norm: 0.101 | params norm: 11609.246 | params rms: 0.062860 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 23:23:10.622073 | finish at 2026-10-10 10:39:13 + [2026-10-04 11:17:31] iteration 777/ 6855 | consumed samples: 99456 | elapsed time per iteration (ms): 88641.6 | throughput per GPU (TFLOP/s/GPU): 41.5 | MFU 13.29% | FLOPs estimator: workload-aware | logical tokens/s: 189269.9 | valid tokens/s: 189269.9 | physical tokens/s: 189791.1 | loss tokens/s: 148346.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.218523E-01 | lm-global-supervised-tokens: 1.314969E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.359942E-02 | lm-token-buffer-mib: 5.827607E+01 | global_load_balancing_loss: 1.000446E+00 | loss scale: 1.0 | grad norm: 0.049 | params norm: 11609.162 | params rms: 0.062860 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 5:39:23.345637 | finish at 2026-10-10 16:56:55 + [2026-10-04 11:18:59] iteration 778/ 6855 | consumed samples: 99584 | elapsed time per iteration (ms): 87683.8 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.75% | FLOPs estimator: workload-aware | logical tokens/s: 191336.9 | valid tokens/s: 191336.9 | physical tokens/s: 191702.7 | loss tokens/s: 139686.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.091848E-01 | lm-global-supervised-tokens: 1.224824E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.775471E-02 | lm-token-buffer-mib: 5.811057E+01 | global_load_balancing_loss: 1.000226E+00 | loss scale: 1.0 | grad norm: 0.054 | params norm: 11609.076 | params rms: 0.062859 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 4:00:54.711839 | finish at 2026-10-10 15:19:54 + [2026-10-04 11:20:30] iteration 779/ 6855 | consumed samples: 99712 | elapsed time per iteration (ms): 90530.3 | throughput per GPU (TFLOP/s/GPU): 41.7 | MFU 13.36% | FLOPs estimator: workload-aware | logical tokens/s: 185321.2 | valid tokens/s: 185321.2 | physical tokens/s: 185636.8 | loss tokens/s: 136498.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.071325E-01 | lm-global-supervised-tokens: 1.235726E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.154134E-02 | lm-token-buffer-mib: 5.816429E+01 | global_load_balancing_loss: 1.000541E+00 | loss scale: 1.0 | grad norm: 0.042 | params norm: 11608.989 | params rms: 0.062859 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 8:47:42.197814 | finish at 2026-10-10 20:08:12 + [2026-10-04 11:22:14] iteration 780/ 6855 | consumed samples: 99840 | elapsed time per iteration (ms): 104632.5 | throughput per GPU (TFLOP/s/GPU): 34.1 | MFU 10.93% | FLOPs estimator: workload-aware | logical tokens/s: 160344.0 | valid tokens/s: 160344.0 | physical tokens/s: 160684.5 | loss tokens/s: 120938.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.468101E-01 | lm-global-supervised-tokens: 1.265411E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.157172E-02 | lm-token-buffer-mib: 5.819363E+01 | global_load_balancing_loss: 1.000677E+00 | loss scale: 1.0 | grad norm: 0.044 | params norm: 11608.902 | params rms: 0.062858 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 7 days, 8:34:02.363226 | finish at 2026-10-11 19:56:17 + [2026-10-04 11:23:42] iteration 781/ 6855 | consumed samples: 99968 | elapsed time per iteration (ms): 87604.2 | throughput per GPU (TFLOP/s/GPU): 41.6 | MFU 13.34% | FLOPs estimator: workload-aware | logical tokens/s: 191511.5 | valid tokens/s: 191511.5 | physical tokens/s: 192009.2 | loss tokens/s: 145913.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.650304E-01 | lm-global-supervised-tokens: 1.278265E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.369742E-02 | lm-token-buffer-mib: 5.817395E+01 | global_load_balancing_loss: 1.000104E+00 | loss scale: 1.0 | grad norm: 0.064 | params norm: 11608.817 | params rms: 0.062858 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 3:48:27.923143 | finish at 2026-10-10 15:12:10 + [2026-10-04 11:25:05] iteration 782/ 6855 | consumed samples: 100096 | elapsed time per iteration (ms): 83661.5 | throughput per GPU (TFLOP/s/GPU): 43.4 | MFU 13.90% | FLOPs estimator: workload-aware | logical tokens/s: 200536.2 | valid tokens/s: 200536.2 | physical tokens/s: 201070.7 | loss tokens/s: 156480.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.309949E-01 | lm-global-supervised-tokens: 1.309142E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.074469E-02 | lm-token-buffer-mib: 5.811594E+01 | global_load_balancing_loss: 1.000421E+00 | loss scale: 1.0 | grad norm: 0.070 | params norm: 11608.731 | params rms: 0.062858 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 21:07:56.254611 | finish at 2026-10-10 08:33:02 + [2026-10-04 11:26:35] iteration 783/ 6855 | consumed samples: 100224 | elapsed time per iteration (ms): 90022.4 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.40% | FLOPs estimator: workload-aware | logical tokens/s: 186366.6 | valid tokens/s: 186366.6 | physical tokens/s: 186769.6 | loss tokens/s: 135448.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.431152E-01 | lm-global-supervised-tokens: 1.219341E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.394216E-02 | lm-token-buffer-mib: 5.811399E+01 | global_load_balancing_loss: 9.997562E-01 | loss scale: 1.0 | grad norm: 0.039 | params norm: 11608.645 | params rms: 0.062857 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 7:50:15.770445 | finish at 2026-10-10 19:16:51 + [2026-10-04 11:28:01] iteration 784/ 6855 | consumed samples: 100352 | elapsed time per iteration (ms): 85864.4 | throughput per GPU (TFLOP/s/GPU): 42.8 | MFU 13.71% | FLOPs estimator: workload-aware | logical tokens/s: 195391.3 | valid tokens/s: 195391.3 | physical tokens/s: 195906.4 | loss tokens/s: 147481.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.893966E-01 | lm-global-supervised-tokens: 1.266337E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.097343E-02 | lm-token-buffer-mib: 5.812752E+01 | global_load_balancing_loss: 1.000061E+00 | loss scale: 1.0 | grad norm: 0.046 | params norm: 11608.556 | params rms: 0.062857 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 0:48:02.978837 | finish at 2026-10-10 12:16:04 + [2026-10-04 11:29:29] iteration 785/ 6855 | consumed samples: 100480 | elapsed time per iteration (ms): 87330.6 | throughput per GPU (TFLOP/s/GPU): 41.5 | MFU 13.30% | FLOPs estimator: workload-aware | logical tokens/s: 192111.1 | valid tokens/s: 192111.1 | physical tokens/s: 192713.8 | loss tokens/s: 149708.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.546009E-01 | lm-global-supervised-tokens: 1.307411E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.864594E-02 | lm-token-buffer-mib: 5.806920E+01 | global_load_balancing_loss: 9.997916E-01 | loss scale: 1.0 | grad norm: 0.088 | params norm: 11608.468 | params rms: 0.062856 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 3:14:56.724775 | finish at 2026-10-10 14:44:25 + [2026-10-04 11:30:56] iteration 786/ 6855 | consumed samples: 100608 | elapsed time per iteration (ms): 86896.0 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.53% | FLOPs estimator: workload-aware | logical tokens/s: 193072.2 | valid tokens/s: 193072.2 | physical tokens/s: 193512.5 | loss tokens/s: 141263.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.595467E-01 | lm-global-supervised-tokens: 1.227527E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.413501E-02 | lm-token-buffer-mib: 5.813734E+01 | global_load_balancing_loss: 1.000673E+00 | loss scale: 1.0 | grad norm: 0.108 | params norm: 11608.380 | params rms: 0.062856 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 2:29:31.745308 | finish at 2026-10-10 14:00:27 + [2026-10-04 11:32:26] iteration 787/ 6855 | consumed samples: 100736 | elapsed time per iteration (ms): 90853.4 | throughput per GPU (TFLOP/s/GPU): 40.6 | MFU 13.01% | FLOPs estimator: workload-aware | logical tokens/s: 184661.7 | valid tokens/s: 184661.7 | physical tokens/s: 185144.6 | loss tokens/s: 131778.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.428252E-01 | lm-global-supervised-tokens: 1.197255E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.222691E-02 | lm-token-buffer-mib: 5.814441E+01 | global_load_balancing_loss: 1.001414E+00 | loss scale: 1.0 | grad norm: 0.052 | params norm: 11608.291 | params rms: 0.062855 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 9:08:18.572930 | finish at 2026-10-10 20:40:45 + [2026-10-04 11:33:56] iteration 788/ 6855 | consumed samples: 100864 | elapsed time per iteration (ms): 89293.7 | throughput per GPU (TFLOP/s/GPU): 41.4 | MFU 13.28% | FLOPs estimator: workload-aware | logical tokens/s: 187887.7 | valid tokens/s: 187887.7 | physical tokens/s: 188403.2 | loss tokens/s: 138390.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.201675E-01 | lm-global-supervised-tokens: 1.235742E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.263986E-02 | lm-token-buffer-mib: 5.833417E+01 | global_load_balancing_loss: 1.000225E+00 | loss scale: 1.0 | grad norm: 0.072 | params norm: 11608.201 | params rms: 0.062855 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 6:29:04.885010 | finish at 2026-10-10 18:03:01 + [2026-10-04 11:35:23] iteration 789/ 6855 | consumed samples: 100992 | elapsed time per iteration (ms): 87736.0 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.40% | FLOPs estimator: workload-aware | logical tokens/s: 191223.0 | valid tokens/s: 191223.0 | physical tokens/s: 191722.9 | loss tokens/s: 148278.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.148706E-01 | lm-global-supervised-tokens: 1.300932E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.333757E-02 | lm-token-buffer-mib: 5.811274E+01 | global_load_balancing_loss: 9.991164E-01 | loss scale: 1.0 | grad norm: 0.142 | params norm: 11608.112 | params rms: 0.062854 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 3:50:06.399928 | finish at 2026-10-10 15:25:30 + [2026-10-04 11:37:10] iteration 790/ 6855 | consumed samples: 101120 | elapsed time per iteration (ms): 106811.0 | throughput per GPU (TFLOP/s/GPU): 34.2 | MFU 10.96% | FLOPs estimator: workload-aware | logical tokens/s: 157073.4 | valid tokens/s: 157073.3 | physical tokens/s: 157475.1 | loss tokens/s: 120431.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.823164E-01 | lm-global-supervised-tokens: 1.286339E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.444447E-02 | lm-token-buffer-mib: 5.822630E+01 | global_load_balancing_loss: 9.997225E-01 | loss scale: 1.0 | grad norm: 0.191 | params norm: 11608.023 | params rms: 0.062854 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 7 days, 11:56:48.802420 | finish at 2026-10-11 23:33:59 + [2026-10-04 11:38:38] iteration 791/ 6855 | consumed samples: 101248 | elapsed time per iteration (ms): 87292.7 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.77% | FLOPs estimator: workload-aware | logical tokens/s: 192194.4 | valid tokens/s: 192194.4 | physical tokens/s: 192597.2 | loss tokens/s: 143661.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.714994E-01 | lm-global-supervised-tokens: 1.254059E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.499511E-02 | lm-token-buffer-mib: 5.804311E+01 | global_load_balancing_loss: 1.001277E+00 | loss scale: 1.0 | grad norm: 0.097 | params norm: 11607.932 | params rms: 0.062853 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 3:02:23.164440 | finish at 2026-10-10 14:41:01 + [2026-10-04 11:40:04] iteration 792/ 6855 | consumed samples: 101376 | elapsed time per iteration (ms): 86690.4 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.74% | FLOPs estimator: workload-aware | logical tokens/s: 193530.0 | valid tokens/s: 193530.0 | physical tokens/s: 193816.9 | loss tokens/s: 143755.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.832377E-01 | lm-global-supervised-tokens: 1.246219E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.315566E-02 | lm-token-buffer-mib: 5.808751E+01 | global_load_balancing_loss: 1.000339E+00 | loss scale: 1.0 | grad norm: 0.055 | params norm: 11607.841 | params rms: 0.062853 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 2:00:03.979790 | finish at 2026-10-10 13:40:08 + [2026-10-04 11:41:30] iteration 793/ 6855 | consumed samples: 101504 | elapsed time per iteration (ms): 85804.6 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.58% | FLOPs estimator: workload-aware | logical tokens/s: 195527.4 | valid tokens/s: 195527.4 | physical tokens/s: 196013.9 | loss tokens/s: 146499.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.624443E-01 | lm-global-supervised-tokens: 1.257034E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.169804E-02 | lm-token-buffer-mib: 5.806234E+01 | global_load_balancing_loss: 1.000076E+00 | loss scale: 1.0 | grad norm: 0.088 | params norm: 11607.752 | params rms: 0.062852 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 0:29:07.433172 | finish at 2026-10-10 12:10:38 + [2026-10-04 11:42:56] iteration 794/ 6855 | consumed samples: 101632 | elapsed time per iteration (ms): 85709.8 | throughput per GPU (TFLOP/s/GPU): 43.4 | MFU 13.92% | FLOPs estimator: workload-aware | logical tokens/s: 195744.4 | valid tokens/s: 195744.4 | physical tokens/s: 196250.2 | loss tokens/s: 137767.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.522179E-01 | lm-global-supervised-tokens: 1.180804E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.157573E-02 | lm-token-buffer-mib: 5.807130E+01 | global_load_balancing_loss: 9.995346E-01 | loss scale: 1.0 | grad norm: 0.153 | params norm: 11607.665 | params rms: 0.062852 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 0:18:06.909973 | finish at 2026-10-10 12:01:03 + [2026-10-04 11:44:23] iteration 795/ 6855 | consumed samples: 101760 | elapsed time per iteration (ms): 87517.2 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.58% | FLOPs estimator: workload-aware | logical tokens/s: 191701.8 | valid tokens/s: 191701.8 | physical tokens/s: 192106.8 | loss tokens/s: 141358.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.202821E-01 | lm-global-supervised-tokens: 1.237126E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.143989E-02 | lm-token-buffer-mib: 5.806418E+01 | global_load_balancing_loss: 9.993998E-01 | loss scale: 1.0 | grad norm: 0.049 | params norm: 11607.576 | params rms: 0.062851 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 3:19:13.987784 | finish at 2026-10-10 15:03:37 + [2026-10-04 11:45:48] iteration 796/ 6855 | consumed samples: 101888 | elapsed time per iteration (ms): 84868.5 | throughput per GPU (TFLOP/s/GPU): 43.8 | MFU 14.05% | FLOPs estimator: workload-aware | logical tokens/s: 197684.7 | valid tokens/s: 197684.7 | physical tokens/s: 198124.5 | loss tokens/s: 148806.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.919142E-01 | lm-global-supervised-tokens: 1.262895E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.833262E-02 | lm-token-buffer-mib: 5.808705E+01 | global_load_balancing_loss: 1.000339E+00 | loss scale: 1.0 | grad norm: 0.114 | params norm: 11607.488 | params rms: 0.062851 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 22:50:18.094118 | finish at 2026-10-10 10:36:06 + [2026-10-04 11:47:14] iteration 797/ 6855 | consumed samples: 102016 | elapsed time per iteration (ms): 85782.6 | throughput per GPU (TFLOP/s/GPU): 42.8 | MFU 13.72% | FLOPs estimator: workload-aware | logical tokens/s: 195577.8 | valid tokens/s: 195577.8 | physical tokens/s: 195997.0 | loss tokens/s: 151567.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.787960E-01 | lm-global-supervised-tokens: 1.300185E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.348983E-02 | lm-token-buffer-mib: 5.812215E+01 | global_load_balancing_loss: 1.000093E+00 | loss scale: 1.0 | grad norm: 0.056 | params norm: 11607.400 | params rms: 0.062850 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 0:21:11.198337 | finish at 2026-10-10 12:08:25 + [2026-10-04 11:48:38] iteration 798/ 6855 | consumed samples: 102144 | elapsed time per iteration (ms): 83950.6 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.60% | FLOPs estimator: workload-aware | logical tokens/s: 199846.2 | valid tokens/s: 199846.2 | physical tokens/s: 200314.3 | loss tokens/s: 153181.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.716940E-01 | lm-global-supervised-tokens: 1.285970E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.755625E-02 | lm-token-buffer-mib: 5.811800E+01 | global_load_balancing_loss: 1.002315E+00 | loss scale: 1.0 | grad norm: 0.060 | params norm: 11607.313 | params rms: 0.062850 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 21:14:48.886179 | finish at 2026-10-10 09:03:27 + [2026-10-04 11:50:04] iteration 799/ 6855 | consumed samples: 102272 | elapsed time per iteration (ms): 86131.4 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.49% | FLOPs estimator: workload-aware | logical tokens/s: 194785.3 | valid tokens/s: 194785.3 | physical tokens/s: 195226.9 | loss tokens/s: 148222.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.856526E-01 | lm-global-supervised-tokens: 1.276660E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.165145E-02 | lm-token-buffer-mib: 5.811867E+01 | global_load_balancing_loss: 1.001857E+00 | loss scale: 1.0 | grad norm: 0.084 | params norm: 11607.226 | params rms: 0.062849 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 0:53:31.894779 | finish at 2026-10-10 12:43:36 + [2026-10-04 11:51:49] iteration 800/ 6855 | consumed samples: 102400 | elapsed time per iteration (ms): 105041.4 | throughput per GPU (TFLOP/s/GPU): 35.1 | MFU 11.25% | FLOPs estimator: workload-aware | logical tokens/s: 159720.0 | valid tokens/s: 159720.0 | physical tokens/s: 159936.1 | loss tokens/s: 121888.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.549841E-01 | lm-global-supervised-tokens: 1.280330E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.764643E-02 | lm-token-buffer-mib: 5.811601E+01 | global_load_balancing_loss: 1.000598E+00 | loss scale: 1.0 | grad norm: 0.056 | params norm: 11607.138 | params rms: 0.062849 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 7 days, 8:40:25.857185 | finish at 2026-10-11 20:32:15 +times across ranks (ms): + save-checkpoint: + rank 0: 24251.55 + rank 1: 24251.53 + rank 2: 24251.73 + rank 3: 24251.54 + rank 4: 24251.55 + rank 5: 24251.54 + rank 6: 24251.52 + rank 7: 24251.55 + rank 8: 24251.52 + rank 9: 24251.51 + rank 10: 24251.53 + rank 11: 24251.52 + rank 12: 24251.51 + rank 13: 24251.54 + rank 14: 24251.52 + rank 15: 24251.52 + rank 16: 24253.49 + rank 17: 24253.53 + rank 18: 24253.49 + rank 19: 24253.50 + rank 20: 24253.48 + rank 21: 24253.50 + rank 22: 24253.48 + rank 23: 24253.49 + rank 24: 24251.14 + rank 25: 24251.16 + rank 26: 24251.19 + rank 27: 24251.18 + rank 28: 24251.12 + rank 29: 24251.13 + rank 30: 24251.14 + rank 31: 24251.30 + rank 32: 24251.17 + rank 33: 24251.18 + rank 34: 24251.14 + rank 35: 24251.15 + rank 36: 24251.14 + rank 37: 24251.12 + rank 38: 24251.14 + rank 39: 24251.15 + rank 40: 24251.58 + rank 41: 24251.56 + rank 42: 24251.55 + rank 43: 24251.57 + rank 44: 24251.55 + rank 45: 24251.59 + rank 46: 24251.55 + rank 47: 24251.56 + rank 48: 24251.62 + rank 49: 24251.63 + rank 50: 24251.69 + rank 51: 24251.61 + rank 52: 24251.63 + rank 53: 24251.60 + rank 54: 24251.62 + rank 55: 24251.61 + rank 56: 24251.62 + rank 57: 24251.60 + rank 58: 24251.58 + rank 59: 24251.58 + rank 60: 24251.58 + rank 61: 24251.62 + rank 62: 24251.60 + rank 63: 24251.60 + [2026-10-04 11:54:14] iteration 801/ 6855 | consumed samples: 102528 | elapsed time per iteration (ms): 118021.3 | throughput per GPU (TFLOP/s/GPU): 31.2 | MFU 10.01% | FLOPs estimator: workload-aware | logical tokens/s: 142153.7 | valid tokens/s: 142153.6 | physical tokens/s: 142528.3 | loss tokens/s: 103022.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.074906E-01 | lm-global-supervised-tokens: 1.215889E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.366990E+00 | lm-token-buffer-mib: 5.841123E+01 | global_load_balancing_loss: 1.000657E+00 | loss scale: 1.0 | grad norm: 0.109 | params norm: 11607.051 | params rms: 0.062848 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 8 days, 6:28:20.833755 | finish at 2026-10-12 18:22:35 + [2026-10-04 11:55:47] iteration 802/ 6855 | consumed samples: 102656 | elapsed time per iteration (ms): 92929.0 | throughput per GPU (TFLOP/s/GPU): 39.1 | MFU 12.55% | FLOPs estimator: workload-aware | logical tokens/s: 180537.7 | valid tokens/s: 180537.7 | physical tokens/s: 180906.7 | loss tokens/s: 138685.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.592864E-01 | lm-global-supervised-tokens: 1.288789E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.976845E-02 | lm-token-buffer-mib: 5.820721E+01 | global_load_balancing_loss: 9.985531E-01 | loss scale: 1.0 | grad norm: 0.088 | params norm: 11606.964 | params rms: 0.062848 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 12:14:58.983849 | finish at 2026-10-11 00:10:46 + [2026-10-04 11:57:15] iteration 803/ 6855 | consumed samples: 102784 | elapsed time per iteration (ms): 88557.1 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.51% | FLOPs estimator: workload-aware | logical tokens/s: 189450.3 | valid tokens/s: 189450.3 | physical tokens/s: 190045.0 | loss tokens/s: 136160.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.413423E-01 | lm-global-supervised-tokens: 1.205800E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.290795E-02 | lm-token-buffer-mib: 5.819352E+01 | global_load_balancing_loss: 9.988232E-01 | loss scale: 1.0 | grad norm: 0.109 | params norm: 11606.874 | params rms: 0.062847 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 4:52:27.383413 | finish at 2026-10-10 16:49:43 + [2026-10-04 11:58:43] iteration 804/ 6855 | consumed samples: 102912 | elapsed time per iteration (ms): 87885.2 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.59% | FLOPs estimator: workload-aware | logical tokens/s: 190899.2 | valid tokens/s: 190899.2 | physical tokens/s: 191354.0 | loss tokens/s: 146282.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.816949E-01 | lm-global-supervised-tokens: 1.285603E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.447568E-02 | lm-token-buffer-mib: 5.815988E+01 | global_load_balancing_loss: 9.994567E-01 | loss scale: 1.0 | grad norm: 0.112 | params norm: 11606.786 | params rms: 0.062847 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 3:43:13.418919 | finish at 2026-10-10 15:41:56 + [2026-10-04 12:00:09] iteration 805/ 6855 | consumed samples: 103040 | elapsed time per iteration (ms): 86243.5 | throughput per GPU (TFLOP/s/GPU): 41.7 | MFU 13.36% | FLOPs estimator: workload-aware | logical tokens/s: 194533.2 | valid tokens/s: 194533.2 | physical tokens/s: 194895.4 | loss tokens/s: 147124.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.693070E-01 | lm-global-supervised-tokens: 1.268854E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.512706E-02 | lm-token-buffer-mib: 5.812240E+01 | global_load_balancing_loss: 1.000944E+00 | loss scale: 1.0 | grad norm: 0.064 | params norm: 11606.698 | params rms: 0.062846 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 0:56:12.889364 | finish at 2026-10-10 12:56:22 + [2026-10-04 12:01:38] iteration 806/ 6855 | consumed samples: 103168 | elapsed time per iteration (ms): 88802.9 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.67% | FLOPs estimator: workload-aware | logical tokens/s: 188926.3 | valid tokens/s: 188926.3 | physical tokens/s: 189340.4 | loss tokens/s: 132711.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.235495E-01 | lm-global-supervised-tokens: 1.178514E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.296909E-02 | lm-token-buffer-mib: 5.813004E+01 | global_load_balancing_loss: 1.001343E+00 | loss scale: 1.0 | grad norm: 0.090 | params norm: 11606.611 | params rms: 0.062846 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 5:12:48.568054 | finish at 2026-10-10 17:14:27 + [2026-10-04 12:03:05] iteration 807/ 6855 | consumed samples: 103296 | elapsed time per iteration (ms): 87021.7 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.67% | FLOPs estimator: workload-aware | logical tokens/s: 192793.4 | valid tokens/s: 192793.4 | physical tokens/s: 193128.5 | loss tokens/s: 141630.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.903797E-01 | lm-global-supervised-tokens: 1.232493E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.260589E-02 | lm-token-buffer-mib: 5.804893E+01 | global_load_balancing_loss: 1.001925E+00 | loss scale: 1.0 | grad norm: 0.049 | params norm: 11606.523 | params rms: 0.062846 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 2:11:47.251122 | finish at 2026-10-10 14:14:52 + [2026-10-04 12:04:31] iteration 808/ 6855 | consumed samples: 103424 | elapsed time per iteration (ms): 85523.5 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.76% | FLOPs estimator: workload-aware | logical tokens/s: 196170.8 | valid tokens/s: 196170.8 | physical tokens/s: 196550.7 | loss tokens/s: 154055.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.353181E-01 | lm-global-supervised-tokens: 1.317540E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.147313E-02 | lm-token-buffer-mib: 5.819582E+01 | global_load_balancing_loss: 9.994833E-01 | loss scale: 1.0 | grad norm: 0.052 | params norm: 11606.434 | params rms: 0.062845 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 23:39:20.424078 | finish at 2026-10-10 11:43:51 + [2026-10-04 12:05:55] iteration 809/ 6855 | consumed samples: 103552 | elapsed time per iteration (ms): 84156.7 | throughput per GPU (TFLOP/s/GPU): 43.7 | MFU 14.02% | FLOPs estimator: workload-aware | logical tokens/s: 199356.6 | valid tokens/s: 199356.6 | physical tokens/s: 200024.1 | loss tokens/s: 149140.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.734168E-01 | lm-global-supervised-tokens: 1.255120E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.291746E-02 | lm-token-buffer-mib: 5.814707E+01 | global_load_balancing_loss: 9.986361E-01 | loss scale: 1.0 | grad norm: 0.060 | params norm: 11606.348 | params rms: 0.062845 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 21:20:11.279279 | finish at 2026-10-10 09:26:06 + [2026-10-04 12:07:44] iteration 810/ 6855 | consumed samples: 103680 | elapsed time per iteration (ms): 108967.4 | throughput per GPU (TFLOP/s/GPU): 34.1 | MFU 10.94% | FLOPs estimator: workload-aware | logical tokens/s: 153965.5 | valid tokens/s: 153965.5 | physical tokens/s: 154405.1 | loss tokens/s: 111463.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.937408E-01 | lm-global-supervised-tokens: 1.214586E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.204273E-02 | lm-token-buffer-mib: 5.817955E+01 | global_load_balancing_loss: 9.992078E-01 | loss scale: 1.0 | grad norm: 0.043 | params norm: 11606.262 | params rms: 0.062844 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 7 days, 14:58:27.805177 | finish at 2026-10-12 03:06:12 + [2026-10-04 12:09:13] iteration 811/ 6855 | consumed samples: 103808 | elapsed time per iteration (ms): 89242.3 | throughput per GPU (TFLOP/s/GPU): 41.4 | MFU 13.28% | FLOPs estimator: workload-aware | logical tokens/s: 187995.7 | valid tokens/s: 187995.7 | physical tokens/s: 188554.1 | loss tokens/s: 140962.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.903234E-01 | lm-global-supervised-tokens: 1.257983E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.509136E-02 | lm-token-buffer-mib: 5.822566E+01 | global_load_balancing_loss: 9.984750E-01 | loss scale: 1.0 | grad norm: 0.114 | params norm: 11606.174 | params rms: 0.062844 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 5:49:40.720783 | finish at 2026-10-10 17:58:54 + [2026-10-04 12:10:44] iteration 812/ 6855 | consumed samples: 103936 | elapsed time per iteration (ms): 91247.5 | throughput per GPU (TFLOP/s/GPU): 40.7 | MFU 13.06% | FLOPs estimator: workload-aware | logical tokens/s: 183864.5 | valid tokens/s: 183864.5 | physical tokens/s: 184290.2 | loss tokens/s: 136540.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.320921E-01 | lm-global-supervised-tokens: 1.245899E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.491870E-02 | lm-token-buffer-mib: 5.814884E+01 | global_load_balancing_loss: 1.000809E+00 | loss scale: 1.0 | grad norm: 0.057 | params norm: 11606.086 | params rms: 0.062843 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 9:10:08.387139 | finish at 2026-10-10 21:20:53 + [2026-10-04 12:12:10] iteration 813/ 6855 | consumed samples: 104064 | elapsed time per iteration (ms): 86039.0 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.45% | FLOPs estimator: workload-aware | logical tokens/s: 194995.4 | valid tokens/s: 194995.3 | physical tokens/s: 195564.3 | loss tokens/s: 154875.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.759307E-01 | lm-global-supervised-tokens: 1.332538E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.109099E-02 | lm-token-buffer-mib: 5.824828E+01 | global_load_balancing_loss: 1.002655E+00 | loss scale: 1.0 | grad norm: 0.139 | params norm: 11606.000 | params rms: 0.062843 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 0:24:07.851405 | finish at 2026-10-10 12:36:18 + [2026-10-04 12:13:42] iteration 814/ 6855 | consumed samples: 104192 | elapsed time per iteration (ms): 92131.2 | throughput per GPU (TFLOP/s/GPU): 40.7 | MFU 13.04% | FLOPs estimator: workload-aware | logical tokens/s: 182100.8 | valid tokens/s: 182100.7 | physical tokens/s: 182327.2 | loss tokens/s: 133129.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.966276E-01 | lm-global-supervised-tokens: 1.226535E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.457089E-02 | lm-token-buffer-mib: 5.819609E+01 | global_load_balancing_loss: 1.002310E+00 | loss scale: 1.0 | grad norm: 0.052 | params norm: 11605.916 | params rms: 0.062842 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 10:36:04.784175 | finish at 2026-10-10 22:49:47 + [2026-10-04 12:15:09] iteration 815/ 6855 | consumed samples: 104320 | elapsed time per iteration (ms): 86394.1 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.41% | FLOPs estimator: workload-aware | logical tokens/s: 194193.2 | valid tokens/s: 194193.2 | physical tokens/s: 194789.2 | loss tokens/s: 149247.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.460441E-01 | lm-global-supervised-tokens: 1.289412E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.153566E-02 | lm-token-buffer-mib: 5.822148E+01 | global_load_balancing_loss: 1.001246E+00 | loss scale: 1.0 | grad norm: 0.379 | params norm: 11605.832 | params rms: 0.062842 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 0:57:00.304661 | finish at 2026-10-10 13:12:09 + [2026-10-04 12:16:33] iteration 816/ 6855 | consumed samples: 104448 | elapsed time per iteration (ms): 84517.7 | throughput per GPU (TFLOP/s/GPU): 43.8 | MFU 14.03% | FLOPs estimator: workload-aware | logical tokens/s: 198504.7 | valid tokens/s: 198504.7 | physical tokens/s: 198881.4 | loss tokens/s: 149950.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.961121E-01 | lm-global-supervised-tokens: 1.267346E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.197038E-02 | lm-token-buffer-mib: 5.809470E+01 | global_load_balancing_loss: 1.000118E+00 | loss scale: 1.0 | grad norm: 0.194 | params norm: 11605.747 | params rms: 0.062841 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 21:46:42.191346 | finish at 2026-10-10 10:03:16 + [2026-10-04 12:17:59] iteration 817/ 6855 | consumed samples: 104576 | elapsed time per iteration (ms): 85988.8 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.62% | FLOPs estimator: workload-aware | logical tokens/s: 195108.3 | valid tokens/s: 195108.3 | physical tokens/s: 195585.4 | loss tokens/s: 148907.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.920634E-01 | lm-global-supervised-tokens: 1.280435E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.210056E-02 | lm-token-buffer-mib: 5.815087E+01 | global_load_balancing_loss: 9.995932E-01 | loss scale: 1.0 | grad norm: 0.307 | params norm: 11605.662 | params rms: 0.062841 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 0:13:20.081022 | finish at 2026-10-10 12:31:19 + [2026-10-04 12:19:28] iteration 818/ 6855 | consumed samples: 104704 | elapsed time per iteration (ms): 88382.5 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.44% | FLOPs estimator: workload-aware | logical tokens/s: 189824.4 | valid tokens/s: 189824.4 | physical tokens/s: 190156.6 | loss tokens/s: 136772.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 5.990223E-01 | lm-global-supervised-tokens: 1.208826E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.324280E-02 | lm-token-buffer-mib: 5.807367E+01 | global_load_balancing_loss: 9.993002E-01 | loss scale: 1.0 | grad norm: 0.138 | params norm: 11605.576 | params rms: 0.062840 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 4:12:45.146340 | finish at 2026-10-10 16:32:13 + [2026-10-04 12:20:51] iteration 819/ 6855 | consumed samples: 104832 | elapsed time per iteration (ms): 83602.7 | throughput per GPU (TFLOP/s/GPU): 43.7 | MFU 14.00% | FLOPs estimator: workload-aware | logical tokens/s: 200677.9 | valid tokens/s: 200677.9 | physical tokens/s: 201115.1 | loss tokens/s: 151112.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.329698E-01 | lm-global-supervised-tokens: 1.263340E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.600286E-02 | lm-token-buffer-mib: 5.811572E+01 | global_load_balancing_loss: 9.986327E-01 | loss scale: 1.0 | grad norm: 0.189 | params norm: 11605.491 | params rms: 0.062840 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 20:10:25.828094 | finish at 2026-10-10 08:31:17 + [2026-10-04 12:22:39] iteration 820/ 6855 | consumed samples: 104960 | elapsed time per iteration (ms): 107792.5 | throughput per GPU (TFLOP/s/GPU): 34.8 | MFU 11.14% | FLOPs estimator: workload-aware | logical tokens/s: 155643.4 | valid tokens/s: 155643.4 | physical tokens/s: 156050.8 | loss tokens/s: 114239.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.054470E-01 | lm-global-supervised-tokens: 1.231412E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.868851E-02 | lm-token-buffer-mib: 5.822826E+01 | global_load_balancing_loss: 9.990577E-01 | loss scale: 1.0 | grad norm: 0.193 | params norm: 11605.404 | params rms: 0.062839 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 7 days, 12:42:07.600924 | finish at 2026-10-12 01:04:47 + [2026-10-04 12:24:05] iteration 821/ 6855 | consumed samples: 105088 | elapsed time per iteration (ms): 86268.5 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.55% | FLOPs estimator: workload-aware | logical tokens/s: 194476.6 | valid tokens/s: 194476.6 | physical tokens/s: 194959.6 | loss tokens/s: 148643.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.118705E-01 | lm-global-supervised-tokens: 1.282328E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.541528E-02 | lm-token-buffer-mib: 5.811391E+01 | global_load_balancing_loss: 1.000752E+00 | loss scale: 1.0 | grad norm: 0.139 | params norm: 11605.317 | params rms: 0.062839 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 0:35:43.948275 | finish at 2026-10-10 12:59:49 + [2026-10-04 12:25:29] iteration 822/ 6855 | consumed samples: 105216 | elapsed time per iteration (ms): 83456.2 | throughput per GPU (TFLOP/s/GPU): 44.0 | MFU 14.10% | FLOPs estimator: workload-aware | logical tokens/s: 201029.4 | valid tokens/s: 201029.4 | physical tokens/s: 201511.4 | loss tokens/s: 147139.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.331597E-01 | lm-global-supervised-tokens: 1.227966E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.156644E-02 | lm-token-buffer-mib: 5.815896E+01 | global_load_balancing_loss: 1.001109E+00 | loss scale: 1.0 | grad norm: 0.104 | params norm: 11605.232 | params rms: 0.062839 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 19:51:31.344521 | finish at 2026-10-10 08:17:00 + [2026-10-04 12:26:58] iteration 823/ 6855 | consumed samples: 105344 | elapsed time per iteration (ms): 88791.8 | throughput per GPU (TFLOP/s/GPU): 41.3 | MFU 13.22% | FLOPs estimator: workload-aware | logical tokens/s: 188949.5 | valid tokens/s: 188949.5 | physical tokens/s: 189453.3 | loss tokens/s: 147733.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.733686E-01 | lm-global-supervised-tokens: 1.311751E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.151827E-02 | lm-token-buffer-mib: 5.808582E+01 | global_load_balancing_loss: 9.997174E-01 | loss scale: 1.0 | grad norm: 0.114 | params norm: 11605.148 | params rms: 0.062838 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 4:46:32.268604 | finish at 2026-10-10 17:13:30 + [2026-10-04 12:28:23] iteration 824/ 6855 | consumed samples: 105472 | elapsed time per iteration (ms): 85287.8 | throughput per GPU (TFLOP/s/GPU): 43.9 | MFU 14.06% | FLOPs estimator: workload-aware | logical tokens/s: 196712.5 | valid tokens/s: 196712.5 | physical tokens/s: 197061.0 | loss tokens/s: 136528.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.766964E-01 | lm-global-supervised-tokens: 1.164425E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.187680E-02 | lm-token-buffer-mib: 5.811602E+01 | global_load_balancing_loss: 1.001936E+00 | loss scale: 1.0 | grad norm: 0.105 | params norm: 11605.064 | params rms: 0.062838 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 22:52:50.683421 | finish at 2026-10-10 11:21:14 + [2026-10-04 12:29:48] iteration 825/ 6855 | consumed samples: 105600 | elapsed time per iteration (ms): 85077.7 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.76% | FLOPs estimator: workload-aware | logical tokens/s: 197198.5 | valid tokens/s: 197198.4 | physical tokens/s: 197591.5 | loss tokens/s: 150068.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.700519E-01 | lm-global-supervised-tokens: 1.276750E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.303351E-02 | lm-token-buffer-mib: 5.823236E+01 | global_load_balancing_loss: 9.997916E-01 | loss scale: 1.0 | grad norm: 0.116 | params norm: 11604.978 | params rms: 0.062837 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 22:30:18.804989 | finish at 2026-10-10 11:00:07 + [2026-10-04 12:31:15] iteration 826/ 6855 | consumed samples: 105728 | elapsed time per iteration (ms): 87061.9 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.41% | FLOPs estimator: workload-aware | logical tokens/s: 192704.1 | valid tokens/s: 192704.1 | physical tokens/s: 193238.7 | loss tokens/s: 143048.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.840893E-01 | lm-global-supervised-tokens: 1.245408E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.573115E-02 | lm-token-buffer-mib: 5.819880E+01 | global_load_balancing_loss: 1.000736E+00 | loss scale: 1.0 | grad norm: 0.103 | params norm: 11604.892 | params rms: 0.062837 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 1:48:16.138440 | finish at 2026-10-10 14:19:31 + [2026-10-04 12:32:45] iteration 827/ 6855 | consumed samples: 105856 | elapsed time per iteration (ms): 89859.8 | throughput per GPU (TFLOP/s/GPU): 41.3 | MFU 13.24% | FLOPs estimator: workload-aware | logical tokens/s: 186704.4 | valid tokens/s: 186704.4 | physical tokens/s: 187046.7 | loss tokens/s: 135428.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.662176E-01 | lm-global-supervised-tokens: 1.216960E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.370972E-02 | lm-token-buffer-mib: 5.807790E+01 | global_load_balancing_loss: 1.001078E+00 | loss scale: 1.0 | grad norm: 0.058 | params norm: 11604.809 | params rms: 0.062836 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 6:27:54.619185 | finish at 2026-10-10 19:00:40 + [2026-10-04 12:34:14] iteration 828/ 6855 | consumed samples: 105984 | elapsed time per iteration (ms): 89276.8 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.47% | FLOPs estimator: workload-aware | logical tokens/s: 187923.7 | valid tokens/s: 187923.7 | physical tokens/s: 188591.8 | loss tokens/s: 137772.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.167899E-01 | lm-global-supervised-tokens: 1.229987E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.209545E-02 | lm-token-buffer-mib: 5.833518E+01 | global_load_balancing_loss: 9.995529E-01 | loss scale: 1.0 | grad norm: 0.152 | params norm: 11604.724 | params rms: 0.062836 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 5:27:51.013013 | finish at 2026-10-10 18:02:05 + [2026-10-04 12:35:43] iteration 829/ 6855 | consumed samples: 106112 | elapsed time per iteration (ms): 88468.3 | throughput per GPU (TFLOP/s/GPU): 41.2 | MFU 13.21% | FLOPs estimator: workload-aware | logical tokens/s: 189640.5 | valid tokens/s: 189640.5 | physical tokens/s: 190156.6 | loss tokens/s: 141325.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.400363E-01 | lm-global-supervised-tokens: 1.250281E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.473796E-02 | lm-token-buffer-mib: 5.823252E+01 | global_load_balancing_loss: 1.000050E+00 | loss scale: 1.0 | grad norm: 0.074 | params norm: 11604.637 | params rms: 0.062835 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 4:05:09.868674 | finish at 2026-10-10 16:40:53 + [2026-10-04 12:37:31] iteration 830/ 6855 | consumed samples: 106240 | elapsed time per iteration (ms): 108627.9 | throughput per GPU (TFLOP/s/GPU): 34.7 | MFU 11.14% | FLOPs estimator: workload-aware | logical tokens/s: 154446.4 | valid tokens/s: 154446.4 | physical tokens/s: 154839.5 | loss tokens/s: 111615.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.042577E-01 | lm-global-supervised-tokens: 1.212460E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.178918E-02 | lm-token-buffer-mib: 5.812757E+01 | global_load_balancing_loss: 9.998186E-01 | loss scale: 1.0 | grad norm: 0.108 | params norm: 11604.552 | params rms: 0.062835 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 7 days, 13:48:03.030730 | finish at 2026-10-12 02:25:34 + [2026-10-04 12:38:59] iteration 831/ 6855 | consumed samples: 106368 | elapsed time per iteration (ms): 87629.3 | throughput per GPU (TFLOP/s/GPU): 41.6 | MFU 13.34% | FLOPs estimator: workload-aware | logical tokens/s: 191456.6 | valid tokens/s: 191456.6 | physical tokens/s: 191953.3 | loss tokens/s: 138162.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.452674E-01 | lm-global-supervised-tokens: 1.210706E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.399173E-02 | lm-token-buffer-mib: 5.818690E+01 | global_load_balancing_loss: 9.996524E-01 | loss scale: 1.0 | grad norm: 0.082 | params norm: 11604.468 | params rms: 0.062834 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 2:37:58.605171 | finish at 2026-10-10 15:16:58 + [2026-10-04 12:40:29] iteration 832/ 6855 | consumed samples: 106496 | elapsed time per iteration (ms): 89894.2 | throughput per GPU (TFLOP/s/GPU): 41.7 | MFU 13.37% | FLOPs estimator: workload-aware | logical tokens/s: 186632.8 | valid tokens/s: 186632.8 | physical tokens/s: 187037.3 | loss tokens/s: 136556.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.977264E-01 | lm-global-supervised-tokens: 1.227567E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.653978E-02 | lm-token-buffer-mib: 5.810117E+01 | global_load_balancing_loss: 9.998860E-01 | loss scale: 1.0 | grad norm: 0.109 | params norm: 11604.381 | params rms: 0.062834 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 6:23:52.877693 | finish at 2026-10-10 19:04:22 + [2026-10-04 12:41:54] iteration 833/ 6855 | consumed samples: 106624 | elapsed time per iteration (ms): 85514.1 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.79% | FLOPs estimator: workload-aware | logical tokens/s: 196192.0 | valid tokens/s: 196192.0 | physical tokens/s: 196591.6 | loss tokens/s: 140943.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.531612E-01 | lm-global-supervised-tokens: 1.205268E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.264836E-02 | lm-token-buffer-mib: 5.807565E+01 | global_load_balancing_loss: 1.000543E+00 | loss scale: 1.0 | grad norm: 0.052 | params norm: 11604.294 | params rms: 0.062833 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 23:02:46.001103 | finish at 2026-10-10 11:44:40 + [2026-10-04 12:43:17] iteration 834/ 6855 | consumed samples: 106752 | elapsed time per iteration (ms): 82231.7 | throughput per GPU (TFLOP/s/GPU): 43.5 | MFU 13.96% | FLOPs estimator: workload-aware | logical tokens/s: 204023.7 | valid tokens/s: 204023.7 | physical tokens/s: 204501.0 | loss tokens/s: 155982.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.757447E-01 | lm-global-supervised-tokens: 1.282673E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.271486E-02 | lm-token-buffer-mib: 5.832858E+01 | global_load_balancing_loss: 9.998251E-01 | loss scale: 1.0 | grad norm: 0.045 | params norm: 11604.209 | params rms: 0.062833 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 17:31:57.040956 | finish at 2026-10-10 06:15:14 + [2026-10-04 12:44:43] iteration 835/ 6855 | consumed samples: 106880 | elapsed time per iteration (ms): 86665.9 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.77% | FLOPs estimator: workload-aware | logical tokens/s: 193585.0 | valid tokens/s: 193585.0 | physical tokens/s: 193944.5 | loss tokens/s: 145645.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.564053E-01 | lm-global-supervised-tokens: 1.262246E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.542836E-02 | lm-token-buffer-mib: 5.820958E+01 | global_load_balancing_loss: 1.000413E+00 | loss scale: 1.0 | grad norm: 0.080 | params norm: 11604.124 | params rms: 0.062833 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 0:55:28.597388 | finish at 2026-10-10 13:40:12 + [2026-10-04 12:46:10] iteration 836/ 6855 | consumed samples: 107008 | elapsed time per iteration (ms): 86475.1 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.56% | FLOPs estimator: workload-aware | logical tokens/s: 194012.1 | valid tokens/s: 194012.1 | physical tokens/s: 194502.7 | loss tokens/s: 145118.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.994570E-01 | lm-global-supervised-tokens: 1.254913E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.452927E-02 | lm-token-buffer-mib: 5.833743E+01 | global_load_balancing_loss: 1.001126E+00 | loss scale: 1.0 | grad norm: 0.050 | params norm: 11604.040 | params rms: 0.062832 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 0:34:53.487944 | finish at 2026-10-10 13:21:03 + [2026-10-04 12:47:35] iteration 837/ 6855 | consumed samples: 107136 | elapsed time per iteration (ms): 84887.4 | throughput per GPU (TFLOP/s/GPU): 43.3 | MFU 13.87% | FLOPs estimator: workload-aware | logical tokens/s: 197640.8 | valid tokens/s: 197640.7 | physical tokens/s: 198296.0 | loss tokens/s: 142225.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.635532E-01 | lm-global-supervised-tokens: 1.207316E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.163971E-02 | lm-token-buffer-mib: 5.807247E+01 | global_load_balancing_loss: 1.000577E+00 | loss scale: 1.0 | grad norm: 0.105 | params norm: 11603.955 | params rms: 0.062832 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 21:54:12.424323 | finish at 2026-10-10 10:41:47 + [2026-10-04 12:49:04] iteration 838/ 6855 | consumed samples: 107264 | elapsed time per iteration (ms): 89216.1 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.50% | FLOPs estimator: workload-aware | logical tokens/s: 188050.9 | valid tokens/s: 188050.9 | physical tokens/s: 188530.5 | loss tokens/s: 137533.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.779320E-01 | lm-global-supervised-tokens: 1.227017E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.268477E-02 | lm-token-buffer-mib: 5.809669E+01 | global_load_balancing_loss: 1.001384E+00 | loss scale: 1.0 | grad norm: 0.058 | params norm: 11603.871 | params rms: 0.062831 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 5:06:53.082767 | finish at 2026-10-10 17:55:57 + [2026-10-04 12:50:28] iteration 839/ 6855 | consumed samples: 107392 | elapsed time per iteration (ms): 84623.7 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.60% | FLOPs estimator: workload-aware | logical tokens/s: 198255.6 | valid tokens/s: 198255.6 | physical tokens/s: 198683.0 | loss tokens/s: 144557.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.026319E-01 | lm-global-supervised-tokens: 1.223301E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.372328E-02 | lm-token-buffer-mib: 5.812675E+01 | global_load_balancing_loss: 9.998158E-01 | loss scale: 1.0 | grad norm: 0.042 | params norm: 11603.788 | params rms: 0.062831 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 21:24:56.332092 | finish at 2026-10-10 10:15:25 + [2026-10-04 12:52:20] iteration 840/ 6855 | consumed samples: 107520 | elapsed time per iteration (ms): 111310.1 | throughput per GPU (TFLOP/s/GPU): 33.4 | MFU 10.72% | FLOPs estimator: workload-aware | logical tokens/s: 150724.2 | valid tokens/s: 150724.2 | physical tokens/s: 151114.6 | loss tokens/s: 111058.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.355600E-01 | lm-global-supervised-tokens: 1.236192E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.603978E-02 | lm-token-buffer-mib: 5.812819E+01 | global_load_balancing_loss: 1.000701E+00 | loss scale: 1.0 | grad norm: 0.100 | params norm: 11603.704 | params rms: 0.062830 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 7 days, 17:58:50.217555 | finish at 2026-10-12 06:51:10 +times across ranks (ms): + save-checkpoint: + rank 0: 25727.19 + rank 1: 25727.18 + rank 2: 25727.23 + rank 3: 25727.19 + rank 4: 25727.20 + rank 5: 25727.17 + rank 6: 25727.15 + rank 7: 25727.16 + rank 8: 25727.04 + rank 9: 25726.99 + rank 10: 25727.01 + rank 11: 25727.00 + rank 12: 25727.02 + rank 13: 25727.00 + rank 14: 25727.04 + rank 15: 25726.98 + rank 16: 25726.57 + rank 17: 25726.57 + rank 18: 25726.55 + rank 19: 25726.58 + rank 20: 25726.54 + rank 21: 25726.58 + rank 22: 25726.55 + rank 23: 25726.54 + rank 24: 25727.55 + rank 25: 25727.51 + rank 26: 25727.54 + rank 27: 25727.60 + rank 28: 25727.51 + rank 29: 25727.86 + rank 30: 25727.98 + rank 31: 25727.54 + rank 32: 25726.90 + rank 33: 25726.89 + rank 34: 25726.89 + rank 35: 25726.88 + rank 36: 25726.99 + rank 37: 25726.83 + rank 38: 25726.85 + rank 39: 25726.87 + rank 40: 25727.05 + rank 41: 25727.01 + rank 42: 25727.02 + rank 43: 25727.03 + rank 44: 25727.04 + rank 45: 25727.02 + rank 46: 25726.90 + rank 47: 25726.87 + rank 48: 25726.82 + rank 49: 25726.81 + rank 50: 25726.93 + rank 51: 25726.64 + rank 52: 25726.88 + rank 53: 25726.81 + rank 54: 25726.68 + rank 55: 25726.80 + rank 56: 25726.33 + rank 57: 25726.33 + rank 58: 25726.28 + rank 59: 25726.33 + rank 60: 25726.29 + rank 61: 25726.33 + rank 62: 25726.28 + rank 63: 25726.33 + [2026-10-04 12:54:18] iteration 841/ 6855 | consumed samples: 107648 | elapsed time per iteration (ms): 92371.1 | throughput per GPU (TFLOP/s/GPU): 40.3 | MFU 12.92% | FLOPs estimator: workload-aware | logical tokens/s: 181627.9 | valid tokens/s: 181627.9 | physical tokens/s: 181991.1 | loss tokens/s: 130661.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.648956E-01 | lm-global-supervised-tokens: 1.206935E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 9.249241E-02 | lm-token-buffer-mib: 5.805186E+01 | global_load_balancing_loss: 9.998378E-01 | loss scale: 1.0 | grad norm: 0.097 | params norm: 11603.620 | params rms: 0.062830 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 10:18:39.726268 | finish at 2026-10-10 23:12:58 + [2026-10-04 12:55:50] iteration 842/ 6855 | consumed samples: 107776 | elapsed time per iteration (ms): 92093.2 | throughput per GPU (TFLOP/s/GPU): 39.5 | MFU 12.65% | FLOPs estimator: workload-aware | logical tokens/s: 182176.3 | valid tokens/s: 182176.3 | physical tokens/s: 182537.0 | loss tokens/s: 131363.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.605424E-01 | lm-global-supervised-tokens: 1.209764E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.394521E-02 | lm-token-buffer-mib: 5.816525E+01 | global_load_balancing_loss: 9.993998E-01 | loss scale: 1.0 | grad norm: 0.072 | params norm: 11603.534 | params rms: 0.062829 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 9:49:16.325393 | finish at 2026-10-10 22:45:06 + [2026-10-04 12:57:13] iteration 843/ 6855 | consumed samples: 107904 | elapsed time per iteration (ms): 83346.9 | throughput per GPU (TFLOP/s/GPU): 43.9 | MFU 14.07% | FLOPs estimator: workload-aware | logical tokens/s: 201293.6 | valid tokens/s: 201293.6 | physical tokens/s: 201766.7 | loss tokens/s: 155045.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.751191E-01 | lm-global-supervised-tokens: 1.292256E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.651044E-02 | lm-token-buffer-mib: 5.804459E+01 | global_load_balancing_loss: 9.991781E-01 | loss scale: 1.0 | grad norm: 0.153 | params norm: 11603.449 | params rms: 0.062829 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 19:11:21.502516 | finish at 2026-10-10 08:08:35 + [2026-10-04 12:58:38] iteration 844/ 6855 | consumed samples: 108032 | elapsed time per iteration (ms): 85020.9 | throughput per GPU (TFLOP/s/GPU): 43.5 | MFU 13.93% | FLOPs estimator: workload-aware | logical tokens/s: 197329.8 | valid tokens/s: 197329.8 | physical tokens/s: 197687.6 | loss tokens/s: 147192.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.076794E-01 | lm-global-supervised-tokens: 1.251444E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.182044E-02 | lm-token-buffer-mib: 5.804716E+01 | global_load_balancing_loss: 9.994991E-01 | loss scale: 1.0 | grad norm: 0.044 | params norm: 11603.365 | params rms: 0.062828 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 21:57:40.383467 | finish at 2026-10-10 10:56:19 + [2026-10-04 13:00:04] iteration 845/ 6855 | consumed samples: 108160 | elapsed time per iteration (ms): 86016.4 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.38% | FLOPs estimator: workload-aware | logical tokens/s: 195046.4 | valid tokens/s: 195046.4 | physical tokens/s: 195662.1 | loss tokens/s: 147751.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.026937E-01 | lm-global-supervised-tokens: 1.270904E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.558935E-02 | lm-token-buffer-mib: 5.814633E+01 | global_load_balancing_loss: 1.000569E+00 | loss scale: 1.0 | grad norm: 0.118 | params norm: 11603.283 | params rms: 0.062828 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 23:35:58.312404 | finish at 2026-10-10 12:36:03 + [2026-10-04 13:01:33] iteration 846/ 6855 | consumed samples: 108288 | elapsed time per iteration (ms): 88266.3 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.50% | FLOPs estimator: workload-aware | logical tokens/s: 190074.3 | valid tokens/s: 190074.3 | physical tokens/s: 190438.4 | loss tokens/s: 135892.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.282696E-01 | lm-global-supervised-tokens: 1.199477E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.589042E-02 | lm-token-buffer-mib: 5.806528E+01 | global_load_balancing_loss: 1.000385E+00 | loss scale: 1.0 | grad norm: 0.104 | params norm: 11603.199 | params rms: 0.062828 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 3:19:52.351205 | finish at 2026-10-10 16:21:25 + [2026-10-04 13:03:01] iteration 847/ 6855 | consumed samples: 108416 | elapsed time per iteration (ms): 87858.3 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.69% | FLOPs estimator: workload-aware | logical tokens/s: 190956.6 | valid tokens/s: 190956.6 | physical tokens/s: 191499.1 | loss tokens/s: 136635.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.686959E-01 | lm-global-supervised-tokens: 1.200456E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.591356E-02 | lm-token-buffer-mib: 5.828573E+01 | global_load_balancing_loss: 1.000103E+00 | loss scale: 1.0 | grad norm: 0.078 | params norm: 11603.113 | params rms: 0.062827 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 2:37:32.573116 | finish at 2026-10-10 15:40:33 + [2026-10-04 13:04:27] iteration 848/ 6855 | consumed samples: 108544 | elapsed time per iteration (ms): 86900.5 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.64% | FLOPs estimator: workload-aware | logical tokens/s: 193062.0 | valid tokens/s: 193062.0 | physical tokens/s: 193544.7 | loss tokens/s: 142119.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.961869E-01 | lm-global-supervised-tokens: 1.235029E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.100134E-02 | lm-token-buffer-mib: 5.806631E+01 | global_load_balancing_loss: 9.998069E-01 | loss scale: 1.0 | grad norm: 0.081 | params norm: 11603.027 | params rms: 0.062827 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 1:00:11.375464 | finish at 2026-10-10 14:04:39 + [2026-10-04 13:05:52] iteration 849/ 6855 | consumed samples: 108672 | elapsed time per iteration (ms): 84467.4 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.78% | FLOPs estimator: workload-aware | logical tokens/s: 198623.2 | valid tokens/s: 198623.1 | physical tokens/s: 199128.9 | loss tokens/s: 149480.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.473193E-01 | lm-global-supervised-tokens: 1.262622E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.327155E-02 | lm-token-buffer-mib: 5.823689E+01 | global_load_balancing_loss: 1.000964E+00 | loss scale: 1.0 | grad norm: 0.181 | params norm: 11602.941 | params rms: 0.062826 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 20:55:11.363790 | finish at 2026-10-10 10:01:03 + [2026-10-04 13:07:43] iteration 850/ 6855 | consumed samples: 108800 | elapsed time per iteration (ms): 110643.1 | throughput per GPU (TFLOP/s/GPU): 33.5 | MFU 10.74% | FLOPs estimator: workload-aware | logical tokens/s: 151632.9 | valid tokens/s: 151632.9 | physical tokens/s: 152038.1 | loss tokens/s: 112443.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.251170E-01 | lm-global-supervised-tokens: 1.244107E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.214380E-02 | lm-token-buffer-mib: 5.809756E+01 | global_load_balancing_loss: 1.000059E+00 | loss scale: 1.0 | grad norm: 0.265 | params norm: 11602.853 | params rms: 0.062826 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 7 days, 16:33:32.074778 | finish at 2026-10-12 05:41:15 + [2026-10-04 13:09:10] iteration 851/ 6855 | consumed samples: 108928 | elapsed time per iteration (ms): 87817.3 | throughput per GPU (TFLOP/s/GPU): 41.4 | MFU 13.28% | FLOPs estimator: workload-aware | logical tokens/s: 191045.9 | valid tokens/s: 191045.9 | physical tokens/s: 191604.9 | loss tokens/s: 145162.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.770874E-01 | lm-global-supervised-tokens: 1.274779E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.337688E-02 | lm-token-buffer-mib: 5.825163E+01 | global_load_balancing_loss: 1.000614E+00 | loss scale: 1.0 | grad norm: 0.063 | params norm: 11602.765 | params rms: 0.062825 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 2:27:35.063962 | finish at 2026-10-10 15:36:45 + [2026-10-04 13:10:38] iteration 852/ 6855 | consumed samples: 109056 | elapsed time per iteration (ms): 87459.1 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.54% | FLOPs estimator: workload-aware | logical tokens/s: 191828.6 | valid tokens/s: 191828.6 | physical tokens/s: 192326.7 | loss tokens/s: 145003.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.369114E-01 | lm-global-supervised-tokens: 1.268190E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.166164E-02 | lm-token-buffer-mib: 5.816464E+01 | global_load_balancing_loss: 1.001355E+00 | loss scale: 1.0 | grad norm: 0.161 | params norm: 11602.680 | params rms: 0.062825 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 1:50:16.851400 | finish at 2026-10-10 15:00:55 + [2026-10-04 13:12:05] iteration 853/ 6855 | consumed samples: 109184 | elapsed time per iteration (ms): 87208.3 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.55% | FLOPs estimator: workload-aware | logical tokens/s: 192380.8 | valid tokens/s: 192380.8 | physical tokens/s: 192797.2 | loss tokens/s: 138253.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.481996E-01 | lm-global-supervised-tokens: 1.205682E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.527249E-02 | lm-token-buffer-mib: 5.810186E+01 | global_load_balancing_loss: 1.000442E+00 | loss scale: 1.0 | grad norm: 0.040 | params norm: 11602.596 | params rms: 0.062824 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 1:23:44.414759 | finish at 2026-10-10 14:35:49 + [2026-10-04 13:13:32] iteration 854/ 6855 | consumed samples: 109312 | elapsed time per iteration (ms): 86981.7 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.43% | FLOPs estimator: workload-aware | logical tokens/s: 192881.6 | valid tokens/s: 192881.6 | physical tokens/s: 193426.9 | loss tokens/s: 149972.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.466343E-01 | lm-global-supervised-tokens: 1.304483E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.700450E-02 | lm-token-buffer-mib: 5.826362E+01 | global_load_balancing_loss: 1.000291E+00 | loss scale: 1.0 | grad norm: 0.198 | params norm: 11602.512 | params rms: 0.062824 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 0:59:36.973902 | finish at 2026-10-10 14:13:09 + [2026-10-04 13:15:02] iteration 855/ 6855 | consumed samples: 109440 | elapsed time per iteration (ms): 90227.2 | throughput per GPU (TFLOP/s/GPU): 41.4 | MFU 13.28% | FLOPs estimator: workload-aware | logical tokens/s: 185943.6 | valid tokens/s: 185943.6 | physical tokens/s: 186412.2 | loss tokens/s: 136657.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.599748E-01 | lm-global-supervised-tokens: 1.233021E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.905249E-02 | lm-token-buffer-mib: 5.813715E+01 | global_load_balancing_loss: 9.999963E-01 | loss scale: 1.0 | grad norm: 0.143 | params norm: 11602.426 | params rms: 0.062823 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 6:22:43.348961 | finish at 2026-10-10 19:37:46 + [2026-10-04 13:16:28] iteration 856/ 6855 | consumed samples: 109568 | elapsed time per iteration (ms): 85795.9 | throughput per GPU (TFLOP/s/GPU): 43.3 | MFU 13.88% | FLOPs estimator: workload-aware | logical tokens/s: 195547.4 | valid tokens/s: 195547.4 | physical tokens/s: 195886.8 | loss tokens/s: 145103.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.194906E-01 | lm-global-supervised-tokens: 1.244929E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.278764E-02 | lm-token-buffer-mib: 5.803109E+01 | global_load_balancing_loss: 9.993142E-01 | loss scale: 1.0 | grad norm: 0.125 | params norm: 11602.342 | params rms: 0.062823 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 22:58:09.836443 | finish at 2026-10-10 12:14:38 + [2026-10-04 13:17:55] iteration 857/ 6855 | consumed samples: 109696 | elapsed time per iteration (ms): 87277.7 | throughput per GPU (TFLOP/s/GPU): 44.4 | MFU 14.23% | FLOPs estimator: workload-aware | logical tokens/s: 192227.6 | valid tokens/s: 192227.6 | physical tokens/s: 192600.6 | loss tokens/s: 142806.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.329687E-01 | lm-global-supervised-tokens: 1.246385E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.109594E-02 | lm-token-buffer-mib: 5.813906E+01 | global_load_balancing_loss: 9.996470E-01 | loss scale: 1.0 | grad norm: 0.069 | params norm: 11602.260 | params rms: 0.062822 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 1:24:51.758687 | finish at 2026-10-10 14:42:47 + [2026-10-04 13:19:25] iteration 858/ 6855 | consumed samples: 109824 | elapsed time per iteration (ms): 89772.2 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.47% | FLOPs estimator: workload-aware | logical tokens/s: 186886.3 | valid tokens/s: 186886.3 | physical tokens/s: 187252.6 | loss tokens/s: 131630.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.504560E-01 | lm-global-supervised-tokens: 1.181680E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.463546E-02 | lm-token-buffer-mib: 5.812475E+01 | global_load_balancing_loss: 1.001261E+00 | loss scale: 1.0 | grad norm: 0.046 | params norm: 11602.176 | params rms: 0.062822 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 5:32:44.041323 | finish at 2026-10-10 18:52:09 + [2026-10-04 13:20:55] iteration 859/ 6855 | consumed samples: 109952 | elapsed time per iteration (ms): 89387.0 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.64% | FLOPs estimator: workload-aware | logical tokens/s: 187692.0 | valid tokens/s: 187692.0 | physical tokens/s: 188092.8 | loss tokens/s: 139982.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.593641E-01 | lm-global-supervised-tokens: 1.251257E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.389637E-02 | lm-token-buffer-mib: 5.808323E+01 | global_load_balancing_loss: 9.999619E-01 | loss scale: 1.0 | grad norm: 0.122 | params norm: 11602.093 | params rms: 0.062822 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 4:52:44.193753 | finish at 2026-10-10 18:13:39 + [2026-10-04 13:22:42] iteration 860/ 6855 | consumed samples: 110080 | elapsed time per iteration (ms): 107186.7 | throughput per GPU (TFLOP/s/GPU): 34.2 | MFU 10.97% | FLOPs estimator: workload-aware | logical tokens/s: 156523.0 | valid tokens/s: 156523.0 | physical tokens/s: 156849.1 | loss tokens/s: 115780.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.393429E-01 | lm-global-supervised-tokens: 1.241014E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.340801E-02 | lm-token-buffer-mib: 5.809809E+01 | global_load_balancing_loss: 1.000421E+00 | loss scale: 1.0 | grad norm: 0.050 | params norm: 11602.010 | params rms: 0.062821 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 7 days, 10:29:44.338599 | finish at 2026-10-11 23:52:26 + [2026-10-04 13:24:09] iteration 861/ 6855 | consumed samples: 110208 | elapsed time per iteration (ms): 87601.9 | throughput per GPU (TFLOP/s/GPU): 42.8 | MFU 13.72% | FLOPs estimator: workload-aware | logical tokens/s: 191516.3 | valid tokens/s: 191516.3 | physical tokens/s: 191773.7 | loss tokens/s: 139421.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.017313E-01 | lm-global-supervised-tokens: 1.221359E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.625848E-02 | lm-token-buffer-mib: 5.802735E+01 | global_load_balancing_loss: 1.000105E+00 | loss scale: 1.0 | grad norm: 0.099 | params norm: 11601.928 | params rms: 0.062821 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 1:51:26.003566 | finish at 2026-10-10 15:15:35 + [2026-10-04 13:25:36] iteration 862/ 6855 | consumed samples: 110336 | elapsed time per iteration (ms): 86399.4 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.58% | FLOPs estimator: workload-aware | logical tokens/s: 194181.3 | valid tokens/s: 194181.3 | physical tokens/s: 194931.0 | loss tokens/s: 143512.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.835675E-01 | lm-global-supervised-tokens: 1.239935E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.111873E-02 | lm-token-buffer-mib: 5.815429E+01 | global_load_balancing_loss: 1.001175E+00 | loss scale: 1.0 | grad norm: 0.089 | params norm: 11601.847 | params rms: 0.062820 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 23:49:51.518444 | finish at 2026-10-10 13:15:27 + [2026-10-04 13:27:04] iteration 863/ 6855 | consumed samples: 110464 | elapsed time per iteration (ms): 87984.0 | throughput per GPU (TFLOP/s/GPU): 41.3 | MFU 13.24% | FLOPs estimator: workload-aware | logical tokens/s: 190684.6 | valid tokens/s: 190684.6 | physical tokens/s: 191138.7 | loss tokens/s: 142601.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.484457E-01 | lm-global-supervised-tokens: 1.254661E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.314291E-02 | lm-token-buffer-mib: 5.809788E+01 | global_load_balancing_loss: 9.993915E-01 | loss scale: 1.0 | grad norm: 0.305 | params norm: 11601.766 | params rms: 0.062820 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 2:26:40.174950 | finish at 2026-10-10 15:53:44 + [2026-10-04 13:28:31] iteration 864/ 6855 | consumed samples: 110592 | elapsed time per iteration (ms): 87357.7 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.55% | FLOPs estimator: workload-aware | logical tokens/s: 192051.7 | valid tokens/s: 192051.7 | physical tokens/s: 192538.0 | loss tokens/s: 146583.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.906655E-01 | lm-global-supervised-tokens: 1.280522E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.317381E-02 | lm-token-buffer-mib: 5.814268E+01 | global_load_balancing_loss: 9.997522E-01 | loss scale: 1.0 | grad norm: 0.106 | params norm: 11601.684 | params rms: 0.062819 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 1:22:40.198467 | finish at 2026-10-10 14:51:11 + [2026-10-04 13:29:59] iteration 865/ 6855 | consumed samples: 110720 | elapsed time per iteration (ms): 87668.3 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.75% | FLOPs estimator: workload-aware | logical tokens/s: 191371.0 | valid tokens/s: 191371.0 | physical tokens/s: 192019.0 | loss tokens/s: 144183.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.061726E-01 | lm-global-supervised-tokens: 1.264035E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.237283E-02 | lm-token-buffer-mib: 5.846267E+01 | global_load_balancing_loss: 9.990801E-01 | loss scale: 1.0 | grad norm: 0.250 | params norm: 11601.602 | params rms: 0.062819 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 1:52:13.297853 | finish at 2026-10-10 15:22:12 + [2026-10-04 13:31:25] iteration 866/ 6855 | consumed samples: 110848 | elapsed time per iteration (ms): 86378.1 | throughput per GPU (TFLOP/s/GPU): 41.7 | MFU 13.37% | FLOPs estimator: workload-aware | logical tokens/s: 194230.0 | valid tokens/s: 194230.0 | physical tokens/s: 194569.4 | loss tokens/s: 148888.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.678090E-01 | lm-global-supervised-tokens: 1.286066E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.236423E-02 | lm-token-buffer-mib: 5.806164E+01 | global_load_balancing_loss: 9.999959E-01 | loss scale: 1.0 | grad norm: 0.045 | params norm: 11601.522 | params rms: 0.062818 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 23:41:58.264781 | finish at 2026-10-10 13:13:23 + [2026-10-04 13:32:51] iteration 867/ 6855 | consumed samples: 110976 | elapsed time per iteration (ms): 85648.6 | throughput per GPU (TFLOP/s/GPU): 43.3 | MFU 13.86% | FLOPs estimator: workload-aware | logical tokens/s: 195883.9 | valid tokens/s: 195883.9 | physical tokens/s: 196415.5 | loss tokens/s: 143981.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.462042E-01 | lm-global-supervised-tokens: 1.233182E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.442269E-02 | lm-token-buffer-mib: 5.815539E+01 | global_load_balancing_loss: 1.000269E+00 | loss scale: 1.0 | grad norm: 0.154 | params norm: 11601.443 | params rms: 0.062818 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 22:27:43.546154 | finish at 2026-10-10 12:00:34 + [2026-10-04 13:34:18] iteration 868/ 6855 | consumed samples: 111104 | elapsed time per iteration (ms): 87678.5 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.49% | FLOPs estimator: workload-aware | logical tokens/s: 191348.4 | valid tokens/s: 191348.4 | physical tokens/s: 191771.3 | loss tokens/s: 146389.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.782234E-01 | lm-global-supervised-tokens: 1.283517E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.141234E-02 | lm-token-buffer-mib: 5.809859E+01 | global_load_balancing_loss: 1.000116E+00 | loss scale: 1.0 | grad norm: 0.070 | params norm: 11601.362 | params rms: 0.062818 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 1:48:51.190542 | finish at 2026-10-10 15:23:10 + [2026-10-04 13:35:48] iteration 869/ 6855 | consumed samples: 111232 | elapsed time per iteration (ms): 90002.1 | throughput per GPU (TFLOP/s/GPU): 41.3 | MFU 13.23% | FLOPs estimator: workload-aware | logical tokens/s: 186408.9 | valid tokens/s: 186408.9 | physical tokens/s: 186746.1 | loss tokens/s: 136951.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.627457E-01 | lm-global-supervised-tokens: 1.232596E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.377705E-02 | lm-token-buffer-mib: 5.819524E+01 | global_load_balancing_loss: 1.001173E+00 | loss scale: 1.0 | grad norm: 0.097 | params norm: 11601.281 | params rms: 0.062817 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 5:39:12.416410 | finish at 2026-10-10 19:15:01 + [2026-10-04 13:37:35] iteration 870/ 6855 | consumed samples: 111360 | elapsed time per iteration (ms): 106238.8 | throughput per GPU (TFLOP/s/GPU): 33.9 | MFU 10.86% | FLOPs estimator: workload-aware | logical tokens/s: 157919.5 | valid tokens/s: 157919.5 | physical tokens/s: 158312.9 | loss tokens/s: 121638.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.074605E-01 | lm-global-supervised-tokens: 1.292278E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.154287E-02 | lm-token-buffer-mib: 5.809307E+01 | global_load_balancing_loss: 1.001573E+00 | loss scale: 1.0 | grad norm: 0.159 | params norm: 11601.204 | params rms: 0.062817 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 7 days, 8:37:19.359559 | finish at 2026-10-11 22:14:54 + [2026-10-04 13:39:00] iteration 871/ 6855 | consumed samples: 111488 | elapsed time per iteration (ms): 85170.1 | throughput per GPU (TFLOP/s/GPU): 43.1 | MFU 13.82% | FLOPs estimator: workload-aware | logical tokens/s: 196984.3 | valid tokens/s: 196984.3 | physical tokens/s: 197328.7 | loss tokens/s: 145841.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.634569E-01 | lm-global-supervised-tokens: 1.242134E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.366892E-02 | lm-token-buffer-mib: 5.818707E+01 | global_load_balancing_loss: 1.000696E+00 | loss scale: 1.0 | grad norm: 0.148 | params norm: 11601.124 | params rms: 0.062816 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 21:34:18.173569 | finish at 2026-10-10 11:13:18 + [2026-10-04 13:40:25] iteration 872/ 6855 | consumed samples: 111616 | elapsed time per iteration (ms): 85382.6 | throughput per GPU (TFLOP/s/GPU): 43.7 | MFU 14.01% | FLOPs estimator: workload-aware | logical tokens/s: 196493.9 | valid tokens/s: 196493.9 | physical tokens/s: 196963.7 | loss tokens/s: 152428.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.046549E-01 | lm-global-supervised-tokens: 1.301470E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.209244E-02 | lm-token-buffer-mib: 5.807184E+01 | global_load_balancing_loss: 9.999710E-01 | loss scale: 1.0 | grad norm: 0.125 | params norm: 11601.045 | params rms: 0.062816 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 21:54:03.835171 | finish at 2026-10-10 11:34:29 + [2026-10-04 13:41:56] iteration 873/ 6855 | consumed samples: 111744 | elapsed time per iteration (ms): 90421.5 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.56% | FLOPs estimator: workload-aware | logical tokens/s: 185544.5 | valid tokens/s: 185544.5 | physical tokens/s: 185906.4 | loss tokens/s: 129897.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.437047E-01 | lm-global-supervised-tokens: 1.174556E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.184865E-02 | lm-token-buffer-mib: 5.805444E+01 | global_load_balancing_loss: 9.999710E-01 | loss scale: 1.0 | grad norm: 0.064 | params norm: 11600.964 | params rms: 0.062815 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 6:15:01.492674 | finish at 2026-10-10 19:56:57 + [2026-10-04 13:43:23] iteration 874/ 6855 | consumed samples: 111872 | elapsed time per iteration (ms): 87157.1 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.64% | FLOPs estimator: workload-aware | logical tokens/s: 192492.9 | valid tokens/s: 192492.9 | physical tokens/s: 193169.1 | loss tokens/s: 137949.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.710268E-01 | lm-global-supervised-tokens: 1.202330E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.294926E-02 | lm-token-buffer-mib: 5.804135E+01 | global_load_balancing_loss: 1.000561E+00 | loss scale: 1.0 | grad norm: 0.331 | params norm: 11600.883 | params rms: 0.062815 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 0:48:06.690451 | finish at 2026-10-10 14:31:29 + [2026-10-04 13:44:49] iteration 875/ 6855 | consumed samples: 112000 | elapsed time per iteration (ms): 86167.4 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.61% | FLOPs estimator: workload-aware | logical tokens/s: 194704.5 | valid tokens/s: 194704.5 | physical tokens/s: 195364.4 | loss tokens/s: 145396.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.380855E-01 | lm-global-supervised-tokens: 1.252847E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.196995E-02 | lm-token-buffer-mib: 5.805000E+01 | global_load_balancing_loss: 9.997851E-01 | loss scale: 1.0 | grad norm: 0.106 | params norm: 11600.803 | params rms: 0.062815 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 23:08:00.816054 | finish at 2026-10-10 12:52:50 + [2026-10-04 13:46:19] iteration 876/ 6855 | consumed samples: 112128 | elapsed time per iteration (ms): 90508.3 | throughput per GPU (TFLOP/s/GPU): 41.4 | MFU 13.27% | FLOPs estimator: workload-aware | logical tokens/s: 185366.3 | valid tokens/s: 185366.3 | physical tokens/s: 185593.0 | loss tokens/s: 140488.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.065423E-01 | lm-global-supervised-tokens: 1.271532E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.159598E-02 | lm-token-buffer-mib: 5.806146E+01 | global_load_balancing_loss: 9.995472E-01 | loss scale: 1.0 | grad norm: 0.109 | params norm: 11600.723 | params rms: 0.062814 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 6:19:08.839568 | finish at 2026-10-10 20:05:28 + [2026-10-04 13:47:53] iteration 877/ 6855 | consumed samples: 112256 | elapsed time per iteration (ms): 93228.9 | throughput per GPU (TFLOP/s/GPU): 40.8 | MFU 13.07% | FLOPs estimator: workload-aware | logical tokens/s: 179956.2 | valid tokens/s: 179956.2 | physical tokens/s: 180328.5 | loss tokens/s: 134109.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.080538E-01 | lm-global-supervised-tokens: 1.250289E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.411703E-02 | lm-token-buffer-mib: 5.807634E+01 | global_load_balancing_loss: 1.000019E+00 | loss scale: 1.0 | grad norm: 0.184 | params norm: 11600.642 | params rms: 0.062814 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 10:48:42.653265 | finish at 2026-10-11 00:36:35 + [2026-10-04 13:49:19] iteration 878/ 6855 | consumed samples: 112384 | elapsed time per iteration (ms): 86729.9 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.61% | FLOPs estimator: workload-aware | logical tokens/s: 193441.4 | valid tokens/s: 193441.4 | physical tokens/s: 193837.9 | loss tokens/s: 143182.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.656023E-01 | lm-global-supervised-tokens: 1.241822E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.536862E-02 | lm-token-buffer-mib: 5.809464E+01 | global_load_balancing_loss: 9.997161E-01 | loss scale: 1.0 | grad norm: 0.091 | params norm: 11600.559 | params rms: 0.062813 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 23:59:44.906583 | finish at 2026-10-10 13:49:04 + [2026-10-04 13:50:50] iteration 879/ 6855 | consumed samples: 112512 | elapsed time per iteration (ms): 90492.1 | throughput per GPU (TFLOP/s/GPU): 41.4 | MFU 13.27% | FLOPs estimator: workload-aware | logical tokens/s: 185399.3 | valid tokens/s: 185399.3 | physical tokens/s: 185791.7 | loss tokens/s: 138599.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.025035E-01 | lm-global-supervised-tokens: 1.254214E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.169751E-02 | lm-token-buffer-mib: 5.826760E+01 | global_load_balancing_loss: 9.999419E-01 | loss scale: 1.0 | grad norm: 0.165 | params norm: 11600.476 | params rms: 0.062813 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 6:13:01.068861 | finish at 2026-10-10 20:03:51 + [2026-10-04 13:52:37] iteration 880/ 6855 | consumed samples: 112640 | elapsed time per iteration (ms): 107424.7 | throughput per GPU (TFLOP/s/GPU): 34.2 | MFU 10.96% | FLOPs estimator: workload-aware | logical tokens/s: 156176.3 | valid tokens/s: 156176.3 | physical tokens/s: 156427.4 | loss tokens/s: 112786.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.882929E-01 | lm-global-supervised-tokens: 1.211604E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.267889E-02 | lm-token-buffer-mib: 5.809595E+01 | global_load_balancing_loss: 1.000054E+00 | loss scale: 1.0 | grad norm: 0.119 | params norm: 11600.393 | params rms: 0.062812 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 7 days, 10:17:42.420231 | finish at 2026-10-12 00:10:20 +times across ranks (ms): + save-checkpoint: + rank 0: 24224.62 + rank 1: 24224.62 + rank 2: 24224.62 + rank 3: 24224.65 + rank 4: 24224.62 + rank 5: 24224.60 + rank 6: 24224.59 + rank 7: 24224.69 + rank 8: 24224.21 + rank 9: 24224.15 + rank 10: 24224.17 + rank 11: 24224.15 + rank 12: 24224.18 + rank 13: 24224.19 + rank 14: 24224.18 + rank 15: 24224.17 + rank 16: 24224.24 + rank 17: 24224.24 + rank 18: 24224.23 + rank 19: 24224.23 + rank 20: 24224.27 + rank 21: 24224.24 + rank 22: 24224.24 + rank 23: 24224.22 + rank 24: 24223.86 + rank 25: 24223.96 + rank 26: 24223.88 + rank 27: 24223.90 + rank 28: 24223.84 + rank 29: 24223.90 + rank 30: 24223.85 + rank 31: 24223.86 + rank 32: 24223.60 + rank 33: 24223.60 + rank 34: 24223.60 + rank 35: 24223.58 + rank 36: 24223.54 + rank 37: 24223.55 + rank 38: 24223.58 + rank 39: 24223.56 + rank 40: 24223.42 + rank 41: 24223.42 + rank 42: 24223.46 + rank 43: 24223.41 + rank 44: 24223.38 + rank 45: 24223.40 + rank 46: 24223.41 + rank 47: 24223.39 + rank 48: 24223.81 + rank 49: 24223.82 + rank 50: 24223.81 + rank 51: 24223.82 + rank 52: 24223.84 + rank 53: 24223.79 + rank 54: 24223.80 + rank 55: 24223.79 + rank 56: 24224.49 + rank 57: 24224.47 + rank 58: 24224.46 + rank 59: 24224.46 + rank 60: 24224.45 + rank 61: 24224.49 + rank 62: 24224.46 + rank 63: 24224.50 + [2026-10-04 13:54:38] iteration 881/ 6855 | consumed samples: 112768 | elapsed time per iteration (ms): 95977.0 | throughput per GPU (TFLOP/s/GPU): 37.9 | MFU 12.16% | FLOPs estimator: workload-aware | logical tokens/s: 174804.4 | valid tokens/s: 174804.4 | physical tokens/s: 175309.7 | loss tokens/s: 131101.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.939401E-01 | lm-global-supervised-tokens: 1.258275E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 7.878658E-01 | lm-token-buffer-mib: 5.816950E+01 | global_load_balancing_loss: 9.996018E-01 | loss scale: 1.0 | grad norm: 0.119 | params norm: 11600.312 | params rms: 0.062812 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 15:16:06.400009 | finish at 2026-10-11 05:10:44 + [2026-10-04 13:56:11] iteration 882/ 6855 | consumed samples: 112896 | elapsed time per iteration (ms): 92864.8 | throughput per GPU (TFLOP/s/GPU): 39.6 | MFU 12.70% | FLOPs estimator: workload-aware | logical tokens/s: 180662.0 | valid tokens/s: 180662.0 | physical tokens/s: 181292.0 | loss tokens/s: 135474.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.871748E-01 | lm-global-supervised-tokens: 1.258082E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.399472E-02 | lm-token-buffer-mib: 5.885193E+01 | global_load_balancing_loss: 1.000235E+00 | loss scale: 1.0 | grad norm: 0.099 | params norm: 11600.229 | params rms: 0.062811 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 10:04:41.539975 | finish at 2026-10-11 00:00:52 + [2026-10-04 13:57:36] iteration 883/ 6855 | consumed samples: 113024 | elapsed time per iteration (ms): 85674.0 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.53% | FLOPs estimator: workload-aware | logical tokens/s: 195825.8 | valid tokens/s: 195825.8 | physical tokens/s: 196542.4 | loss tokens/s: 147425.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.716969E-01 | lm-global-supervised-tokens: 1.263056E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.266921E-02 | lm-token-buffer-mib: 5.823822E+01 | global_load_balancing_loss: 1.000888E+00 | loss scale: 1.0 | grad norm: 0.070 | params norm: 11600.148 | params rms: 0.062811 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 22:07:25.257717 | finish at 2026-10-10 12:05:01 + [2026-10-04 13:59:07] iteration 884/ 6855 | consumed samples: 113152 | elapsed time per iteration (ms): 90348.8 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.56% | FLOPs estimator: workload-aware | logical tokens/s: 185693.3 | valid tokens/s: 185693.3 | physical tokens/s: 186205.9 | loss tokens/s: 126449.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.145948E-01 | lm-global-supervised-tokens: 1.142455E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.496335E-02 | lm-token-buffer-mib: 5.814996E+01 | global_load_balancing_loss: 1.001167E+00 | loss scale: 1.0 | grad norm: 0.060 | params norm: 11600.068 | params rms: 0.062811 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 5:51:12.550647 | finish at 2026-10-10 19:50:19 + [2026-10-04 14:00:34] iteration 885/ 6855 | consumed samples: 113280 | elapsed time per iteration (ms): 87271.8 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.62% | FLOPs estimator: workload-aware | logical tokens/s: 192240.8 | valid tokens/s: 192240.8 | physical tokens/s: 192828.9 | loss tokens/s: 140369.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.738644E-01 | lm-global-supervised-tokens: 1.225033E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.424021E-02 | lm-token-buffer-mib: 5.832513E+01 | global_load_balancing_loss: 1.000803E+00 | loss scale: 1.0 | grad norm: 0.076 | params norm: 11599.990 | params rms: 0.062810 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 0:43:32.650516 | finish at 2026-10-10 14:44:06 + [2026-10-04 14:02:00] iteration 886/ 6855 | consumed samples: 113408 | elapsed time per iteration (ms): 86001.1 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.55% | FLOPs estimator: workload-aware | logical tokens/s: 195080.9 | valid tokens/s: 195080.9 | physical tokens/s: 195539.1 | loss tokens/s: 141035.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.456341E-01 | lm-global-supervised-tokens: 1.212919E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.194113E-02 | lm-token-buffer-mib: 5.809249E+01 | global_load_balancing_loss: 1.001020E+00 | loss scale: 1.0 | grad norm: 0.046 | params norm: 11599.910 | params rms: 0.062810 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 22:35:40.544931 | finish at 2026-10-10 12:37:40 + [2026-10-04 14:03:25] iteration 887/ 6855 | consumed samples: 113536 | elapsed time per iteration (ms): 85399.8 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.77% | FLOPs estimator: workload-aware | logical tokens/s: 196454.6 | valid tokens/s: 196454.6 | physical tokens/s: 197102.4 | loss tokens/s: 145931.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.514984E-01 | lm-global-supervised-tokens: 1.246254E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.274898E-02 | lm-token-buffer-mib: 5.815896E+01 | global_load_balancing_loss: 9.998997E-01 | loss scale: 1.0 | grad norm: 0.069 | params norm: 11599.829 | params rms: 0.062809 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 21:34:26.267159 | finish at 2026-10-10 11:37:51 + [2026-10-04 14:04:55] iteration 888/ 6855 | consumed samples: 113664 | elapsed time per iteration (ms): 89431.7 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.77% | FLOPs estimator: workload-aware | logical tokens/s: 187597.7 | valid tokens/s: 187597.7 | physical tokens/s: 187955.0 | loss tokens/s: 137066.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.002186E-01 | lm-global-supervised-tokens: 1.225812E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.411122E-02 | lm-token-buffer-mib: 5.821009E+01 | global_load_balancing_loss: 1.000525E+00 | loss scale: 1.0 | grad norm: 0.035 | params norm: 11599.749 | params rms: 0.062809 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 4:13:59.059123 | finish at 2026-10-10 18:18:54 + [2026-10-04 14:06:26] iteration 889/ 6855 | consumed samples: 113792 | elapsed time per iteration (ms): 91178.2 | throughput per GPU (TFLOP/s/GPU): 40.9 | MFU 13.12% | FLOPs estimator: workload-aware | logical tokens/s: 184004.3 | valid tokens/s: 184004.3 | physical tokens/s: 184357.4 | loss tokens/s: 132892.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.174041E-01 | lm-global-supervised-tokens: 1.211691E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.570508E-02 | lm-token-buffer-mib: 5.804879E+01 | global_load_balancing_loss: 9.991940E-01 | loss scale: 1.0 | grad norm: 0.073 | params norm: 11599.669 | params rms: 0.062808 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 7:06:09.074386 | finish at 2026-10-10 21:12:35 + [2026-10-04 14:08:13] iteration 890/ 6855 | consumed samples: 113920 | elapsed time per iteration (ms): 107662.5 | throughput per GPU (TFLOP/s/GPU): 33.5 | MFU 10.74% | FLOPs estimator: workload-aware | logical tokens/s: 155831.2 | valid tokens/s: 155831.2 | physical tokens/s: 156289.6 | loss tokens/s: 117866.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.280015E-01 | lm-global-supervised-tokens: 1.268985E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.336187E-02 | lm-token-buffer-mib: 5.824062E+01 | global_load_balancing_loss: 9.993164E-01 | loss scale: 1.0 | grad norm: 0.056 | params norm: 11599.587 | params rms: 0.062808 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 7 days, 10:23:26.969792 | finish at 2026-10-12 00:31:40 + [2026-10-04 14:09:42] iteration 891/ 6855 | consumed samples: 114048 | elapsed time per iteration (ms): 88007.6 | throughput per GPU (TFLOP/s/GPU): 41.7 | MFU 13.38% | FLOPs estimator: workload-aware | logical tokens/s: 190633.2 | valid tokens/s: 190633.2 | physical tokens/s: 191139.6 | loss tokens/s: 142045.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.869453E-01 | lm-global-supervised-tokens: 1.250112E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.353674E-02 | lm-token-buffer-mib: 5.804025E+01 | global_load_balancing_loss: 1.000012E+00 | loss scale: 1.0 | grad norm: 0.053 | params norm: 11599.507 | params rms: 0.062808 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 1:47:57.429191 | finish at 2026-10-10 15:57:39 + [2026-10-04 14:11:12] iteration 892/ 6855 | consumed samples: 114176 | elapsed time per iteration (ms): 90793.0 | throughput per GPU (TFLOP/s/GPU): 41.0 | MFU 13.16% | FLOPs estimator: workload-aware | logical tokens/s: 184785.0 | valid tokens/s: 184785.0 | physical tokens/s: 185210.7 | loss tokens/s: 127788.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.655701E-01 | lm-global-supervised-tokens: 1.160227E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.178043E-02 | lm-token-buffer-mib: 5.811259E+01 | global_load_balancing_loss: 9.996611E-01 | loss scale: 1.0 | grad norm: 0.082 | params norm: 11599.429 | params rms: 0.062807 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 6:23:18.931862 | finish at 2026-10-10 20:34:31 + [2026-10-04 14:12:38] iteration 893/ 6855 | consumed samples: 114304 | elapsed time per iteration (ms): 85899.6 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.46% | FLOPs estimator: workload-aware | logical tokens/s: 195311.8 | valid tokens/s: 195311.8 | physical tokens/s: 195839.9 | loss tokens/s: 138659.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.286228E-01 | lm-global-supervised-tokens: 1.191076E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.342922E-02 | lm-token-buffer-mib: 5.809109E+01 | global_load_balancing_loss: 1.000295E+00 | loss scale: 1.0 | grad norm: 0.071 | params norm: 11599.351 | params rms: 0.062807 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 22:15:33.706770 | finish at 2026-10-10 12:28:12 + [2026-10-04 14:14:02] iteration 894/ 6855 | consumed samples: 114432 | elapsed time per iteration (ms): 84081.1 | throughput per GPU (TFLOP/s/GPU): 43.3 | MFU 13.89% | FLOPs estimator: workload-aware | logical tokens/s: 199535.7 | valid tokens/s: 199535.7 | physical tokens/s: 200178.9 | loss tokens/s: 150437.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.682580E-01 | lm-global-supervised-tokens: 1.264892E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.247562E-02 | lm-token-buffer-mib: 5.805699E+01 | global_load_balancing_loss: 1.000309E+00 | loss scale: 1.0 | grad norm: 0.077 | params norm: 11599.274 | params rms: 0.062806 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 19:13:27.481080 | finish at 2026-10-10 09:27:30 + [2026-10-04 14:15:30] iteration 895/ 6855 | consumed samples: 114560 | elapsed time per iteration (ms): 87256.9 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.59% | FLOPs estimator: workload-aware | logical tokens/s: 192273.6 | valid tokens/s: 192273.6 | physical tokens/s: 192803.2 | loss tokens/s: 143489.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.432948E-01 | lm-global-supervised-tokens: 1.252043E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.158209E-02 | lm-token-buffer-mib: 5.813149E+01 | global_load_balancing_loss: 1.001234E+00 | loss scale: 1.0 | grad norm: 0.110 | params norm: 11599.194 | params rms: 0.062806 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 0:27:31.012173 | finish at 2026-10-10 14:43:01 + [2026-10-04 14:16:56] iteration 896/ 6855 | consumed samples: 114688 | elapsed time per iteration (ms): 86457.2 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.55% | FLOPs estimator: workload-aware | logical tokens/s: 194051.4 | valid tokens/s: 194051.4 | physical tokens/s: 194523.5 | loss tokens/s: 144260.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.463429E-01 | lm-global-supervised-tokens: 1.247234E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.703297E-02 | lm-token-buffer-mib: 5.811826E+01 | global_load_balancing_loss: 1.001151E+00 | loss scale: 1.0 | grad norm: 0.130 | params norm: 11599.114 | params rms: 0.062805 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 23:06:38.286032 | finish at 2026-10-10 13:23:34 + [2026-10-04 14:18:22] iteration 897/ 6855 | consumed samples: 114816 | elapsed time per iteration (ms): 85773.8 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.61% | FLOPs estimator: workload-aware | logical tokens/s: 195597.0 | valid tokens/s: 195597.0 | physical tokens/s: 196194.7 | loss tokens/s: 149927.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.523612E-01 | lm-global-supervised-tokens: 1.285984E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.308075E-02 | lm-token-buffer-mib: 5.811284E+01 | global_load_balancing_loss: 1.000886E+00 | loss scale: 1.0 | grad norm: 0.120 | params norm: 11599.032 | params rms: 0.062805 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 21:57:20.473083 | finish at 2026-10-10 12:15:42 + [2026-10-04 14:19:50] iteration 898/ 6855 | consumed samples: 114944 | elapsed time per iteration (ms): 88572.6 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.41% | FLOPs estimator: workload-aware | logical tokens/s: 189417.3 | valid tokens/s: 189417.3 | physical tokens/s: 190064.9 | loss tokens/s: 141957.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.063345E-01 | lm-global-supervised-tokens: 1.257355E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.193548E-02 | lm-token-buffer-mib: 5.813849E+01 | global_load_balancing_loss: 1.000406E+00 | loss scale: 1.0 | grad norm: 0.141 | params norm: 11598.951 | params rms: 0.062805 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 2:33:46.967590 | finish at 2026-10-10 16:53:37 + [2026-10-04 14:21:18] iteration 899/ 6855 | consumed samples: 115072 | elapsed time per iteration (ms): 88009.8 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.60% | FLOPs estimator: workload-aware | logical tokens/s: 190628.5 | valid tokens/s: 190628.5 | physical tokens/s: 191013.6 | loss tokens/s: 137059.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.863287E-01 | lm-global-supervised-tokens: 1.206260E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.546087E-02 | lm-token-buffer-mib: 5.813799E+01 | global_load_balancing_loss: 9.997115E-01 | loss scale: 1.0 | grad norm: 0.040 | params norm: 11598.868 | params rms: 0.062804 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 1:36:26.352925 | finish at 2026-10-10 15:57:45 + [2026-10-04 14:23:08] iteration 900/ 6855 | consumed samples: 115200 | elapsed time per iteration (ms): 109815.6 | throughput per GPU (TFLOP/s/GPU): 33.3 | MFU 10.66% | FLOPs estimator: workload-aware | logical tokens/s: 152775.6 | valid tokens/s: 152775.6 | physical tokens/s: 153027.1 | loss tokens/s: 115831.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.615521E-01 | lm-global-supervised-tokens: 1.272007E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.380154E-02 | lm-token-buffer-mib: 5.809113E+01 | global_load_balancing_loss: 9.998122E-01 | loss scale: 1.0 | grad norm: 0.081 | params norm: 11598.785 | params rms: 0.062804 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 7 days, 13:39:12.063628 | finish at 2026-10-12 04:02:20 + [2026-10-04 14:24:41] iteration 901/ 6855 | consumed samples: 115328 | elapsed time per iteration (ms): 92564.2 | throughput per GPU (TFLOP/s/GPU): 41.4 | MFU 13.26% | FLOPs estimator: workload-aware | logical tokens/s: 181249.2 | valid tokens/s: 181249.2 | physical tokens/s: 181564.4 | loss tokens/s: 135370.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.051409E-01 | lm-global-supervised-tokens: 1.253042E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.514729E-02 | lm-token-buffer-mib: 5.808237E+01 | global_load_balancing_loss: 9.992121E-01 | loss scale: 1.0 | grad norm: 0.117 | params norm: 11598.703 | params rms: 0.062803 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 9:05:26.995091 | finish at 2026-10-10 23:30:08 + [2026-10-04 14:26:08] iteration 902/ 6855 | consumed samples: 115456 | elapsed time per iteration (ms): 86851.0 | throughput per GPU (TFLOP/s/GPU): 43.6 | MFU 13.97% | FLOPs estimator: workload-aware | logical tokens/s: 193171.5 | valid tokens/s: 193171.5 | physical tokens/s: 193651.3 | loss tokens/s: 139440.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.543892E-01 | lm-global-supervised-tokens: 1.211053E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.156164E-02 | lm-token-buffer-mib: 5.816534E+01 | global_load_balancing_loss: 9.996247E-01 | loss scale: 1.0 | grad norm: 0.077 | params norm: 11598.620 | params rms: 0.062803 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 23:37:03.837361 | finish at 2026-10-10 14:03:11 + [2026-10-04 14:27:37] iteration 903/ 6855 | consumed samples: 115584 | elapsed time per iteration (ms): 89273.8 | throughput per GPU (TFLOP/s/GPU): 41.7 | MFU 13.36% | FLOPs estimator: workload-aware | logical tokens/s: 187929.9 | valid tokens/s: 187929.9 | physical tokens/s: 188379.6 | loss tokens/s: 135811.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.210352E-01 | lm-global-supervised-tokens: 1.212436E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.524544E-02 | lm-token-buffer-mib: 5.811037E+01 | global_load_balancing_loss: 9.990647E-01 | loss scale: 1.0 | grad norm: 0.046 | params norm: 11598.538 | params rms: 0.062802 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 3:35:57.516495 | finish at 2026-10-10 18:03:34 + [2026-10-04 14:29:05] iteration 904/ 6855 | consumed samples: 115712 | elapsed time per iteration (ms): 88539.0 | throughput per GPU (TFLOP/s/GPU): 40.7 | MFU 13.03% | FLOPs estimator: workload-aware | logical tokens/s: 189488.5 | valid tokens/s: 189488.5 | physical tokens/s: 190073.2 | loss tokens/s: 146044.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.876294E-01 | lm-global-supervised-tokens: 1.293062E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.303309E-02 | lm-token-buffer-mib: 5.808777E+01 | global_load_balancing_loss: 9.997665E-01 | loss scale: 1.0 | grad norm: 0.110 | params norm: 11598.455 | params rms: 0.062802 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 2:21:35.687104 | finish at 2026-10-10 16:50:41 + [2026-10-04 14:30:30] iteration 905/ 6855 | consumed samples: 115840 | elapsed time per iteration (ms): 84424.9 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.61% | FLOPs estimator: workload-aware | logical tokens/s: 198723.3 | valid tokens/s: 198723.3 | physical tokens/s: 199348.5 | loss tokens/s: 147964.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.230319E-01 | lm-global-supervised-tokens: 1.249184E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.318985E-02 | lm-token-buffer-mib: 5.825834E+01 | global_load_balancing_loss: 1.001310E+00 | loss scale: 1.0 | grad norm: 0.139 | params norm: 11598.370 | params rms: 0.062801 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 19:32:07.894306 | finish at 2026-10-10 10:02:38 + [2026-10-04 14:31:56] iteration 906/ 6855 | consumed samples: 115968 | elapsed time per iteration (ms): 85734.7 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.57% | FLOPs estimator: workload-aware | logical tokens/s: 195687.5 | valid tokens/s: 195687.5 | physical tokens/s: 196110.6 | loss tokens/s: 148425.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.900374E-01 | lm-global-supervised-tokens: 1.272520E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.162941E-02 | lm-token-buffer-mib: 5.813042E+01 | global_load_balancing_loss: 1.001759E+00 | loss scale: 1.0 | grad norm: 0.241 | params norm: 11598.285 | params rms: 0.062801 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 21:40:35.881853 | finish at 2026-10-10 12:12:31 + [2026-10-04 14:33:21] iteration 907/ 6855 | consumed samples: 116096 | elapsed time per iteration (ms): 85153.9 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.50% | FLOPs estimator: workload-aware | logical tokens/s: 197022.4 | valid tokens/s: 197022.4 | physical tokens/s: 197724.7 | loss tokens/s: 148346.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.075906E-01 | lm-global-supervised-tokens: 1.263224E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.477793E-02 | lm-token-buffer-mib: 5.819191E+01 | global_load_balancing_loss: 1.001646E+00 | loss scale: 1.0 | grad norm: 0.095 | params norm: 11598.198 | params rms: 0.062800 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 20:41:35.181100 | finish at 2026-10-10 11:14:56 + [2026-10-04 14:34:46] iteration 908/ 6855 | consumed samples: 116224 | elapsed time per iteration (ms): 85647.1 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.49% | FLOPs estimator: workload-aware | logical tokens/s: 195886.9 | valid tokens/s: 195886.9 | physical tokens/s: 196337.1 | loss tokens/s: 150189.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.869017E-01 | lm-global-supervised-tokens: 1.286326E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.680041E-02 | lm-token-buffer-mib: 5.815186E+01 | global_load_balancing_loss: 1.000210E+00 | loss scale: 1.0 | grad norm: 0.074 | params norm: 11598.112 | params rms: 0.062800 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 21:29:03.490692 | finish at 2026-10-10 12:03:50 + [2026-10-04 14:36:11] iteration 909/ 6855 | consumed samples: 116352 | elapsed time per iteration (ms): 84163.1 | throughput per GPU (TFLOP/s/GPU): 43.1 | MFU 13.82% | FLOPs estimator: workload-aware | logical tokens/s: 199341.1 | valid tokens/s: 199341.1 | physical tokens/s: 199941.1 | loss tokens/s: 151724.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.042606E-01 | lm-global-supervised-tokens: 1.276960E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.294507E-02 | lm-token-buffer-mib: 5.814865E+01 | global_load_balancing_loss: 9.998034E-01 | loss scale: 1.0 | grad norm: 0.178 | params norm: 11598.026 | params rms: 0.062800 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 19:00:33.843660 | finish at 2026-10-10 09:36:44 + [2026-10-04 14:38:02] iteration 910/ 6855 | consumed samples: 116480 | elapsed time per iteration (ms): 111151.8 | throughput per GPU (TFLOP/s/GPU): 33.7 | MFU 10.79% | FLOPs estimator: workload-aware | logical tokens/s: 150938.9 | valid tokens/s: 150938.9 | physical tokens/s: 151325.9 | loss tokens/s: 109677.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.216099E-01 | lm-global-supervised-tokens: 1.219084E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.663320E-02 | lm-token-buffer-mib: 5.819582E+01 | global_load_balancing_loss: 9.990323E-01 | loss scale: 1.0 | grad norm: 0.276 | params norm: 11597.939 | params rms: 0.062799 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 7 days, 15:33:17.451925 | finish at 2026-10-12 06:11:19 + [2026-10-04 14:39:26] iteration 911/ 6855 | consumed samples: 116608 | elapsed time per iteration (ms): 84460.5 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.68% | FLOPs estimator: workload-aware | logical tokens/s: 198639.2 | valid tokens/s: 198639.2 | physical tokens/s: 199239.8 | loss tokens/s: 151232.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.968137E-01 | lm-global-supervised-tokens: 1.277316E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.331561E-02 | lm-token-buffer-mib: 5.810296E+01 | global_load_balancing_loss: 1.000651E+00 | loss scale: 1.0 | grad norm: 0.111 | params norm: 11597.851 | params rms: 0.062799 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 19:27:13.488358 | finish at 2026-10-10 10:06:40 + [2026-10-04 14:40:53] iteration 912/ 6855 | consumed samples: 116736 | elapsed time per iteration (ms): 86460.3 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.40% | FLOPs estimator: workload-aware | logical tokens/s: 194044.9 | valid tokens/s: 194044.9 | physical tokens/s: 194441.5 | loss tokens/s: 145465.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.647314E-01 | lm-global-supervised-tokens: 1.257702E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.166189E-02 | lm-token-buffer-mib: 5.822677E+01 | global_load_balancing_loss: 9.993723E-01 | loss scale: 1.0 | grad norm: 0.099 | params norm: 11597.765 | params rms: 0.062798 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 22:43:53.649146 | finish at 2026-10-10 13:24:46 + [2026-10-04 14:42:19] iteration 913/ 6855 | consumed samples: 116864 | elapsed time per iteration (ms): 86524.8 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.67% | FLOPs estimator: workload-aware | logical tokens/s: 193900.0 | valid tokens/s: 193900.0 | physical tokens/s: 194251.9 | loss tokens/s: 141772.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.013532E-01 | lm-global-supervised-tokens: 1.226681E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.143841E-02 | lm-token-buffer-mib: 5.811705E+01 | global_load_balancing_loss: 9.998392E-01 | loss scale: 1.0 | grad norm: 0.055 | params norm: 11597.680 | params rms: 0.062798 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 22:48:50.612722 | finish at 2026-10-10 13:31:10 + [2026-10-04 14:43:45] iteration 914/ 6855 | consumed samples: 116992 | elapsed time per iteration (ms): 86224.4 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.69% | FLOPs estimator: workload-aware | logical tokens/s: 194575.8 | valid tokens/s: 194575.7 | physical tokens/s: 194871.4 | loss tokens/s: 139044.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.472090E-01 | lm-global-supervised-tokens: 1.198901E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.444260E-02 | lm-token-buffer-mib: 5.805739E+01 | global_load_balancing_loss: 1.000886E+00 | loss scale: 1.0 | grad norm: 0.085 | params norm: 11597.594 | params rms: 0.062797 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 22:17:39.349046 | finish at 2026-10-10 13:01:25 + [2026-10-04 14:45:15] iteration 915/ 6855 | consumed samples: 117120 | elapsed time per iteration (ms): 89899.4 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.69% | FLOPs estimator: workload-aware | logical tokens/s: 186622.0 | valid tokens/s: 186622.0 | physical tokens/s: 187050.1 | loss tokens/s: 135686.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.607327E-01 | lm-global-supervised-tokens: 1.219817E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.310235E-02 | lm-token-buffer-mib: 5.820160E+01 | global_load_balancing_loss: 1.000498E+00 | loss scale: 1.0 | grad norm: 0.109 | params norm: 11597.509 | params rms: 0.062797 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 4:20:02.621498 | finish at 2026-10-10 19:05:18 + [2026-10-04 14:46:43] iteration 916/ 6855 | consumed samples: 117248 | elapsed time per iteration (ms): 87915.7 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.63% | FLOPs estimator: workload-aware | logical tokens/s: 190832.9 | valid tokens/s: 190832.9 | physical tokens/s: 191314.4 | loss tokens/s: 146305.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.257760E-01 | lm-global-supervised-tokens: 1.286253E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.283834E-02 | lm-token-buffer-mib: 5.822680E+01 | global_load_balancing_loss: 1.000124E+00 | loss scale: 1.0 | grad norm: 0.041 | params norm: 11597.424 | params rms: 0.062796 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 1:02:11.611089 | finish at 2026-10-10 15:48:55 + [2026-10-04 14:48:09] iteration 917/ 6855 | consumed samples: 117376 | elapsed time per iteration (ms): 85436.7 | throughput per GPU (TFLOP/s/GPU): 43.3 | MFU 13.89% | FLOPs estimator: workload-aware | logical tokens/s: 196370.2 | valid tokens/s: 196370.2 | physical tokens/s: 197039.2 | loss tokens/s: 141862.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.760512E-01 | lm-global-supervised-tokens: 1.212026E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.534211E-02 | lm-token-buffer-mib: 5.820575E+01 | global_load_balancing_loss: 9.994769E-01 | loss scale: 1.0 | grad norm: 0.112 | params norm: 11597.339 | params rms: 0.062796 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 20:55:22.833587 | finish at 2026-10-10 11:43:31 + [2026-10-04 14:49:36] iteration 918/ 6855 | consumed samples: 117504 | elapsed time per iteration (ms): 87622.0 | throughput per GPU (TFLOP/s/GPU): 42.8 | MFU 13.72% | FLOPs estimator: workload-aware | logical tokens/s: 191472.5 | valid tokens/s: 191472.5 | physical tokens/s: 191887.1 | loss tokens/s: 138387.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.926953E-01 | lm-global-supervised-tokens: 1.212579E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.159912E-02 | lm-token-buffer-mib: 5.805901E+01 | global_load_balancing_loss: 1.000113E+00 | loss scale: 1.0 | grad norm: 0.038 | params norm: 11597.255 | params rms: 0.062795 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 0:30:12.061586 | finish at 2026-10-10 15:19:48 + [2026-10-04 14:51:05] iteration 919/ 6855 | consumed samples: 117632 | elapsed time per iteration (ms): 89152.1 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.45% | FLOPs estimator: workload-aware | logical tokens/s: 188186.3 | valid tokens/s: 188186.3 | physical tokens/s: 188635.3 | loss tokens/s: 132453.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.607096E-01 | lm-global-supervised-tokens: 1.180848E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.276153E-02 | lm-token-buffer-mib: 5.816749E+01 | global_load_balancing_loss: 1.000760E+00 | loss scale: 1.0 | grad norm: 0.082 | params norm: 11597.171 | params rms: 0.062795 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 3:00:06.754307 | finish at 2026-10-10 17:51:12 + [2026-10-04 14:52:52] iteration 920/ 6855 | consumed samples: 117760 | elapsed time per iteration (ms): 106856.6 | throughput per GPU (TFLOP/s/GPU): 34.3 | MFU 10.99% | FLOPs estimator: workload-aware | logical tokens/s: 157005.5 | valid tokens/s: 157005.5 | physical tokens/s: 157488.8 | loss tokens/s: 116825.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.803670E-01 | lm-global-supervised-tokens: 1.248358E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.292502E-02 | lm-token-buffer-mib: 5.823298E+01 | global_load_balancing_loss: 1.000213E+00 | loss scale: 1.0 | grad norm: 0.086 | params norm: 11597.086 | params rms: 0.062794 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 7 days, 8:09:54.185882 | finish at 2026-10-11 23:02:46 +times across ranks (ms): + save-checkpoint: + rank 0: 24450.30 + rank 1: 24450.25 + rank 2: 24450.33 + rank 3: 24450.29 + rank 4: 24450.29 + rank 5: 24450.26 + rank 6: 24450.21 + rank 7: 24450.28 + rank 8: 24450.17 + rank 9: 24450.09 + rank 10: 24450.10 + rank 11: 24450.14 + rank 12: 24450.12 + rank 13: 24450.12 + rank 14: 24450.01 + rank 15: 24450.07 + rank 16: 24449.68 + rank 17: 24449.65 + rank 18: 24449.61 + rank 19: 24449.66 + rank 20: 24449.61 + rank 21: 24449.70 + rank 22: 24449.63 + rank 23: 24449.61 + rank 24: 24450.56 + rank 25: 24451.37 + rank 26: 24450.62 + rank 27: 24450.62 + rank 28: 24450.50 + rank 29: 24450.56 + rank 30: 24450.48 + rank 31: 24450.58 + rank 32: 24450.94 + rank 33: 24451.07 + rank 34: 24450.97 + rank 35: 24450.97 + rank 36: 24450.90 + rank 37: 24450.85 + rank 38: 24450.91 + rank 39: 24450.90 + rank 40: 24450.49 + rank 41: 24450.41 + rank 42: 24450.46 + rank 43: 24450.48 + rank 44: 24450.39 + rank 45: 24450.42 + rank 46: 24450.43 + rank 47: 24450.39 + rank 48: 24451.12 + rank 49: 24451.27 + rank 50: 24451.23 + rank 51: 24451.11 + rank 52: 24451.32 + rank 53: 24451.26 + rank 54: 24451.27 + rank 55: 24451.31 + rank 56: 24450.41 + rank 57: 24450.39 + rank 58: 24450.38 + rank 59: 24450.35 + rank 60: 24450.36 + rank 61: 24450.39 + rank 62: 24450.38 + rank 63: 24450.39 + [2026-10-04 14:54:54] iteration 921/ 6855 | consumed samples: 117888 | elapsed time per iteration (ms): 97217.1 | throughput per GPU (TFLOP/s/GPU): 38.1 | MFU 12.20% | FLOPs estimator: workload-aware | logical tokens/s: 172574.1 | valid tokens/s: 172574.1 | physical tokens/s: 172804.1 | loss tokens/s: 127090.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.836168E-01 | lm-global-supervised-tokens: 1.235534E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.259060E-01 | lm-token-buffer-mib: 5.810600E+01 | global_load_balancing_loss: 9.996921E-01 | loss scale: 1.0 | grad norm: 0.047 | params norm: 11597.002 | params rms: 0.062794 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 16:14:45.975148 | finish at 2026-10-11 07:09:40 + [2026-10-04 14:56:26] iteration 922/ 6855 | consumed samples: 118016 | elapsed time per iteration (ms): 91600.1 | throughput per GPU (TFLOP/s/GPU): 40.3 | MFU 12.92% | FLOPs estimator: workload-aware | logical tokens/s: 183156.4 | valid tokens/s: 183156.4 | physical tokens/s: 183784.4 | loss tokens/s: 140703.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.245380E-01 | lm-global-supervised-tokens: 1.288850E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.984767E-02 | lm-token-buffer-mib: 5.817168E+01 | global_load_balancing_loss: 9.999495E-01 | loss scale: 1.0 | grad norm: 0.104 | params norm: 11596.919 | params rms: 0.062794 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 6:57:43.420416 | finish at 2026-10-10 21:54:09 + [2026-10-04 14:57:53] iteration 923/ 6855 | consumed samples: 118144 | elapsed time per iteration (ms): 87704.9 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.54% | FLOPs estimator: workload-aware | logical tokens/s: 191291.8 | valid tokens/s: 191291.8 | physical tokens/s: 191861.1 | loss tokens/s: 140631.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.545392E-01 | lm-global-supervised-tokens: 1.233410E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.175987E-02 | lm-token-buffer-mib: 5.813529E+01 | global_load_balancing_loss: 1.001493E+00 | loss scale: 1.0 | grad norm: 0.171 | params norm: 11596.836 | params rms: 0.062793 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 0:31:05.220868 | finish at 2026-10-10 15:28:59 + [2026-10-04 14:59:20] iteration 924/ 6855 | consumed samples: 118272 | elapsed time per iteration (ms): 86925.4 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.59% | FLOPs estimator: workload-aware | logical tokens/s: 193006.8 | valid tokens/s: 193006.7 | physical tokens/s: 193591.8 | loss tokens/s: 142677.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.665355E-01 | lm-global-supervised-tokens: 1.240232E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.144351E-02 | lm-token-buffer-mib: 5.813240E+01 | global_load_balancing_loss: 1.001030E+00 | loss scale: 1.0 | grad norm: 0.076 | params norm: 11596.751 | params rms: 0.062793 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 23:12:34.681847 | finish at 2026-10-10 14:11:55 + [2026-10-04 15:00:47] iteration 925/ 6855 | consumed samples: 118400 | elapsed time per iteration (ms): 86311.4 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.66% | FLOPs estimator: workload-aware | logical tokens/s: 194379.9 | valid tokens/s: 194379.9 | physical tokens/s: 194836.9 | loss tokens/s: 143661.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.699413E-01 | lm-global-supervised-tokens: 1.239967E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.483253E-02 | lm-token-buffer-mib: 5.817883E+01 | global_load_balancing_loss: 1.000825E+00 | loss scale: 1.0 | grad norm: 0.047 | params norm: 11596.669 | params rms: 0.062792 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 22:10:26.878734 | finish at 2026-10-10 13:11:14 + [2026-10-04 15:02:15] iteration 926/ 6855 | consumed samples: 118528 | elapsed time per iteration (ms): 88756.3 | throughput per GPU (TFLOP/s/GPU): 41.6 | MFU 13.34% | FLOPs estimator: workload-aware | logical tokens/s: 189024.6 | valid tokens/s: 189024.6 | physical tokens/s: 189440.3 | loss tokens/s: 139925.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.613399E-01 | lm-global-supervised-tokens: 1.241925E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.405965E-02 | lm-token-buffer-mib: 5.814352E+01 | global_load_balancing_loss: 1.000159E+00 | loss scale: 1.0 | grad norm: 0.040 | params norm: 11596.587 | params rms: 0.062792 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 2:10:36.395149 | finish at 2026-10-10 17:12:52 + [2026-10-04 15:03:45] iteration 927/ 6855 | consumed samples: 118656 | elapsed time per iteration (ms): 89326.7 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.39% | FLOPs estimator: workload-aware | logical tokens/s: 187817.9 | valid tokens/s: 187817.9 | physical tokens/s: 188407.6 | loss tokens/s: 145820.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.186816E-01 | lm-global-supervised-tokens: 1.302564E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.487607E-02 | lm-token-buffer-mib: 5.805505E+01 | global_load_balancing_loss: 1.000187E+00 | loss scale: 1.0 | grad norm: 0.049 | params norm: 11596.504 | params rms: 0.062791 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 3:05:28.586981 | finish at 2026-10-10 18:09:13 + [2026-10-04 15:05:09] iteration 928/ 6855 | consumed samples: 118784 | elapsed time per iteration (ms): 84578.9 | throughput per GPU (TFLOP/s/GPU): 43.5 | MFU 13.95% | FLOPs estimator: workload-aware | logical tokens/s: 198361.7 | valid tokens/s: 198361.7 | physical tokens/s: 198733.5 | loss tokens/s: 143500.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.582780E-01 | lm-global-supervised-tokens: 1.213708E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.704763E-02 | lm-token-buffer-mib: 5.803519E+01 | global_load_balancing_loss: 9.993806E-01 | loss scale: 1.0 | grad norm: 0.032 | params norm: 11596.423 | params rms: 0.062791 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 19:14:59.077296 | finish at 2026-10-10 10:20:08 + [2026-10-04 15:06:36] iteration 929/ 6855 | consumed samples: 118912 | elapsed time per iteration (ms): 87025.4 | throughput per GPU (TFLOP/s/GPU): 41.7 | MFU 13.37% | FLOPs estimator: workload-aware | logical tokens/s: 192785.4 | valid tokens/s: 192785.4 | physical tokens/s: 193256.0 | loss tokens/s: 144246.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.118826E-01 | lm-global-supervised-tokens: 1.255307E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.642881E-02 | lm-token-buffer-mib: 5.807497E+01 | global_load_balancing_loss: 9.990121E-01 | loss scale: 1.0 | grad norm: 0.099 | params norm: 11596.342 | params rms: 0.062790 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 23:15:12.271281 | finish at 2026-10-10 14:21:49 + [2026-10-04 15:08:24] iteration 930/ 6855 | consumed samples: 119040 | elapsed time per iteration (ms): 107687.9 | throughput per GPU (TFLOP/s/GPU): 33.9 | MFU 10.85% | FLOPs estimator: workload-aware | logical tokens/s: 155794.5 | valid tokens/s: 155794.5 | physical tokens/s: 156240.5 | loss tokens/s: 114772.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.760976E-01 | lm-global-supervised-tokens: 1.235964E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.322537E-02 | lm-token-buffer-mib: 5.816222E+01 | global_load_balancing_loss: 9.986078E-01 | loss scale: 1.0 | grad norm: 0.140 | params norm: 11596.260 | params rms: 0.062790 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 7 days, 9:14:10.964695 | finish at 2026-10-12 00:22:35 + [2026-10-04 15:09:54] iteration 931/ 6855 | consumed samples: 119168 | elapsed time per iteration (ms): 90321.4 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.54% | FLOPs estimator: workload-aware | logical tokens/s: 185750.0 | valid tokens/s: 185750.0 | physical tokens/s: 186213.9 | loss tokens/s: 135523.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.625217E-01 | lm-global-supervised-tokens: 1.224069E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.260312E-02 | lm-token-buffer-mib: 5.805369E+01 | global_load_balancing_loss: 1.000337E+00 | loss scale: 1.0 | grad norm: 0.033 | params norm: 11596.175 | params rms: 0.062790 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 4:37:43.717525 | finish at 2026-10-10 19:47:38 + [2026-10-04 15:11:26] iteration 932/ 6855 | consumed samples: 119296 | elapsed time per iteration (ms): 92005.0 | throughput per GPU (TFLOP/s/GPU): 40.3 | MFU 12.91% | FLOPs estimator: workload-aware | logical tokens/s: 182350.5 | valid tokens/s: 182350.5 | physical tokens/s: 182777.2 | loss tokens/s: 137708.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.955258E-01 | lm-global-supervised-tokens: 1.266989E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.199741E-02 | lm-token-buffer-mib: 5.832638E+01 | global_load_balancing_loss: 1.000674E+00 | loss scale: 1.0 | grad norm: 0.134 | params norm: 11596.090 | params rms: 0.062789 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 7:22:25.715941 | finish at 2026-10-10 22:33:52 + [2026-10-04 15:12:53] iteration 933/ 6855 | consumed samples: 119424 | elapsed time per iteration (ms): 86190.2 | throughput per GPU (TFLOP/s/GPU): 41.5 | MFU 13.31% | FLOPs estimator: workload-aware | logical tokens/s: 194653.3 | valid tokens/s: 194653.2 | physical tokens/s: 195240.5 | loss tokens/s: 145526.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.614562E-01 | lm-global-supervised-tokens: 1.254293E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.319964E-02 | lm-token-buffer-mib: 5.815044E+01 | global_load_balancing_loss: 1.002060E+00 | loss scale: 1.0 | grad norm: 0.079 | params norm: 11596.008 | params rms: 0.062789 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 21:46:58.120674 | finish at 2026-10-10 12:59:51 + [2026-10-04 15:14:22] iteration 934/ 6855 | consumed samples: 119552 | elapsed time per iteration (ms): 89293.8 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.56% | FLOPs estimator: workload-aware | logical tokens/s: 187887.1 | valid tokens/s: 187887.1 | physical tokens/s: 188180.5 | loss tokens/s: 137747.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.113264E-01 | lm-global-supervised-tokens: 1.230000E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.264028E-02 | lm-token-buffer-mib: 5.805799E+01 | global_load_balancing_loss: 1.001002E+00 | loss scale: 1.0 | grad norm: 0.047 | params norm: 11595.929 | params rms: 0.062788 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 2:51:48.347676 | finish at 2026-10-10 18:06:10 + [2026-10-04 15:15:52] iteration 935/ 6855 | consumed samples: 119680 | elapsed time per iteration (ms): 90456.6 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.57% | FLOPs estimator: workload-aware | logical tokens/s: 185472.5 | valid tokens/s: 185472.5 | physical tokens/s: 186002.2 | loss tokens/s: 132801.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.315625E-01 | lm-global-supervised-tokens: 1.201274E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.298631E-02 | lm-token-buffer-mib: 5.813515E+01 | global_load_balancing_loss: 1.000810E+00 | loss scale: 1.0 | grad norm: 0.058 | params norm: 11595.848 | params rms: 0.062788 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 4:45:02.996902 | finish at 2026-10-10 20:00:55 + [2026-10-04 15:17:16] iteration 936/ 6855 | consumed samples: 119808 | elapsed time per iteration (ms): 83972.5 | throughput per GPU (TFLOP/s/GPU): 43.5 | MFU 13.95% | FLOPs estimator: workload-aware | logical tokens/s: 199793.6 | valid tokens/s: 199793.6 | physical tokens/s: 200183.3 | loss tokens/s: 153579.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.396597E-01 | lm-global-supervised-tokens: 1.289646E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.506492E-02 | lm-token-buffer-mib: 5.812170E+01 | global_load_balancing_loss: 1.000211E+00 | loss scale: 1.0 | grad norm: 0.037 | params norm: 11595.767 | params rms: 0.062787 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 18:03:53.134869 | finish at 2026-10-10 09:21:09 + [2026-10-04 15:18:44] iteration 937/ 6855 | consumed samples: 119936 | elapsed time per iteration (ms): 87652.9 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.63% | FLOPs estimator: workload-aware | logical tokens/s: 191404.4 | valid tokens/s: 191404.3 | physical tokens/s: 191902.6 | loss tokens/s: 139371.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.842911E-01 | lm-global-supervised-tokens: 1.221629E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.371002E-02 | lm-token-buffer-mib: 5.821315E+01 | global_load_balancing_loss: 9.999041E-01 | loss scale: 1.0 | grad norm: 0.053 | params norm: 11595.688 | params rms: 0.062787 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 0:05:29.722400 | finish at 2026-10-10 15:24:14 + [2026-10-04 15:20:10] iteration 938/ 6855 | consumed samples: 120064 | elapsed time per iteration (ms): 86247.9 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.80% | FLOPs estimator: workload-aware | logical tokens/s: 194523.0 | valid tokens/s: 194522.9 | physical tokens/s: 195073.3 | loss tokens/s: 142642.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.548592E-01 | lm-global-supervised-tokens: 1.230266E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.251294E-02 | lm-token-buffer-mib: 5.813496E+01 | global_load_balancing_loss: 9.991307E-01 | loss scale: 1.0 | grad norm: 0.032 | params norm: 11595.610 | params rms: 0.062786 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 21:45:29.105088 | finish at 2026-10-10 13:05:39 + [2026-10-04 15:21:38] iteration 939/ 6855 | consumed samples: 120192 | elapsed time per iteration (ms): 88048.7 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.53% | FLOPs estimator: workload-aware | logical tokens/s: 190544.0 | valid tokens/s: 190544.0 | physical tokens/s: 190832.6 | loss tokens/s: 143128.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.847486E-01 | lm-global-supervised-tokens: 1.260229E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.164082E-02 | lm-token-buffer-mib: 5.811193E+01 | global_load_balancing_loss: 9.993201E-01 | loss scale: 1.0 | grad norm: 0.050 | params norm: 11595.528 | params rms: 0.062786 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 0:41:36.047696 | finish at 2026-10-10 16:03:14 + [2026-10-04 15:23:26] iteration 940/ 6855 | consumed samples: 120320 | elapsed time per iteration (ms): 107731.8 | throughput per GPU (TFLOP/s/GPU): 34.5 | MFU 11.07% | FLOPs estimator: workload-aware | logical tokens/s: 155731.0 | valid tokens/s: 155731.0 | physical tokens/s: 156076.9 | loss tokens/s: 117516.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.245162E-01 | lm-global-supervised-tokens: 1.266028E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.435983E-02 | lm-token-buffer-mib: 5.821494E+01 | global_load_balancing_loss: 9.999230E-01 | loss scale: 1.0 | grad norm: 0.035 | params norm: 11595.447 | params rms: 0.062786 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 7 days, 9:00:33.744135 | finish at 2026-10-12 00:24:00 + [2026-10-04 15:24:54] iteration 941/ 6855 | consumed samples: 120448 | elapsed time per iteration (ms): 88282.9 | throughput per GPU (TFLOP/s/GPU): 41.4 | MFU 13.28% | FLOPs estimator: workload-aware | logical tokens/s: 190038.7 | valid tokens/s: 190038.7 | physical tokens/s: 190478.5 | loss tokens/s: 149513.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.406282E-01 | lm-global-supervised-tokens: 1.319952E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.316422E-02 | lm-token-buffer-mib: 5.817366E+01 | global_load_balancing_loss: 9.993389E-01 | loss scale: 1.0 | grad norm: 0.047 | params norm: 11595.367 | params rms: 0.062785 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 1:01:45.134612 | finish at 2026-10-10 16:26:39 + [2026-10-04 15:26:21] iteration 942/ 6855 | consumed samples: 120576 | elapsed time per iteration (ms): 86404.9 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.47% | FLOPs estimator: workload-aware | logical tokens/s: 194169.7 | valid tokens/s: 194169.7 | physical tokens/s: 194591.6 | loss tokens/s: 150521.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.120777E-01 | lm-global-supervised-tokens: 1.300579E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.523538E-02 | lm-token-buffer-mib: 5.808052E+01 | global_load_balancing_loss: 1.001042E+00 | loss scale: 1.0 | grad norm: 0.045 | params norm: 11595.287 | params rms: 0.062785 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 21:55:12.136074 | finish at 2026-10-10 13:21:33 + [2026-10-04 15:27:46] iteration 943/ 6855 | consumed samples: 120704 | elapsed time per iteration (ms): 85108.4 | throughput per GPU (TFLOP/s/GPU): 43.3 | MFU 13.89% | FLOPs estimator: workload-aware | logical tokens/s: 197127.5 | valid tokens/s: 197127.4 | physical tokens/s: 197657.9 | loss tokens/s: 144896.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.729720E-01 | lm-global-supervised-tokens: 1.233192E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.225458E-02 | lm-token-buffer-mib: 5.808463E+01 | global_load_balancing_loss: 1.001351E+00 | loss scale: 1.0 | grad norm: 0.087 | params norm: 11595.204 | params rms: 0.062784 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 19:46:00.927677 | finish at 2026-10-10 11:13:47 + [2026-10-04 15:29:10] iteration 944/ 6855 | consumed samples: 120832 | elapsed time per iteration (ms): 84105.2 | throughput per GPU (TFLOP/s/GPU): 45.0 | MFU 14.42% | FLOPs estimator: workload-aware | logical tokens/s: 199478.9 | valid tokens/s: 199478.9 | physical tokens/s: 199927.0 | loss tokens/s: 142542.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.063720E-01 | lm-global-supervised-tokens: 1.198856E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.160424E-02 | lm-token-buffer-mib: 5.808722E+01 | global_load_balancing_loss: 1.001434E+00 | loss scale: 1.0 | grad norm: 0.217 | params norm: 11595.122 | params rms: 0.062784 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 18:05:45.617659 | finish at 2026-10-10 09:34:56 + [2026-10-04 15:30:40] iteration 945/ 6855 | consumed samples: 120960 | elapsed time per iteration (ms): 89931.9 | throughput per GPU (TFLOP/s/GPU): 41.1 | MFU 13.17% | FLOPs estimator: workload-aware | logical tokens/s: 186554.5 | valid tokens/s: 186554.5 | physical tokens/s: 187000.6 | loss tokens/s: 136408.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.802465E-01 | lm-global-supervised-tokens: 1.226747E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.219054E-02 | lm-token-buffer-mib: 5.810065E+01 | global_load_balancing_loss: 1.001220E+00 | loss scale: 1.0 | grad norm: 0.278 | params norm: 11595.040 | params rms: 0.062783 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 3:38:17.405148 | finish at 2026-10-10 19:08:57 + [2026-10-04 15:32:03] iteration 946/ 6855 | consumed samples: 121088 | elapsed time per iteration (ms): 83382.7 | throughput per GPU (TFLOP/s/GPU): 43.3 | MFU 13.88% | FLOPs estimator: workload-aware | logical tokens/s: 201207.2 | valid tokens/s: 201207.2 | physical tokens/s: 201641.4 | loss tokens/s: 152127.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.989031E-01 | lm-global-supervised-tokens: 1.268478E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.174174E-02 | lm-token-buffer-mib: 5.810474E+01 | global_load_balancing_loss: 9.996278E-01 | loss scale: 1.0 | grad norm: 0.155 | params norm: 11594.957 | params rms: 0.062783 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 16:51:48.568518 | finish at 2026-10-10 08:23:52 + [2026-10-04 15:33:34] iteration 947/ 6855 | consumed samples: 121216 | elapsed time per iteration (ms): 90496.8 | throughput per GPU (TFLOP/s/GPU): 41.4 | MFU 13.26% | FLOPs estimator: workload-aware | logical tokens/s: 185390.2 | valid tokens/s: 185390.2 | physical tokens/s: 185827.3 | loss tokens/s: 135989.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.944744E-01 | lm-global-supervised-tokens: 1.230660E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.266060E-02 | lm-token-buffer-mib: 5.804341E+01 | global_load_balancing_loss: 9.979329E-01 | loss scale: 1.0 | grad norm: 0.056 | params norm: 11594.875 | params rms: 0.062782 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 4:30:54.943362 | finish at 2026-10-10 20:04:29 + [2026-10-04 15:35:00] iteration 948/ 6855 | consumed samples: 121344 | elapsed time per iteration (ms): 86409.3 | throughput per GPU (TFLOP/s/GPU): 41.4 | MFU 13.27% | FLOPs estimator: workload-aware | logical tokens/s: 194159.0 | valid tokens/s: 194159.0 | physical tokens/s: 194728.0 | loss tokens/s: 150207.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.434058E-01 | lm-global-supervised-tokens: 1.297935E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.890821E-02 | lm-token-buffer-mib: 5.821490E+01 | global_load_balancing_loss: 9.986236E-01 | loss scale: 1.0 | grad norm: 0.099 | params norm: 11594.793 | params rms: 0.062782 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 21:46:59.958143 | finish at 2026-10-10 13:22:00 + [2026-10-04 15:36:29] iteration 949/ 6855 | consumed samples: 121472 | elapsed time per iteration (ms): 88731.2 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.55% | FLOPs estimator: workload-aware | logical tokens/s: 189079.1 | valid tokens/s: 189079.1 | physical tokens/s: 189500.9 | loss tokens/s: 139429.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.797717E-01 | lm-global-supervised-tokens: 1.237178E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.683186E-02 | lm-token-buffer-mib: 5.806241E+01 | global_load_balancing_loss: 9.999890E-01 | loss scale: 1.0 | grad norm: 0.046 | params norm: 11594.711 | params rms: 0.062782 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 1:34:06.365697 | finish at 2026-10-10 17:10:35 + [2026-10-04 15:38:16] iteration 950/ 6855 | consumed samples: 121600 | elapsed time per iteration (ms): 107408.3 | throughput per GPU (TFLOP/s/GPU): 34.2 | MFU 10.97% | FLOPs estimator: workload-aware | logical tokens/s: 156200.2 | valid tokens/s: 156200.2 | physical tokens/s: 156593.7 | loss tokens/s: 118338.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.149482E-01 | lm-global-supervised-tokens: 1.271058E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.563043E-02 | lm-token-buffer-mib: 5.816166E+01 | global_load_balancing_loss: 1.000940E+00 | loss scale: 1.0 | grad norm: 0.063 | params norm: 11594.628 | params rms: 0.062781 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 7 days, 8:10:46.213777 | finish at 2026-10-11 23:49:03 + [2026-10-04 15:39:42] iteration 951/ 6855 | consumed samples: 121728 | elapsed time per iteration (ms): 86130.0 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.67% | FLOPs estimator: workload-aware | logical tokens/s: 194788.7 | valid tokens/s: 194788.7 | physical tokens/s: 195138.7 | loss tokens/s: 144141.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.560802E-01 | lm-global-supervised-tokens: 1.241493E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.156225E-02 | lm-token-buffer-mib: 5.809179E+01 | global_load_balancing_loss: 1.002471E+00 | loss scale: 1.0 | grad norm: 0.044 | params norm: 11594.545 | params rms: 0.062781 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 21:15:11.439034 | finish at 2026-10-10 12:54:54 + [2026-10-04 15:41:11] iteration 952/ 6855 | consumed samples: 121856 | elapsed time per iteration (ms): 88543.5 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.60% | FLOPs estimator: workload-aware | logical tokens/s: 189479.9 | valid tokens/s: 189479.9 | physical tokens/s: 189812.8 | loss tokens/s: 135613.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.628285E-01 | lm-global-supervised-tokens: 1.200772E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.140572E-02 | lm-token-buffer-mib: 5.804955E+01 | global_load_balancing_loss: 1.001951E+00 | loss scale: 1.0 | grad norm: 0.048 | params norm: 11594.460 | params rms: 0.062780 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 1:11:12.157742 | finish at 2026-10-10 16:52:23 + [2026-10-04 15:42:36] iteration 953/ 6855 | consumed samples: 121984 | elapsed time per iteration (ms): 85365.4 | throughput per GPU (TFLOP/s/GPU): 43.6 | MFU 13.97% | FLOPs estimator: workload-aware | logical tokens/s: 196533.9 | valid tokens/s: 196533.9 | physical tokens/s: 196871.9 | loss tokens/s: 141852.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.796126E-01 | lm-global-supervised-tokens: 1.210929E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.207398E-02 | lm-token-buffer-mib: 5.817749E+01 | global_load_balancing_loss: 9.997178E-01 | loss scale: 1.0 | grad norm: 0.037 | params norm: 11594.375 | params rms: 0.062780 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 19:57:06.636277 | finish at 2026-10-10 11:39:43 + [2026-10-04 15:44:00] iteration 954/ 6855 | consumed samples: 122112 | elapsed time per iteration (ms): 83781.9 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.58% | FLOPs estimator: workload-aware | logical tokens/s: 200248.3 | valid tokens/s: 200248.3 | physical tokens/s: 200945.4 | loss tokens/s: 148467.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.264521E-01 | lm-global-supervised-tokens: 1.243884E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.276905E-02 | lm-token-buffer-mib: 5.827864E+01 | global_load_balancing_loss: 9.988740E-01 | loss scale: 1.0 | grad norm: 0.057 | params norm: 11594.289 | params rms: 0.062779 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 17:19:56.831094 | finish at 2026-10-10 09:03:57 + [2026-10-04 15:45:32] iteration 955/ 6855 | consumed samples: 122240 | elapsed time per iteration (ms): 91633.3 | throughput per GPU (TFLOP/s/GPU): 40.5 | MFU 12.98% | FLOPs estimator: workload-aware | logical tokens/s: 183090.9 | valid tokens/s: 183090.9 | physical tokens/s: 183545.4 | loss tokens/s: 133733.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.163178E-01 | lm-global-supervised-tokens: 1.225441E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.302151E-02 | lm-token-buffer-mib: 5.808181E+01 | global_load_balancing_loss: 9.993600E-01 | loss scale: 1.0 | grad norm: 0.061 | params norm: 11594.204 | params rms: 0.062779 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 6:10:36.298776 | finish at 2026-10-10 21:56:08 + [2026-10-04 15:46:59] iteration 956/ 6855 | consumed samples: 122368 | elapsed time per iteration (ms): 87502.9 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.51% | FLOPs estimator: workload-aware | logical tokens/s: 191732.6 | valid tokens/s: 191732.5 | physical tokens/s: 192237.5 | loss tokens/s: 141762.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.929650E-01 | lm-global-supervised-tokens: 1.240462E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 3.312533E-02 | lm-token-buffer-mib: 5.809370E+01 | global_load_balancing_loss: 9.993504E-01 | loss scale: 1.0 | grad norm: 0.041 | params norm: 11594.118 | params rms: 0.062778 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 23:22:59.706279 | finish at 2026-10-10 15:09:59 + [2026-10-04 15:48:30] iteration 957/ 6855 | consumed samples: 122496 | elapsed time per iteration (ms): 90560.6 | throughput per GPU (TFLOP/s/GPU): 41.0 | MFU 13.14% | FLOPs estimator: workload-aware | logical tokens/s: 185259.6 | valid tokens/s: 185259.6 | physical tokens/s: 185830.1 | loss tokens/s: 133938.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.939536E-01 | lm-global-supervised-tokens: 1.212956E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.659928E-02 | lm-token-buffer-mib: 5.810876E+01 | global_load_balancing_loss: 9.997996E-01 | loss scale: 1.0 | grad norm: 0.102 | params norm: 11594.029 | params rms: 0.062778 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 4:22:06.143436 | finish at 2026-10-10 20:10:36 + [2026-10-04 15:49:59] iteration 958/ 6855 | consumed samples: 122624 | elapsed time per iteration (ms): 89367.0 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.44% | FLOPs estimator: workload-aware | logical tokens/s: 187733.4 | valid tokens/s: 187733.4 | physical tokens/s: 188240.2 | loss tokens/s: 131475.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.229812E-01 | lm-global-supervised-tokens: 1.174960E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.304187E-02 | lm-token-buffer-mib: 5.815858E+01 | global_load_balancing_loss: 1.001600E+00 | loss scale: 1.0 | grad norm: 0.060 | params norm: 11593.942 | params rms: 0.062777 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 2:23:17.227726 | finish at 2026-10-10 18:13:16 + [2026-10-04 15:51:23] iteration 959/ 6855 | consumed samples: 122752 | elapsed time per iteration (ms): 83496.0 | throughput per GPU (TFLOP/s/GPU): 43.8 | MFU 14.04% | FLOPs estimator: workload-aware | logical tokens/s: 200933.6 | valid tokens/s: 200933.6 | physical tokens/s: 201415.2 | loss tokens/s: 152091.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.813361E-01 | lm-global-supervised-tokens: 1.269903E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.170300E-02 | lm-token-buffer-mib: 5.814905E+01 | global_load_balancing_loss: 1.000727E+00 | loss scale: 1.0 | grad norm: 0.062 | params norm: 11593.857 | params rms: 0.062777 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 16:44:52.241995 | finish at 2026-10-10 08:36:15 + [2026-10-04 15:53:10] iteration 960/ 6855 | consumed samples: 122880 | elapsed time per iteration (ms): 106841.7 | throughput per GPU (TFLOP/s/GPU): 34.0 | MFU 10.90% | FLOPs estimator: workload-aware | logical tokens/s: 157028.4 | valid tokens/s: 157028.4 | physical tokens/s: 157480.9 | loss tokens/s: 114242.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.890431E-01 | lm-global-supervised-tokens: 1.220590E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.222239E-02 | lm-token-buffer-mib: 5.809930E+01 | global_load_balancing_loss: 1.001842E+00 | loss scale: 1.0 | grad norm: 0.053 | params norm: 11593.775 | params rms: 0.062777 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 7 days, 6:57:12.036992 | finish at 2026-10-11 22:50:22 +times across ranks (ms): + save-checkpoint: + rank 0: 24216.47 + rank 1: 24216.47 + rank 2: 24216.50 + rank 3: 24216.53 + rank 4: 24216.52 + rank 5: 24216.49 + rank 6: 24216.49 + rank 7: 24216.50 + rank 8: 24216.72 + rank 9: 24216.70 + rank 10: 24216.70 + rank 11: 24216.71 + rank 12: 24216.71 + rank 13: 24216.69 + rank 14: 24216.71 + rank 15: 24216.71 + rank 16: 24216.05 + rank 17: 24216.04 + rank 18: 24216.01 + rank 19: 24216.03 + rank 20: 24216.00 + rank 21: 24216.03 + rank 22: 24216.01 + rank 23: 24216.00 + rank 24: 24216.59 + rank 25: 24216.57 + rank 26: 24216.58 + rank 27: 24216.61 + rank 28: 24216.55 + rank 29: 24216.57 + rank 30: 24216.59 + rank 31: 24216.58 + rank 32: 24216.82 + rank 33: 24216.83 + rank 34: 24216.81 + rank 35: 24216.80 + rank 36: 24216.79 + rank 37: 24216.79 + rank 38: 24216.80 + rank 39: 24216.78 + rank 40: 24216.81 + rank 41: 24216.76 + rank 42: 24216.81 + rank 43: 24216.81 + rank 44: 24216.76 + rank 45: 24216.77 + rank 46: 24216.80 + rank 47: 24216.76 + rank 48: 24218.42 + rank 49: 24218.44 + rank 50: 24218.41 + rank 51: 24218.44 + rank 52: 24218.43 + rank 53: 24218.41 + rank 54: 24218.42 + rank 55: 24218.44 + rank 56: 24216.61 + rank 57: 24216.60 + rank 58: 24216.56 + rank 59: 24216.57 + rank 60: 24216.58 + rank 61: 24216.59 + rank 62: 24216.57 + rank 63: 24216.58 + [2026-10-04 15:55:25] iteration 961/ 6855 | consumed samples: 123008 | elapsed time per iteration (ms): 110678.5 | throughput per GPU (TFLOP/s/GPU): 33.0 | MFU 10.58% | FLOPs estimator: workload-aware | logical tokens/s: 151584.6 | valid tokens/s: 151584.6 | physical tokens/s: 151964.0 | loss tokens/s: 110889.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.651568E-01 | lm-global-supervised-tokens: 1.227307E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 7.626119E-01 | lm-token-buffer-mib: 5.823757E+01 | global_load_balancing_loss: 1.000224E+00 | loss scale: 1.0 | grad norm: 0.043 | params norm: 11593.693 | params rms: 0.062776 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 7 days, 13:12:19.254284 | finish at 2026-10-12 05:07:44 + [2026-10-04 15:56:59] iteration 962/ 6855 | consumed samples: 123136 | elapsed time per iteration (ms): 94453.0 | throughput per GPU (TFLOP/s/GPU): 39.8 | MFU 12.74% | FLOPs estimator: workload-aware | logical tokens/s: 177624.5 | valid tokens/s: 177624.5 | physical tokens/s: 178027.9 | loss tokens/s: 125773.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.282476E-01 | lm-global-supervised-tokens: 1.187966E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 3.669773E-02 | lm-token-buffer-mib: 5.817453E+01 | global_load_balancing_loss: 1.001071E+00 | loss scale: 1.0 | grad norm: 0.072 | params norm: 11593.611 | params rms: 0.062776 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 10:36:51.374855 | finish at 2026-10-11 02:33:51 + [2026-10-04 15:58:26] iteration 963/ 6855 | consumed samples: 123264 | elapsed time per iteration (ms): 86378.1 | throughput per GPU (TFLOP/s/GPU): 43.7 | MFU 13.99% | FLOPs estimator: workload-aware | logical tokens/s: 194228.6 | valid tokens/s: 194228.6 | physical tokens/s: 194708.8 | loss tokens/s: 141447.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.623473E-01 | lm-global-supervised-tokens: 1.221795E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.704114E-02 | lm-token-buffer-mib: 5.811499E+01 | global_load_balancing_loss: 9.987615E-01 | loss scale: 1.0 | grad norm: 0.068 | params norm: 11593.528 | params rms: 0.062775 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 21:22:19.805457 | finish at 2026-10-10 13:20:45 + [2026-10-04 15:59:49] iteration 964/ 6855 | consumed samples: 123392 | elapsed time per iteration (ms): 83609.5 | throughput per GPU (TFLOP/s/GPU): 43.6 | MFU 13.97% | FLOPs estimator: workload-aware | logical tokens/s: 200661.1 | valid tokens/s: 200661.1 | physical tokens/s: 201013.6 | loss tokens/s: 149922.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.634542E-01 | lm-global-supervised-tokens: 1.253497E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.799104E-02 | lm-token-buffer-mib: 5.812357E+01 | global_load_balancing_loss: 9.992819E-01 | loss scale: 1.0 | grad norm: 0.052 | params norm: 11593.446 | params rms: 0.062775 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 16:49:03.800056 | finish at 2026-10-10 08:48:53 + [2026-10-04 16:01:18] iteration 965/ 6855 | consumed samples: 123520 | elapsed time per iteration (ms): 89267.7 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.49% | FLOPs estimator: workload-aware | logical tokens/s: 187942.5 | valid tokens/s: 187942.5 | physical tokens/s: 188452.8 | loss tokens/s: 134639.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.995618E-01 | lm-global-supervised-tokens: 1.201900E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.693879E-02 | lm-token-buffer-mib: 5.804709E+01 | global_load_balancing_loss: 9.992507E-01 | loss scale: 1.0 | grad norm: 0.064 | params norm: 11593.366 | params rms: 0.062774 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 2:03:06.872110 | finish at 2026-10-10 18:04:25 + [2026-10-04 16:02:42] iteration 966/ 6855 | consumed samples: 123648 | elapsed time per iteration (ms): 84009.5 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.70% | FLOPs estimator: workload-aware | logical tokens/s: 199706.2 | valid tokens/s: 199706.2 | physical tokens/s: 200317.1 | loss tokens/s: 153510.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.755966E-01 | lm-global-supervised-tokens: 1.289637E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.160459E-02 | lm-token-buffer-mib: 5.821672E+01 | global_load_balancing_loss: 9.995670E-01 | loss scale: 1.0 | grad norm: 0.107 | params norm: 11593.286 | params rms: 0.062774 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 17:25:31.725222 | finish at 2026-10-10 09:28:14 + [2026-10-04 16:04:06] iteration 967/ 6855 | consumed samples: 123776 | elapsed time per iteration (ms): 83238.8 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.76% | FLOPs estimator: workload-aware | logical tokens/s: 201555.0 | valid tokens/s: 201555.0 | physical tokens/s: 202277.7 | loss tokens/s: 158989.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.073764E-01 | lm-global-supervised-tokens: 1.323407E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.648725E-02 | lm-token-buffer-mib: 5.812485E+01 | global_load_balancing_loss: 1.000277E+00 | loss scale: 1.0 | grad norm: 0.201 | params norm: 11593.207 | params rms: 0.062773 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 16:08:30.218933 | finish at 2026-10-10 08:12:36 + [2026-10-04 16:05:32] iteration 968/ 6855 | consumed samples: 123904 | elapsed time per iteration (ms): 86172.6 | throughput per GPU (TFLOP/s/GPU): 43.3 | MFU 13.87% | FLOPs estimator: workload-aware | logical tokens/s: 194693.1 | valid tokens/s: 194693.0 | physical tokens/s: 195115.5 | loss tokens/s: 147364.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.048500E-01 | lm-global-supervised-tokens: 1.269880E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.172203E-02 | lm-token-buffer-mib: 5.805025E+01 | global_load_balancing_loss: 1.001446E+00 | loss scale: 1.0 | grad norm: 0.272 | params norm: 11593.128 | params rms: 0.062773 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 20:54:58.160946 | finish at 2026-10-10 13:00:30 + [2026-10-04 16:06:58] iteration 969/ 6855 | consumed samples: 124032 | elapsed time per iteration (ms): 86241.9 | throughput per GPU (TFLOP/s/GPU): 42.8 | MFU 13.71% | FLOPs estimator: workload-aware | logical tokens/s: 194535.6 | valid tokens/s: 194535.6 | physical tokens/s: 194955.7 | loss tokens/s: 145538.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.720031E-01 | lm-global-supervised-tokens: 1.255151E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.269163E-02 | lm-token-buffer-mib: 5.815649E+01 | global_load_balancing_loss: 1.002042E+00 | loss scale: 1.0 | grad norm: 0.099 | params norm: 11593.047 | params rms: 0.062773 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 21:00:19.654807 | finish at 2026-10-10 13:07:18 + [2026-10-04 16:08:48] iteration 970/ 6855 | consumed samples: 124160 | elapsed time per iteration (ms): 110267.7 | throughput per GPU (TFLOP/s/GPU): 33.3 | MFU 10.69% | FLOPs estimator: workload-aware | logical tokens/s: 152149.4 | valid tokens/s: 152149.4 | physical tokens/s: 152471.1 | loss tokens/s: 112818.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.116625E-01 | lm-global-supervised-tokens: 1.244020E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.182263E-02 | lm-token-buffer-mib: 5.823615E+01 | global_load_balancing_loss: 1.001924E+00 | loss scale: 1.0 | grad norm: 0.175 | params norm: 11592.967 | params rms: 0.062772 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 7 days, 12:15:25.596648 | finish at 2026-10-12 04:24:14 + [2026-10-04 16:10:19] iteration 971/ 6855 | consumed samples: 124288 | elapsed time per iteration (ms): 90891.0 | throughput per GPU (TFLOP/s/GPU): 41.1 | MFU 13.19% | FLOPs estimator: workload-aware | logical tokens/s: 184585.6 | valid tokens/s: 184585.6 | physical tokens/s: 184952.7 | loss tokens/s: 134187.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.066547E-01 | lm-global-supervised-tokens: 1.219645E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.071158E-02 | lm-token-buffer-mib: 5.823792E+01 | global_load_balancing_loss: 1.000676E+00 | loss scale: 1.0 | grad norm: 0.068 | params norm: 11592.888 | params rms: 0.062772 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 4:33:22.783073 | finish at 2026-10-10 20:43:42 + [2026-10-04 16:11:47] iteration 972/ 6855 | consumed samples: 124416 | elapsed time per iteration (ms): 88041.0 | throughput per GPU (TFLOP/s/GPU): 41.0 | MFU 13.15% | FLOPs estimator: workload-aware | logical tokens/s: 190560.1 | valid tokens/s: 190560.1 | physical tokens/s: 190993.8 | loss tokens/s: 146891.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.591969E-01 | lm-global-supervised-tokens: 1.293248E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.153443E-02 | lm-token-buffer-mib: 5.822707E+01 | global_load_balancing_loss: 9.985626E-01 | loss scale: 1.0 | grad norm: 0.220 | params norm: 11592.809 | params rms: 0.062771 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 23:52:25.059282 | finish at 2026-10-10 16:04:12 + [2026-10-04 16:13:17] iteration 973/ 6855 | consumed samples: 124544 | elapsed time per iteration (ms): 89889.9 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.52% | FLOPs estimator: workload-aware | logical tokens/s: 186641.8 | valid tokens/s: 186641.8 | physical tokens/s: 187171.0 | loss tokens/s: 131348.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.689205E-01 | lm-global-supervised-tokens: 1.180687E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.168166E-02 | lm-token-buffer-mib: 5.817177E+01 | global_load_balancing_loss: 9.990421E-01 | loss scale: 1.0 | grad norm: 0.166 | params norm: 11592.731 | params rms: 0.062771 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 2:52:12.299061 | finish at 2026-10-10 19:05:29 + [2026-10-04 16:14:43] iteration 974/ 6855 | consumed samples: 124672 | elapsed time per iteration (ms): 85721.0 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.52% | FLOPs estimator: workload-aware | logical tokens/s: 195718.3 | valid tokens/s: 195718.3 | physical tokens/s: 196183.4 | loss tokens/s: 143992.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.147381E-01 | lm-global-supervised-tokens: 1.234316E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.380403E-02 | lm-token-buffer-mib: 5.813000E+01 | global_load_balancing_loss: 9.992922E-01 | loss scale: 1.0 | grad norm: 0.106 | params norm: 11592.652 | params rms: 0.062770 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 20:02:05.245610 | finish at 2026-10-10 12:16:48 + [2026-10-04 16:16:11] iteration 975/ 6855 | consumed samples: 124800 | elapsed time per iteration (ms): 88040.5 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.63% | FLOPs estimator: workload-aware | logical tokens/s: 190562.1 | valid tokens/s: 190562.1 | physical tokens/s: 191171.5 | loss tokens/s: 131782.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.711460E-01 | lm-global-supervised-tokens: 1.160222E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.298065E-02 | lm-token-buffer-mib: 5.803909E+01 | global_load_balancing_loss: 9.990323E-01 | loss scale: 1.0 | grad norm: 0.060 | params norm: 11592.574 | params rms: 0.062770 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 23:47:58.161993 | finish at 2026-10-10 16:04:09 + [2026-10-04 16:17:44] iteration 976/ 6855 | consumed samples: 124928 | elapsed time per iteration (ms): 93167.5 | throughput per GPU (TFLOP/s/GPU): 41.2 | MFU 13.19% | FLOPs estimator: workload-aware | logical tokens/s: 180075.2 | valid tokens/s: 180075.2 | physical tokens/s: 180363.7 | loss tokens/s: 137067.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.191151E-01 | lm-global-supervised-tokens: 1.277019E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.319546E-02 | lm-token-buffer-mib: 5.815528E+01 | global_load_balancing_loss: 9.997673E-01 | loss scale: 1.0 | grad norm: 0.049 | params norm: 11592.495 | params rms: 0.062770 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 8:08:51.588241 | finish at 2026-10-11 00:26:36 + [2026-10-04 16:19:09] iteration 977/ 6855 | consumed samples: 125056 | elapsed time per iteration (ms): 85329.7 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.49% | FLOPs estimator: workload-aware | logical tokens/s: 196615.8 | valid tokens/s: 196615.8 | physical tokens/s: 197177.5 | loss tokens/s: 151897.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.735714E-01 | lm-global-supervised-tokens: 1.296133E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.502408E-02 | lm-token-buffer-mib: 5.812685E+01 | global_load_balancing_loss: 9.998927E-01 | loss scale: 1.0 | grad norm: 0.139 | params norm: 11592.415 | params rms: 0.062769 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 19:19:27.833614 | finish at 2026-10-10 11:38:37 + [2026-10-04 16:20:37] iteration 978/ 6855 | consumed samples: 125184 | elapsed time per iteration (ms): 87861.2 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.56% | FLOPs estimator: workload-aware | logical tokens/s: 190950.7 | valid tokens/s: 190950.7 | physical tokens/s: 191348.4 | loss tokens/s: 136208.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.552991E-01 | lm-global-supervised-tokens: 1.196741E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.780063E-02 | lm-token-buffer-mib: 5.813943E+01 | global_load_balancing_loss: 1.001784E+00 | loss scale: 1.0 | grad norm: 0.123 | params norm: 11592.334 | params rms: 0.062769 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 23:26:00.246331 | finish at 2026-10-10 15:46:38 + [2026-10-04 16:22:05] iteration 979/ 6855 | consumed samples: 125312 | elapsed time per iteration (ms): 87397.1 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.54% | FLOPs estimator: workload-aware | logical tokens/s: 191965.2 | valid tokens/s: 191965.1 | physical tokens/s: 192529.4 | loss tokens/s: 138879.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.404443E-01 | lm-global-supervised-tokens: 1.213765E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.437115E-02 | lm-token-buffer-mib: 5.804822E+01 | global_load_balancing_loss: 1.001755E+00 | loss scale: 1.0 | grad norm: 0.117 | params norm: 11592.253 | params rms: 0.062768 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 22:39:05.401463 | finish at 2026-10-10 15:01:10 + [2026-10-04 16:23:54] iteration 980/ 6855 | consumed samples: 125440 | elapsed time per iteration (ms): 109671.1 | throughput per GPU (TFLOP/s/GPU): 34.5 | MFU 11.06% | FLOPs estimator: workload-aware | logical tokens/s: 152977.5 | valid tokens/s: 152977.4 | physical tokens/s: 153591.6 | loss tokens/s: 110607.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.891876E-01 | lm-global-supervised-tokens: 1.213049E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.449140E-02 | lm-token-buffer-mib: 5.836178E+01 | global_load_balancing_loss: 1.001255E+00 | loss scale: 1.0 | grad norm: 0.109 | params norm: 11592.172 | params rms: 0.062768 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 7 days, 10:58:37.552239 | finish at 2026-10-12 03:22:32 + [2026-10-04 16:25:19] iteration 981/ 6855 | consumed samples: 125568 | elapsed time per iteration (ms): 84222.0 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.67% | FLOPs estimator: workload-aware | logical tokens/s: 199202.2 | valid tokens/s: 199202.2 | physical tokens/s: 199670.3 | loss tokens/s: 147310.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.575808E-01 | lm-global-supervised-tokens: 1.240682E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.308910E-02 | lm-token-buffer-mib: 5.813718E+01 | global_load_balancing_loss: 1.000530E+00 | loss scale: 1.0 | grad norm: 0.047 | params norm: 11592.091 | params rms: 0.062767 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 17:25:20.252792 | finish at 2026-10-10 09:50:39 + [2026-10-04 16:26:47] iteration 982/ 6855 | consumed samples: 125696 | elapsed time per iteration (ms): 88356.6 | throughput per GPU (TFLOP/s/GPU): 42.8 | MFU 13.72% | FLOPs estimator: workload-aware | logical tokens/s: 189880.0 | valid tokens/s: 189880.0 | physical tokens/s: 190323.6 | loss tokens/s: 141408.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.241057E-01 | lm-global-supervised-tokens: 1.249433E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.105750E-02 | lm-token-buffer-mib: 5.820395E+01 | global_load_balancing_loss: 9.998336E-01 | loss scale: 1.0 | grad norm: 0.092 | params norm: 11592.009 | params rms: 0.062767 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 0:08:38.410087 | finish at 2026-10-10 16:35:25 + [2026-10-04 16:28:14] iteration 983/ 6855 | consumed samples: 125824 | elapsed time per iteration (ms): 87022.6 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.49% | FLOPs estimator: workload-aware | logical tokens/s: 192791.3 | valid tokens/s: 192791.3 | physical tokens/s: 193328.9 | loss tokens/s: 143619.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.483967E-01 | lm-global-supervised-tokens: 1.249812E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.306441E-02 | lm-token-buffer-mib: 5.810461E+01 | global_load_balancing_loss: 9.988186E-01 | loss scale: 1.0 | grad norm: 0.055 | params norm: 11591.928 | params rms: 0.062767 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 21:56:36.603222 | finish at 2026-10-10 14:24:51 + [2026-10-04 16:29:47] iteration 984/ 6855 | consumed samples: 125952 | elapsed time per iteration (ms): 92757.1 | throughput per GPU (TFLOP/s/GPU): 41.5 | MFU 13.30% | FLOPs estimator: workload-aware | logical tokens/s: 180872.7 | valid tokens/s: 180872.7 | physical tokens/s: 181204.7 | loss tokens/s: 123164.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.268933E-01 | lm-global-supervised-tokens: 1.142440E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.422641E-02 | lm-token-buffer-mib: 5.803736E+01 | global_load_balancing_loss: 9.994144E-01 | loss scale: 1.0 | grad norm: 0.115 | params norm: 11591.845 | params rms: 0.062766 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 7:16:16.681362 | finish at 2026-10-10 23:46:03 + [2026-10-04 16:31:13] iteration 985/ 6855 | consumed samples: 126080 | elapsed time per iteration (ms): 86733.6 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.62% | FLOPs estimator: workload-aware | logical tokens/s: 193433.8 | valid tokens/s: 193433.8 | physical tokens/s: 193910.2 | loss tokens/s: 139078.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.641987E-01 | lm-global-supervised-tokens: 1.206276E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.728420E-02 | lm-token-buffer-mib: 5.806924E+01 | global_load_balancing_loss: 1.000569E+00 | loss scale: 1.0 | grad norm: 0.048 | params norm: 11591.761 | params rms: 0.062766 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 21:25:26.069076 | finish at 2026-10-10 13:56:40 + [2026-10-04 16:32:38] iteration 986/ 6855 | consumed samples: 126208 | elapsed time per iteration (ms): 84892.3 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.68% | FLOPs estimator: workload-aware | logical tokens/s: 197629.1 | valid tokens/s: 197629.1 | physical tokens/s: 198067.3 | loss tokens/s: 144272.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.501029E-01 | lm-global-supervised-tokens: 1.224761E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.247092E-02 | lm-token-buffer-mib: 5.808811E+01 | global_load_balancing_loss: 1.000653E+00 | loss scale: 1.0 | grad norm: 0.111 | params norm: 11591.679 | params rms: 0.062765 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 18:23:53.136140 | finish at 2026-10-10 10:56:31 + [2026-10-04 16:34:05] iteration 987/ 6855 | consumed samples: 126336 | elapsed time per iteration (ms): 86406.2 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.52% | FLOPs estimator: workload-aware | logical tokens/s: 194166.6 | valid tokens/s: 194166.6 | physical tokens/s: 194666.2 | loss tokens/s: 148136.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.256392E-01 | lm-global-supervised-tokens: 1.279991E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.138908E-02 | lm-token-buffer-mib: 5.810594E+01 | global_load_balancing_loss: 1.001159E+00 | loss scale: 1.0 | grad norm: 0.106 | params norm: 11591.598 | params rms: 0.062765 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 20:50:31.693125 | finish at 2026-10-10 13:24:36 + [2026-10-04 16:35:32] iteration 988/ 6855 | consumed samples: 126464 | elapsed time per iteration (ms): 87721.8 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.67% | FLOPs estimator: workload-aware | logical tokens/s: 191253.8 | valid tokens/s: 191253.8 | physical tokens/s: 191651.1 | loss tokens/s: 136205.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.522163E-01 | lm-global-supervised-tokens: 1.194818E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.402405E-02 | lm-token-buffer-mib: 5.807130E+01 | global_load_balancing_loss: 1.000809E+00 | loss scale: 1.0 | grad norm: 0.186 | params norm: 11591.517 | params rms: 0.062764 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 22:57:43.977371 | finish at 2026-10-10 15:33:16 + [2026-10-04 16:36:58] iteration 989/ 6855 | consumed samples: 126592 | elapsed time per iteration (ms): 85973.9 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.59% | FLOPs estimator: workload-aware | logical tokens/s: 195142.5 | valid tokens/s: 195142.5 | physical tokens/s: 195715.4 | loss tokens/s: 148781.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.494337E-01 | lm-global-supervised-tokens: 1.279128E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.164137E-02 | lm-token-buffer-mib: 5.813393E+01 | global_load_balancing_loss: 9.999835E-01 | loss scale: 1.0 | grad norm: 0.066 | params norm: 11591.435 | params rms: 0.062764 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 20:05:22.816751 | finish at 2026-10-10 12:42:21 + [2026-10-04 16:38:42] iteration 990/ 6855 | consumed samples: 126720 | elapsed time per iteration (ms): 104023.3 | throughput per GPU (TFLOP/s/GPU): 34.7 | MFU 11.13% | FLOPs estimator: workload-aware | logical tokens/s: 161283.0 | valid tokens/s: 161283.0 | physical tokens/s: 161764.8 | loss tokens/s: 126040.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.756149E-01 | lm-global-supervised-tokens: 1.311119E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.288884E-02 | lm-token-buffer-mib: 5.822693E+01 | global_load_balancing_loss: 9.988548E-01 | loss scale: 1.0 | grad norm: 0.269 | params norm: 11591.354 | params rms: 0.062763 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 7 days, 1:28:16.719825 | finish at 2026-10-11 18:06:59 + [2026-10-04 16:40:10] iteration 991/ 6855 | consumed samples: 126848 | elapsed time per iteration (ms): 87417.3 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.42% | FLOPs estimator: workload-aware | logical tokens/s: 191920.5 | valid tokens/s: 191920.5 | physical tokens/s: 192638.0 | loss tokens/s: 145405.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.292956E-01 | lm-global-supervised-tokens: 1.271093E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.459824E-02 | lm-token-buffer-mib: 5.830593E+01 | global_load_balancing_loss: 9.990070E-01 | loss scale: 1.0 | grad norm: 0.161 | params norm: 11591.272 | params rms: 0.062763 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 22:23:34.999582 | finish at 2026-10-10 15:03:45 + [2026-10-04 16:41:35] iteration 992/ 6855 | consumed samples: 126976 | elapsed time per iteration (ms): 85048.6 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.66% | FLOPs estimator: workload-aware | logical tokens/s: 197265.9 | valid tokens/s: 197265.9 | physical tokens/s: 197878.5 | loss tokens/s: 144445.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.192766E-01 | lm-global-supervised-tokens: 1.228491E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.987909E-02 | lm-token-buffer-mib: 5.803383E+01 | global_load_balancing_loss: 1.000520E+00 | loss scale: 1.0 | grad norm: 0.123 | params norm: 11591.190 | params rms: 0.062763 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 18:30:39.713707 | finish at 2026-10-10 11:12:15 + [2026-10-04 16:43:00] iteration 993/ 6855 | consumed samples: 127104 | elapsed time per iteration (ms): 85405.0 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.74% | FLOPs estimator: workload-aware | logical tokens/s: 196442.3 | valid tokens/s: 196442.3 | physical tokens/s: 196873.5 | loss tokens/s: 143624.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.950337E-01 | lm-global-supervised-tokens: 1.226624E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.107215E-02 | lm-token-buffer-mib: 5.810162E+01 | global_load_balancing_loss: 1.001289E+00 | loss scale: 1.0 | grad norm: 0.080 | params norm: 11591.107 | params rms: 0.062762 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 19:04:04.114025 | finish at 2026-10-10 11:47:04 + [2026-10-04 16:44:26] iteration 994/ 6855 | consumed samples: 127232 | elapsed time per iteration (ms): 85347.4 | throughput per GPU (TFLOP/s/GPU): 43.6 | MFU 13.96% | FLOPs estimator: workload-aware | logical tokens/s: 196575.2 | valid tokens/s: 196575.2 | physical tokens/s: 197024.2 | loss tokens/s: 150490.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.148795E-01 | lm-global-supervised-tokens: 1.284398E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.298714E-02 | lm-token-buffer-mib: 5.817069E+01 | global_load_balancing_loss: 9.999245E-01 | loss scale: 1.0 | grad norm: 0.089 | params norm: 11591.025 | params rms: 0.062762 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 18:57:01.200539 | finish at 2026-10-10 11:41:27 + [2026-10-04 16:45:52] iteration 995/ 6855 | consumed samples: 127360 | elapsed time per iteration (ms): 86070.6 | throughput per GPU (TFLOP/s/GPU): 43.2 | MFU 13.84% | FLOPs estimator: workload-aware | logical tokens/s: 194923.5 | valid tokens/s: 194923.5 | physical tokens/s: 195387.1 | loss tokens/s: 139452.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.746897E-01 | lm-global-supervised-tokens: 1.200275E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.121840E-02 | lm-token-buffer-mib: 5.837717E+01 | global_load_balancing_loss: 1.001474E+00 | loss scale: 1.0 | grad norm: 0.043 | params norm: 11590.942 | params rms: 0.062761 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 20:06:13.734355 | finish at 2026-10-10 12:52:05 + [2026-10-04 16:47:15] iteration 996/ 6855 | consumed samples: 127488 | elapsed time per iteration (ms): 83342.4 | throughput per GPU (TFLOP/s/GPU): 43.7 | MFU 14.00% | FLOPs estimator: workload-aware | logical tokens/s: 201304.5 | valid tokens/s: 201304.5 | physical tokens/s: 201748.0 | loss tokens/s: 147589.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.635090E-01 | lm-global-supervised-tokens: 1.230048E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.252464E-02 | lm-token-buffer-mib: 5.816250E+01 | global_load_balancing_loss: 1.000720E+00 | loss scale: 1.0 | grad norm: 0.126 | params norm: 11590.857 | params rms: 0.062761 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 15:38:22.999107 | finish at 2026-10-10 08:25:38 + [2026-10-04 16:48:42] iteration 997/ 6855 | consumed samples: 127616 | elapsed time per iteration (ms): 87152.1 | throughput per GPU (TFLOP/s/GPU): 41.6 | MFU 13.33% | FLOPs estimator: workload-aware | logical tokens/s: 192504.3 | valid tokens/s: 192504.3 | physical tokens/s: 192939.3 | loss tokens/s: 146604.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.605052E-01 | lm-global-supervised-tokens: 1.277692E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.321176E-02 | lm-token-buffer-mib: 5.822205E+01 | global_load_balancing_loss: 1.001401E+00 | loss scale: 1.0 | grad norm: 0.053 | params norm: 11590.771 | params rms: 0.062760 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 21:48:57.158731 | finish at 2026-10-10 14:37:39 + [2026-10-04 16:50:09] iteration 998/ 6855 | consumed samples: 127744 | elapsed time per iteration (ms): 86637.1 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.79% | FLOPs estimator: workload-aware | logical tokens/s: 193649.4 | valid tokens/s: 193649.4 | physical tokens/s: 194118.5 | loss tokens/s: 140111.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.791847E-01 | lm-global-supervised-tokens: 1.213880E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.158728E-02 | lm-token-buffer-mib: 5.803794E+01 | global_load_balancing_loss: 9.989887E-01 | loss scale: 1.0 | grad norm: 0.062 | params norm: 11590.688 | params rms: 0.062760 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 20:57:13.293506 | finish at 2026-10-10 13:47:22 + [2026-10-04 16:51:41] iteration 999/ 6855 | consumed samples: 127872 | elapsed time per iteration (ms): 91733.4 | throughput per GPU (TFLOP/s/GPU): 40.4 | MFU 12.95% | FLOPs estimator: workload-aware | logical tokens/s: 182890.3 | valid tokens/s: 182890.3 | physical tokens/s: 183193.1 | loss tokens/s: 133101.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.599225E-01 | lm-global-supervised-tokens: 1.220988E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.610419E-02 | lm-token-buffer-mib: 5.807767E+01 | global_load_balancing_loss: 9.982352E-01 | loss scale: 1.0 | grad norm: 0.066 | params norm: 11590.604 | params rms: 0.062759 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 5:13:10.980904 | finish at 2026-10-10 22:04:52 + [2026-10-04 16:53:26] iteration 1000/ 6855 | consumed samples: 128000 | elapsed time per iteration (ms): 105264.0 | throughput per GPU (TFLOP/s/GPU): 34.2 | MFU 10.96% | FLOPs estimator: workload-aware | logical tokens/s: 159382.0 | valid tokens/s: 159382.0 | physical tokens/s: 159735.0 | loss tokens/s: 114316.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 5.859841E-01 | lm-global-supervised-tokens: 1.203342E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.734661E-02 | lm-token-buffer-mib: 5.803349E+01 | global_load_balancing_loss: 9.993649E-01 | loss scale: 1.0 | grad norm: 0.033 | params norm: 11590.521 | params rms: 0.062759 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 7 days, 3:12:00.573069 | finish at 2026-10-11 20:05:26 +times across ranks (ms): + save-checkpoint: + rank 0: 24023.43 + rank 1: 24023.44 + rank 2: 24023.43 + rank 3: 24023.43 + rank 4: 24023.45 + rank 5: 24023.42 + rank 6: 24023.40 + rank 7: 24023.48 + rank 8: 24023.58 + rank 9: 24023.57 + rank 10: 24023.56 + rank 11: 24023.57 + rank 12: 24023.58 + rank 13: 24023.57 + rank 14: 24023.57 + rank 15: 24023.57 + rank 16: 24023.58 + rank 17: 24023.66 + rank 18: 24023.59 + rank 19: 24023.61 + rank 20: 24023.59 + rank 21: 24023.70 + rank 22: 24023.58 + rank 23: 24023.58 + rank 24: 24023.45 + rank 25: 24023.43 + rank 26: 24023.46 + rank 27: 24023.49 + rank 28: 24023.40 + rank 29: 24023.45 + rank 30: 24023.46 + rank 31: 24023.45 + rank 32: 24023.42 + rank 33: 24023.43 + rank 34: 24023.41 + rank 35: 24023.39 + rank 36: 24023.38 + rank 37: 24023.38 + rank 38: 24023.41 + rank 39: 24023.37 + rank 40: 24023.19 + rank 41: 24023.21 + rank 42: 24023.18 + rank 43: 24023.19 + rank 44: 24023.17 + rank 45: 24023.20 + rank 46: 24023.18 + rank 47: 24023.21 + rank 48: 24023.29 + rank 49: 24023.30 + rank 50: 24023.27 + rank 51: 24023.30 + rank 52: 24023.29 + rank 53: 24023.27 + rank 54: 24023.27 + rank 55: 24023.28 + rank 56: 24023.46 + rank 57: 24023.44 + rank 58: 24023.43 + rank 59: 24023.43 + rank 60: 24023.43 + rank 61: 24023.44 + rank 62: 24023.43 + rank 63: 24023.44 + [2026-10-04 16:55:37] iteration 1001/ 6855 | consumed samples: 128128 | elapsed time per iteration (ms): 106897.7 | throughput per GPU (TFLOP/s/GPU): 34.7 | MFU 11.13% | FLOPs estimator: workload-aware | logical tokens/s: 156945.6 | valid tokens/s: 156945.6 | physical tokens/s: 157377.5 | loss tokens/s: 116392.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.708149E-01 | lm-global-supervised-tokens: 1.244204E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.586964E+00 | lm-token-buffer-mib: 5.831086E+01 | global_load_balancing_loss: 9.990421E-01 | loss scale: 1.0 | grad norm: 0.093 | params norm: 11590.439 | params rms: 0.062758 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 7 days, 5:49:39.358134 | finish at 2026-10-11 22:45:16 +times across ranks (ms): + forward-backward: + rank 0: 104031.98 + rank 1: 104029.23 + rank 2: 104032.14 + rank 3: 104033.27 + rank 4: 104031.10 + rank 5: 104033.32 + rank 6: 104030.89 + rank 7: 104030.56 + rank 8: 104028.29 + rank 9: 104026.41 + rank 10: 104028.23 + rank 11: 104030.07 + rank 12: 104027.80 + rank 13: 104030.60 + rank 14: 104027.33 + rank 15: 104027.56 + rank 16: 104024.76 + rank 17: 104023.88 + rank 18: 104025.11 + rank 19: 104026.43 + rank 20: 104023.87 + rank 21: 104027.14 + rank 22: 104024.49 + rank 23: 104024.48 + rank 24: 104023.15 + rank 25: 104021.48 + rank 26: 104023.34 + rank 27: 104023.38 + rank 28: 104023.52 + rank 29: 104024.61 + rank 30: 104024.22 + rank 31: 104022.86 + rank 32: 104020.03 + rank 33: 104017.64 + rank 34: 104017.77 + rank 35: 104021.12 + rank 36: 104017.84 + rank 37: 104019.23 + rank 38: 104019.04 + rank 39: 104019.02 + rank 40: 104022.23 + rank 41: 104018.45 + rank 42: 104021.01 + rank 43: 104021.21 + rank 44: 104016.69 + rank 45: 104017.94 + rank 46: 104022.75 + rank 47: 104021.93 + rank 48: 104028.23 + rank 49: 104025.16 + rank 50: 104026.14 + rank 51: 104027.58 + rank 52: 104023.58 + rank 53: 104025.34 + rank 54: 104028.53 + rank 55: 104028.97 + rank 56: 104032.66 + rank 57: 104028.38 + rank 58: 104030.79 + rank 59: 104029.98 + rank 60: 104028.27 + rank 61: 104029.25 + rank 62: 104032.48 + rank 63: 104031.73 + forward-compute: + rank 0: 51870.12 + rank 1: 51870.75 + rank 2: 51874.82 + rank 3: 51874.12 + rank 4: 51872.63 + rank 5: 51873.80 + rank 6: 51871.13 + rank 7: 51876.65 + rank 8: 50910.50 + rank 9: 50907.70 + rank 10: 50910.00 + rank 11: 50919.55 + rank 12: 50908.44 + rank 13: 50909.21 + rank 14: 50911.02 + rank 15: 50909.41 + rank 16: 46000.14 + rank 17: 45999.09 + rank 18: 46003.28 + rank 19: 46009.66 + rank 20: 45999.54 + rank 21: 46009.78 + rank 22: 46004.11 + rank 23: 46005.05 + rank 24: 49795.90 + rank 25: 49795.15 + rank 26: 49806.92 + rank 27: 49800.97 + rank 28: 49811.00 + rank 29: 49810.85 + rank 30: 49805.87 + rank 31: 49805.30 + rank 32: 53372.90 + rank 33: 53370.38 + rank 34: 53372.61 + rank 35: 53373.04 + rank 36: 53370.44 + rank 37: 53371.78 + rank 38: 53372.33 + rank 39: 53372.84 + rank 40: 47938.28 + rank 41: 47935.12 + rank 42: 47942.62 + rank 43: 47942.47 + rank 44: 47939.96 + rank 45: 47942.37 + rank 46: 47941.78 + rank 47: 47943.60 + rank 48: 47223.20 + rank 49: 47221.49 + rank 50: 47224.08 + rank 51: 47226.04 + rank 52: 47225.48 + rank 53: 47231.14 + rank 54: 47225.64 + rank 55: 47226.99 + rank 56: 52479.41 + rank 57: 52486.07 + rank 58: 52490.22 + rank 59: 52488.97 + rank 60: 52489.47 + rank 61: 52491.89 + rank 62: 52490.67 + rank 63: 52493.30 + backward-compute: + rank 0: 52117.27 + rank 1: 52114.73 + rank 2: 52111.80 + rank 3: 52120.82 + rank 4: 52118.05 + rank 5: 52119.31 + rank 6: 52118.78 + rank 7: 52112.48 + rank 8: 53083.17 + rank 9: 53083.95 + rank 10: 53082.18 + rank 11: 53073.16 + rank 12: 53082.62 + rank 13: 53085.27 + rank 14: 53079.87 + rank 15: 53081.66 + rank 16: 57255.06 + rank 17: 57255.13 + rank 18: 57253.12 + rank 19: 57245.87 + rank 20: 57255.81 + rank 21: 57248.03 + rank 22: 57248.53 + rank 23: 57251.20 + rank 24: 54183.87 + rank 25: 54179.17 + rank 26: 54173.27 + rank 27: 54182.03 + rank 28: 54167.82 + rank 29: 54170.36 + rank 30: 54173.21 + rank 31: 54176.93 + rank 32: 50603.11 + rank 33: 50610.54 + rank 34: 50602.12 + rank 35: 50605.07 + rank 36: 50606.16 + rank 37: 50607.69 + rank 38: 50608.60 + rank 39: 50605.18 + rank 40: 56047.19 + rank 41: 56049.59 + rank 42: 56041.86 + rank 43: 56043.89 + rank 44: 56041.14 + rank 45: 56041.79 + rank 46: 56045.16 + rank 47: 56042.87 + rank 48: 56765.70 + rank 49: 56766.32 + rank 50: 56765.36 + rank 51: 56766.78 + rank 52: 56760.18 + rank 53: 56759.61 + rank 54: 56764.95 + rank 55: 56766.22 + rank 56: 51515.99 + rank 57: 51505.46 + rank 58: 51500.76 + rank 59: 51507.30 + rank 60: 51501.62 + rank 61: 51500.88 + rank 62: 51503.38 + rank 63: 51499.46 + batch-generator: + rank 0: 2317.96 + rank 1: 2315.07 + rank 2: 2317.76 + rank 3: 2316.88 + rank 4: 2313.80 + rank 5: 2316.26 + rank 6: 2318.82 + rank 7: 2316.94 + rank 8: 23.64 + rank 9: 24.54 + rank 10: 23.55 + rank 11: 23.48 + rank 12: 22.89 + rank 13: 24.80 + rank 14: 24.28 + rank 15: 23.78 + rank 16: 26.01 + rank 17: 24.78 + rank 18: 23.82 + rank 19: 23.30 + rank 20: 22.15 + rank 21: 24.08 + rank 22: 28.47 + rank 23: 22.76 + rank 24: 34.25 + rank 25: 33.87 + rank 26: 34.85 + rank 27: 35.52 + rank 28: 33.39 + rank 29: 36.37 + rank 30: 34.30 + rank 31: 35.72 + rank 32: 34.35 + rank 33: 30.02 + rank 34: 36.12 + rank 35: 33.54 + rank 36: 33.89 + rank 37: 32.24 + rank 38: 32.87 + rank 39: 30.58 + rank 40: 25.22 + rank 41: 24.99 + rank 42: 24.78 + rank 43: 24.17 + rank 44: 23.61 + rank 45: 24.58 + rank 46: 23.23 + rank 47: 24.55 + rank 48: 34.57 + rank 49: 29.42 + rank 50: 28.88 + rank 51: 24.95 + rank 52: 27.07 + rank 53: 22.15 + rank 54: 30.96 + rank 55: 28.07 + rank 56: 23.72 + rank 57: 24.51 + rank 58: 24.43 + rank 59: 22.32 + rank 60: 23.08 + rank 61: 24.33 + rank 62: 23.36 + rank 63: 26.38 + embedding-grads-all-reduce: + rank 0: 0.02 + rank 1: 0.03 + rank 2: 0.03 + rank 3: 0.02 + rank 4: 0.02 + rank 5: 0.03 + rank 6: 0.02 + rank 7: 0.03 + rank 8: 0.02 + rank 9: 0.03 + rank 10: 0.03 + rank 11: 0.02 + rank 12: 0.03 + rank 13: 0.02 + rank 14: 0.02 + rank 15: 0.02 + rank 16: 0.03 + rank 17: 0.03 + rank 18: 0.03 + rank 19: 0.02 + rank 20: 0.03 + rank 21: 0.03 + rank 22: 0.03 + rank 23: 0.02 + rank 24: 0.03 + rank 25: 0.03 + rank 26: 0.03 + rank 27: 0.04 + rank 28: 0.03 + rank 29: 0.04 + rank 30: 0.04 + rank 31: 0.04 + rank 32: 0.03 + rank 33: 0.03 + rank 34: 0.04 + rank 35: 0.03 + rank 36: 0.03 + rank 37: 0.03 + rank 38: 0.03 + rank 39: 0.03 + rank 40: 0.03 + rank 41: 0.03 + rank 42: 0.03 + rank 43: 0.03 + rank 44: 0.03 + rank 45: 0.03 + rank 46: 0.03 + rank 47: 0.03 + rank 48: 0.03 + rank 49: 0.03 + rank 50: 0.03 + rank 51: 0.03 + rank 52: 0.03 + rank 53: 0.03 + rank 54: 0.03 + rank 55: 0.03 + rank 56: 0.03 + rank 57: 0.03 + rank 58: 0.02 + rank 59: 0.02 + rank 60: 0.03 + rank 61: 0.03 + rank 62: 0.03 + rank 63: 0.03 + all-grads-sync: + rank 0: 1.03 + rank 1: 0.77 + rank 2: 0.78 + rank 3: 0.81 + rank 4: 0.87 + rank 5: 0.81 + rank 6: 0.80 + rank 7: 0.81 + rank 8: 0.75 + rank 9: 0.75 + rank 10: 0.88 + rank 11: 0.84 + rank 12: 0.79 + rank 13: 0.80 + rank 14: 0.79 + rank 15: 0.77 + rank 16: 0.79 + rank 17: 1.01 + rank 18: 0.75 + rank 19: 0.76 + rank 20: 0.89 + rank 21: 0.75 + rank 22: 0.77 + rank 23: 0.99 + rank 24: 1.37 + rank 25: 1.39 + rank 26: 1.39 + rank 27: 1.19 + rank 28: 1.46 + rank 29: 1.25 + rank 30: 1.82 + rank 31: 1.28 + rank 32: 1.18 + rank 33: 1.15 + rank 34: 1.31 + rank 35: 1.21 + rank 36: 1.14 + rank 37: 1.22 + rank 38: 1.13 + rank 39: 1.20 + rank 40: 0.82 + rank 41: 0.81 + rank 42: 0.77 + rank 43: 0.73 + rank 44: 0.75 + rank 45: 0.81 + rank 46: 0.75 + rank 47: 0.81 + rank 48: 1.03 + rank 49: 1.08 + rank 50: 1.02 + rank 51: 1.13 + rank 52: 1.10 + rank 53: 0.81 + rank 54: 1.01 + rank 55: 1.00 + rank 56: 1.06 + rank 57: 0.74 + rank 58: 0.76 + rank 59: 0.74 + rank 60: 0.99 + rank 61: 1.09 + rank 62: 0.77 + rank 63: 0.76 + params-all-gather: + rank 0: 0.02 + rank 1: 0.02 + rank 2: 0.02 + rank 3: 0.02 + rank 4: 0.02 + rank 5: 0.02 + rank 6: 0.02 + rank 7: 0.02 + rank 8: 0.02 + rank 9: 0.02 + rank 10: 0.02 + rank 11: 0.02 + rank 12: 0.02 + rank 13: 0.02 + rank 14: 0.02 + rank 15: 0.02 + rank 16: 0.02 + rank 17: 0.02 + rank 18: 0.02 + rank 19: 0.02 + rank 20: 0.02 + rank 21: 0.02 + rank 22: 0.02 + rank 23: 0.02 + rank 24: 0.03 + rank 25: 0.03 + rank 26: 0.03 + rank 27: 0.03 + rank 28: 0.04 + rank 29: 0.03 + rank 30: 0.03 + rank 31: 0.04 + rank 32: 0.02 + rank 33: 0.04 + rank 34: 0.03 + rank 35: 0.02 + rank 36: 0.02 + rank 37: 0.02 + rank 38: 0.02 + rank 39: 0.02 + rank 40: 0.02 + rank 41: 0.03 + rank 42: 0.02 + rank 43: 0.02 + rank 44: 0.02 + rank 45: 0.02 + rank 46: 0.02 + rank 47: 0.02 + rank 48: 0.03 + rank 49: 0.03 + rank 50: 0.03 + rank 51: 0.03 + rank 52: 0.03 + rank 53: 0.03 + rank 54: 0.02 + rank 55: 0.03 + rank 56: 0.02 + rank 57: 0.02 + rank 58: 0.02 + rank 59: 0.02 + rank 60: 0.02 + rank 61: 0.02 + rank 62: 0.02 + rank 63: 0.02 + optimizer-copy-to-main-grad: + rank 0: 2.32 + rank 1: 1.83 + rank 2: 1.73 + rank 3: 2.16 + rank 4: 1.76 + rank 5: 1.93 + rank 6: 1.93 + rank 7: 1.90 + rank 8: 1.78 + rank 9: 2.13 + rank 10: 1.73 + rank 11: 1.74 + rank 12: 1.86 + rank 13: 1.84 + rank 14: 1.88 + rank 15: 2.01 + rank 16: 2.54 + rank 17: 2.14 + rank 18: 2.22 + rank 19: 2.19 + rank 20: 2.18 + rank 21: 2.31 + rank 22: 2.57 + rank 23: 2.30 + rank 24: 1.91 + rank 25: 2.19 + rank 26: 1.94 + rank 27: 2.07 + rank 28: 2.37 + rank 29: 2.45 + rank 30: 2.27 + rank 31: 1.98 + rank 32: 2.16 + rank 33: 1.93 + rank 34: 2.04 + rank 35: 2.38 + rank 36: 2.15 + rank 37: 2.04 + rank 38: 2.29 + rank 39: 1.90 + rank 40: 1.79 + rank 41: 1.86 + rank 42: 1.92 + rank 43: 1.98 + rank 44: 1.86 + rank 45: 2.06 + rank 46: 1.80 + rank 47: 1.74 + rank 48: 2.08 + rank 49: 1.86 + rank 50: 2.35 + rank 51: 1.81 + rank 52: 1.98 + rank 53: 1.75 + rank 54: 1.78 + rank 55: 1.82 + rank 56: 1.77 + rank 57: 2.42 + rank 58: 1.87 + rank 59: 1.66 + rank 60: 1.83 + rank 61: 1.70 + rank 62: 1.86 + rank 63: 1.72 + optimizer-inner-step: + rank 0: 42.30 + rank 1: 42.25 + rank 2: 41.97 + rank 3: 42.16 + rank 4: 41.99 + rank 5: 42.08 + rank 6: 42.45 + rank 7: 42.41 + rank 8: 42.13 + rank 9: 42.22 + rank 10: 42.05 + rank 11: 42.15 + rank 12: 42.12 + rank 13: 42.36 + rank 14: 42.08 + rank 15: 42.02 + rank 16: 42.49 + rank 17: 42.21 + rank 18: 42.26 + rank 19: 42.25 + rank 20: 42.26 + rank 21: 42.07 + rank 22: 42.59 + rank 23: 42.34 + rank 24: 41.92 + rank 25: 42.17 + rank 26: 42.05 + rank 27: 42.29 + rank 28: 46.05 + rank 29: 42.04 + rank 30: 41.99 + rank 31: 42.35 + rank 32: 42.20 + rank 33: 41.98 + rank 34: 42.12 + rank 35: 47.85 + rank 36: 42.26 + rank 37: 42.21 + rank 38: 42.18 + rank 39: 42.00 + rank 40: 42.10 + rank 41: 42.23 + rank 42: 42.30 + rank 43: 42.35 + rank 44: 41.93 + rank 45: 42.53 + rank 46: 42.17 + rank 47: 41.83 + rank 48: 43.14 + rank 49: 42.01 + rank 50: 53.77 + rank 51: 41.86 + rank 52: 42.23 + rank 53: 42.01 + rank 54: 42.24 + rank 55: 42.14 + rank 56: 41.75 + rank 57: 47.51 + rank 58: 42.11 + rank 59: 42.03 + rank 60: 42.32 + rank 61: 41.89 + rank 62: 42.07 + rank 63: 41.91 + optimizer-copy-main-to-model-params: + rank 0: 4.42 + rank 1: 3.87 + rank 2: 3.76 + rank 3: 4.32 + rank 4: 3.76 + rank 5: 3.96 + rank 6: 3.95 + rank 7: 4.02 + rank 8: 3.87 + rank 9: 4.44 + rank 10: 3.71 + rank 11: 3.73 + rank 12: 3.95 + rank 13: 3.82 + rank 14: 4.01 + rank 15: 4.27 + rank 16: 4.31 + rank 17: 3.97 + rank 18: 4.05 + rank 19: 4.01 + rank 20: 4.03 + rank 21: 4.24 + rank 22: 4.58 + rank 23: 4.17 + rank 24: 3.87 + rank 25: 4.19 + rank 26: 3.91 + rank 27: 4.05 + rank 28: 4.36 + rank 29: 4.36 + rank 30: 4.33 + rank 31: 3.97 + rank 32: 4.14 + rank 33: 3.87 + rank 34: 4.00 + rank 35: 4.52 + rank 36: 4.27 + rank 37: 4.06 + rank 38: 4.52 + rank 39: 3.88 + rank 40: 3.72 + rank 41: 3.97 + rank 42: 3.94 + rank 43: 3.99 + rank 44: 3.96 + rank 45: 4.23 + rank 46: 3.74 + rank 47: 3.75 + rank 48: 4.18 + rank 49: 3.93 + rank 50: 4.89 + rank 51: 3.93 + rank 52: 4.21 + rank 53: 3.87 + rank 54: 3.87 + rank 55: 3.89 + rank 56: 3.79 + rank 57: 4.72 + rank 58: 3.89 + rank 59: 3.77 + rank 60: 3.77 + rank 61: 3.66 + rank 62: 3.88 + rank 63: 3.71 + optimizer: + rank 0: 53.96 + rank 1: 53.11 + rank 2: 52.68 + rank 3: 53.63 + rank 4: 52.67 + rank 5: 53.05 + rank 6: 53.45 + rank 7: 53.45 + rank 8: 53.02 + rank 9: 53.88 + rank 10: 52.77 + rank 11: 52.84 + rank 12: 53.10 + rank 13: 53.19 + rank 14: 53.15 + rank 15: 53.46 + rank 16: 54.22 + rank 17: 53.47 + rank 18: 53.61 + rank 19: 53.51 + rank 20: 53.55 + rank 21: 53.70 + rank 22: 54.67 + rank 23: 53.85 + rank 24: 52.81 + rank 25: 53.45 + rank 26: 52.98 + rank 27: 53.39 + rank 28: 57.58 + rank 29: 53.53 + rank 30: 53.45 + rank 31: 53.34 + rank 32: 53.35 + rank 33: 52.76 + rank 34: 53.11 + rank 35: 59.53 + rank 36: 53.57 + rank 37: 53.28 + rank 38: 53.83 + rank 39: 52.81 + rank 40: 52.81 + rank 41: 53.25 + rank 42: 53.30 + rank 43: 53.39 + rank 44: 52.91 + rank 45: 53.95 + rank 46: 52.85 + rank 47: 52.50 + rank 48: 54.33 + rank 49: 52.90 + rank 50: 66.01 + rank 51: 52.77 + rank 52: 53.50 + rank 53: 52.81 + rank 54: 53.06 + rank 55: 52.99 + rank 56: 52.52 + rank 57: 59.85 + rank 58: 52.99 + rank 59: 52.71 + rank 60: 53.04 + rank 61: 52.49 + rank 62: 52.94 + rank 63: 52.54 + interval-time: + rank 0: 0.74 + rank 1: 0.75 + rank 2: 0.74 + rank 3: 0.74 + rank 4: 0.76 + rank 5: 0.74 + rank 6: 0.75 + rank 7: 0.84 + rank 8: 0.73 + rank 9: 0.73 + rank 10: 0.74 + rank 11: 0.72 + rank 12: 0.74 + rank 13: 0.73 + rank 14: 0.73 + rank 15: 0.73 + rank 16: 0.73 + rank 17: 0.74 + rank 18: 0.74 + rank 19: 0.74 + rank 20: 0.72 + rank 21: 0.75 + rank 22: 0.72 + rank 23: 0.80 + rank 24: 0.89 + rank 25: 0.88 + rank 26: 0.90 + rank 27: 0.91 + rank 28: 0.85 + rank 29: 0.89 + rank 30: 0.84 + rank 31: 0.97 + rank 32: 0.85 + rank 33: 0.83 + rank 34: 0.85 + rank 35: 0.82 + rank 36: 0.89 + rank 37: 0.80 + rank 38: 0.83 + rank 39: 0.91 + rank 40: 0.75 + rank 41: 0.73 + rank 42: 0.73 + rank 43: 0.75 + rank 44: 0.73 + rank 45: 0.73 + rank 46: 0.74 + rank 47: 0.73 + rank 48: 0.83 + rank 49: 0.82 + rank 50: 0.81 + rank 51: 0.83 + rank 52: 0.83 + rank 53: 0.80 + rank 54: 0.82 + rank 55: 0.93 + rank 56: 0.76 + rank 57: 0.74 + rank 58: 0.74 + rank 59: 0.74 + rank 60: 0.72 + rank 61: 0.73 + rank 62: 0.80 + rank 63: 3.16 + [2026-10-04 16:57:10] iteration 1002/ 6855 | consumed samples: 128256 | elapsed time per iteration (ms): 92574.6 | throughput per GPU (TFLOP/s/GPU): 39.9 | MFU 12.79% | FLOPs estimator: workload-aware | logical tokens/s: 181229.2 | valid tokens/s: 181229.2 | physical tokens/s: 181630.4 | loss tokens/s: 134969.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.142592E-01 | lm-global-supervised-tokens: 1.249474E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.635752E-02 | lm-token-buffer-mib: 5.832584E+01 | global_load_balancing_loss: 1.000340E+00 | loss scale: 1.0 | grad norm: 0.044 | params norm: 11590.357 | params rms: 0.062758 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 6:30:38.854598 | finish at 2026-10-10 23:27:49 +times across ranks (ms): + forward-backward: + rank 0: 87805.85 + rank 1: 87887.05 + rank 2: 87881.61 + rank 3: 87886.51 + rank 4: 87887.50 + rank 5: 87886.02 + rank 6: 87884.82 + rank 7: 87884.36 + rank 8: 87801.48 + rank 9: 87883.91 + rank 10: 87878.58 + rank 11: 87883.33 + rank 12: 87883.49 + rank 13: 87883.40 + rank 14: 87881.83 + rank 15: 87880.62 + rank 16: 87797.94 + rank 17: 87880.76 + rank 18: 87874.19 + rank 19: 87879.05 + rank 20: 87879.99 + rank 21: 87880.36 + rank 22: 87879.08 + rank 23: 87877.77 + rank 24: 87796.80 + rank 25: 87878.66 + rank 26: 87874.41 + rank 27: 87877.95 + rank 28: 87879.12 + rank 29: 87879.07 + rank 30: 87877.20 + rank 31: 87875.52 + rank 32: 87795.64 + rank 33: 87877.30 + rank 34: 87871.05 + rank 35: 87875.48 + rank 36: 87875.80 + rank 37: 87874.95 + rank 38: 87876.60 + rank 39: 87875.07 + rank 40: 87795.87 + rank 41: 87876.67 + rank 42: 87870.39 + rank 43: 87874.77 + rank 44: 87874.08 + rank 45: 87872.12 + rank 46: 87877.13 + rank 47: 87875.42 + rank 48: 87802.88 + rank 49: 87883.95 + rank 50: 87876.36 + rank 51: 87881.99 + rank 52: 87879.20 + rank 53: 87879.46 + rank 54: 87882.91 + rank 55: 87882.81 + rank 56: 87805.04 + rank 57: 87886.58 + rank 58: 87880.20 + rank 59: 87883.84 + rank 60: 87884.55 + rank 61: 87882.54 + rank 62: 87886.91 + rank 63: 87885.41 + forward-compute: + rank 0: 34041.96 + rank 1: 34130.64 + rank 2: 34129.34 + rank 3: 34130.81 + rank 4: 34132.89 + rank 5: 34129.06 + rank 6: 34129.59 + rank 7: 34133.51 + rank 8: 34589.10 + rank 9: 34672.59 + rank 10: 34666.09 + rank 11: 34670.71 + rank 12: 34674.30 + rank 13: 34675.59 + rank 14: 34673.57 + rank 15: 34673.76 + rank 16: 34055.39 + rank 17: 34127.15 + rank 18: 34119.85 + rank 19: 34125.12 + rank 20: 34124.79 + rank 21: 34125.13 + rank 22: 34124.81 + rank 23: 34125.38 + rank 24: 33344.67 + rank 25: 33435.61 + rank 26: 33429.61 + rank 27: 33431.93 + rank 28: 33447.83 + rank 29: 33440.93 + rank 30: 33441.25 + rank 31: 33437.04 + rank 32: 32121.82 + rank 33: 32208.75 + rank 34: 32196.42 + rank 35: 32206.18 + rank 36: 32208.46 + rank 37: 32211.04 + rank 38: 32214.44 + rank 39: 32203.29 + rank 40: 36007.01 + rank 41: 36088.73 + rank 42: 36083.61 + rank 43: 36088.23 + rank 44: 36089.53 + rank 45: 36089.81 + rank 46: 36090.42 + rank 47: 36088.91 + rank 48: 33318.24 + rank 49: 33404.30 + rank 50: 33399.64 + rank 51: 33402.68 + rank 52: 33411.22 + rank 53: 33410.60 + rank 54: 33403.79 + rank 55: 33404.94 + rank 56: 33137.03 + rank 57: 33222.02 + rank 58: 33219.41 + rank 59: 33233.22 + rank 60: 33234.45 + rank 61: 33232.98 + rank 62: 33226.88 + rank 63: 33228.11 + backward-compute: + rank 0: 53726.67 + rank 1: 53725.09 + rank 2: 53710.24 + rank 3: 53719.74 + rank 4: 53717.47 + rank 5: 53719.98 + rank 6: 53719.05 + rank 7: 53715.53 + rank 8: 53177.83 + rank 9: 53178.00 + rank 10: 53170.11 + rank 11: 53179.14 + rank 12: 53174.26 + rank 13: 53173.10 + rank 14: 53172.39 + rank 15: 53173.47 + rank 16: 53708.57 + rank 17: 53718.73 + rank 18: 53719.67 + rank 19: 53718.51 + rank 20: 53721.39 + rank 21: 53720.12 + rank 22: 53720.16 + rank 23: 53717.42 + rank 24: 54412.41 + rank 25: 54403.36 + rank 26: 54403.10 + rank 27: 54405.37 + rank 28: 54392.84 + rank 29: 54386.54 + rank 30: 54393.90 + rank 31: 54397.22 + rank 32: 55631.29 + rank 33: 55630.65 + rank 34: 55627.49 + rank 35: 55629.32 + rank 36: 55628.17 + rank 37: 55624.41 + rank 38: 55620.96 + rank 39: 55632.77 + rank 40: 51751.96 + rank 41: 51754.79 + rank 42: 51753.21 + rank 43: 51750.01 + rank 44: 51749.71 + rank 45: 51746.92 + rank 46: 51752.48 + rank 47: 51751.50 + rank 48: 54449.65 + rank 49: 54444.22 + rank 50: 54442.44 + rank 51: 54442.48 + rank 52: 54432.64 + rank 53: 54435.92 + rank 54: 54444.82 + rank 55: 54443.08 + rank 56: 54631.41 + rank 57: 54629.35 + rank 58: 54620.20 + rank 59: 54618.92 + rank 60: 54615.42 + rank 61: 54614.75 + rank 62: 54627.45 + rank 63: 54623.87 + batch-generator: + rank 0: 24.33 + rank 1: 20.65 + rank 2: 24.31 + rank 3: 23.61 + rank 4: 24.30 + rank 5: 22.55 + rank 6: 22.86 + rank 7: 23.33 + rank 8: 23.56 + rank 9: 22.06 + rank 10: 23.25 + rank 11: 23.65 + rank 12: 23.32 + rank 13: 22.98 + rank 14: 23.11 + rank 15: 22.48 + rank 16: 23.38 + rank 17: 21.61 + rank 18: 21.89 + rank 19: 21.22 + rank 20: 20.95 + rank 21: 21.40 + rank 22: 22.25 + rank 23: 21.59 + rank 24: 33.06 + rank 25: 30.59 + rank 26: 37.46 + rank 27: 37.42 + rank 28: 32.35 + rank 29: 39.25 + rank 30: 34.03 + rank 31: 34.10 + rank 32: 36.65 + rank 33: 32.39 + rank 34: 40.55 + rank 35: 35.42 + rank 36: 36.10 + rank 37: 38.79 + rank 38: 35.79 + rank 39: 34.87 + rank 40: 23.60 + rank 41: 23.72 + rank 42: 25.34 + rank 43: 24.19 + rank 44: 22.60 + rank 45: 23.85 + rank 46: 23.05 + rank 47: 23.49 + rank 48: 28.94 + rank 49: 31.46 + rank 50: 29.14 + rank 51: 30.26 + rank 52: 25.06 + rank 53: 23.79 + rank 54: 29.79 + rank 55: 28.70 + rank 56: 26.44 + rank 57: 24.86 + rank 58: 25.23 + rank 59: 21.56 + rank 60: 24.65 + rank 61: 24.74 + rank 62: 22.84 + rank 63: 23.83 + embedding-grads-all-reduce: + rank 0: 0.02 + rank 1: 0.02 + rank 2: 0.02 + rank 3: 0.02 + rank 4: 0.02 + rank 5: 0.02 + rank 6: 0.02 + rank 7: 0.03 + rank 8: 0.02 + rank 9: 0.02 + rank 10: 0.03 + rank 11: 0.03 + rank 12: 0.03 + rank 13: 0.02 + rank 14: 0.03 + rank 15: 0.02 + rank 16: 0.03 + rank 17: 0.02 + rank 18: 0.02 + rank 19: 0.02 + rank 20: 0.03 + rank 21: 0.03 + rank 22: 0.03 + rank 23: 0.03 + rank 24: 0.04 + rank 25: 0.03 + rank 26: 0.04 + rank 27: 0.04 + rank 28: 0.03 + rank 29: 0.04 + rank 30: 0.04 + rank 31: 0.04 + rank 32: 0.04 + rank 33: 0.03 + rank 34: 0.04 + rank 35: 0.03 + rank 36: 0.03 + rank 37: 0.03 + rank 38: 0.03 + rank 39: 0.03 + rank 40: 0.03 + rank 41: 0.03 + rank 42: 0.03 + rank 43: 0.03 + rank 44: 0.03 + rank 45: 0.03 + rank 46: 0.02 + rank 47: 0.03 + rank 48: 0.04 + rank 49: 0.03 + rank 50: 0.02 + rank 51: 0.03 + rank 52: 0.03 + rank 53: 0.03 + rank 54: 0.03 + rank 55: 0.03 + rank 56: 0.03 + rank 57: 0.03 + rank 58: 0.03 + rank 59: 0.02 + rank 60: 0.03 + rank 61: 0.03 + rank 62: 0.02 + rank 63: 0.03 + all-grads-sync: + rank 0: 1.31 + rank 1: 0.82 + rank 2: 0.79 + rank 3: 0.97 + rank 4: 0.82 + rank 5: 0.87 + rank 6: 0.89 + rank 7: 0.80 + rank 8: 0.82 + rank 9: 0.77 + rank 10: 1.15 + rank 11: 0.80 + rank 12: 0.81 + rank 13: 0.77 + rank 14: 1.10 + rank 15: 1.06 + rank 16: 0.77 + rank 17: 0.75 + rank 18: 0.76 + rank 19: 0.75 + rank 20: 0.79 + rank 21: 0.81 + rank 22: 0.81 + rank 23: 0.98 + rank 24: 1.50 + rank 25: 1.14 + rank 26: 1.66 + rank 27: 1.45 + rank 28: 1.26 + rank 29: 1.34 + rank 30: 1.28 + rank 31: 1.29 + rank 32: 1.24 + rank 33: 1.10 + rank 34: 1.29 + rank 35: 1.41 + rank 36: 1.63 + rank 37: 1.17 + rank 38: 1.22 + rank 39: 1.17 + rank 40: 0.79 + rank 41: 0.78 + rank 42: 0.83 + rank 43: 0.83 + rank 44: 0.90 + rank 45: 0.81 + rank 46: 0.81 + rank 47: 0.92 + rank 48: 0.99 + rank 49: 1.43 + rank 50: 1.08 + rank 51: 1.41 + rank 52: 0.85 + rank 53: 1.04 + rank 54: 0.97 + rank 55: 1.09 + rank 56: 0.89 + rank 57: 0.94 + rank 58: 0.80 + rank 59: 0.75 + rank 60: 0.83 + rank 61: 1.02 + rank 62: 0.82 + rank 63: 0.78 + params-all-gather: + rank 0: 0.02 + rank 1: 0.02 + rank 2: 0.02 + rank 3: 0.02 + rank 4: 0.02 + rank 5: 0.02 + rank 6: 0.02 + rank 7: 0.02 + rank 8: 0.02 + rank 9: 0.02 + rank 10: 0.02 + rank 11: 0.02 + rank 12: 0.02 + rank 13: 0.02 + rank 14: 0.02 + rank 15: 0.02 + rank 16: 0.02 + rank 17: 0.03 + rank 18: 0.02 + rank 19: 0.02 + rank 20: 0.02 + rank 21: 0.02 + rank 22: 0.02 + rank 23: 0.02 + rank 24: 0.04 + rank 25: 0.03 + rank 26: 0.04 + rank 27: 0.03 + rank 28: 0.03 + rank 29: 0.03 + rank 30: 0.03 + rank 31: 0.04 + rank 32: 0.03 + rank 33: 0.02 + rank 34: 0.02 + rank 35: 0.02 + rank 36: 0.03 + rank 37: 0.02 + rank 38: 0.02 + rank 39: 0.02 + rank 40: 0.02 + rank 41: 0.02 + rank 42: 0.02 + rank 43: 0.02 + rank 44: 0.03 + rank 45: 0.02 + rank 46: 0.03 + rank 47: 0.02 + rank 48: 0.03 + rank 49: 0.03 + rank 50: 0.03 + rank 51: 0.02 + rank 52: 0.03 + rank 53: 0.02 + rank 54: 0.03 + rank 55: 0.02 + rank 56: 0.02 + rank 57: 0.02 + rank 58: 0.02 + rank 59: 0.02 + rank 60: 0.02 + rank 61: 0.02 + rank 62: 0.02 + rank 63: 0.02 + optimizer-copy-to-main-grad: + rank 0: 2.17 + rank 1: 1.81 + rank 2: 1.67 + rank 3: 2.06 + rank 4: 1.68 + rank 5: 1.79 + rank 6: 1.85 + rank 7: 1.88 + rank 8: 1.77 + rank 9: 2.10 + rank 10: 1.73 + rank 11: 1.73 + rank 12: 1.74 + rank 13: 1.67 + rank 14: 1.88 + rank 15: 1.97 + rank 16: 2.35 + rank 17: 2.07 + rank 18: 2.14 + rank 19: 2.09 + rank 20: 2.17 + rank 21: 2.27 + rank 22: 2.59 + rank 23: 2.35 + rank 24: 1.96 + rank 25: 2.06 + rank 26: 1.93 + rank 27: 2.06 + rank 28: 2.15 + rank 29: 2.39 + rank 30: 2.29 + rank 31: 1.98 + rank 32: 2.23 + rank 33: 1.87 + rank 34: 1.98 + rank 35: 2.36 + rank 36: 2.04 + rank 37: 2.01 + rank 38: 2.48 + rank 39: 1.92 + rank 40: 1.74 + rank 41: 2.80 + rank 42: 1.81 + rank 43: 1.87 + rank 44: 1.89 + rank 45: 2.02 + rank 46: 1.73 + rank 47: 1.71 + rank 48: 1.92 + rank 49: 1.84 + rank 50: 2.31 + rank 51: 1.82 + rank 52: 1.94 + rank 53: 1.75 + rank 54: 1.76 + rank 55: 1.80 + rank 56: 1.73 + rank 57: 2.34 + rank 58: 1.87 + rank 59: 1.75 + rank 60: 1.72 + rank 61: 1.65 + rank 62: 1.87 + rank 63: 1.70 + optimizer-inner-step: + rank 0: 42.07 + rank 1: 42.00 + rank 2: 41.69 + rank 3: 41.85 + rank 4: 41.67 + rank 5: 41.81 + rank 6: 42.19 + rank 7: 42.07 + rank 8: 41.86 + rank 9: 41.94 + rank 10: 41.74 + rank 11: 41.91 + rank 12: 41.86 + rank 13: 42.13 + rank 14: 41.86 + rank 15: 41.83 + rank 16: 42.27 + rank 17: 41.92 + rank 18: 42.01 + rank 19: 41.98 + rank 20: 41.97 + rank 21: 41.86 + rank 22: 42.35 + rank 23: 42.02 + rank 24: 41.49 + rank 25: 41.85 + rank 26: 41.78 + rank 27: 42.00 + rank 28: 46.01 + rank 29: 41.65 + rank 30: 41.67 + rank 31: 42.01 + rank 32: 41.90 + rank 33: 41.80 + rank 34: 41.89 + rank 35: 47.60 + rank 36: 41.93 + rank 37: 41.92 + rank 38: 41.94 + rank 39: 41.84 + rank 40: 41.86 + rank 41: 41.93 + rank 42: 42.01 + rank 43: 42.08 + rank 44: 41.66 + rank 45: 42.21 + rank 46: 41.90 + rank 47: 41.56 + rank 48: 42.52 + rank 49: 41.76 + rank 50: 53.55 + rank 51: 41.56 + rank 52: 41.92 + rank 53: 41.79 + rank 54: 41.98 + rank 55: 41.85 + rank 56: 41.52 + rank 57: 47.31 + rank 58: 41.80 + rank 59: 41.73 + rank 60: 41.87 + rank 61: 41.65 + rank 62: 41.79 + rank 63: 41.68 + optimizer-copy-main-to-model-params: + rank 0: 4.38 + rank 1: 3.85 + rank 2: 3.76 + rank 3: 4.31 + rank 4: 3.70 + rank 5: 3.90 + rank 6: 3.95 + rank 7: 4.04 + rank 8: 3.89 + rank 9: 4.41 + rank 10: 3.74 + rank 11: 3.82 + rank 12: 3.94 + rank 13: 3.77 + rank 14: 4.04 + rank 15: 4.20 + rank 16: 4.40 + rank 17: 4.00 + rank 18: 4.07 + rank 19: 4.02 + rank 20: 4.01 + rank 21: 4.27 + rank 22: 4.61 + rank 23: 4.19 + rank 24: 3.90 + rank 25: 4.16 + rank 26: 3.88 + rank 27: 4.02 + rank 28: 4.31 + rank 29: 4.31 + rank 30: 4.28 + rank 31: 3.95 + rank 32: 4.10 + rank 33: 3.89 + rank 34: 3.98 + rank 35: 4.54 + rank 36: 4.21 + rank 37: 4.08 + rank 38: 4.58 + rank 39: 3.96 + rank 40: 3.78 + rank 41: 3.97 + rank 42: 3.92 + rank 43: 4.06 + rank 44: 3.96 + rank 45: 4.22 + rank 46: 3.74 + rank 47: 3.72 + rank 48: 4.11 + rank 49: 3.93 + rank 50: 4.90 + rank 51: 3.93 + rank 52: 4.21 + rank 53: 3.85 + rank 54: 3.85 + rank 55: 3.86 + rank 56: 3.84 + rank 57: 4.79 + rank 58: 3.96 + rank 59: 3.78 + rank 60: 3.76 + rank 61: 3.72 + rank 62: 3.93 + rank 63: 3.76 + optimizer: + rank 0: 53.66 + rank 1: 52.83 + rank 2: 52.38 + rank 3: 53.36 + rank 4: 52.28 + rank 5: 52.74 + rank 6: 53.19 + rank 7: 53.18 + rank 8: 52.75 + rank 9: 53.55 + rank 10: 52.48 + rank 11: 52.67 + rank 12: 52.82 + rank 13: 52.87 + rank 14: 52.95 + rank 15: 53.15 + rank 16: 54.06 + rank 17: 53.19 + rank 18: 53.35 + rank 19: 53.23 + rank 20: 53.21 + rank 21: 53.47 + rank 22: 54.45 + rank 23: 53.55 + rank 24: 52.49 + rank 25: 53.10 + rank 26: 52.66 + rank 27: 53.09 + rank 28: 57.61 + rank 29: 53.18 + rank 30: 53.16 + rank 31: 52.99 + rank 32: 53.00 + rank 33: 52.59 + rank 34: 52.82 + rank 35: 59.28 + rank 36: 53.18 + rank 37: 52.99 + rank 38: 53.66 + rank 39: 52.72 + rank 40: 52.64 + rank 41: 53.00 + rank 42: 52.97 + rank 43: 53.22 + rank 44: 52.68 + rank 45: 53.61 + rank 46: 52.61 + rank 47: 52.25 + rank 48: 53.69 + rank 49: 52.66 + rank 50: 65.81 + rank 51: 52.46 + rank 52: 53.19 + rank 53: 52.57 + rank 54: 52.77 + rank 55: 52.66 + rank 56: 52.32 + rank 57: 59.71 + rank 58: 52.77 + rank 59: 52.42 + rank 60: 52.72 + rank 61: 52.29 + rank 62: 52.71 + rank 63: 52.34 + interval-time: + rank 0: 0.81 + rank 1: 0.83 + rank 2: 0.77 + rank 3: 0.73 + rank 4: 0.79 + rank 5: 0.80 + rank 6: 0.73 + rank 7: 0.82 + rank 8: 0.75 + rank 9: 0.72 + rank 10: 0.81 + rank 11: 0.72 + rank 12: 0.81 + rank 13: 0.72 + rank 14: 0.82 + rank 15: 0.73 + rank 16: 0.74 + rank 17: 0.82 + rank 18: 0.85 + rank 19: 0.76 + rank 20: 0.82 + rank 21: 0.82 + rank 22: 0.76 + rank 23: 0.82 + rank 24: 1.46 + rank 25: 1.30 + rank 26: 1.38 + rank 27: 1.38 + rank 28: 1.47 + rank 29: 1.49 + rank 30: 1.38 + rank 31: 1.47 + rank 32: 0.94 + rank 33: 0.93 + rank 34: 0.94 + rank 35: 0.81 + rank 36: 0.90 + rank 37: 0.89 + rank 38: 0.84 + rank 39: 0.91 + rank 40: 0.75 + rank 41: 0.81 + rank 42: 0.78 + rank 43: 0.87 + rank 44: 0.82 + rank 45: 0.81 + rank 46: 0.74 + rank 47: 0.82 + rank 48: 0.97 + rank 49: 0.93 + rank 50: 0.82 + rank 51: 0.90 + rank 52: 0.92 + rank 53: 0.81 + rank 54: 0.92 + rank 55: 0.92 + rank 56: 0.90 + rank 57: 0.77 + rank 58: 0.76 + rank 59: 0.82 + rank 60: 0.80 + rank 61: 0.76 + rank 62: 0.82 + rank 63: 3.20 + [2026-10-04 16:58:35] iteration 1003/ 6855 | consumed samples: 128384 | elapsed time per iteration (ms): 85724.4 | throughput per GPU (TFLOP/s/GPU): 43.2 | MFU 13.83% | FLOPs estimator: workload-aware | logical tokens/s: 195710.3 | valid tokens/s: 195710.3 | physical tokens/s: 196200.6 | loss tokens/s: 147959.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.666963E-01 | lm-global-supervised-tokens: 1.268371E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.181147E-02 | lm-token-buffer-mib: 5.810174E+01 | global_load_balancing_loss: 1.001059E+00 | loss scale: 1.0 | grad norm: 0.125 | params norm: 11590.275 | params rms: 0.062758 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 19:20:59.419267 | finish at 2026-10-10 12:19:35 +times across ranks (ms): + forward-backward: + rank 0: 85511.67 + rank 1: 85512.53 + rank 2: 85513.92 + rank 3: 85513.04 + rank 4: 85514.70 + rank 5: 85514.29 + rank 6: 85511.98 + rank 7: 85511.88 + rank 8: 85508.34 + rank 9: 85509.52 + rank 10: 85510.20 + rank 11: 85509.89 + rank 12: 85510.94 + rank 13: 85511.41 + rank 14: 85508.40 + rank 15: 85507.59 + rank 16: 85504.93 + rank 17: 85506.02 + rank 18: 85506.80 + rank 19: 85506.20 + rank 20: 85508.26 + rank 21: 85508.00 + rank 22: 85504.83 + rank 23: 85504.65 + rank 24: 85516.12 + rank 25: 85514.23 + rank 26: 85514.35 + rank 27: 85513.75 + rank 28: 85514.84 + rank 29: 85514.27 + rank 30: 85514.11 + rank 31: 85516.52 + rank 32: 85499.16 + rank 33: 85499.96 + rank 34: 85499.79 + rank 35: 85499.62 + rank 36: 85501.53 + rank 37: 85500.28 + rank 38: 85500.81 + rank 39: 85500.21 + rank 40: 85502.20 + rank 41: 85502.31 + rank 42: 85502.02 + rank 43: 85500.64 + rank 44: 85501.31 + rank 45: 85499.75 + rank 46: 85503.85 + rank 47: 85503.30 + rank 48: 85512.52 + rank 49: 85509.16 + rank 50: 85507.62 + rank 51: 85507.69 + rank 52: 85508.33 + rank 53: 85507.09 + rank 54: 85510.54 + rank 55: 85510.12 + rank 56: 85512.20 + rank 57: 85512.53 + rank 58: 85512.09 + rank 59: 85510.78 + rank 60: 85511.37 + rank 61: 85510.28 + rank 62: 85513.60 + rank 63: 85512.96 + forward-compute: + rank 0: 31892.91 + rank 1: 31891.39 + rank 2: 31893.46 + rank 3: 31892.45 + rank 4: 31896.79 + rank 5: 31895.83 + rank 6: 31896.98 + rank 7: 31905.78 + rank 8: 32497.19 + rank 9: 32506.18 + rank 10: 32504.62 + rank 11: 32504.53 + rank 12: 32505.39 + rank 13: 32506.78 + rank 14: 32502.58 + rank 15: 32501.20 + rank 16: 34861.98 + rank 17: 34862.81 + rank 18: 34863.32 + rank 19: 34863.26 + rank 20: 34864.57 + rank 21: 34863.81 + rank 22: 34863.83 + rank 23: 34864.38 + rank 24: 28806.58 + rank 25: 28817.52 + rank 26: 28818.08 + rank 27: 28815.03 + rank 28: 28828.07 + rank 29: 28826.43 + rank 30: 28821.83 + rank 31: 28820.33 + rank 32: 30240.38 + rank 33: 30241.85 + rank 34: 30242.63 + rank 35: 30245.88 + rank 36: 30247.46 + rank 37: 30249.91 + rank 38: 30248.43 + rank 39: 30245.96 + rank 40: 30079.26 + rank 41: 30082.04 + rank 42: 30085.16 + rank 43: 30085.01 + rank 44: 30083.75 + rank 45: 30088.35 + rank 46: 30088.61 + rank 47: 30087.61 + rank 48: 32727.69 + rank 49: 32729.95 + rank 50: 32731.67 + rank 51: 32730.43 + rank 52: 32734.75 + rank 53: 32734.56 + rank 54: 32731.46 + rank 55: 32730.50 + rank 56: 32671.28 + rank 57: 32676.64 + rank 58: 32677.32 + rank 59: 32670.67 + rank 60: 32676.43 + rank 61: 32680.81 + rank 62: 32674.76 + rank 63: 32681.75 + backward-compute: + rank 0: 53583.64 + rank 1: 53589.46 + rank 2: 53583.46 + rank 3: 53584.34 + rank 4: 53582.91 + rank 5: 53584.11 + rank 6: 53581.54 + rank 7: 53570.28 + rank 8: 52978.53 + rank 9: 52969.23 + rank 10: 52971.74 + rank 11: 52970.75 + rank 12: 52971.04 + rank 13: 52970.60 + rank 14: 52972.52 + rank 15: 52974.16 + rank 16: 50607.52 + rank 17: 50607.13 + rank 18: 50608.91 + rank 19: 50608.00 + rank 20: 50608.88 + rank 21: 50607.56 + rank 22: 50606.68 + rank 23: 50606.70 + rank 24: 56652.29 + rank 25: 56634.82 + rank 26: 56640.54 + rank 27: 56641.78 + rank 28: 56632.60 + rank 29: 56626.98 + rank 30: 56634.45 + rank 31: 56636.75 + rank 32: 55218.55 + rank 33: 55222.81 + rank 34: 55218.00 + rank 35: 55215.29 + rank 36: 55216.05 + rank 37: 55214.69 + rank 38: 55208.34 + rank 39: 55216.59 + rank 40: 55391.08 + rank 41: 55389.76 + rank 42: 55384.75 + rank 43: 55380.39 + rank 44: 55384.45 + rank 45: 55378.88 + rank 46: 55384.27 + rank 47: 55382.86 + rank 48: 52737.17 + rank 49: 52742.11 + rank 50: 52740.07 + rank 51: 52740.25 + rank 52: 52735.33 + rank 53: 52738.09 + rank 54: 52743.27 + rank 55: 52745.02 + rank 56: 52806.57 + rank 57: 52801.34 + rank 58: 52800.17 + rank 59: 52807.90 + rank 60: 52800.81 + rank 61: 52796.27 + rank 62: 52804.50 + rank 63: 52797.79 + batch-generator: + rank 0: 29.56 + rank 1: 29.68 + rank 2: 30.90 + rank 3: 28.89 + rank 4: 33.19 + rank 5: 30.91 + rank 6: 32.16 + rank 7: 30.23 + rank 8: 22.49 + rank 9: 21.01 + rank 10: 21.53 + rank 11: 21.33 + rank 12: 22.01 + rank 13: 22.14 + rank 14: 22.54 + rank 15: 21.49 + rank 16: 20.54 + rank 17: 22.00 + rank 18: 21.70 + rank 19: 21.65 + rank 20: 20.86 + rank 21: 21.70 + rank 22: 21.15 + rank 23: 20.74 + rank 24: 36.82 + rank 25: 36.25 + rank 26: 37.75 + rank 27: 37.12 + rank 28: 38.88 + rank 29: 40.37 + rank 30: 35.65 + rank 31: 38.27 + rank 32: 35.65 + rank 33: 30.53 + rank 34: 33.61 + rank 35: 32.51 + rank 36: 32.61 + rank 37: 31.56 + rank 38: 34.71 + rank 39: 31.63 + rank 40: 23.53 + rank 41: 21.62 + rank 42: 22.03 + rank 43: 22.67 + rank 44: 21.76 + rank 45: 22.31 + rank 46: 22.03 + rank 47: 22.13 + rank 48: 43.95 + rank 49: 23.15 + rank 50: 25.81 + rank 51: 29.93 + rank 52: 33.36 + rank 53: 22.82 + rank 54: 21.99 + rank 55: 28.34 + rank 56: 24.03 + rank 57: 23.71 + rank 58: 23.21 + rank 59: 21.22 + rank 60: 22.25 + rank 61: 23.25 + rank 62: 21.15 + rank 63: 21.97 + embedding-grads-all-reduce: + rank 0: 0.02 + rank 1: 0.02 + rank 2: 0.02 + rank 3: 0.03 + rank 4: 0.02 + rank 5: 0.03 + rank 6: 0.02 + rank 7: 0.02 + rank 8: 0.03 + rank 9: 0.02 + rank 10: 0.03 + rank 11: 0.02 + rank 12: 0.03 + rank 13: 0.02 + rank 14: 0.03 + rank 15: 0.02 + rank 16: 0.03 + rank 17: 0.03 + rank 18: 0.03 + rank 19: 0.03 + rank 20: 0.03 + rank 21: 0.02 + rank 22: 0.03 + rank 23: 0.02 + rank 24: 0.04 + rank 25: 0.03 + rank 26: 0.04 + rank 27: 0.04 + rank 28: 0.04 + rank 29: 0.04 + rank 30: 0.03 + rank 31: 0.04 + rank 32: 0.04 + rank 33: 0.03 + rank 34: 0.04 + rank 35: 0.03 + rank 36: 0.03 + rank 37: 0.03 + rank 38: 0.03 + rank 39: 0.03 + rank 40: 0.02 + rank 41: 0.02 + rank 42: 0.03 + rank 43: 0.03 + rank 44: 0.03 + rank 45: 0.02 + rank 46: 0.02 + rank 47: 0.03 + rank 48: 0.05 + rank 49: 0.03 + rank 50: 0.03 + rank 51: 0.03 + rank 52: 0.04 + rank 53: 0.03 + rank 54: 0.03 + rank 55: 0.03 + rank 56: 0.03 + rank 57: 0.03 + rank 58: 0.03 + rank 59: 0.02 + rank 60: 0.03 + rank 61: 0.03 + rank 62: 0.02 + rank 63: 0.03 + all-grads-sync: + rank 0: 0.96 + rank 1: 0.79 + rank 2: 0.84 + rank 3: 1.04 + rank 4: 0.88 + rank 5: 1.01 + rank 6: 1.04 + rank 7: 0.84 + rank 8: 0.81 + rank 9: 0.93 + rank 10: 0.78 + rank 11: 0.96 + rank 12: 0.78 + rank 13: 0.80 + rank 14: 1.01 + rank 15: 0.84 + rank 16: 0.78 + rank 17: 0.82 + rank 18: 0.76 + rank 19: 0.80 + rank 20: 0.78 + rank 21: 0.85 + rank 22: 0.77 + rank 23: 0.82 + rank 24: 1.34 + rank 25: 1.76 + rank 26: 1.23 + rank 27: 1.23 + rank 28: 1.38 + rank 29: 1.15 + rank 30: 1.29 + rank 31: 1.24 + rank 32: 1.62 + rank 33: 1.25 + rank 34: 1.27 + rank 35: 1.25 + rank 36: 1.22 + rank 37: 1.12 + rank 38: 1.76 + rank 39: 1.18 + rank 40: 0.81 + rank 41: 0.77 + rank 42: 0.78 + rank 43: 0.83 + rank 44: 0.98 + rank 45: 0.80 + rank 46: 0.77 + rank 47: 0.95 + rank 48: 2.01 + rank 49: 0.83 + rank 50: 0.90 + rank 51: 0.99 + rank 52: 1.08 + rank 53: 0.86 + rank 54: 1.03 + rank 55: 1.10 + rank 56: 0.91 + rank 57: 0.93 + rank 58: 0.81 + rank 59: 0.81 + rank 60: 0.95 + rank 61: 0.83 + rank 62: 0.79 + rank 63: 0.81 + params-all-gather: + rank 0: 0.02 + rank 1: 0.02 + rank 2: 0.02 + rank 3: 0.02 + rank 4: 0.02 + rank 5: 0.02 + rank 6: 0.02 + rank 7: 0.02 + rank 8: 0.02 + rank 9: 0.02 + rank 10: 0.02 + rank 11: 0.02 + rank 12: 0.02 + rank 13: 0.02 + rank 14: 0.02 + rank 15: 0.02 + rank 16: 0.02 + rank 17: 0.02 + rank 18: 0.02 + rank 19: 0.02 + rank 20: 0.02 + rank 21: 0.02 + rank 22: 0.02 + rank 23: 0.02 + rank 24: 0.04 + rank 25: 0.03 + rank 26: 0.03 + rank 27: 0.04 + rank 28: 0.04 + rank 29: 0.03 + rank 30: 0.03 + rank 31: 0.03 + rank 32: 0.02 + rank 33: 0.02 + rank 34: 0.02 + rank 35: 0.03 + rank 36: 0.03 + rank 37: 0.02 + rank 38: 0.03 + rank 39: 0.02 + rank 40: 0.02 + rank 41: 0.02 + rank 42: 0.02 + rank 43: 0.02 + rank 44: 0.02 + rank 45: 0.02 + rank 46: 0.02 + rank 47: 0.02 + rank 48: 0.03 + rank 49: 0.03 + rank 50: 0.03 + rank 51: 0.03 + rank 52: 0.03 + rank 53: 0.03 + rank 54: 0.03 + rank 55: 0.03 + rank 56: 0.02 + rank 57: 0.02 + rank 58: 0.02 + rank 59: 0.02 + rank 60: 0.02 + rank 61: 0.02 + rank 62: 0.02 + rank 63: 0.02 + optimizer-copy-to-main-grad: + rank 0: 2.12 + rank 1: 1.79 + rank 2: 1.72 + rank 3: 2.08 + rank 4: 1.78 + rank 5: 1.82 + rank 6: 1.86 + rank 7: 1.94 + rank 8: 1.71 + rank 9: 2.07 + rank 10: 1.76 + rank 11: 1.71 + rank 12: 1.78 + rank 13: 1.76 + rank 14: 1.85 + rank 15: 1.94 + rank 16: 2.36 + rank 17: 2.06 + rank 18: 2.13 + rank 19: 2.14 + rank 20: 2.13 + rank 21: 2.32 + rank 22: 2.50 + rank 23: 2.25 + rank 24: 1.84 + rank 25: 2.21 + rank 26: 2.04 + rank 27: 2.18 + rank 28: 2.31 + rank 29: 2.30 + rank 30: 2.20 + rank 31: 2.01 + rank 32: 2.06 + rank 33: 1.97 + rank 34: 2.03 + rank 35: 2.30 + rank 36: 2.06 + rank 37: 2.01 + rank 38: 2.47 + rank 39: 1.82 + rank 40: 1.75 + rank 41: 1.81 + rank 42: 1.81 + rank 43: 1.84 + rank 44: 1.84 + rank 45: 2.04 + rank 46: 1.75 + rank 47: 1.65 + rank 48: 3.02 + rank 49: 1.85 + rank 50: 2.32 + rank 51: 1.81 + rank 52: 1.98 + rank 53: 1.77 + rank 54: 1.80 + rank 55: 1.79 + rank 56: 1.75 + rank 57: 2.32 + rank 58: 1.87 + rank 59: 1.70 + rank 60: 1.69 + rank 61: 1.66 + rank 62: 1.83 + rank 63: 1.73 + optimizer-inner-step: + rank 0: 41.89 + rank 1: 41.75 + rank 2: 41.49 + rank 3: 41.70 + rank 4: 41.49 + rank 5: 41.60 + rank 6: 41.96 + rank 7: 41.92 + rank 8: 41.72 + rank 9: 41.69 + rank 10: 41.62 + rank 11: 41.73 + rank 12: 41.70 + rank 13: 41.92 + rank 14: 41.68 + rank 15: 41.55 + rank 16: 42.07 + rank 17: 41.78 + rank 18: 41.84 + rank 19: 41.79 + rank 20: 41.82 + rank 21: 41.70 + rank 22: 42.17 + rank 23: 41.89 + rank 24: 41.40 + rank 25: 41.97 + rank 26: 41.68 + rank 27: 42.66 + rank 28: 47.05 + rank 29: 41.62 + rank 30: 41.61 + rank 31: 43.81 + rank 32: 41.83 + rank 33: 41.57 + rank 34: 41.69 + rank 35: 47.62 + rank 36: 41.78 + rank 37: 41.77 + rank 38: 42.63 + rank 39: 41.58 + rank 40: 41.76 + rank 41: 41.82 + rank 42: 41.86 + rank 43: 41.86 + rank 44: 41.51 + rank 45: 42.06 + rank 46: 41.70 + rank 47: 41.37 + rank 48: 44.44 + rank 49: 41.62 + rank 50: 53.29 + rank 51: 41.42 + rank 52: 41.74 + rank 53: 41.59 + rank 54: 41.83 + rank 55: 41.67 + rank 56: 41.33 + rank 57: 47.21 + rank 58: 41.64 + rank 59: 41.53 + rank 60: 41.84 + rank 61: 41.47 + rank 62: 41.57 + rank 63: 41.51 + optimizer-copy-main-to-model-params: + rank 0: 4.36 + rank 1: 3.86 + rank 2: 3.77 + rank 3: 4.26 + rank 4: 3.69 + rank 5: 3.87 + rank 6: 3.91 + rank 7: 3.99 + rank 8: 3.82 + rank 9: 4.40 + rank 10: 3.80 + rank 11: 3.73 + rank 12: 3.91 + rank 13: 3.81 + rank 14: 3.98 + rank 15: 4.18 + rank 16: 4.38 + rank 17: 3.99 + rank 18: 4.00 + rank 19: 4.04 + rank 20: 3.96 + rank 21: 4.28 + rank 22: 4.61 + rank 23: 4.14 + rank 24: 3.94 + rank 25: 4.48 + rank 26: 4.45 + rank 27: 4.28 + rank 28: 4.65 + rank 29: 4.79 + rank 30: 4.50 + rank 31: 4.11 + rank 32: 4.08 + rank 33: 3.91 + rank 34: 4.01 + rank 35: 4.61 + rank 36: 4.29 + rank 37: 4.07 + rank 38: 4.60 + rank 39: 3.86 + rank 40: 3.75 + rank 41: 3.93 + rank 42: 3.90 + rank 43: 4.03 + rank 44: 3.94 + rank 45: 4.23 + rank 46: 3.75 + rank 47: 3.70 + rank 48: 4.42 + rank 49: 3.87 + rank 50: 4.90 + rank 51: 3.94 + rank 52: 4.24 + rank 53: 3.85 + rank 54: 3.87 + rank 55: 3.87 + rank 56: 3.80 + rank 57: 4.75 + rank 58: 3.90 + rank 59: 3.69 + rank 60: 3.71 + rank 61: 3.70 + rank 62: 3.90 + rank 63: 3.72 + optimizer: + rank 0: 54.34 + rank 1: 53.48 + rank 2: 53.11 + rank 3: 53.99 + rank 4: 53.00 + rank 5: 53.39 + rank 6: 53.81 + rank 7: 53.85 + rank 8: 53.39 + rank 9: 54.16 + rank 10: 53.25 + rank 11: 53.26 + rank 12: 53.49 + rank 13: 53.58 + rank 14: 53.55 + rank 15: 53.71 + rank 16: 54.69 + rank 17: 53.89 + rank 18: 53.97 + rank 19: 53.94 + rank 20: 53.87 + rank 21: 54.20 + rank 22: 55.13 + rank 23: 54.20 + rank 24: 53.18 + rank 25: 54.37 + rank 26: 53.98 + rank 27: 54.85 + rank 28: 59.76 + rank 29: 54.41 + rank 30: 54.08 + rank 31: 55.81 + rank 32: 53.77 + rank 33: 53.27 + rank 34: 53.51 + rank 35: 60.25 + rank 36: 53.99 + rank 37: 53.70 + rank 38: 55.25 + rank 39: 53.24 + rank 40: 53.31 + rank 41: 53.63 + rank 42: 53.63 + rank 43: 53.81 + rank 44: 53.33 + rank 45: 54.30 + rank 46: 53.24 + rank 47: 52.88 + rank 48: 56.83 + rank 49: 53.32 + rank 50: 66.45 + rank 51: 53.19 + rank 52: 53.92 + rank 53: 53.24 + rank 54: 53.50 + rank 55: 53.36 + rank 56: 52.99 + rank 57: 60.45 + rank 58: 53.39 + rank 59: 52.99 + rank 60: 53.37 + rank 61: 52.96 + rank 62: 53.31 + rank 63: 52.99 + interval-time: + rank 0: 0.98 + rank 1: 1.11 + rank 2: 0.98 + rank 3: 0.94 + rank 4: 1.01 + rank 5: 1.08 + rank 6: 1.02 + rank 7: 1.06 + rank 8: 0.82 + rank 9: 0.73 + rank 10: 0.80 + rank 11: 0.73 + rank 12: 0.82 + rank 13: 0.80 + rank 14: 0.73 + rank 15: 0.72 + rank 16: 0.73 + rank 17: 0.81 + rank 18: 0.79 + rank 19: 0.75 + rank 20: 0.80 + rank 21: 0.83 + rank 22: 0.73 + rank 23: 0.80 + rank 24: 1.02 + rank 25: 0.96 + rank 26: 0.89 + rank 27: 1.02 + rank 28: 0.94 + rank 29: 1.05 + rank 30: 0.86 + rank 31: 0.99 + rank 32: 0.94 + rank 33: 0.93 + rank 34: 0.93 + rank 35: 0.81 + rank 36: 0.89 + rank 37: 0.88 + rank 38: 0.97 + rank 39: 0.88 + rank 40: 0.74 + rank 41: 0.81 + rank 42: 0.79 + rank 43: 0.83 + rank 44: 0.82 + rank 45: 0.80 + rank 46: 0.74 + rank 47: 0.81 + rank 48: 1.18 + rank 49: 0.95 + rank 50: 0.83 + rank 51: 0.92 + rank 52: 0.96 + rank 53: 0.83 + rank 54: 0.92 + rank 55: 0.95 + rank 56: 0.87 + rank 57: 0.75 + rank 58: 0.77 + rank 59: 0.82 + rank 60: 0.81 + rank 61: 0.75 + rank 62: 0.82 + rank 63: 3.04 + [2026-10-04 17:00:02] iteration 1004/ 6855 | consumed samples: 128512 | elapsed time per iteration (ms): 86887.3 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.60% | FLOPs estimator: workload-aware | logical tokens/s: 193091.1 | valid tokens/s: 193091.1 | physical tokens/s: 193552.5 | loss tokens/s: 142824.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.560136E-01 | lm-global-supervised-tokens: 1.240965E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.244428E-02 | lm-token-buffer-mib: 5.806170E+01 | global_load_balancing_loss: 1.001949E+00 | loss scale: 1.0 | grad norm: 0.095 | params norm: 11590.191 | params rms: 0.062757 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 21:12:57.789078 | finish at 2026-10-10 14:13:00 +times across ranks (ms): + forward-backward: + rank 0: 86669.73 + rank 1: 86671.72 + rank 2: 86671.08 + rank 3: 86671.54 + rank 4: 86673.88 + rank 5: 86673.26 + rank 6: 86669.62 + rank 7: 86671.10 + rank 8: 86666.74 + rank 9: 86669.12 + rank 10: 86667.38 + rank 11: 86668.69 + rank 12: 86669.79 + rank 13: 86669.53 + rank 14: 86666.52 + rank 15: 86666.94 + rank 16: 86662.88 + rank 17: 86665.37 + rank 18: 86663.69 + rank 19: 86664.65 + rank 20: 86666.45 + rank 21: 86665.99 + rank 22: 86662.81 + rank 23: 86663.77 + rank 24: 86661.67 + rank 25: 86664.30 + rank 26: 86663.38 + rank 27: 86662.97 + rank 28: 86665.35 + rank 29: 86665.09 + rank 30: 86663.23 + rank 31: 86662.52 + rank 32: 86658.39 + rank 33: 86659.34 + rank 34: 86658.30 + rank 35: 86659.20 + rank 36: 86660.35 + rank 37: 86659.20 + rank 38: 86659.87 + rank 39: 86660.48 + rank 40: 86661.10 + rank 41: 86662.12 + rank 42: 86660.23 + rank 43: 86660.05 + rank 44: 86660.20 + rank 45: 86658.94 + rank 46: 86662.12 + rank 47: 86662.12 + rank 48: 86670.38 + rank 49: 86668.90 + rank 50: 86666.94 + rank 51: 86666.45 + rank 52: 86666.27 + rank 53: 86665.96 + rank 54: 86668.44 + rank 55: 86669.43 + rank 56: 86670.37 + rank 57: 86671.65 + rank 58: 86669.85 + rank 59: 86669.12 + rank 60: 86669.83 + rank 61: 86667.92 + rank 62: 86671.54 + rank 63: 86672.72 + forward-compute: + rank 0: 31828.65 + rank 1: 31832.23 + rank 2: 31834.28 + rank 3: 31833.50 + rank 4: 31837.51 + rank 5: 31836.11 + rank 6: 31832.35 + rank 7: 31839.59 + rank 8: 30974.53 + rank 9: 30971.65 + rank 10: 30971.62 + rank 11: 30969.55 + rank 12: 30973.22 + rank 13: 30974.36 + rank 14: 30969.28 + rank 15: 30972.99 + rank 16: 33336.48 + rank 17: 33323.80 + rank 18: 33323.43 + rank 19: 33325.86 + rank 20: 33327.70 + rank 21: 33329.07 + rank 22: 33330.46 + rank 23: 33331.70 + rank 24: 33986.70 + rank 25: 33991.15 + rank 26: 33991.68 + rank 27: 33990.94 + rank 28: 33998.89 + rank 29: 33998.21 + rank 30: 34168.37 + rank 31: 33996.33 + rank 32: 32799.14 + rank 33: 32805.60 + rank 34: 32814.30 + rank 35: 32803.17 + rank 36: 32809.53 + rank 37: 32811.44 + rank 38: 32820.89 + rank 39: 32814.20 + rank 40: 35914.04 + rank 41: 35914.12 + rank 42: 35915.43 + rank 43: 35914.20 + rank 44: 35915.55 + rank 45: 35914.49 + rank 46: 35914.64 + rank 47: 35915.03 + rank 48: 34047.19 + rank 49: 34055.55 + rank 50: 34054.09 + rank 51: 34056.52 + rank 52: 34062.61 + rank 53: 34061.54 + rank 54: 34059.08 + rank 55: 34060.93 + rank 56: 33311.11 + rank 57: 33313.61 + rank 58: 33305.20 + rank 59: 33305.25 + rank 60: 33310.72 + rank 61: 33316.79 + rank 62: 33317.38 + rank 63: 33317.38 + backward-compute: + rank 0: 54807.95 + rank 1: 54808.99 + rank 2: 54802.98 + rank 3: 54805.01 + rank 4: 54799.61 + rank 5: 54803.27 + rank 6: 54802.75 + rank 7: 54797.71 + rank 8: 55655.86 + rank 9: 55663.61 + rank 10: 55659.27 + rank 11: 55662.81 + rank 12: 55662.33 + rank 13: 55660.51 + rank 14: 55662.77 + rank 15: 55661.61 + rank 16: 53291.88 + rank 17: 53308.66 + rank 18: 53307.41 + rank 19: 53304.38 + rank 20: 53303.15 + rank 21: 53303.53 + rank 22: 53299.53 + rank 23: 53299.04 + rank 24: 52633.47 + rank 25: 52628.39 + rank 26: 52630.53 + rank 27: 52631.21 + rank 28: 52626.52 + rank 29: 52619.21 + rank 30: 52447.65 + rank 31: 52624.48 + rank 32: 53818.57 + rank 33: 53815.44 + rank 34: 53804.61 + rank 35: 53816.67 + rank 36: 53815.25 + rank 37: 53808.91 + rank 38: 53797.21 + rank 39: 53807.27 + rank 40: 50710.27 + rank 41: 50713.46 + rank 42: 50711.77 + rank 43: 50710.08 + rank 44: 50710.70 + rank 45: 50711.38 + rank 46: 50713.09 + rank 47: 50712.12 + rank 48: 52576.38 + rank 49: 52574.85 + rank 50: 52574.62 + rank 51: 52574.50 + rank 52: 52566.04 + rank 53: 52567.14 + rank 54: 52572.98 + rank 55: 52570.78 + rank 56: 53324.46 + rank 57: 53322.53 + rank 58: 53329.67 + rank 59: 53332.24 + rank 60: 53326.77 + rank 61: 53317.98 + rank 62: 53321.19 + rank 63: 53320.21 + batch-generator: + rank 0: 24.20 + rank 1: 22.14 + rank 2: 24.01 + rank 3: 23.66 + rank 4: 24.84 + rank 5: 24.68 + rank 6: 23.28 + rank 7: 25.63 + rank 8: 23.03 + rank 9: 22.09 + rank 10: 26.35 + rank 11: 22.88 + rank 12: 23.73 + rank 13: 22.97 + rank 14: 22.48 + rank 15: 23.17 + rank 16: 20.41 + rank 17: 21.02 + rank 18: 21.07 + rank 19: 21.22 + rank 20: 19.82 + rank 21: 21.97 + rank 22: 21.21 + rank 23: 21.54 + rank 24: 31.58 + rank 25: 33.69 + rank 26: 34.04 + rank 27: 35.68 + rank 28: 30.75 + rank 29: 34.00 + rank 30: 34.43 + rank 31: 33.12 + rank 32: 38.46 + rank 33: 31.98 + rank 34: 32.77 + rank 35: 33.02 + rank 36: 29.46 + rank 37: 32.80 + rank 38: 34.15 + rank 39: 30.78 + rank 40: 23.10 + rank 41: 21.84 + rank 42: 22.33 + rank 43: 23.04 + rank 44: 21.73 + rank 45: 22.29 + rank 46: 22.16 + rank 47: 22.64 + rank 48: 43.18 + rank 49: 34.04 + rank 50: 27.79 + rank 51: 32.33 + rank 52: 35.08 + rank 53: 26.11 + rank 54: 32.08 + rank 55: 31.26 + rank 56: 23.67 + rank 57: 22.68 + rank 58: 21.44 + rank 59: 21.58 + rank 60: 21.73 + rank 61: 21.33 + rank 62: 20.38 + rank 63: 20.52 + embedding-grads-all-reduce: + rank 0: 0.02 + rank 1: 0.02 + rank 2: 0.02 + rank 3: 0.03 + rank 4: 0.02 + rank 5: 0.03 + rank 6: 0.02 + rank 7: 0.03 + rank 8: 0.02 + rank 9: 0.03 + rank 10: 0.02 + rank 11: 0.02 + rank 12: 0.03 + rank 13: 0.02 + rank 14: 0.02 + rank 15: 0.02 + rank 16: 0.03 + rank 17: 0.03 + rank 18: 0.03 + rank 19: 0.02 + rank 20: 0.03 + rank 21: 0.03 + rank 22: 0.02 + rank 23: 0.03 + rank 24: 0.04 + rank 25: 0.03 + rank 26: 0.04 + rank 27: 0.04 + rank 28: 0.03 + rank 29: 0.04 + rank 30: 0.11 + rank 31: 0.04 + rank 32: 0.03 + rank 33: 0.03 + rank 34: 0.03 + rank 35: 0.04 + rank 36: 0.03 + rank 37: 0.03 + rank 38: 0.03 + rank 39: 0.03 + rank 40: 0.03 + rank 41: 0.02 + rank 42: 0.03 + rank 43: 0.03 + rank 44: 0.02 + rank 45: 0.03 + rank 46: 0.02 + rank 47: 0.03 + rank 48: 0.04 + rank 49: 0.03 + rank 50: 0.03 + rank 51: 0.03 + rank 52: 0.03 + rank 53: 0.03 + rank 54: 0.03 + rank 55: 0.03 + rank 56: 0.02 + rank 57: 0.03 + rank 58: 0.02 + rank 59: 0.02 + rank 60: 0.02 + rank 61: 0.02 + rank 62: 0.03 + rank 63: 0.03 + all-grads-sync: + rank 0: 0.81 + rank 1: 0.78 + rank 2: 0.93 + rank 3: 0.94 + rank 4: 1.01 + rank 5: 1.23 + rank 6: 0.82 + rank 7: 0.79 + rank 8: 0.92 + rank 9: 0.78 + rank 10: 0.76 + rank 11: 0.93 + rank 12: 0.90 + rank 13: 0.80 + rank 14: 0.96 + rank 15: 0.85 + rank 16: 0.77 + rank 17: 0.77 + rank 18: 0.81 + rank 19: 0.78 + rank 20: 1.19 + rank 21: 0.77 + rank 22: 0.85 + rank 23: 0.78 + rank 24: 1.25 + rank 25: 1.27 + rank 26: 1.18 + rank 27: 1.21 + rank 28: 1.27 + rank 29: 1.24 + rank 30: 1.47 + rank 31: 1.27 + rank 32: 1.34 + rank 33: 1.16 + rank 34: 1.50 + rank 35: 1.16 + rank 36: 1.10 + rank 37: 1.12 + rank 38: 1.44 + rank 39: 1.14 + rank 40: 0.78 + rank 41: 0.81 + rank 42: 1.01 + rank 43: 0.86 + rank 44: 0.91 + rank 45: 0.75 + rank 46: 0.80 + rank 47: 0.85 + rank 48: 1.51 + rank 49: 1.14 + rank 50: 1.45 + rank 51: 1.02 + rank 52: 0.82 + rank 53: 1.17 + rank 54: 1.03 + rank 55: 0.99 + rank 56: 0.88 + rank 57: 0.95 + rank 58: 0.82 + rank 59: 0.75 + rank 60: 0.89 + rank 61: 0.78 + rank 62: 0.83 + rank 63: 1.11 + params-all-gather: + rank 0: 0.02 + rank 1: 0.02 + rank 2: 0.02 + rank 3: 0.02 + rank 4: 0.02 + rank 5: 0.02 + rank 6: 0.02 + rank 7: 0.02 + rank 8: 0.02 + rank 9: 0.02 + rank 10: 0.02 + rank 11: 0.02 + rank 12: 0.02 + rank 13: 0.02 + rank 14: 0.02 + rank 15: 0.02 + rank 16: 0.02 + rank 17: 0.02 + rank 18: 0.02 + rank 19: 0.02 + rank 20: 0.02 + rank 21: 0.02 + rank 22: 0.02 + rank 23: 0.02 + rank 24: 0.03 + rank 25: 0.03 + rank 26: 0.04 + rank 27: 0.04 + rank 28: 0.03 + rank 29: 0.03 + rank 30: 0.03 + rank 31: 0.03 + rank 32: 0.02 + rank 33: 0.02 + rank 34: 0.02 + rank 35: 0.02 + rank 36: 0.02 + rank 37: 0.03 + rank 38: 0.03 + rank 39: 0.02 + rank 40: 0.02 + rank 41: 0.02 + rank 42: 0.02 + rank 43: 0.02 + rank 44: 0.02 + rank 45: 0.02 + rank 46: 0.02 + rank 47: 0.02 + rank 48: 0.03 + rank 49: 0.03 + rank 50: 0.03 + rank 51: 0.02 + rank 52: 0.03 + rank 53: 0.03 + rank 54: 0.02 + rank 55: 0.03 + rank 56: 0.02 + rank 57: 0.02 + rank 58: 0.02 + rank 59: 0.02 + rank 60: 0.02 + rank 61: 0.02 + rank 62: 0.02 + rank 63: 0.02 + optimizer-copy-to-main-grad: + rank 0: 2.16 + rank 1: 1.52 + rank 2: 1.81 + rank 3: 2.07 + rank 4: 1.71 + rank 5: 1.95 + rank 6: 1.89 + rank 7: 1.87 + rank 8: 1.77 + rank 9: 2.19 + rank 10: 1.75 + rank 11: 1.72 + rank 12: 1.78 + rank 13: 1.76 + rank 14: 1.87 + rank 15: 1.98 + rank 16: 2.35 + rank 17: 2.07 + rank 18: 2.14 + rank 19: 2.09 + rank 20: 2.17 + rank 21: 2.32 + rank 22: 2.57 + rank 23: 2.32 + rank 24: 1.93 + rank 25: 2.21 + rank 26: 1.98 + rank 27: 2.01 + rank 28: 2.20 + rank 29: 2.19 + rank 30: 2.40 + rank 31: 2.02 + rank 32: 1.93 + rank 33: 1.92 + rank 34: 1.98 + rank 35: 2.23 + rank 36: 2.13 + rank 37: 1.93 + rank 38: 2.24 + rank 39: 1.90 + rank 40: 1.72 + rank 41: 1.85 + rank 42: 1.79 + rank 43: 1.82 + rank 44: 1.81 + rank 45: 1.97 + rank 46: 1.73 + rank 47: 1.67 + rank 48: 2.54 + rank 49: 1.87 + rank 50: 2.41 + rank 51: 1.82 + rank 52: 1.95 + rank 53: 1.77 + rank 54: 1.77 + rank 55: 1.80 + rank 56: 1.74 + rank 57: 2.35 + rank 58: 1.82 + rank 59: 1.68 + rank 60: 1.70 + rank 61: 1.66 + rank 62: 1.81 + rank 63: 1.71 + optimizer-inner-step: + rank 0: 44.25 + rank 1: 44.11 + rank 2: 43.98 + rank 3: 44.08 + rank 4: 43.90 + rank 5: 43.98 + rank 6: 44.39 + rank 7: 44.34 + rank 8: 49.44 + rank 9: 51.84 + rank 10: 47.34 + rank 11: 47.82 + rank 12: 49.13 + rank 13: 48.96 + rank 14: 49.04 + rank 15: 48.95 + rank 16: 46.23 + rank 17: 45.95 + rank 18: 45.99 + rank 19: 46.00 + rank 20: 45.97 + rank 21: 45.87 + rank 22: 46.34 + rank 23: 46.03 + rank 24: 45.40 + rank 25: 45.67 + rank 26: 45.74 + rank 27: 45.76 + rank 28: 46.17 + rank 29: 45.55 + rank 30: 45.52 + rank 31: 45.97 + rank 32: 45.92 + rank 33: 45.69 + rank 34: 45.87 + rank 35: 47.83 + rank 36: 46.01 + rank 37: 45.98 + rank 38: 45.76 + rank 39: 45.76 + rank 40: 45.90 + rank 41: 45.93 + rank 42: 45.96 + rank 43: 46.07 + rank 44: 45.66 + rank 45: 46.24 + rank 46: 45.91 + rank 47: 45.51 + rank 48: 45.91 + rank 49: 45.76 + rank 50: 55.94 + rank 51: 45.64 + rank 52: 45.88 + rank 53: 45.80 + rank 54: 45.99 + rank 55: 45.86 + rank 56: 45.54 + rank 57: 47.04 + rank 58: 45.88 + rank 59: 45.72 + rank 60: 46.07 + rank 61: 45.61 + rank 62: 45.80 + rank 63: 45.66 + optimizer-copy-main-to-model-params: + rank 0: 4.36 + rank 1: 3.85 + rank 2: 3.72 + rank 3: 4.31 + rank 4: 3.74 + rank 5: 3.94 + rank 6: 3.99 + rank 7: 4.00 + rank 8: 3.87 + rank 9: 4.46 + rank 10: 3.85 + rank 11: 3.79 + rank 12: 4.02 + rank 13: 3.79 + rank 14: 4.06 + rank 15: 4.23 + rank 16: 4.40 + rank 17: 4.03 + rank 18: 4.04 + rank 19: 4.02 + rank 20: 4.02 + rank 21: 4.26 + rank 22: 4.60 + rank 23: 4.21 + rank 24: 3.93 + rank 25: 4.22 + rank 26: 3.95 + rank 27: 4.29 + rank 28: 4.32 + rank 29: 4.35 + rank 30: 4.32 + rank 31: 4.18 + rank 32: 4.05 + rank 33: 3.90 + rank 34: 4.02 + rank 35: 4.50 + rank 36: 4.24 + rank 37: 4.07 + rank 38: 4.50 + rank 39: 3.92 + rank 40: 3.71 + rank 41: 4.01 + rank 42: 3.88 + rank 43: 4.02 + rank 44: 3.97 + rank 45: 4.27 + rank 46: 3.73 + rank 47: 3.77 + rank 48: 4.34 + rank 49: 3.90 + rank 50: 5.09 + rank 51: 4.02 + rank 52: 4.23 + rank 53: 3.89 + rank 54: 3.87 + rank 55: 3.89 + rank 56: 3.79 + rank 57: 4.76 + rank 58: 3.91 + rank 59: 3.74 + rank 60: 3.70 + rank 61: 3.68 + rank 62: 3.86 + rank 63: 3.69 + optimizer: + rank 0: 58.73 + rank 1: 57.86 + rank 2: 57.52 + rank 3: 58.45 + rank 4: 57.45 + rank 5: 57.82 + rank 6: 58.31 + rank 7: 58.28 + rank 8: 61.45 + rank 9: 64.63 + rank 10: 59.31 + rank 11: 59.69 + rank 12: 61.27 + rank 13: 60.83 + rank 14: 61.30 + rank 15: 61.44 + rank 16: 59.11 + rank 17: 58.34 + rank 18: 58.40 + rank 19: 58.32 + rank 20: 58.31 + rank 21: 58.55 + rank 22: 59.52 + rank 23: 58.65 + rank 24: 57.57 + rank 25: 58.26 + rank 26: 57.82 + rank 27: 58.44 + rank 28: 58.83 + rank 29: 58.30 + rank 30: 58.26 + rank 31: 58.31 + rank 32: 58.07 + rank 33: 57.60 + rank 34: 57.95 + rank 35: 60.55 + rank 36: 58.35 + rank 37: 58.13 + rank 38: 58.49 + rank 39: 57.69 + rank 40: 57.66 + rank 41: 58.09 + rank 42: 58.00 + rank 43: 58.21 + rank 44: 57.75 + rank 45: 58.78 + rank 46: 57.67 + rank 47: 57.33 + rank 48: 58.40 + rank 49: 57.69 + rank 50: 70.14 + rank 51: 57.70 + rank 52: 58.26 + rank 53: 57.70 + rank 54: 57.87 + rank 55: 57.76 + rank 56: 57.35 + rank 57: 60.46 + rank 58: 57.84 + rank 59: 57.45 + rank 60: 57.81 + rank 61: 57.30 + rank 62: 57.69 + rank 63: 57.34 + interval-time: + rank 0: 0.74 + rank 1: 0.82 + rank 2: 0.79 + rank 3: 0.73 + rank 4: 0.81 + rank 5: 0.82 + rank 6: 0.74 + rank 7: 0.83 + rank 8: 0.83 + rank 9: 0.74 + rank 10: 0.82 + rank 11: 0.74 + rank 12: 0.82 + rank 13: 0.78 + rank 14: 0.74 + rank 15: 0.74 + rank 16: 0.75 + rank 17: 0.82 + rank 18: 0.84 + rank 19: 0.96 + rank 20: 0.82 + rank 21: 0.84 + rank 22: 0.75 + rank 23: 0.83 + rank 24: 0.97 + rank 25: 0.95 + rank 26: 0.88 + rank 27: 0.99 + rank 28: 0.92 + rank 29: 0.95 + rank 30: 0.83 + rank 31: 0.97 + rank 32: 0.93 + rank 33: 0.92 + rank 34: 0.93 + rank 35: 0.81 + rank 36: 0.88 + rank 37: 0.90 + rank 38: 0.83 + rank 39: 0.90 + rank 40: 0.75 + rank 41: 0.85 + rank 42: 0.82 + rank 43: 0.85 + rank 44: 0.84 + rank 45: 0.86 + rank 46: 0.75 + rank 47: 0.85 + rank 48: 11.35 + rank 49: 11.36 + rank 50: 11.33 + rank 51: 11.33 + rank 52: 11.33 + rank 53: 11.34 + rank 54: 11.34 + rank 55: 11.36 + rank 56: 0.80 + rank 57: 0.73 + rank 58: 0.74 + rank 59: 0.81 + rank 60: 0.79 + rank 61: 0.74 + rank 62: 0.77 + rank 63: 3.10 + [2026-10-04 17:01:33] iteration 1005/ 6855 | consumed samples: 128640 | elapsed time per iteration (ms): 90349.6 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.49% | FLOPs estimator: workload-aware | logical tokens/s: 185692.1 | valid tokens/s: 185692.1 | physical tokens/s: 186144.4 | loss tokens/s: 129075.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.974060E-01 | lm-global-supervised-tokens: 1.166192E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.850388E-02 | lm-token-buffer-mib: 5.832231E+01 | global_load_balancing_loss: 1.002087E+00 | loss scale: 1.0 | grad norm: 0.079 | params norm: 11590.108 | params rms: 0.062757 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 2:49:05.002842 | finish at 2026-10-10 19:50:38 +times across ranks (ms): + forward-backward: + rank 0: 90107.09 + rank 1: 90107.85 + rank 2: 90108.72 + rank 3: 90109.07 + rank 4: 90109.23 + rank 5: 90108.73 + rank 6: 90106.67 + rank 7: 90107.08 + rank 8: 90103.55 + rank 9: 90104.41 + rank 10: 90103.95 + rank 11: 90106.23 + rank 12: 90106.20 + rank 13: 90106.16 + rank 14: 90103.07 + rank 15: 90103.29 + rank 16: 90099.44 + rank 17: 90101.16 + rank 18: 90101.50 + rank 19: 90102.47 + rank 20: 90101.98 + rank 21: 90102.65 + rank 22: 90100.52 + rank 23: 90099.86 + rank 24: 90098.93 + rank 25: 90100.52 + rank 26: 90100.21 + rank 27: 90101.61 + rank 28: 90100.80 + rank 29: 90099.80 + rank 30: 90100.42 + rank 31: 90098.88 + rank 32: 90097.38 + rank 33: 90097.49 + rank 34: 90094.74 + rank 35: 90097.45 + rank 36: 90096.54 + rank 37: 90095.61 + rank 38: 90097.30 + rank 39: 90096.61 + rank 40: 90097.30 + rank 41: 90097.72 + rank 42: 90097.00 + rank 43: 90097.39 + rank 44: 90095.51 + rank 45: 90094.79 + rank 46: 90099.16 + rank 47: 90098.52 + rank 48: 90109.36 + rank 49: 90109.23 + rank 50: 90109.16 + rank 51: 90109.90 + rank 52: 90109.52 + rank 53: 90108.62 + rank 54: 90109.94 + rank 55: 90110.38 + rank 56: 90106.34 + rank 57: 90107.38 + rank 58: 90106.57 + rank 59: 90106.31 + rank 60: 90106.15 + rank 61: 90104.95 + rank 62: 90108.59 + rank 63: 90108.03 + forward-compute: + rank 0: 40125.98 + rank 1: 40125.39 + rank 2: 40130.11 + rank 3: 40126.98 + rank 4: 40128.25 + rank 5: 40127.48 + rank 6: 40129.16 + rank 7: 40130.08 + rank 8: 35270.37 + rank 9: 35274.35 + rank 10: 35271.13 + rank 11: 35273.79 + rank 12: 35274.48 + rank 13: 35272.30 + rank 14: 35269.70 + rank 15: 35270.11 + rank 16: 38378.89 + rank 17: 38382.14 + rank 18: 38380.50 + rank 19: 38380.85 + rank 20: 38382.30 + rank 21: 38382.72 + rank 22: 38380.04 + rank 23: 38378.45 + rank 24: 31744.41 + rank 25: 31749.41 + rank 26: 31753.54 + rank 27: 31750.24 + rank 28: 31755.74 + rank 29: 31749.28 + rank 30: 31753.46 + rank 31: 31747.64 + rank 32: 35496.45 + rank 33: 35505.84 + rank 34: 35505.92 + rank 35: 35509.00 + rank 36: 35516.33 + rank 37: 35518.28 + rank 38: 35518.46 + rank 39: 35510.02 + rank 40: 34346.42 + rank 41: 34341.09 + rank 42: 34341.95 + rank 43: 34341.96 + rank 44: 34343.46 + rank 45: 34340.79 + rank 46: 34346.53 + rank 47: 34345.07 + rank 48: 34676.79 + rank 49: 34684.21 + rank 50: 34684.71 + rank 51: 34684.13 + rank 52: 34691.31 + rank 53: 34690.75 + rank 54: 34685.97 + rank 55: 34688.26 + rank 56: 36318.62 + rank 57: 36321.64 + rank 58: 36328.38 + rank 59: 36322.86 + rank 60: 36330.39 + rank 61: 36331.58 + rank 62: 36325.67 + rank 63: 36343.55 + backward-compute: + rank 0: 49943.65 + rank 1: 49949.18 + rank 2: 49939.29 + rank 3: 49947.26 + rank 4: 49942.84 + rank 5: 49946.46 + rank 6: 49941.58 + rank 7: 49940.87 + rank 8: 54796.00 + rank 9: 54796.13 + rank 10: 54797.83 + rank 11: 54797.43 + rank 12: 54797.03 + rank 13: 54798.72 + rank 14: 54799.95 + rank 15: 54798.12 + rank 16: 51684.33 + rank 17: 51681.94 + rank 18: 51687.39 + rank 19: 51687.20 + rank 20: 51685.79 + rank 21: 51683.75 + rank 22: 51685.20 + rank 23: 51686.30 + rank 24: 58316.67 + rank 25: 58310.21 + rank 26: 58307.04 + rank 27: 58310.82 + rank 28: 58307.27 + rank 29: 58310.24 + rank 30: 58304.07 + rank 31: 58313.71 + rank 32: 54558.49 + rank 33: 54551.02 + rank 34: 54550.06 + rank 35: 54547.17 + rank 36: 54540.33 + rank 37: 54537.87 + rank 38: 54536.41 + rank 39: 54547.19 + rank 40: 55713.54 + rank 41: 55724.02 + rank 42: 55721.87 + rank 43: 55722.38 + rank 44: 55718.92 + rank 45: 55721.51 + rank 46: 55717.86 + rank 47: 55720.89 + rank 48: 55391.17 + rank 49: 55390.71 + rank 50: 55384.57 + rank 51: 55387.18 + rank 52: 55379.80 + rank 53: 55382.38 + rank 54: 55388.45 + rank 55: 55384.16 + rank 56: 53752.22 + rank 57: 53750.53 + rank 58: 53743.59 + rank 59: 53749.39 + rank 60: 53741.88 + rank 61: 53738.01 + rank 62: 53749.18 + rank 63: 53731.22 + batch-generator: + rank 0: 23.92 + rank 1: 20.86 + rank 2: 23.41 + rank 3: 22.62 + rank 4: 24.05 + rank 5: 23.35 + rank 6: 25.28 + rank 7: 24.51 + rank 8: 22.00 + rank 9: 21.89 + rank 10: 21.58 + rank 11: 22.06 + rank 12: 22.08 + rank 13: 22.25 + rank 14: 23.16 + rank 15: 21.51 + rank 16: 21.00 + rank 17: 21.02 + rank 18: 21.66 + rank 19: 22.33 + rank 20: 21.29 + rank 21: 22.76 + rank 22: 22.29 + rank 23: 21.46 + rank 24: 33.77 + rank 25: 35.12 + rank 26: 34.54 + rank 27: 35.88 + rank 28: 29.98 + rank 29: 35.91 + rank 30: 40.06 + rank 31: 32.71 + rank 32: 35.30 + rank 33: 31.96 + rank 34: 31.93 + rank 35: 30.24 + rank 36: 30.93 + rank 37: 32.10 + rank 38: 32.58 + rank 39: 32.46 + rank 40: 22.37 + rank 41: 22.47 + rank 42: 22.63 + rank 43: 23.26 + rank 44: 21.05 + rank 45: 22.03 + rank 46: 20.41 + rank 47: 22.35 + rank 48: 35.85 + rank 49: 23.90 + rank 50: 31.88 + rank 51: 32.41 + rank 52: 24.14 + rank 53: 23.40 + rank 54: 31.57 + rank 55: 29.99 + rank 56: 23.37 + rank 57: 23.51 + rank 58: 22.58 + rank 59: 21.36 + rank 60: 22.50 + rank 61: 22.64 + rank 62: 20.73 + rank 63: 22.13 + embedding-grads-all-reduce: + rank 0: 0.02 + rank 1: 0.03 + rank 2: 0.02 + rank 3: 0.02 + rank 4: 0.02 + rank 5: 0.02 + rank 6: 0.02 + rank 7: 0.03 + rank 8: 0.02 + rank 9: 0.03 + rank 10: 0.02 + rank 11: 0.03 + rank 12: 0.03 + rank 13: 0.02 + rank 14: 0.02 + rank 15: 0.02 + rank 16: 0.03 + rank 17: 0.02 + rank 18: 0.02 + rank 19: 0.02 + rank 20: 0.03 + rank 21: 0.03 + rank 22: 0.03 + rank 23: 0.02 + rank 24: 0.04 + rank 25: 0.04 + rank 26: 0.04 + rank 27: 0.04 + rank 28: 0.03 + rank 29: 0.03 + rank 30: 0.04 + rank 31: 0.03 + rank 32: 0.03 + rank 33: 0.03 + rank 34: 0.03 + rank 35: 0.03 + rank 36: 0.03 + rank 37: 0.03 + rank 38: 0.03 + rank 39: 0.03 + rank 40: 0.03 + rank 41: 0.02 + rank 42: 0.02 + rank 43: 0.02 + rank 44: 0.03 + rank 45: 0.03 + rank 46: 0.02 + rank 47: 0.03 + rank 48: 0.04 + rank 49: 0.03 + rank 50: 0.03 + rank 51: 0.03 + rank 52: 0.03 + rank 53: 0.03 + rank 54: 0.03 + rank 55: 0.03 + rank 56: 0.02 + rank 57: 0.02 + rank 58: 0.03 + rank 59: 0.02 + rank 60: 0.02 + rank 61: 0.02 + rank 62: 0.02 + rank 63: 0.03 + all-grads-sync: + rank 0: 1.25 + rank 1: 0.77 + rank 2: 0.85 + rank 3: 0.86 + rank 4: 0.83 + rank 5: 0.97 + rank 6: 0.79 + rank 7: 0.79 + rank 8: 1.13 + rank 9: 0.73 + rank 10: 0.77 + rank 11: 0.80 + rank 12: 0.84 + rank 13: 0.93 + rank 14: 0.79 + rank 15: 0.77 + rank 16: 0.80 + rank 17: 0.78 + rank 18: 1.14 + rank 19: 0.98 + rank 20: 0.80 + rank 21: 0.82 + rank 22: 1.26 + rank 23: 0.76 + rank 24: 1.31 + rank 25: 1.31 + rank 26: 1.26 + rank 27: 1.73 + rank 28: 1.04 + rank 29: 0.93 + rank 30: 1.27 + rank 31: 0.95 + rank 32: 0.98 + rank 33: 1.52 + rank 34: 1.16 + rank 35: 1.24 + rank 36: 1.32 + rank 37: 1.19 + rank 38: 1.35 + rank 39: 1.19 + rank 40: 1.01 + rank 41: 0.79 + rank 42: 0.78 + rank 43: 0.88 + rank 44: 0.74 + rank 45: 0.90 + rank 46: 1.01 + rank 47: 0.78 + rank 48: 1.22 + rank 49: 0.82 + rank 50: 1.11 + rank 51: 1.00 + rank 52: 0.84 + rank 53: 0.89 + rank 54: 1.12 + rank 55: 1.21 + rank 56: 0.84 + rank 57: 0.80 + rank 58: 0.82 + rank 59: 0.82 + rank 60: 0.81 + rank 61: 0.80 + rank 62: 0.81 + rank 63: 0.95 + params-all-gather: + rank 0: 0.02 + rank 1: 0.02 + rank 2: 0.02 + rank 3: 0.02 + rank 4: 0.02 + rank 5: 0.02 + rank 6: 0.02 + rank 7: 0.02 + rank 8: 0.02 + rank 9: 0.02 + rank 10: 0.02 + rank 11: 0.02 + rank 12: 0.02 + rank 13: 0.02 + rank 14: 0.02 + rank 15: 0.02 + rank 16: 0.02 + rank 17: 0.03 + rank 18: 0.03 + rank 19: 0.02 + rank 20: 0.02 + rank 21: 0.02 + rank 22: 0.02 + rank 23: 0.02 + rank 24: 0.04 + rank 25: 0.03 + rank 26: 0.04 + rank 27: 0.04 + rank 28: 0.04 + rank 29: 0.03 + rank 30: 0.03 + rank 31: 0.04 + rank 32: 0.03 + rank 33: 0.02 + rank 34: 0.03 + rank 35: 0.03 + rank 36: 0.03 + rank 37: 0.02 + rank 38: 0.03 + rank 39: 0.02 + rank 40: 0.02 + rank 41: 0.02 + rank 42: 0.02 + rank 43: 0.02 + rank 44: 0.02 + rank 45: 0.02 + rank 46: 0.02 + rank 47: 0.02 + rank 48: 0.02 + rank 49: 0.03 + rank 50: 0.03 + rank 51: 0.02 + rank 52: 0.03 + rank 53: 0.03 + rank 54: 0.03 + rank 55: 0.03 + rank 56: 0.02 + rank 57: 0.02 + rank 58: 0.02 + rank 59: 0.02 + rank 60: 0.02 + rank 61: 0.02 + rank 62: 0.02 + rank 63: 0.02 + optimizer-copy-to-main-grad: + rank 0: 2.17 + rank 1: 1.83 + rank 2: 1.71 + rank 3: 2.04 + rank 4: 1.69 + rank 5: 1.82 + rank 6: 1.89 + rank 7: 1.86 + rank 8: 1.79 + rank 9: 2.10 + rank 10: 1.72 + rank 11: 1.73 + rank 12: 1.78 + rank 13: 1.82 + rank 14: 1.85 + rank 15: 1.98 + rank 16: 2.38 + rank 17: 2.08 + rank 18: 2.16 + rank 19: 2.12 + rank 20: 2.09 + rank 21: 2.32 + rank 22: 2.52 + rank 23: 2.27 + rank 24: 1.90 + rank 25: 2.22 + rank 26: 1.98 + rank 27: 2.11 + rank 28: 2.22 + rank 29: 2.20 + rank 30: 2.26 + rank 31: 1.89 + rank 32: 2.09 + rank 33: 1.93 + rank 34: 1.98 + rank 35: 2.27 + rank 36: 2.08 + rank 37: 1.99 + rank 38: 2.49 + rank 39: 2.00 + rank 40: 1.74 + rank 41: 1.79 + rank 42: 1.79 + rank 43: 1.83 + rank 44: 1.79 + rank 45: 1.99 + rank 46: 1.74 + rank 47: 1.66 + rank 48: 2.74 + rank 49: 1.82 + rank 50: 2.32 + rank 51: 1.81 + rank 52: 1.99 + rank 53: 1.76 + rank 54: 1.76 + rank 55: 1.80 + rank 56: 1.76 + rank 57: 2.36 + rank 58: 1.80 + rank 59: 1.66 + rank 60: 1.71 + rank 61: 1.69 + rank 62: 1.81 + rank 63: 1.66 + optimizer-inner-step: + rank 0: 42.86 + rank 1: 42.78 + rank 2: 42.53 + rank 3: 42.66 + rank 4: 42.45 + rank 5: 42.64 + rank 6: 43.00 + rank 7: 42.99 + rank 8: 42.70 + rank 9: 42.68 + rank 10: 42.54 + rank 11: 42.74 + rank 12: 42.67 + rank 13: 42.92 + rank 14: 42.64 + rank 15: 42.59 + rank 16: 43.07 + rank 17: 42.79 + rank 18: 42.81 + rank 19: 42.81 + rank 20: 42.80 + rank 21: 42.64 + rank 22: 43.13 + rank 23: 42.80 + rank 24: 38.97 + rank 25: 43.74 + rank 26: 40.47 + rank 27: 44.42 + rank 28: 47.68 + rank 29: 42.82 + rank 30: 41.81 + rank 31: 43.80 + rank 32: 42.77 + rank 33: 42.52 + rank 34: 42.66 + rank 35: 47.72 + rank 36: 42.14 + rank 37: 42.76 + rank 38: 42.83 + rank 39: 42.69 + rank 40: 42.68 + rank 41: 42.78 + rank 42: 42.88 + rank 43: 42.90 + rank 44: 42.46 + rank 45: 43.05 + rank 46: 42.74 + rank 47: 42.37 + rank 48: 43.27 + rank 49: 42.53 + rank 50: 54.22 + rank 51: 42.39 + rank 52: 42.69 + rank 53: 42.57 + rank 54: 42.74 + rank 55: 42.70 + rank 56: 42.31 + rank 57: 47.13 + rank 58: 42.65 + rank 59: 42.54 + rank 60: 42.93 + rank 61: 42.46 + rank 62: 42.61 + rank 63: 42.45 + optimizer-copy-main-to-model-params: + rank 0: 4.41 + rank 1: 3.90 + rank 2: 3.77 + rank 3: 4.35 + rank 4: 3.77 + rank 5: 3.93 + rank 6: 3.98 + rank 7: 3.97 + rank 8: 3.90 + rank 9: 4.40 + rank 10: 3.80 + rank 11: 3.81 + rank 12: 3.95 + rank 13: 3.76 + rank 14: 4.02 + rank 15: 4.21 + rank 16: 4.45 + rank 17: 4.03 + rank 18: 4.03 + rank 19: 4.07 + rank 20: 3.99 + rank 21: 4.33 + rank 22: 4.59 + rank 23: 4.19 + rank 24: 3.88 + rank 25: 4.26 + rank 26: 3.93 + rank 27: 4.09 + rank 28: 4.35 + rank 29: 4.35 + rank 30: 4.29 + rank 31: 4.06 + rank 32: 4.10 + rank 33: 3.88 + rank 34: 4.00 + rank 35: 4.55 + rank 36: 4.27 + rank 37: 4.06 + rank 38: 4.57 + rank 39: 4.29 + rank 40: 3.75 + rank 41: 3.99 + rank 42: 3.90 + rank 43: 3.98 + rank 44: 3.97 + rank 45: 4.18 + rank 46: 3.73 + rank 47: 3.73 + rank 48: 4.22 + rank 49: 3.88 + rank 50: 4.91 + rank 51: 4.02 + rank 52: 4.23 + rank 53: 3.88 + rank 54: 3.90 + rank 55: 3.87 + rank 56: 3.82 + rank 57: 4.81 + rank 58: 3.89 + rank 59: 3.84 + rank 60: 3.86 + rank 61: 3.71 + rank 62: 3.88 + rank 63: 3.69 + optimizer: + rank 0: 54.76 + rank 1: 53.97 + rank 2: 53.54 + rank 3: 54.48 + rank 4: 53.44 + rank 5: 53.88 + rank 6: 54.31 + rank 7: 54.28 + rank 8: 53.87 + rank 9: 54.58 + rank 10: 53.62 + rank 11: 53.78 + rank 12: 53.92 + rank 13: 53.95 + rank 14: 54.00 + rank 15: 54.20 + rank 16: 55.20 + rank 17: 54.38 + rank 18: 54.39 + rank 19: 54.41 + rank 20: 54.29 + rank 21: 54.62 + rank 22: 55.48 + rank 23: 54.61 + rank 24: 53.61 + rank 25: 58.81 + rank 26: 55.14 + rank 27: 59.37 + rank 28: 62.95 + rank 29: 58.03 + rank 30: 56.96 + rank 31: 58.58 + rank 32: 54.17 + rank 33: 53.60 + rank 34: 53.93 + rank 35: 59.75 + rank 36: 54.43 + rank 37: 54.11 + rank 38: 55.08 + rank 39: 54.21 + rank 40: 53.67 + rank 41: 54.08 + rank 42: 54.09 + rank 43: 54.19 + rank 44: 53.75 + rank 45: 54.68 + rank 46: 53.69 + rank 47: 53.34 + rank 48: 54.92 + rank 49: 53.69 + rank 50: 66.79 + rank 51: 53.67 + rank 52: 54.31 + rank 53: 53.67 + rank 54: 53.90 + rank 55: 53.80 + rank 56: 53.39 + rank 57: 59.83 + rank 58: 53.82 + rank 59: 53.58 + rank 60: 54.02 + rank 61: 53.39 + rank 62: 53.76 + rank 63: 53.34 + interval-time: + rank 0: 0.74 + rank 1: 0.85 + rank 2: 0.75 + rank 3: 0.73 + rank 4: 0.78 + rank 5: 0.83 + rank 6: 0.75 + rank 7: 0.83 + rank 8: 0.85 + rank 9: 0.74 + rank 10: 0.83 + rank 11: 0.77 + rank 12: 0.88 + rank 13: 0.82 + rank 14: 0.79 + rank 15: 0.74 + rank 16: 0.74 + rank 17: 0.83 + rank 18: 0.81 + rank 19: 0.75 + rank 20: 0.80 + rank 21: 0.83 + rank 22: 0.73 + rank 23: 0.80 + rank 24: 0.96 + rank 25: 0.96 + rank 26: 0.90 + rank 27: 1.03 + rank 28: 0.92 + rank 29: 0.93 + rank 30: 0.83 + rank 31: 0.96 + rank 32: 0.93 + rank 33: 0.87 + rank 34: 0.94 + rank 35: 0.80 + rank 36: 0.90 + rank 37: 0.90 + rank 38: 0.94 + rank 39: 0.92 + rank 40: 0.76 + rank 41: 0.83 + rank 42: 0.80 + rank 43: 0.84 + rank 44: 0.81 + rank 45: 0.80 + rank 46: 0.75 + rank 47: 0.89 + rank 48: 0.95 + rank 49: 0.94 + rank 50: 0.86 + rank 51: 0.84 + rank 52: 0.95 + rank 53: 0.86 + rank 54: 0.94 + rank 55: 0.96 + rank 56: 0.86 + rank 57: 0.75 + rank 58: 0.76 + rank 59: 0.82 + rank 60: 0.79 + rank 61: 0.76 + rank 62: 0.81 + rank 63: 3.15 + [2026-10-04 17:03:01] iteration 1006/ 6855 | consumed samples: 128768 | elapsed time per iteration (ms): 87910.5 | throughput per GPU (TFLOP/s/GPU): 41.7 | MFU 13.36% | FLOPs estimator: workload-aware | logical tokens/s: 190843.4 | valid tokens/s: 190843.4 | physical tokens/s: 191346.0 | loss tokens/s: 141279.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.512120E-01 | lm-global-supervised-tokens: 1.241992E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.617031E-02 | lm-token-buffer-mib: 5.834308E+01 | global_load_balancing_loss: 1.000104E+00 | loss scale: 1.0 | grad norm: 0.084 | params norm: 11590.026 | params rms: 0.062756 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 22:49:48.602644 | finish at 2026-10-10 15:52:49 + [2026-10-04 17:04:28] iteration 1007/ 6855 | consumed samples: 128896 | elapsed time per iteration (ms): 87642.8 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.56% | FLOPs estimator: workload-aware | logical tokens/s: 191426.8 | valid tokens/s: 191426.8 | physical tokens/s: 191913.6 | loss tokens/s: 143778.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.874899E-01 | lm-global-supervised-tokens: 1.260116E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.863871E-02 | lm-token-buffer-mib: 5.807923E+01 | global_load_balancing_loss: 1.000497E+00 | loss scale: 1.0 | grad norm: 0.060 | params norm: 11589.943 | params rms: 0.062756 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 22:22:14.830030 | finish at 2026-10-10 15:26:43 + [2026-10-04 17:05:59] iteration 1008/ 6855 | consumed samples: 129024 | elapsed time per iteration (ms): 90800.2 | throughput per GPU (TFLOP/s/GPU): 41.2 | MFU 13.20% | FLOPs estimator: workload-aware | logical tokens/s: 184770.6 | valid tokens/s: 184770.6 | physical tokens/s: 185211.7 | loss tokens/s: 133969.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.013365E-01 | lm-global-supervised-tokens: 1.216449E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.376159E-02 | lm-token-buffer-mib: 5.803142E+01 | global_load_balancing_loss: 9.992886E-01 | loss scale: 1.0 | grad norm: 0.167 | params norm: 11589.860 | params rms: 0.062755 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 3:28:28.833441 | finish at 2026-10-10 20:34:28 + [2026-10-04 17:07:26] iteration 1009/ 6855 | consumed samples: 129152 | elapsed time per iteration (ms): 87333.6 | throughput per GPU (TFLOP/s/GPU): 41.5 | MFU 13.30% | FLOPs estimator: workload-aware | logical tokens/s: 192104.3 | valid tokens/s: 192104.3 | physical tokens/s: 192569.6 | loss tokens/s: 143610.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.561475E-01 | lm-global-supervised-tokens: 1.254204E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.275424E-02 | lm-token-buffer-mib: 5.816880E+01 | global_load_balancing_loss: 9.994656E-01 | loss scale: 1.0 | grad norm: 0.115 | params norm: 11589.779 | params rms: 0.062755 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 21:49:12.129687 | finish at 2026-10-10 14:56:38 + [2026-10-04 17:09:18] iteration 1010/ 6855 | consumed samples: 129280 | elapsed time per iteration (ms): 111919.5 | throughput per GPU (TFLOP/s/GPU): 33.6 | MFU 10.78% | FLOPs estimator: workload-aware | logical tokens/s: 149904.2 | valid tokens/s: 149904.2 | physical tokens/s: 150250.8 | loss tokens/s: 103747.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.873462E-01 | lm-global-supervised-tokens: 1.161136E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.158739E-02 | lm-token-buffer-mib: 5.809059E+01 | global_load_balancing_loss: 9.995151E-01 | loss scale: 1.0 | grad norm: 0.105 | params norm: 11589.700 | params rms: 0.062754 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 7 days, 13:42:49.684404 | finish at 2026-10-12 06:52:08 + [2026-10-04 17:10:48] iteration 1011/ 6855 | consumed samples: 129408 | elapsed time per iteration (ms): 89283.3 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.45% | FLOPs estimator: workload-aware | logical tokens/s: 187909.9 | valid tokens/s: 187909.9 | physical tokens/s: 188295.8 | loss tokens/s: 134844.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.369030E-01 | lm-global-supervised-tokens: 1.203939E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.437096E-02 | lm-token-buffer-mib: 5.807278E+01 | global_load_balancing_loss: 9.993006E-01 | loss scale: 1.0 | grad norm: 0.122 | params norm: 11589.621 | params rms: 0.062754 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 0:56:11.411078 | finish at 2026-10-10 18:06:59 + [2026-10-04 17:12:15] iteration 1012/ 6855 | consumed samples: 129536 | elapsed time per iteration (ms): 87755.8 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.52% | FLOPs estimator: workload-aware | logical tokens/s: 191180.0 | valid tokens/s: 191180.0 | physical tokens/s: 191539.8 | loss tokens/s: 136952.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.827894E-01 | lm-global-supervised-tokens: 1.201840E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.529724E-02 | lm-token-buffer-mib: 5.813976E+01 | global_load_balancing_loss: 9.998291E-01 | loss scale: 1.0 | grad norm: 0.076 | params norm: 11589.542 | params rms: 0.062754 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 22:25:57.422246 | finish at 2026-10-10 15:38:13 + [2026-10-04 17:13:43] iteration 1013/ 6855 | consumed samples: 129664 | elapsed time per iteration (ms): 88212.8 | throughput per GPU (TFLOP/s/GPU): 41.4 | MFU 13.27% | FLOPs estimator: workload-aware | logical tokens/s: 190189.5 | valid tokens/s: 190189.5 | physical tokens/s: 190569.1 | loss tokens/s: 141438.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.107678E-01 | lm-global-supervised-tokens: 1.247668E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 5.863795E-02 | lm-token-buffer-mib: 5.811061E+01 | global_load_balancing_loss: 1.000348E+00 | loss scale: 1.0 | grad norm: 0.107 | params norm: 11589.462 | params rms: 0.062753 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 23:08:59.418713 | finish at 2026-10-10 16:22:43 + [2026-10-04 17:15:12] iteration 1014/ 6855 | consumed samples: 129792 | elapsed time per iteration (ms): 88353.2 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.45% | FLOPs estimator: workload-aware | logical tokens/s: 189887.1 | valid tokens/s: 189887.1 | physical tokens/s: 190341.1 | loss tokens/s: 141961.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.567400E-01 | lm-global-supervised-tokens: 1.254270E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.167967E-02 | lm-token-buffer-mib: 5.809687E+01 | global_load_balancing_loss: 1.000261E+00 | loss scale: 1.0 | grad norm: 0.040 | params norm: 11589.383 | params rms: 0.062753 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 23:21:10.754083 | finish at 2026-10-10 16:36:23 + [2026-10-04 17:16:38] iteration 1015/ 6855 | consumed samples: 129920 | elapsed time per iteration (ms): 85907.3 | throughput per GPU (TFLOP/s/GPU): 41.5 | MFU 13.29% | FLOPs estimator: workload-aware | logical tokens/s: 195294.1 | valid tokens/s: 195294.0 | physical tokens/s: 195899.6 | loss tokens/s: 150514.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.749720E-01 | lm-global-supervised-tokens: 1.293025E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.473450E-02 | lm-token-buffer-mib: 5.832992E+01 | global_load_balancing_loss: 1.001288E+00 | loss scale: 1.0 | grad norm: 0.069 | params norm: 11589.304 | params rms: 0.062752 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 19:21:38.421726 | finish at 2026-10-10 12:38:16 + [2026-10-04 17:18:03] iteration 1016/ 6855 | consumed samples: 130048 | elapsed time per iteration (ms): 84956.8 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.50% | FLOPs estimator: workload-aware | logical tokens/s: 197479.0 | valid tokens/s: 197479.0 | physical tokens/s: 198003.3 | loss tokens/s: 149538.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.789701E-01 | lm-global-supervised-tokens: 1.270431E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.816848E-02 | lm-token-buffer-mib: 5.824619E+01 | global_load_balancing_loss: 1.000377E+00 | loss scale: 1.0 | grad norm: 0.071 | params norm: 11589.227 | params rms: 0.062752 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 17:47:42.738401 | finish at 2026-10-10 11:05:45 + [2026-10-04 17:19:28] iteration 1017/ 6855 | consumed samples: 130176 | elapsed time per iteration (ms): 85502.3 | throughput per GPU (TFLOP/s/GPU): 43.1 | MFU 13.81% | FLOPs estimator: workload-aware | logical tokens/s: 196219.0 | valid tokens/s: 196219.0 | physical tokens/s: 196762.7 | loss tokens/s: 145240.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.938943E-01 | lm-global-supervised-tokens: 1.241838E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.162042E-02 | lm-token-buffer-mib: 5.802911E+01 | global_load_balancing_loss: 1.001284E+00 | loss scale: 1.0 | grad norm: 0.043 | params norm: 11589.151 | params rms: 0.062751 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 18:39:22.378824 | finish at 2026-10-10 11:58:51 + [2026-10-04 17:20:55] iteration 1018/ 6855 | consumed samples: 130304 | elapsed time per iteration (ms): 86881.7 | throughput per GPU (TFLOP/s/GPU): 41.5 | MFU 13.30% | FLOPs estimator: workload-aware | logical tokens/s: 193103.7 | valid tokens/s: 193103.7 | physical tokens/s: 193708.1 | loss tokens/s: 143948.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.465188E-01 | lm-global-supervised-tokens: 1.250651E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.282990E-02 | lm-token-buffer-mib: 5.819580E+01 | global_load_balancing_loss: 1.000029E+00 | loss scale: 1.0 | grad norm: 0.114 | params norm: 11589.075 | params rms: 0.062751 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 20:52:08.691875 | finish at 2026-10-10 14:13:04 + [2026-10-04 17:22:23] iteration 1019/ 6855 | consumed samples: 130432 | elapsed time per iteration (ms): 88233.1 | throughput per GPU (TFLOP/s/GPU): 41.6 | MFU 13.33% | FLOPs estimator: workload-aware | logical tokens/s: 190144.8 | valid tokens/s: 190144.8 | physical tokens/s: 190612.5 | loss tokens/s: 142265.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.492614E-01 | lm-global-supervised-tokens: 1.255256E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.599638E-02 | lm-token-buffer-mib: 5.810901E+01 | global_load_balancing_loss: 1.000722E+00 | loss scale: 1.0 | grad norm: 0.073 | params norm: 11588.998 | params rms: 0.062751 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 23:02:08.557684 | finish at 2026-10-10 16:24:32 + [2026-10-04 17:24:10] iteration 1020/ 6855 | consumed samples: 130560 | elapsed time per iteration (ms): 106974.6 | throughput per GPU (TFLOP/s/GPU): 34.8 | MFU 11.15% | FLOPs estimator: workload-aware | logical tokens/s: 156833.7 | valid tokens/s: 156833.7 | physical tokens/s: 157373.0 | loss tokens/s: 113240.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.014040E-01 | lm-global-supervised-tokens: 1.211388E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.510757E-02 | lm-token-buffer-mib: 5.805811E+01 | global_load_balancing_loss: 1.000574E+00 | loss scale: 1.0 | grad norm: 0.062 | params norm: 11588.920 | params rms: 0.062750 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 7 days, 5:23:16.709368 | finish at 2026-10-11 22:47:27 + [2026-10-04 17:25:39] iteration 1021/ 6855 | consumed samples: 130688 | elapsed time per iteration (ms): 89107.8 | throughput per GPU (TFLOP/s/GPU): 43.6 | MFU 13.98% | FLOPs estimator: workload-aware | logical tokens/s: 188279.2 | valid tokens/s: 188279.2 | physical tokens/s: 188620.4 | loss tokens/s: 136741.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.600261E-01 | lm-global-supervised-tokens: 1.218470E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.821104E-02 | lm-token-buffer-mib: 5.805421E+01 | global_load_balancing_loss: 1.000287E+00 | loss scale: 1.0 | grad norm: 0.075 | params norm: 11588.843 | params rms: 0.062750 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 0:24:15.105535 | finish at 2026-10-10 17:49:55 + [2026-10-04 17:27:10] iteration 1022/ 6855 | consumed samples: 130816 | elapsed time per iteration (ms): 90408.7 | throughput per GPU (TFLOP/s/GPU): 42.8 | MFU 13.71% | FLOPs estimator: workload-aware | logical tokens/s: 185570.4 | valid tokens/s: 185570.4 | physical tokens/s: 185927.1 | loss tokens/s: 131721.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.263971E-01 | lm-global-supervised-tokens: 1.190878E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.222543E-02 | lm-token-buffer-mib: 5.807829E+01 | global_load_balancing_loss: 9.997995E-01 | loss scale: 1.0 | grad norm: 0.046 | params norm: 11588.765 | params rms: 0.062749 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 2:29:14.225177 | finish at 2026-10-10 19:56:24 + [2026-10-04 17:28:39] iteration 1023/ 6855 | consumed samples: 130944 | elapsed time per iteration (ms): 88797.1 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.61% | FLOPs estimator: workload-aware | logical tokens/s: 188938.5 | valid tokens/s: 188938.5 | physical tokens/s: 189458.9 | loss tokens/s: 134410.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.856278E-01 | lm-global-supervised-tokens: 1.193528E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.134642E-02 | lm-token-buffer-mib: 5.824293E+01 | global_load_balancing_loss: 9.996079E-01 | loss scale: 1.0 | grad norm: 0.066 | params norm: 11588.686 | params rms: 0.062749 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 23:51:04.455385 | finish at 2026-10-10 17:19:43 + [2026-10-04 17:30:11] iteration 1024/ 6855 | consumed samples: 131072 | elapsed time per iteration (ms): 92085.8 | throughput per GPU (TFLOP/s/GPU): 40.9 | MFU 13.12% | FLOPs estimator: workload-aware | logical tokens/s: 182190.6 | valid tokens/s: 182190.6 | physical tokens/s: 182510.6 | loss tokens/s: 135154.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.275335E-01 | lm-global-supervised-tokens: 1.244585E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.351662E-02 | lm-token-buffer-mib: 5.808131E+01 | global_load_balancing_loss: 9.985164E-01 | loss scale: 1.0 | grad norm: 0.048 | params norm: 11588.608 | params rms: 0.062749 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 5:09:12.499598 | finish at 2026-10-10 22:39:23 + [2026-10-04 17:31:42] iteration 1025/ 6855 | consumed samples: 131200 | elapsed time per iteration (ms): 91498.7 | throughput per GPU (TFLOP/s/GPU): 40.9 | MFU 13.10% | FLOPs estimator: workload-aware | logical tokens/s: 183359.8 | valid tokens/s: 183359.8 | physical tokens/s: 183729.4 | loss tokens/s: 135174.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.620021E-01 | lm-global-supervised-tokens: 1.236831E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.178023E-02 | lm-token-buffer-mib: 5.810618E+01 | global_load_balancing_loss: 1.000713E+00 | loss scale: 1.0 | grad norm: 0.045 | params norm: 11588.530 | params rms: 0.062748 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 4:10:37.256420 | finish at 2026-10-10 21:42:19 + [2026-10-04 17:33:12] iteration 1026/ 6855 | consumed samples: 131328 | elapsed time per iteration (ms): 89861.2 | throughput per GPU (TFLOP/s/GPU): 41.5 | MFU 13.30% | FLOPs estimator: workload-aware | logical tokens/s: 186701.5 | valid tokens/s: 186701.5 | physical tokens/s: 187448.6 | loss tokens/s: 137667.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.129135E-01 | lm-global-supervised-tokens: 1.237092E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.162391E-02 | lm-token-buffer-mib: 5.821356E+01 | global_load_balancing_loss: 1.000867E+00 | loss scale: 1.0 | grad norm: 0.073 | params norm: 11588.455 | params rms: 0.062748 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 1:30:00.719939 | finish at 2026-10-10 19:03:13 + [2026-10-04 17:34:39] iteration 1027/ 6855 | consumed samples: 131456 | elapsed time per iteration (ms): 87416.2 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.49% | FLOPs estimator: workload-aware | logical tokens/s: 191922.3 | valid tokens/s: 191922.3 | physical tokens/s: 192288.2 | loss tokens/s: 141609.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.170909E-01 | lm-global-supervised-tokens: 1.237893E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.294215E-02 | lm-token-buffer-mib: 5.816979E+01 | global_load_balancing_loss: 1.001478E+00 | loss scale: 1.0 | grad norm: 0.083 | params norm: 11588.382 | params rms: 0.062747 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 21:31:01.645107 | finish at 2026-10-10 15:05:41 + [2026-10-04 17:36:09] iteration 1028/ 6855 | consumed samples: 131584 | elapsed time per iteration (ms): 89836.3 | throughput per GPU (TFLOP/s/GPU): 41.7 | MFU 13.38% | FLOPs estimator: workload-aware | logical tokens/s: 186753.1 | valid tokens/s: 186753.1 | physical tokens/s: 187155.9 | loss tokens/s: 135867.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.355021E-01 | lm-global-supervised-tokens: 1.220584E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.328426E-02 | lm-token-buffer-mib: 5.805460E+01 | global_load_balancing_loss: 1.001136E+00 | loss scale: 1.0 | grad norm: 0.100 | params norm: 11588.307 | params rms: 0.062747 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 1:24:36.201462 | finish at 2026-10-10 19:00:46 + [2026-10-04 17:37:36] iteration 1029/ 6855 | consumed samples: 131712 | elapsed time per iteration (ms): 86862.8 | throughput per GPU (TFLOP/s/GPU): 42.8 | MFU 13.73% | FLOPs estimator: workload-aware | logical tokens/s: 193146.1 | valid tokens/s: 193146.0 | physical tokens/s: 193682.4 | loss tokens/s: 144485.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.438864E-01 | lm-global-supervised-tokens: 1.255045E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.725575E-02 | lm-token-buffer-mib: 5.807190E+01 | global_load_balancing_loss: 1.000973E+00 | loss scale: 1.0 | grad norm: 0.040 | params norm: 11588.231 | params rms: 0.062747 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 20:34:22.651282 | finish at 2026-10-10 14:11:59 + [2026-10-04 17:39:22] iteration 1030/ 6855 | consumed samples: 131840 | elapsed time per iteration (ms): 105581.6 | throughput per GPU (TFLOP/s/GPU): 34.5 | MFU 11.05% | FLOPs estimator: workload-aware | logical tokens/s: 158902.7 | valid tokens/s: 158902.7 | physical tokens/s: 159296.6 | loss tokens/s: 115620.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.454033E-01 | lm-global-supervised-tokens: 1.220739E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.124926E-02 | lm-token-buffer-mib: 5.814149E+01 | global_load_balancing_loss: 9.992077E-01 | loss scale: 1.0 | grad norm: 0.038 | params norm: 11588.156 | params rms: 0.062746 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 7 days, 2:50:12.532234 | finish at 2026-10-11 20:29:34 + [2026-10-04 17:40:46] iteration 1031/ 6855 | consumed samples: 131968 | elapsed time per iteration (ms): 84566.2 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.73% | FLOPs estimator: workload-aware | logical tokens/s: 198390.9 | valid tokens/s: 198390.9 | physical tokens/s: 198864.2 | loss tokens/s: 146694.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.272700E-01 | lm-global-supervised-tokens: 1.240536E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.687791E-02 | lm-token-buffer-mib: 5.814021E+01 | global_load_balancing_loss: 9.986248E-01 | loss scale: 1.0 | grad norm: 0.040 | params norm: 11588.080 | params rms: 0.062746 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 16:48:33.279526 | finish at 2026-10-10 10:29:20 + [2026-10-04 17:42:12] iteration 1032/ 6855 | consumed samples: 132096 | elapsed time per iteration (ms): 85671.3 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.64% | FLOPs estimator: workload-aware | logical tokens/s: 195832.3 | valid tokens/s: 195832.3 | physical tokens/s: 196535.0 | loss tokens/s: 150172.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.352918E-01 | lm-global-supervised-tokens: 1.286547E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.306438E-02 | lm-token-buffer-mib: 5.819074E+01 | global_load_balancing_loss: 9.990929E-01 | loss scale: 1.0 | grad norm: 0.038 | params norm: 11588.004 | params rms: 0.062745 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 18:34:23.843463 | finish at 2026-10-10 12:16:36 + [2026-10-04 17:43:39] iteration 1033/ 6855 | consumed samples: 132224 | elapsed time per iteration (ms): 86753.7 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.79% | FLOPs estimator: workload-aware | logical tokens/s: 193388.2 | valid tokens/s: 193388.2 | physical tokens/s: 193708.6 | loss tokens/s: 139397.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.536260E-01 | lm-global-supervised-tokens: 1.209323E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.299612E-02 | lm-token-buffer-mib: 5.809796E+01 | global_load_balancing_loss: 1.001470E+00 | loss scale: 1.0 | grad norm: 0.034 | params norm: 11587.927 | params rms: 0.062745 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 20:18:00.184476 | finish at 2026-10-10 14:01:39 + [2026-10-04 17:45:03] iteration 1034/ 6855 | consumed samples: 132352 | elapsed time per iteration (ms): 84104.3 | throughput per GPU (TFLOP/s/GPU): 43.5 | MFU 13.94% | FLOPs estimator: workload-aware | logical tokens/s: 199481.0 | valid tokens/s: 199481.0 | physical tokens/s: 199984.6 | loss tokens/s: 149861.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.323236E-01 | lm-global-supervised-tokens: 1.260402E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.617259E-02 | lm-token-buffer-mib: 5.816647E+01 | global_load_balancing_loss: 1.000136E+00 | loss scale: 1.0 | grad norm: 0.034 | params norm: 11587.849 | params rms: 0.062744 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 15:59:31.309459 | finish at 2026-10-10 09:44:34 + [2026-10-04 17:46:28] iteration 1035/ 6855 | consumed samples: 132480 | elapsed time per iteration (ms): 85439.6 | throughput per GPU (TFLOP/s/GPU): 43.2 | MFU 13.84% | FLOPs estimator: workload-aware | logical tokens/s: 196363.3 | valid tokens/s: 196363.3 | physical tokens/s: 196801.7 | loss tokens/s: 150900.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.709145E-01 | lm-global-supervised-tokens: 1.289286E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.124278E-02 | lm-token-buffer-mib: 5.807263E+01 | global_load_balancing_loss: 9.999760E-01 | loss scale: 1.0 | grad norm: 0.036 | params norm: 11587.772 | params rms: 0.062744 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 18:07:38.316536 | finish at 2026-10-10 11:54:07 + [2026-10-04 17:47:52] iteration 1036/ 6855 | consumed samples: 132608 | elapsed time per iteration (ms): 84166.1 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.75% | FLOPs estimator: workload-aware | logical tokens/s: 199334.4 | valid tokens/s: 199334.4 | physical tokens/s: 199883.3 | loss tokens/s: 150314.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.733148E-01 | lm-global-supervised-tokens: 1.265141E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.294632E-02 | lm-token-buffer-mib: 5.807345E+01 | global_load_balancing_loss: 9.997445E-01 | loss scale: 1.0 | grad norm: 0.052 | params norm: 11587.693 | params rms: 0.062744 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 16:02:42.734439 | finish at 2026-10-10 09:50:35 + [2026-10-04 17:49:18] iteration 1037/ 6855 | consumed samples: 132736 | elapsed time per iteration (ms): 85854.7 | throughput per GPU (TFLOP/s/GPU): 42.8 | MFU 13.71% | FLOPs estimator: workload-aware | logical tokens/s: 195413.1 | valid tokens/s: 195413.1 | physical tokens/s: 195946.1 | loss tokens/s: 142593.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.534804E-01 | lm-global-supervised-tokens: 1.224236E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.239038E-02 | lm-token-buffer-mib: 5.802704E+01 | global_load_balancing_loss: 1.000963E+00 | loss scale: 1.0 | grad norm: 0.043 | params norm: 11587.612 | params rms: 0.062743 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 18:45:02.696438 | finish at 2026-10-10 12:34:21 + [2026-10-04 17:50:43] iteration 1038/ 6855 | consumed samples: 132864 | elapsed time per iteration (ms): 84314.2 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.68% | FLOPs estimator: workload-aware | logical tokens/s: 198984.4 | valid tokens/s: 198984.4 | physical tokens/s: 199449.9 | loss tokens/s: 149733.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.566333E-01 | lm-global-supervised-tokens: 1.262463E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.453979E-02 | lm-token-buffer-mib: 5.806757E+01 | global_load_balancing_loss: 1.000817E+00 | loss scale: 1.0 | grad norm: 0.058 | params norm: 11587.530 | params rms: 0.062743 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 16:14:15.741180 | finish at 2026-10-10 10:04:58 + [2026-10-04 17:52:10] iteration 1039/ 6855 | consumed samples: 132992 | elapsed time per iteration (ms): 87848.0 | throughput per GPU (TFLOP/s/GPU): 41.3 | MFU 13.24% | FLOPs estimator: workload-aware | logical tokens/s: 190980.0 | valid tokens/s: 190980.0 | physical tokens/s: 191532.7 | loss tokens/s: 144349.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.624075E-01 | lm-global-supervised-tokens: 1.268079E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.249862E-02 | lm-token-buffer-mib: 5.808865E+01 | global_load_balancing_loss: 1.001365E+00 | loss scale: 1.0 | grad norm: 0.159 | params norm: 11587.448 | params rms: 0.062742 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 21:55:23.868450 | finish at 2026-10-10 15:47:34 + [2026-10-04 17:54:00] iteration 1040/ 6855 | consumed samples: 133120 | elapsed time per iteration (ms): 109967.6 | throughput per GPU (TFLOP/s/GPU): 33.0 | MFU 10.57% | FLOPs estimator: workload-aware | logical tokens/s: 152564.8 | valid tokens/s: 152564.8 | physical tokens/s: 153080.3 | loss tokens/s: 114007.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.821896E-01 | lm-global-supervised-tokens: 1.253713E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.124628E-02 | lm-token-buffer-mib: 5.812562E+01 | global_load_balancing_loss: 1.000749E+00 | loss scale: 1.0 | grad norm: 0.267 | params norm: 11587.367 | params rms: 0.062742 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 7 days, 9:37:41.679194 | finish at 2026-10-12 03:31:42 +times across ranks (ms): + save-checkpoint: + rank 0: 24777.94 + rank 1: 24777.88 + rank 2: 24777.93 + rank 3: 24777.91 + rank 4: 24777.93 + rank 5: 24777.89 + rank 6: 24777.89 + rank 7: 24777.93 + rank 8: 24777.22 + rank 9: 24777.21 + rank 10: 24777.21 + rank 11: 24777.21 + rank 12: 24777.22 + rank 13: 24777.21 + rank 14: 24777.21 + rank 15: 24777.19 + rank 16: 24777.35 + rank 17: 24777.33 + rank 18: 24777.31 + rank 19: 24777.33 + rank 20: 24777.31 + rank 21: 24777.36 + rank 22: 24777.32 + rank 23: 24777.29 + rank 24: 24780.28 + rank 25: 24780.80 + rank 26: 24780.31 + rank 27: 24780.08 + rank 28: 24779.96 + rank 29: 24780.03 + rank 30: 24779.95 + rank 31: 24780.03 + rank 32: 24777.17 + rank 33: 24777.19 + rank 34: 24777.18 + rank 35: 24777.29 + rank 36: 24777.22 + rank 37: 24777.12 + rank 38: 24777.18 + rank 39: 24777.16 + rank 40: 24777.70 + rank 41: 24777.65 + rank 42: 24777.67 + rank 43: 24777.67 + rank 44: 24777.69 + rank 45: 24777.64 + rank 46: 24777.65 + rank 47: 24777.65 + rank 48: 24777.36 + rank 49: 24777.28 + rank 50: 24777.37 + rank 51: 24777.32 + rank 52: 24777.41 + rank 53: 24777.25 + rank 54: 24777.36 + rank 55: 24777.22 + rank 56: 24776.96 + rank 57: 24776.91 + rank 58: 24776.87 + rank 59: 24776.86 + rank 60: 24776.85 + rank 61: 24776.88 + rank 62: 24776.71 + rank 63: 24776.91 + [2026-10-04 17:56:15] iteration 1041/ 6855 | consumed samples: 133248 | elapsed time per iteration (ms): 109092.4 | throughput per GPU (TFLOP/s/GPU): 33.9 | MFU 10.87% | FLOPs estimator: workload-aware | logical tokens/s: 153788.3 | valid tokens/s: 153788.3 | physical tokens/s: 154193.8 | loss tokens/s: 114033.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.005199E-01 | lm-global-supervised-tokens: 1.244013E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.566623E+00 | lm-token-buffer-mib: 5.811021E+01 | global_load_balancing_loss: 1.000230E+00 | loss scale: 1.0 | grad norm: 0.341 | params norm: 11587.286 | params rms: 0.062741 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 7 days, 8:11:02.974224 | finish at 2026-10-12 02:07:18 + [2026-10-04 17:57:46] iteration 1042/ 6855 | consumed samples: 133376 | elapsed time per iteration (ms): 91180.2 | throughput per GPU (TFLOP/s/GPU): 40.0 | MFU 12.81% | FLOPs estimator: workload-aware | logical tokens/s: 184000.3 | valid tokens/s: 184000.3 | physical tokens/s: 184415.5 | loss tokens/s: 141074.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.237653E-01 | lm-global-supervised-tokens: 1.286319E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.417528E-02 | lm-token-buffer-mib: 5.817755E+01 | global_load_balancing_loss: 9.989265E-01 | loss scale: 1.0 | grad norm: 0.280 | params norm: 11587.201 | params rms: 0.062741 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 3:13:50.439428 | finish at 2026-10-10 21:11:36 + [2026-10-04 17:59:13] iteration 1043/ 6855 | consumed samples: 133504 | elapsed time per iteration (ms): 87531.6 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.62% | FLOPs estimator: workload-aware | logical tokens/s: 191669.5 | valid tokens/s: 191669.5 | physical tokens/s: 192043.1 | loss tokens/s: 141312.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.748707E-01 | lm-global-supervised-tokens: 1.236930E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.200561E-02 | lm-token-buffer-mib: 5.816765E+01 | global_load_balancing_loss: 9.994232E-01 | loss scale: 1.0 | grad norm: 0.051 | params norm: 11587.115 | params rms: 0.062740 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 21:18:53.720161 | finish at 2026-10-10 15:18:07 + [2026-10-04 18:00:36] iteration 1044/ 6855 | consumed samples: 133632 | elapsed time per iteration (ms): 82340.7 | throughput per GPU (TFLOP/s/GPU): 44.3 | MFU 14.20% | FLOPs estimator: workload-aware | logical tokens/s: 203753.1 | valid tokens/s: 203753.1 | physical tokens/s: 204254.9 | loss tokens/s: 157172.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.767254E-01 | lm-global-supervised-tokens: 1.294166E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.310387E-02 | lm-token-buffer-mib: 5.809748E+01 | global_load_balancing_loss: 9.995781E-01 | loss scale: 1.0 | grad norm: 0.131 | params norm: 11587.031 | params rms: 0.062740 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 12:54:41.927718 | finish at 2026-10-10 06:55:18 + [2026-10-04 18:02:06] iteration 1045/ 6855 | consumed samples: 133760 | elapsed time per iteration (ms): 90126.8 | throughput per GPU (TFLOP/s/GPU): 41.1 | MFU 13.16% | FLOPs estimator: workload-aware | logical tokens/s: 186150.6 | valid tokens/s: 186150.6 | physical tokens/s: 186687.5 | loss tokens/s: 137633.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.163283E-01 | lm-global-supervised-tokens: 1.240447E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.169250E-02 | lm-token-buffer-mib: 5.816235E+01 | global_load_balancing_loss: 9.992684E-01 | loss scale: 1.0 | grad norm: 0.150 | params norm: 11586.947 | params rms: 0.062740 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 1:27:16.942451 | finish at 2026-10-10 19:29:23 + [2026-10-04 18:03:32] iteration 1046/ 6855 | consumed samples: 133888 | elapsed time per iteration (ms): 86633.9 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.59% | FLOPs estimator: workload-aware | logical tokens/s: 193655.9 | valid tokens/s: 193655.9 | physical tokens/s: 194241.5 | loss tokens/s: 146218.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.583671E-01 | lm-global-supervised-tokens: 1.266746E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.246628E-02 | lm-token-buffer-mib: 5.802827E+01 | global_load_balancing_loss: 1.000898E+00 | loss scale: 1.0 | grad norm: 0.078 | params norm: 11586.862 | params rms: 0.062739 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 19:47:36.513033 | finish at 2026-10-10 13:51:09 + [2026-10-04 18:05:01] iteration 1047/ 6855 | consumed samples: 134016 | elapsed time per iteration (ms): 88597.5 | throughput per GPU (TFLOP/s/GPU): 43.1 | MFU 13.81% | FLOPs estimator: workload-aware | logical tokens/s: 189364.4 | valid tokens/s: 189364.4 | physical tokens/s: 189679.2 | loss tokens/s: 137526.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.867118E-01 | lm-global-supervised-tokens: 1.218447E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.224600E-02 | lm-token-buffer-mib: 5.809276E+01 | global_load_balancing_loss: 1.001572E+00 | loss scale: 1.0 | grad norm: 0.085 | params norm: 11586.778 | params rms: 0.062739 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 22:56:14.226494 | finish at 2026-10-10 17:01:15 + [2026-10-04 18:06:29] iteration 1048/ 6855 | consumed samples: 134144 | elapsed time per iteration (ms): 87842.0 | throughput per GPU (TFLOP/s/GPU): 41.4 | MFU 13.28% | FLOPs estimator: workload-aware | logical tokens/s: 190991.7 | valid tokens/s: 190991.7 | physical tokens/s: 191448.0 | loss tokens/s: 140454.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.242329E-01 | lm-global-supervised-tokens: 1.233783E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.451816E-02 | lm-token-buffer-mib: 5.821867E+01 | global_load_balancing_loss: 9.997790E-01 | loss scale: 1.0 | grad norm: 0.206 | params norm: 11586.695 | params rms: 0.062738 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 21:41:38.294677 | finish at 2026-10-10 15:48:07 + [2026-10-04 18:07:56] iteration 1049/ 6855 | consumed samples: 134272 | elapsed time per iteration (ms): 86840.9 | throughput per GPU (TFLOP/s/GPU): 41.6 | MFU 13.34% | FLOPs estimator: workload-aware | logical tokens/s: 193193.9 | valid tokens/s: 193193.9 | physical tokens/s: 193599.6 | loss tokens/s: 146391.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.817073E-01 | lm-global-supervised-tokens: 1.271276E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.205228E-02 | lm-token-buffer-mib: 5.805481E+01 | global_load_balancing_loss: 1.000784E+00 | loss scale: 1.0 | grad norm: 0.092 | params norm: 11586.612 | params rms: 0.062738 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 20:03:18.476868 | finish at 2026-10-10 14:11:14 + [2026-10-04 18:09:46] iteration 1050/ 6855 | consumed samples: 134400 | elapsed time per iteration (ms): 110041.8 | throughput per GPU (TFLOP/s/GPU): 34.1 | MFU 10.94% | FLOPs estimator: workload-aware | logical tokens/s: 152461.9 | valid tokens/s: 152461.9 | physical tokens/s: 152858.5 | loss tokens/s: 108311.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.033826E-01 | lm-global-supervised-tokens: 1.191879E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.275182E-02 | lm-token-buffer-mib: 5.805254E+01 | global_load_balancing_loss: 1.000176E+00 | loss scale: 1.0 | grad norm: 0.149 | params norm: 11586.528 | params rms: 0.062737 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 7 days, 9:26:32.476126 | finish at 2026-10-12 03:36:18 + [2026-10-04 18:11:11] iteration 1051/ 6855 | consumed samples: 134528 | elapsed time per iteration (ms): 85676.3 | throughput per GPU (TFLOP/s/GPU): 42.8 | MFU 13.71% | FLOPs estimator: workload-aware | logical tokens/s: 195819.6 | valid tokens/s: 195819.6 | physical tokens/s: 196268.7 | loss tokens/s: 143354.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.255772E-01 | lm-global-supervised-tokens: 1.228205E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.419739E-02 | lm-token-buffer-mib: 5.822681E+01 | global_load_balancing_loss: 1.000561E+00 | loss scale: 1.0 | grad norm: 0.095 | params norm: 11586.443 | params rms: 0.062737 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 18:07:44.988100 | finish at 2026-10-10 12:18:56 + [2026-10-04 18:12:40] iteration 1052/ 6855 | consumed samples: 134656 | elapsed time per iteration (ms): 89013.9 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.52% | FLOPs estimator: workload-aware | logical tokens/s: 188478.2 | valid tokens/s: 188478.2 | physical tokens/s: 189000.6 | loss tokens/s: 140020.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.188208E-01 | lm-global-supervised-tokens: 1.246376E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.169539E-02 | lm-token-buffer-mib: 5.808138E+01 | global_load_balancing_loss: 9.999647E-01 | loss scale: 1.0 | grad norm: 0.066 | params norm: 11586.360 | params rms: 0.062736 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 23:29:07.757406 | finish at 2026-10-10 17:41:48 + [2026-10-04 18:14:10] iteration 1053/ 6855 | consumed samples: 134784 | elapsed time per iteration (ms): 89059.9 | throughput per GPU (TFLOP/s/GPU): 41.7 | MFU 13.35% | FLOPs estimator: workload-aware | logical tokens/s: 188380.9 | valid tokens/s: 188380.9 | physical tokens/s: 188835.8 | loss tokens/s: 141423.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.967087E-01 | lm-global-supervised-tokens: 1.259519E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.849126E-02 | lm-token-buffer-mib: 5.813052E+01 | global_load_balancing_loss: 9.991355E-01 | loss scale: 1.0 | grad norm: 0.065 | params norm: 11586.275 | params rms: 0.062736 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 23:32:05.563580 | finish at 2026-10-10 17:46:15 + [2026-10-04 18:15:35] iteration 1054/ 6855 | consumed samples: 134912 | elapsed time per iteration (ms): 85738.3 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.75% | FLOPs estimator: workload-aware | logical tokens/s: 195679.5 | valid tokens/s: 195679.5 | physical tokens/s: 196156.3 | loss tokens/s: 150584.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.512131E-01 | lm-global-supervised-tokens: 1.291084E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.182472E-02 | lm-token-buffer-mib: 5.825521E+01 | global_load_balancing_loss: 9.998986E-01 | loss scale: 1.0 | grad norm: 0.077 | params norm: 11586.191 | params rms: 0.062735 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 18:09:27.621543 | finish at 2026-10-10 12:25:03 + [2026-10-04 18:17:02] iteration 1055/ 6855 | consumed samples: 135040 | elapsed time per iteration (ms): 87008.8 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.63% | FLOPs estimator: workload-aware | logical tokens/s: 192821.7 | valid tokens/s: 192821.7 | physical tokens/s: 193222.1 | loss tokens/s: 143611.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.541844E-01 | lm-global-supervised-tokens: 1.249549E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.535155E-02 | lm-token-buffer-mib: 5.821735E+01 | global_load_balancing_loss: 9.997517E-01 | loss scale: 1.0 | grad norm: 0.047 | params norm: 11586.109 | params rms: 0.062735 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 20:10:51.218557 | finish at 2026-10-10 14:27:54 + [2026-10-04 18:18:30] iteration 1056/ 6855 | consumed samples: 135168 | elapsed time per iteration (ms): 88149.7 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.53% | FLOPs estimator: workload-aware | logical tokens/s: 190325.8 | valid tokens/s: 190325.8 | physical tokens/s: 190708.7 | loss tokens/s: 140757.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.183645E-01 | lm-global-supervised-tokens: 1.240770E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.264368E-02 | lm-token-buffer-mib: 5.809585E+01 | global_load_balancing_loss: 9.994171E-01 | loss scale: 1.0 | grad norm: 0.065 | params norm: 11586.027 | params rms: 0.062735 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 21:59:40.033831 | finish at 2026-10-10 16:18:10 + [2026-10-04 18:19:56] iteration 1057/ 6855 | consumed samples: 135296 | elapsed time per iteration (ms): 85899.0 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.75% | FLOPs estimator: workload-aware | logical tokens/s: 195313.3 | valid tokens/s: 195313.3 | physical tokens/s: 195773.6 | loss tokens/s: 148340.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.028778E-01 | lm-global-supervised-tokens: 1.274232E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.341027E-02 | lm-token-buffer-mib: 5.807449E+01 | global_load_balancing_loss: 1.000464E+00 | loss scale: 1.0 | grad norm: 0.055 | params norm: 11585.944 | params rms: 0.062734 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 18:20:42.274415 | finish at 2026-10-10 12:40:39 + [2026-10-04 18:21:24] iteration 1058/ 6855 | consumed samples: 135424 | elapsed time per iteration (ms): 87888.4 | throughput per GPU (TFLOP/s/GPU): 41.7 | MFU 13.36% | FLOPs estimator: workload-aware | logical tokens/s: 190891.9 | valid tokens/s: 190891.9 | physical tokens/s: 191488.2 | loss tokens/s: 144194.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.831421E-01 | lm-global-supervised-tokens: 1.267299E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.174393E-02 | lm-token-buffer-mib: 5.808961E+01 | global_load_balancing_loss: 1.001182E+00 | loss scale: 1.0 | grad norm: 0.053 | params norm: 11585.864 | params rms: 0.062734 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 21:31:28.936389 | finish at 2026-10-10 15:52:53 + [2026-10-04 18:22:53] iteration 1059/ 6855 | consumed samples: 135552 | elapsed time per iteration (ms): 89265.4 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.49% | FLOPs estimator: workload-aware | logical tokens/s: 187947.2 | valid tokens/s: 187947.2 | physical tokens/s: 188484.4 | loss tokens/s: 143671.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.229179E-01 | lm-global-supervised-tokens: 1.282488E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.371019E-02 | lm-token-buffer-mib: 5.814426E+01 | global_load_balancing_loss: 1.001176E+00 | loss scale: 1.0 | grad norm: 0.076 | params norm: 11585.786 | params rms: 0.062733 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 23:43:02.335396 | finish at 2026-10-10 18:05:56 + [2026-10-04 18:24:42] iteration 1060/ 6855 | consumed samples: 135680 | elapsed time per iteration (ms): 108798.3 | throughput per GPU (TFLOP/s/GPU): 34.9 | MFU 11.18% | FLOPs estimator: workload-aware | logical tokens/s: 154204.6 | valid tokens/s: 154204.6 | physical tokens/s: 154616.4 | loss tokens/s: 110501.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.818570E-01 | lm-global-supervised-tokens: 1.202234E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.048372E-02 | lm-token-buffer-mib: 5.813558E+01 | global_load_balancing_loss: 1.001724E+00 | loss scale: 1.0 | grad norm: 0.075 | params norm: 11585.707 | params rms: 0.062733 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 7 days, 7:08:06.210835 | finish at 2026-10-12 01:32:49 + [2026-10-04 18:26:08] iteration 1061/ 6855 | consumed samples: 135808 | elapsed time per iteration (ms): 85945.4 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.76% | FLOPs estimator: workload-aware | logical tokens/s: 195207.6 | valid tokens/s: 195207.6 | physical tokens/s: 195600.5 | loss tokens/s: 145058.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.647373E-01 | lm-global-supervised-tokens: 1.246708E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.175494E-02 | lm-token-buffer-mib: 5.803410E+01 | global_load_balancing_loss: 9.995825E-01 | loss scale: 1.0 | grad norm: 0.084 | params norm: 11585.626 | params rms: 0.062732 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 18:19:27.624114 | finish at 2026-10-10 12:45:36 + [2026-10-04 18:27:34] iteration 1062/ 6855 | consumed samples: 135936 | elapsed time per iteration (ms): 86018.1 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.68% | FLOPs estimator: workload-aware | logical tokens/s: 195042.3 | valid tokens/s: 195042.2 | physical tokens/s: 195545.2 | loss tokens/s: 144399.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.536018E-01 | lm-global-supervised-tokens: 1.242096E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.234829E-02 | lm-token-buffer-mib: 5.807881E+01 | global_load_balancing_loss: 1.000627E+00 | loss scale: 1.0 | grad norm: 0.055 | params norm: 11585.546 | params rms: 0.062732 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 18:25:02.958403 | finish at 2026-10-10 12:52:37 + [2026-10-04 18:29:02] iteration 1063/ 6855 | consumed samples: 136064 | elapsed time per iteration (ms): 87610.7 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.64% | FLOPs estimator: workload-aware | logical tokens/s: 191496.8 | valid tokens/s: 191496.8 | physical tokens/s: 192150.3 | loss tokens/s: 140128.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.730707E-01 | lm-global-supervised-tokens: 1.227676E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.358397E-02 | lm-token-buffer-mib: 5.823342E+01 | global_load_balancing_loss: 9.988704E-01 | loss scale: 1.0 | grad norm: 0.118 | params norm: 11585.466 | params rms: 0.062732 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 20:57:21.115776 | finish at 2026-10-10 15:26:23 + [2026-10-04 18:30:33] iteration 1064/ 6855 | consumed samples: 136192 | elapsed time per iteration (ms): 91110.9 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.51% | FLOPs estimator: workload-aware | logical tokens/s: 184140.4 | valid tokens/s: 184140.4 | physical tokens/s: 184517.7 | loss tokens/s: 138461.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.265479E-01 | lm-global-supervised-tokens: 1.261537E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.180407E-02 | lm-token-buffer-mib: 5.801553E+01 | global_load_balancing_loss: 9.993979E-01 | loss scale: 1.0 | grad norm: 0.076 | params norm: 11585.386 | params rms: 0.062731 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 2:33:43.174524 | finish at 2026-10-10 21:04:16 + [2026-10-04 18:32:01] iteration 1065/ 6855 | consumed samples: 136320 | elapsed time per iteration (ms): 87581.4 | throughput per GPU (TFLOP/s/GPU): 41.4 | MFU 13.26% | FLOPs estimator: workload-aware | logical tokens/s: 191560.6 | valid tokens/s: 191560.6 | physical tokens/s: 192062.6 | loss tokens/s: 139944.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.606151E-01 | lm-global-supervised-tokens: 1.225657E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.153558E-02 | lm-token-buffer-mib: 5.805050E+01 | global_load_balancing_loss: 9.995678E-01 | loss scale: 1.0 | grad norm: 0.103 | params norm: 11585.305 | params rms: 0.062731 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 20:51:36.171620 | finish at 2026-10-10 15:23:37 + [2026-10-04 18:33:26] iteration 1066/ 6855 | consumed samples: 136448 | elapsed time per iteration (ms): 85847.0 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.43% | FLOPs estimator: workload-aware | logical tokens/s: 195431.5 | valid tokens/s: 195431.5 | physical tokens/s: 195785.2 | loss tokens/s: 144641.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.359752E-01 | lm-global-supervised-tokens: 1.241706E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.164555E-02 | lm-token-buffer-mib: 5.805330E+01 | global_load_balancing_loss: 9.998283E-01 | loss scale: 1.0 | grad norm: 0.111 | params norm: 11585.225 | params rms: 0.062730 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 18:02:48.210556 | finish at 2026-10-10 12:36:15 + [2026-10-04 18:34:51] iteration 1067/ 6855 | consumed samples: 136576 | elapsed time per iteration (ms): 84997.9 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.68% | FLOPs estimator: workload-aware | logical tokens/s: 197383.9 | valid tokens/s: 197383.8 | physical tokens/s: 197895.6 | loss tokens/s: 156599.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.284780E-01 | lm-global-supervised-tokens: 1.331058E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.750261E-02 | lm-token-buffer-mib: 5.819253E+01 | global_load_balancing_loss: 1.000070E+00 | loss scale: 1.0 | grad norm: 0.087 | params norm: 11585.146 | params rms: 0.062730 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 16:39:27.565120 | finish at 2026-10-10 11:14:19 + [2026-10-04 18:36:20] iteration 1068/ 6855 | consumed samples: 136704 | elapsed time per iteration (ms): 88219.6 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.50% | FLOPs estimator: workload-aware | logical tokens/s: 190175.3 | valid tokens/s: 190175.3 | physical tokens/s: 190497.2 | loss tokens/s: 134735.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.474166E-01 | lm-global-supervised-tokens: 1.188632E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.397686E-02 | lm-token-buffer-mib: 5.810229E+01 | global_load_balancing_loss: 1.001488E+00 | loss scale: 1.0 | grad norm: 0.150 | params norm: 11585.068 | params rms: 0.062729 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 21:48:46.612503 | finish at 2026-10-10 16:25:06 + [2026-10-04 18:37:46] iteration 1069/ 6855 | consumed samples: 136832 | elapsed time per iteration (ms): 86002.4 | throughput per GPU (TFLOP/s/GPU): 43.7 | MFU 13.99% | FLOPs estimator: workload-aware | logical tokens/s: 195078.1 | valid tokens/s: 195078.1 | physical tokens/s: 195610.0 | loss tokens/s: 142933.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.178212E-01 | lm-global-supervised-tokens: 1.229257E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.535834E-02 | lm-token-buffer-mib: 5.816577E+01 | global_load_balancing_loss: 1.000955E+00 | loss scale: 1.0 | grad norm: 0.052 | params norm: 11584.989 | params rms: 0.062729 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 18:13:29.770068 | finish at 2026-10-10 12:51:15 + [2026-10-04 18:39:32] iteration 1070/ 6855 | consumed samples: 136960 | elapsed time per iteration (ms): 106513.2 | throughput per GPU (TFLOP/s/GPU): 34.4 | MFU 11.02% | FLOPs estimator: workload-aware | logical tokens/s: 157512.8 | valid tokens/s: 157512.8 | physical tokens/s: 157847.7 | loss tokens/s: 120357.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.433236E-01 | lm-global-supervised-tokens: 1.281968E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.319155E-02 | lm-token-buffer-mib: 5.807827E+01 | global_load_balancing_loss: 1.000164E+00 | loss scale: 1.0 | grad norm: 0.110 | params norm: 11584.908 | params rms: 0.062729 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 7 days, 3:09:38.876051 | finish at 2026-10-11 21:49:11 + [2026-10-04 18:41:03] iteration 1071/ 6855 | consumed samples: 137088 | elapsed time per iteration (ms): 90530.5 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.58% | FLOPs estimator: workload-aware | logical tokens/s: 185321.0 | valid tokens/s: 185321.0 | physical tokens/s: 185931.3 | loss tokens/s: 130755.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.634502E-01 | lm-global-supervised-tokens: 1.183735E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.240989E-02 | lm-token-buffer-mib: 5.805904E+01 | global_load_balancing_loss: 1.000923E+00 | loss scale: 1.0 | grad norm: 0.105 | params norm: 11584.828 | params rms: 0.062728 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 1:27:08.243385 | finish at 2026-10-10 20:08:11 + [2026-10-04 18:42:30] iteration 1072/ 6855 | consumed samples: 137216 | elapsed time per iteration (ms): 87396.4 | throughput per GPU (TFLOP/s/GPU): 43.4 | MFU 13.92% | FLOPs estimator: workload-aware | logical tokens/s: 191966.7 | valid tokens/s: 191966.6 | physical tokens/s: 192368.9 | loss tokens/s: 142940.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.973342E-01 | lm-global-supervised-tokens: 1.249250E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.016667E-02 | lm-token-buffer-mib: 5.812875E+01 | global_load_balancing_loss: 9.994065E-01 | loss scale: 1.0 | grad norm: 0.086 | params norm: 11584.748 | params rms: 0.062728 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 20:23:33.459645 | finish at 2026-10-10 15:06:04 + [2026-10-04 18:43:56] iteration 1073/ 6855 | consumed samples: 137344 | elapsed time per iteration (ms): 85515.1 | throughput per GPU (TFLOP/s/GPU): 42.8 | MFU 13.71% | FLOPs estimator: workload-aware | logical tokens/s: 196189.7 | valid tokens/s: 196189.7 | physical tokens/s: 196563.2 | loss tokens/s: 140789.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.248406E-01 | lm-global-supervised-tokens: 1.203959E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.317294E-02 | lm-token-buffer-mib: 5.804529E+01 | global_load_balancing_loss: 9.999419E-01 | loss scale: 1.0 | grad norm: 0.169 | params norm: 11584.668 | params rms: 0.062727 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 17:20:48.370247 | finish at 2026-10-10 12:04:44 + [2026-10-04 18:45:23] iteration 1074/ 6855 | consumed samples: 137472 | elapsed time per iteration (ms): 87619.7 | throughput per GPU (TFLOP/s/GPU): 42.8 | MFU 13.72% | FLOPs estimator: workload-aware | logical tokens/s: 191477.2 | valid tokens/s: 191477.2 | physical tokens/s: 191858.9 | loss tokens/s: 139439.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.746633E-01 | lm-global-supervised-tokens: 1.221764E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.828893E-02 | lm-token-buffer-mib: 5.816452E+01 | global_load_balancing_loss: 9.995707E-01 | loss scale: 1.0 | grad norm: 0.098 | params norm: 11584.589 | params rms: 0.062727 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 20:42:09.699076 | finish at 2026-10-10 15:27:33 + [2026-10-04 18:46:49] iteration 1075/ 6855 | consumed samples: 137600 | elapsed time per iteration (ms): 85507.7 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.65% | FLOPs estimator: workload-aware | logical tokens/s: 196206.6 | valid tokens/s: 196206.5 | physical tokens/s: 196855.8 | loss tokens/s: 149288.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.729781E-01 | lm-global-supervised-tokens: 1.276533E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.578276E-02 | lm-token-buffer-mib: 5.822579E+01 | global_load_balancing_loss: 9.992581E-01 | loss scale: 1.0 | grad norm: 0.299 | params norm: 11584.509 | params rms: 0.062726 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 17:17:14.544392 | finish at 2026-10-10 12:04:03 + [2026-10-04 18:48:15] iteration 1076/ 6855 | consumed samples: 137728 | elapsed time per iteration (ms): 86604.0 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.48% | FLOPs estimator: workload-aware | logical tokens/s: 193722.2 | valid tokens/s: 193722.1 | physical tokens/s: 194281.3 | loss tokens/s: 143475.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.054938E-01 | lm-global-supervised-tokens: 1.242550E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.435752E-02 | lm-token-buffer-mib: 5.837365E+01 | global_load_balancing_loss: 1.000248E+00 | loss scale: 1.0 | grad norm: 0.096 | params norm: 11584.432 | params rms: 0.062726 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 19:01:24.453149 | finish at 2026-10-10 13:49:40 + [2026-10-04 18:49:40] iteration 1077/ 6855 | consumed samples: 137856 | elapsed time per iteration (ms): 84855.7 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.54% | FLOPs estimator: workload-aware | logical tokens/s: 197713.6 | valid tokens/s: 197713.6 | physical tokens/s: 198307.9 | loss tokens/s: 145647.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.949861E-01 | lm-global-supervised-tokens: 1.235902E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.252988E-02 | lm-token-buffer-mib: 5.822935E+01 | global_load_balancing_loss: 1.000757E+00 | loss scale: 1.0 | grad norm: 0.310 | params norm: 11584.356 | params rms: 0.062726 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 16:11:36.231937 | finish at 2026-10-10 11:01:16 + [2026-10-04 18:51:06] iteration 1078/ 6855 | consumed samples: 137984 | elapsed time per iteration (ms): 85607.0 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.63% | FLOPs estimator: workload-aware | logical tokens/s: 195979.4 | valid tokens/s: 195979.4 | physical tokens/s: 196451.9 | loss tokens/s: 145631.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.818111E-01 | lm-global-supervised-tokens: 1.246710E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.248886E-02 | lm-token-buffer-mib: 5.803355E+01 | global_load_balancing_loss: 1.001133E+00 | loss scale: 1.0 | grad norm: 0.050 | params norm: 11584.279 | params rms: 0.062725 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 17:22:31.924760 | finish at 2026-10-10 12:13:38 + [2026-10-04 18:52:34] iteration 1079/ 6855 | consumed samples: 138112 | elapsed time per iteration (ms): 88585.6 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.46% | FLOPs estimator: workload-aware | logical tokens/s: 189389.9 | valid tokens/s: 189389.9 | physical tokens/s: 189838.8 | loss tokens/s: 146171.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.274153E-01 | lm-global-supervised-tokens: 1.294867E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.216878E-02 | lm-token-buffer-mib: 5.813203E+01 | global_load_balancing_loss: 1.000359E+00 | loss scale: 1.0 | grad norm: 0.295 | params norm: 11584.199 | params rms: 0.062725 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 22:07:50.277668 | finish at 2026-10-10 17:00:25 + [2026-10-04 18:54:24] iteration 1080/ 6855 | consumed samples: 138240 | elapsed time per iteration (ms): 110086.4 | throughput per GPU (TFLOP/s/GPU): 34.1 | MFU 10.92% | FLOPs estimator: workload-aware | logical tokens/s: 152399.8 | valid tokens/s: 152399.8 | physical tokens/s: 152816.2 | loss tokens/s: 106979.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.116220E-01 | lm-global-supervised-tokens: 1.177694E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.555648E-02 | lm-token-buffer-mib: 5.805354E+01 | global_load_balancing_loss: 1.001414E+00 | loss scale: 1.0 | grad norm: 0.072 | params norm: 11584.118 | params rms: 0.062724 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 7 days, 8:35:49.223167 | finish at 2026-10-12 03:30:14 +times across ranks (ms): + save-checkpoint: + rank 0: 25080.00 + rank 1: 25080.15 + rank 2: 25080.19 + rank 3: 25080.15 + rank 4: 25080.19 + rank 5: 25080.14 + rank 6: 25080.15 + rank 7: 25080.17 + rank 8: 25080.75 + rank 9: 25080.69 + rank 10: 25080.69 + rank 11: 25080.74 + rank 12: 25080.72 + rank 13: 25080.72 + rank 14: 25080.71 + rank 15: 25080.70 + rank 16: 25080.51 + rank 17: 25080.54 + rank 18: 25080.47 + rank 19: 25080.50 + rank 20: 25080.47 + rank 21: 25080.51 + rank 22: 25080.57 + rank 23: 25080.49 + rank 24: 25080.62 + rank 25: 25080.75 + rank 26: 25080.62 + rank 27: 25080.55 + rank 28: 25080.64 + rank 29: 25080.63 + rank 30: 25080.61 + rank 31: 25080.58 + rank 32: 25081.33 + rank 33: 25081.40 + rank 34: 25081.32 + rank 35: 25081.30 + rank 36: 25081.13 + rank 37: 25081.29 + rank 38: 25081.16 + rank 39: 25081.26 + rank 40: 25079.97 + rank 41: 25079.97 + rank 42: 25079.97 + rank 43: 25080.03 + rank 44: 25079.95 + rank 45: 25079.96 + rank 46: 25079.96 + rank 47: 25079.95 + rank 48: 25080.54 + rank 49: 25080.54 + rank 50: 25080.51 + rank 51: 25080.41 + rank 52: 25080.61 + rank 53: 25080.49 + rank 54: 25080.62 + rank 55: 25080.60 + rank 56: 25078.82 + rank 57: 25078.81 + rank 58: 25078.79 + rank 59: 25078.82 + rank 60: 25078.79 + rank 61: 25078.82 + rank 62: 25078.81 + rank 63: 25078.82 + [2026-10-04 18:56:35] iteration 1081/ 6855 | consumed samples: 138368 | elapsed time per iteration (ms): 105703.5 | throughput per GPU (TFLOP/s/GPU): 35.0 | MFU 11.21% | FLOPs estimator: workload-aware | logical tokens/s: 158719.1 | valid tokens/s: 158719.1 | physical tokens/s: 159096.1 | loss tokens/s: 115576.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.506351E-01 | lm-global-supervised-tokens: 1.221684E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 7.870927E-01 | lm-token-buffer-mib: 5.808122E+01 | global_load_balancing_loss: 1.000943E+00 | loss scale: 1.0 | grad norm: 0.087 | params norm: 11584.041 | params rms: 0.062724 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 7 days, 1:32:12.175759 | finish at 2026-10-11 20:28:48 + [2026-10-04 18:58:06] iteration 1082/ 6855 | consumed samples: 138496 | elapsed time per iteration (ms): 90853.0 | throughput per GPU (TFLOP/s/GPU): 40.2 | MFU 12.90% | FLOPs estimator: workload-aware | logical tokens/s: 184662.8 | valid tokens/s: 184662.8 | physical tokens/s: 185027.3 | loss tokens/s: 138435.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.393578E-01 | lm-global-supervised-tokens: 1.257728E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.935448E-02 | lm-token-buffer-mib: 5.804208E+01 | global_load_balancing_loss: 9.998280E-01 | loss scale: 1.0 | grad norm: 0.068 | params norm: 11583.963 | params rms: 0.062723 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 1:41:34.463542 | finish at 2026-10-10 20:39:41 + [2026-10-04 18:59:32] iteration 1083/ 6855 | consumed samples: 138624 | elapsed time per iteration (ms): 85801.6 | throughput per GPU (TFLOP/s/GPU): 43.1 | MFU 13.81% | FLOPs estimator: workload-aware | logical tokens/s: 195534.5 | valid tokens/s: 195534.5 | physical tokens/s: 195880.6 | loss tokens/s: 146559.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.902934E-01 | lm-global-supervised-tokens: 1.257500E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.286838E-02 | lm-token-buffer-mib: 5.805431E+01 | global_load_balancing_loss: 9.979754E-01 | loss scale: 1.0 | grad norm: 0.102 | params norm: 11583.884 | params rms: 0.062723 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 17:34:06.650677 | finish at 2026-10-10 12:33:39 + [2026-10-04 19:00:58] iteration 1084/ 6855 | consumed samples: 138752 | elapsed time per iteration (ms): 86370.4 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.51% | FLOPs estimator: workload-aware | logical tokens/s: 194247.2 | valid tokens/s: 194247.2 | physical tokens/s: 194697.6 | loss tokens/s: 146262.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.772695E-01 | lm-global-supervised-tokens: 1.263272E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.163420E-02 | lm-token-buffer-mib: 5.813721E+01 | global_load_balancing_loss: 9.986526E-01 | loss scale: 1.0 | grad norm: 0.075 | params norm: 11583.805 | params rms: 0.062723 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 18:27:23.664805 | finish at 2026-10-10 13:28:22 + [2026-10-04 19:02:28] iteration 1085/ 6855 | consumed samples: 138880 | elapsed time per iteration (ms): 89389.5 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.42% | FLOPs estimator: workload-aware | logical tokens/s: 187686.2 | valid tokens/s: 187686.2 | physical tokens/s: 188151.2 | loss tokens/s: 136842.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.192108E-01 | lm-global-supervised-tokens: 1.223231E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.314068E-02 | lm-token-buffer-mib: 5.831204E+01 | global_load_balancing_loss: 9.987027E-01 | loss scale: 1.0 | grad norm: 0.061 | params norm: 11583.727 | params rms: 0.062722 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 23:16:17.682695 | finish at 2026-10-10 18:18:46 + [2026-10-04 19:03:57] iteration 1086/ 6855 | consumed samples: 139008 | elapsed time per iteration (ms): 89032.6 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.64% | FLOPs estimator: workload-aware | logical tokens/s: 188438.3 | valid tokens/s: 188438.3 | physical tokens/s: 188777.5 | loss tokens/s: 138533.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.044861E-01 | lm-global-supervised-tokens: 1.233397E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.168261E-02 | lm-token-buffer-mib: 5.810301E+01 | global_load_balancing_loss: 9.996459E-01 | loss scale: 1.0 | grad norm: 0.080 | params norm: 11583.649 | params rms: 0.062722 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 22:40:29.272538 | finish at 2026-10-10 17:44:26 + [2026-10-04 19:05:25] iteration 1087/ 6855 | consumed samples: 139136 | elapsed time per iteration (ms): 87609.8 | throughput per GPU (TFLOP/s/GPU): 41.7 | MFU 13.36% | FLOPs estimator: workload-aware | logical tokens/s: 191498.8 | valid tokens/s: 191498.8 | physical tokens/s: 192128.3 | loss tokens/s: 138832.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.418322E-01 | lm-global-supervised-tokens: 1.216312E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.933918E-02 | lm-token-buffer-mib: 5.813297E+01 | global_load_balancing_loss: 1.001140E+00 | loss scale: 1.0 | grad norm: 0.187 | params norm: 11583.573 | params rms: 0.062721 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 20:22:13.266470 | finish at 2026-10-10 15:27:38 + [2026-10-04 19:06:49] iteration 1088/ 6855 | consumed samples: 139264 | elapsed time per iteration (ms): 84224.8 | throughput per GPU (TFLOP/s/GPU): 43.3 | MFU 13.89% | FLOPs estimator: workload-aware | logical tokens/s: 199195.5 | valid tokens/s: 199195.5 | physical tokens/s: 199717.3 | loss tokens/s: 154316.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.632667E-01 | lm-global-supervised-tokens: 1.299729E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.550841E-02 | lm-token-buffer-mib: 5.818753E+01 | global_load_balancing_loss: 1.001790E+00 | loss scale: 1.0 | grad norm: 0.064 | params norm: 11583.496 | params rms: 0.062721 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 14:55:24.489607 | finish at 2026-10-10 10:02:13 + [2026-10-04 19:08:18] iteration 1089/ 6855 | consumed samples: 139392 | elapsed time per iteration (ms): 88979.8 | throughput per GPU (TFLOP/s/GPU): 40.9 | MFU 13.12% | FLOPs estimator: workload-aware | logical tokens/s: 188550.5 | valid tokens/s: 188550.4 | physical tokens/s: 189154.5 | loss tokens/s: 141059.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.148589E-01 | lm-global-supervised-tokens: 1.255146E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.572268E-02 | lm-token-buffer-mib: 5.813422E+01 | global_load_balancing_loss: 1.002651E+00 | loss scale: 1.0 | grad norm: 0.134 | params norm: 11583.418 | params rms: 0.062720 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 22:30:57.435987 | finish at 2026-10-10 17:39:15 + [2026-10-04 19:10:05] iteration 1090/ 6855 | consumed samples: 139520 | elapsed time per iteration (ms): 107343.4 | throughput per GPU (TFLOP/s/GPU): 33.7 | MFU 10.81% | FLOPs estimator: workload-aware | logical tokens/s: 156294.4 | valid tokens/s: 156294.4 | physical tokens/s: 156721.9 | loss tokens/s: 115757.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.074725E-01 | lm-global-supervised-tokens: 1.242575E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.469782E-02 | lm-token-buffer-mib: 5.814154E+01 | global_load_balancing_loss: 1.002463E+00 | loss scale: 1.0 | grad norm: 0.065 | params norm: 11583.340 | params rms: 0.062720 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 7 days, 3:53:54.514079 | finish at 2026-10-11 23:04:00 + [2026-10-04 19:11:34] iteration 1091/ 6855 | consumed samples: 139648 | elapsed time per iteration (ms): 88788.2 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.43% | FLOPs estimator: workload-aware | logical tokens/s: 188957.4 | valid tokens/s: 188957.4 | physical tokens/s: 189327.6 | loss tokens/s: 140977.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.366102E-01 | lm-global-supervised-tokens: 1.251711E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.148249E-02 | lm-token-buffer-mib: 5.812243E+01 | global_load_balancing_loss: 1.001123E+00 | loss scale: 1.0 | grad norm: 0.087 | params norm: 11583.260 | params rms: 0.062720 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 22:09:35.046808 | finish at 2026-10-10 17:21:09 + [2026-10-04 19:13:07] iteration 1092/ 6855 | consumed samples: 139776 | elapsed time per iteration (ms): 92748.6 | throughput per GPU (TFLOP/s/GPU): 39.9 | MFU 12.79% | FLOPs estimator: workload-aware | logical tokens/s: 180888.4 | valid tokens/s: 180888.4 | physical tokens/s: 181400.7 | loss tokens/s: 132599.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.666659E-01 | lm-global-supervised-tokens: 1.229846E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.166316E-02 | lm-token-buffer-mib: 5.814615E+01 | global_load_balancing_loss: 9.986775E-01 | loss scale: 1.0 | grad norm: 0.086 | params norm: 11583.181 | params rms: 0.062719 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 4:28:30.111761 | finish at 2026-10-10 23:41:37 + [2026-10-04 19:14:38] iteration 1093/ 6855 | consumed samples: 139904 | elapsed time per iteration (ms): 91225.1 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.54% | FLOPs estimator: workload-aware | logical tokens/s: 183909.3 | valid tokens/s: 183909.3 | physical tokens/s: 184220.1 | loss tokens/s: 133709.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.249901E-01 | lm-global-supervised-tokens: 1.219764E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.114807E-02 | lm-token-buffer-mib: 5.825140E+01 | global_load_balancing_loss: 9.961635E-01 | loss scale: 1.0 | grad norm: 0.053 | params norm: 11583.100 | params rms: 0.062719 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 2:00:39.298439 | finish at 2026-10-10 21:15:17 + [2026-10-04 19:16:07] iteration 1094/ 6855 | consumed samples: 140032 | elapsed time per iteration (ms): 89230.9 | throughput per GPU (TFLOP/s/GPU): 41.7 | MFU 13.36% | FLOPs estimator: workload-aware | logical tokens/s: 188020.2 | valid tokens/s: 188020.1 | physical tokens/s: 188403.4 | loss tokens/s: 147273.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.352144E-01 | lm-global-supervised-tokens: 1.314134E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.112999E-02 | lm-token-buffer-mib: 5.806264E+01 | global_load_balancing_loss: 9.980490E-01 | loss scale: 1.0 | grad norm: 0.067 | params norm: 11583.018 | params rms: 0.062718 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 22:47:39.422586 | finish at 2026-10-10 18:03:46 + [2026-10-04 19:17:31] iteration 1095/ 6855 | consumed samples: 140160 | elapsed time per iteration (ms): 84100.9 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.52% | FLOPs estimator: workload-aware | logical tokens/s: 199487.6 | valid tokens/s: 199487.6 | physical tokens/s: 199991.2 | loss tokens/s: 149763.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.606149E-01 | lm-global-supervised-tokens: 1.259521E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.448811E-02 | lm-token-buffer-mib: 5.828713E+01 | global_load_balancing_loss: 1.001093E+00 | loss scale: 1.0 | grad norm: 0.072 | params norm: 11582.938 | params rms: 0.062718 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 14:33:41.080627 | finish at 2026-10-10 09:51:12 + [2026-10-04 19:18:57] iteration 1096/ 6855 | consumed samples: 140288 | elapsed time per iteration (ms): 85817.8 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.65% | FLOPs estimator: workload-aware | logical tokens/s: 195497.7 | valid tokens/s: 195497.7 | physical tokens/s: 195922.5 | loss tokens/s: 144030.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.123565E-01 | lm-global-supervised-tokens: 1.236041E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.378947E-02 | lm-token-buffer-mib: 5.805022E+01 | global_load_balancing_loss: 1.000313E+00 | loss scale: 1.0 | grad norm: 0.078 | params norm: 11582.859 | params rms: 0.062717 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 17:17:04.950743 | finish at 2026-10-10 12:36:02 + [2026-10-04 19:20:22] iteration 1097/ 6855 | consumed samples: 140416 | elapsed time per iteration (ms): 84748.7 | throughput per GPU (TFLOP/s/GPU): 43.7 | MFU 14.00% | FLOPs estimator: workload-aware | logical tokens/s: 197963.9 | valid tokens/s: 197963.9 | physical tokens/s: 198451.2 | loss tokens/s: 148931.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.796988E-01 | lm-global-supervised-tokens: 1.262172E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.614535E-02 | lm-token-buffer-mib: 5.813464E+01 | global_load_balancing_loss: 1.001621E+00 | loss scale: 1.0 | grad norm: 0.082 | params norm: 11582.778 | params rms: 0.062717 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 15:33:02.981097 | finish at 2026-10-10 10:53:25 + [2026-10-04 19:21:47] iteration 1098/ 6855 | consumed samples: 140544 | elapsed time per iteration (ms): 85297.8 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.51% | FLOPs estimator: workload-aware | logical tokens/s: 196689.1 | valid tokens/s: 196689.1 | physical tokens/s: 197223.9 | loss tokens/s: 146033.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.400511E-01 | lm-global-supervised-tokens: 1.245633E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.161559E-02 | lm-token-buffer-mib: 5.817124E+01 | global_load_balancing_loss: 1.002071E+00 | loss scale: 1.0 | grad norm: 0.127 | params norm: 11582.697 | params rms: 0.062717 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 16:24:19.624384 | finish at 2026-10-10 11:46:07 + [2026-10-04 19:23:15] iteration 1099/ 6855 | consumed samples: 140672 | elapsed time per iteration (ms): 87908.9 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.49% | FLOPs estimator: workload-aware | logical tokens/s: 190847.4 | valid tokens/s: 190847.4 | physical tokens/s: 191347.7 | loss tokens/s: 138906.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.765315E-01 | lm-global-supervised-tokens: 1.221110E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.154675E-02 | lm-token-buffer-mib: 5.809681E+01 | global_load_balancing_loss: 1.002708E+00 | loss scale: 1.0 | grad norm: 0.041 | params norm: 11582.615 | params rms: 0.062716 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 20:33:23.758902 | finish at 2026-10-10 15:56:39 + [2026-10-04 19:25:02] iteration 1100/ 6855 | consumed samples: 140800 | elapsed time per iteration (ms): 107536.3 | throughput per GPU (TFLOP/s/GPU): 33.9 | MFU 10.88% | FLOPs estimator: workload-aware | logical tokens/s: 156013.9 | valid tokens/s: 156013.9 | physical tokens/s: 156528.2 | loss tokens/s: 115564.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.008871E-01 | lm-global-supervised-tokens: 1.242733E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.149259E-02 | lm-token-buffer-mib: 5.832961E+01 | global_load_balancing_loss: 9.998766E-01 | loss scale: 1.0 | grad norm: 0.127 | params norm: 11582.533 | params rms: 0.062716 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 7 days, 3:54:31.256618 | finish at 2026-10-11 23:19:34 + [2026-10-04 19:26:33] iteration 1101/ 6855 | consumed samples: 140928 | elapsed time per iteration (ms): 90803.5 | throughput per GPU (TFLOP/s/GPU): 41.3 | MFU 13.25% | FLOPs estimator: workload-aware | logical tokens/s: 184763.4 | valid tokens/s: 184763.4 | physical tokens/s: 185191.2 | loss tokens/s: 137351.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.833259E-01 | lm-global-supervised-tokens: 1.247195E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.240061E-02 | lm-token-buffer-mib: 5.809863E+01 | global_load_balancing_loss: 9.997330E-01 | loss scale: 1.0 | grad norm: 0.036 | params norm: 11582.449 | params rms: 0.062715 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 1:08:03.093075 | finish at 2026-10-10 20:34:36 + [2026-10-04 19:28:01] iteration 1102/ 6855 | consumed samples: 141056 | elapsed time per iteration (ms): 87596.9 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.46% | FLOPs estimator: workload-aware | logical tokens/s: 191526.9 | valid tokens/s: 191526.9 | physical tokens/s: 191924.5 | loss tokens/s: 148701.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.539350E-01 | lm-global-supervised-tokens: 1.302577E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.243646E-02 | lm-token-buffer-mib: 5.811764E+01 | global_load_balancing_loss: 9.994831E-01 | loss scale: 1.0 | grad norm: 0.092 | params norm: 11582.366 | params rms: 0.062715 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 19:59:04.818857 | finish at 2026-10-10 15:27:06 + [2026-10-04 19:29:32] iteration 1103/ 6855 | consumed samples: 141184 | elapsed time per iteration (ms): 91085.9 | throughput per GPU (TFLOP/s/GPU): 40.6 | MFU 13.00% | FLOPs estimator: workload-aware | logical tokens/s: 184191.1 | valid tokens/s: 184191.1 | physical tokens/s: 184865.8 | loss tokens/s: 133962.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.237420E-01 | lm-global-supervised-tokens: 1.220213E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.452892E-02 | lm-token-buffer-mib: 5.818931E+01 | global_load_balancing_loss: 9.999636E-01 | loss scale: 1.0 | grad norm: 0.036 | params norm: 11582.283 | params rms: 0.062714 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 1:32:06.146563 | finish at 2026-10-10 21:01:38 + [2026-10-04 19:31:00] iteration 1104/ 6855 | consumed samples: 141312 | elapsed time per iteration (ms): 88298.5 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.50% | FLOPs estimator: workload-aware | logical tokens/s: 190005.5 | valid tokens/s: 190005.5 | physical tokens/s: 190396.6 | loss tokens/s: 140914.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.823785E-01 | lm-global-supervised-tokens: 1.244251E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.563891E-02 | lm-token-buffer-mib: 5.804893E+01 | global_load_balancing_loss: 9.977188E-01 | loss scale: 1.0 | grad norm: 0.073 | params norm: 11582.201 | params rms: 0.062714 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 21:03:24.506571 | finish at 2026-10-10 16:34:25 + [2026-10-04 19:32:29] iteration 1105/ 6855 | consumed samples: 141440 | elapsed time per iteration (ms): 88290.6 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.52% | FLOPs estimator: workload-aware | logical tokens/s: 190022.4 | valid tokens/s: 190022.4 | physical tokens/s: 190680.8 | loss tokens/s: 140858.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.979272E-01 | lm-global-supervised-tokens: 1.243646E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.302811E-02 | lm-token-buffer-mib: 5.825014E+01 | global_load_balancing_loss: 9.991988E-01 | loss scale: 1.0 | grad norm: 0.061 | params norm: 11582.118 | params rms: 0.062713 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 21:01:11.198487 | finish at 2026-10-10 16:33:40 + [2026-10-04 19:33:57] iteration 1106/ 6855 | consumed samples: 141568 | elapsed time per iteration (ms): 88049.9 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.58% | FLOPs estimator: workload-aware | logical tokens/s: 190541.4 | valid tokens/s: 190541.4 | physical tokens/s: 190991.1 | loss tokens/s: 139061.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.657621E-01 | lm-global-supervised-tokens: 1.224439E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.149628E-02 | lm-token-buffer-mib: 5.805208E+01 | global_load_balancing_loss: 1.000422E+00 | loss scale: 1.0 | grad norm: 0.076 | params norm: 11582.036 | params rms: 0.062713 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 20:36:39.081016 | finish at 2026-10-10 16:10:36 + [2026-10-04 19:35:24] iteration 1107/ 6855 | consumed samples: 141696 | elapsed time per iteration (ms): 87420.7 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.43% | FLOPs estimator: workload-aware | logical tokens/s: 191913.0 | valid tokens/s: 191913.0 | physical tokens/s: 192379.4 | loss tokens/s: 140332.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.361439E-01 | lm-global-supervised-tokens: 1.226793E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.178284E-02 | lm-token-buffer-mib: 5.814207E+01 | global_load_balancing_loss: 1.000408E+00 | loss scale: 1.0 | grad norm: 0.061 | params norm: 11581.956 | params rms: 0.062713 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 19:34:54.392326 | finish at 2026-10-10 15:10:18 + [2026-10-04 19:36:52] iteration 1108/ 6855 | consumed samples: 141824 | elapsed time per iteration (ms): 88419.8 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.62% | FLOPs estimator: workload-aware | logical tokens/s: 189745.0 | valid tokens/s: 189745.0 | physical tokens/s: 190232.2 | loss tokens/s: 138920.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.949533E-01 | lm-global-supervised-tokens: 1.228332E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.188160E-02 | lm-token-buffer-mib: 5.828917E+01 | global_load_balancing_loss: 1.000931E+00 | loss scale: 1.0 | grad norm: 0.092 | params norm: 11581.875 | params rms: 0.062712 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 21:09:08.567554 | finish at 2026-10-10 16:46:01 + [2026-10-04 19:38:21] iteration 1109/ 6855 | consumed samples: 141952 | elapsed time per iteration (ms): 88373.0 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.55% | FLOPs estimator: workload-aware | logical tokens/s: 189845.5 | valid tokens/s: 189845.5 | physical tokens/s: 190342.7 | loss tokens/s: 140132.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.880111E-01 | lm-global-supervised-tokens: 1.238392E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.663852E-02 | lm-token-buffer-mib: 5.828237E+01 | global_load_balancing_loss: 1.001713E+00 | loss scale: 1.0 | grad norm: 0.057 | params norm: 11581.796 | params rms: 0.062712 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 21:03:11.260203 | finish at 2026-10-10 16:41:32 + [2026-10-04 19:40:09] iteration 1110/ 6855 | consumed samples: 142080 | elapsed time per iteration (ms): 107860.9 | throughput per GPU (TFLOP/s/GPU): 34.0 | MFU 10.91% | FLOPs estimator: workload-aware | logical tokens/s: 155544.7 | valid tokens/s: 155544.7 | physical tokens/s: 155805.2 | loss tokens/s: 117232.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.476346E-01 | lm-global-supervised-tokens: 1.264479E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.473842E-02 | lm-token-buffer-mib: 5.804842E+01 | global_load_balancing_loss: 1.000431E+00 | loss scale: 1.0 | grad norm: 0.038 | params norm: 11581.717 | params rms: 0.062711 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 7 days, 4:07:40.731729 | finish at 2026-10-11 23:47:49 + [2026-10-04 19:41:33] iteration 1111/ 6855 | consumed samples: 142208 | elapsed time per iteration (ms): 84664.3 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.62% | FLOPs estimator: workload-aware | logical tokens/s: 198160.7 | valid tokens/s: 198160.7 | physical tokens/s: 198711.0 | loss tokens/s: 144643.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.249468E-01 | lm-global-supervised-tokens: 1.224615E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.159029E-02 | lm-token-buffer-mib: 5.815748E+01 | global_load_balancing_loss: 1.000169E+00 | loss scale: 1.0 | grad norm: 0.189 | params norm: 11581.638 | params rms: 0.062711 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 15:05:11.832123 | finish at 2026-10-10 10:46:45 + [2026-10-04 19:42:59] iteration 1112/ 6855 | consumed samples: 142336 | elapsed time per iteration (ms): 85756.8 | throughput per GPU (TFLOP/s/GPU): 41.7 | MFU 13.38% | FLOPs estimator: workload-aware | logical tokens/s: 195636.9 | valid tokens/s: 195636.9 | physical tokens/s: 196273.1 | loss tokens/s: 140759.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.264552E-01 | lm-global-supervised-tokens: 1.207108E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.348182E-02 | lm-token-buffer-mib: 5.840321E+01 | global_load_balancing_loss: 9.998528E-01 | loss scale: 1.0 | grad norm: 0.253 | params norm: 11581.558 | params rms: 0.062710 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 16:48:21.062418 | finish at 2026-10-10 12:31:20 + [2026-10-04 19:44:23] iteration 1113/ 6855 | consumed samples: 142464 | elapsed time per iteration (ms): 84214.8 | throughput per GPU (TFLOP/s/GPU): 43.6 | MFU 13.96% | FLOPs estimator: workload-aware | logical tokens/s: 199217.5 | valid tokens/s: 199217.5 | physical tokens/s: 199594.7 | loss tokens/s: 141293.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.211270E-01 | lm-global-supervised-tokens: 1.189902E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.131038E-02 | lm-token-buffer-mib: 5.813965E+01 | global_load_balancing_loss: 1.000939E+00 | loss scale: 1.0 | grad norm: 0.057 | params norm: 11581.479 | params rms: 0.062710 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 14:19:21.397345 | finish at 2026-10-10 10:03:45 + [2026-10-04 19:45:46] iteration 1114/ 6855 | consumed samples: 142592 | elapsed time per iteration (ms): 83142.4 | throughput per GPU (TFLOP/s/GPU): 43.1 | MFU 13.82% | FLOPs estimator: workload-aware | logical tokens/s: 201787.3 | valid tokens/s: 201787.3 | physical tokens/s: 202306.4 | loss tokens/s: 151875.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.648407E-01 | lm-global-supervised-tokens: 1.262725E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.175674E-02 | lm-token-buffer-mib: 5.811362E+01 | global_load_balancing_loss: 9.995145E-01 | loss scale: 1.0 | grad norm: 0.185 | params norm: 11581.400 | params rms: 0.062710 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 12:35:20.566458 | finish at 2026-10-10 08:21:07 + [2026-10-04 19:47:10] iteration 1115/ 6855 | consumed samples: 142720 | elapsed time per iteration (ms): 83182.5 | throughput per GPU (TFLOP/s/GPU): 43.4 | MFU 13.91% | FLOPs estimator: workload-aware | logical tokens/s: 201691.0 | valid tokens/s: 201691.0 | physical tokens/s: 202063.5 | loss tokens/s: 153560.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.651788E-01 | lm-global-supervised-tokens: 1.277351E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.304360E-02 | lm-token-buffer-mib: 5.813767E+01 | global_load_balancing_loss: 9.988081E-01 | loss scale: 1.0 | grad norm: 0.048 | params norm: 11581.323 | params rms: 0.062709 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 12:37:47.829890 | finish at 2026-10-10 08:24:57 + [2026-10-04 19:48:36] iteration 1116/ 6855 | consumed samples: 142848 | elapsed time per iteration (ms): 86400.8 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.41% | FLOPs estimator: workload-aware | logical tokens/s: 194178.4 | valid tokens/s: 194178.4 | physical tokens/s: 194943.5 | loss tokens/s: 139097.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.866472E-01 | lm-global-supervised-tokens: 1.201814E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.794265E-02 | lm-token-buffer-mib: 5.832470E+01 | global_load_balancing_loss: 9.998166E-01 | loss scale: 1.0 | grad norm: 0.191 | params norm: 11581.242 | params rms: 0.062709 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 17:44:14.356703 | finish at 2026-10-10 13:32:50 + [2026-10-04 19:50:04] iteration 1117/ 6855 | consumed samples: 142976 | elapsed time per iteration (ms): 88364.2 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.40% | FLOPs estimator: workload-aware | logical tokens/s: 189864.1 | valid tokens/s: 189864.1 | physical tokens/s: 190263.4 | loss tokens/s: 139572.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.973637E-01 | lm-global-supervised-tokens: 1.233322E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.638211E-02 | lm-token-buffer-mib: 5.822961E+01 | global_load_balancing_loss: 1.000048E+00 | loss scale: 1.0 | grad norm: 0.044 | params norm: 11581.161 | params rms: 0.062708 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 20:50:33.509388 | finish at 2026-10-10 16:40:38 + [2026-10-04 19:51:32] iteration 1118/ 6855 | consumed samples: 143104 | elapsed time per iteration (ms): 87954.8 | throughput per GPU (TFLOP/s/GPU): 43.3 | MFU 13.87% | FLOPs estimator: workload-aware | logical tokens/s: 190747.1 | valid tokens/s: 190747.1 | physical tokens/s: 191130.8 | loss tokens/s: 143444.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.600862E-01 | lm-global-supervised-tokens: 1.261665E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.285338E-02 | lm-token-buffer-mib: 5.811283E+01 | global_load_balancing_loss: 9.986975E-01 | loss scale: 1.0 | grad norm: 0.118 | params norm: 11581.081 | params rms: 0.062708 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 20:09:56.584386 | finish at 2026-10-10 16:01:29 + [2026-10-04 19:52:58] iteration 1119/ 6855 | consumed samples: 143232 | elapsed time per iteration (ms): 86066.3 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.77% | FLOPs estimator: workload-aware | logical tokens/s: 194933.6 | valid tokens/s: 194933.6 | physical tokens/s: 195378.3 | loss tokens/s: 147085.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.541674E-01 | lm-global-supervised-tokens: 1.265910E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.155740E-02 | lm-token-buffer-mib: 5.814644E+01 | global_load_balancing_loss: 1.001374E+00 | loss scale: 1.0 | grad norm: 0.042 | params norm: 11581.001 | params rms: 0.062707 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 17:07:56.258022 | finish at 2026-10-10 13:00:55 + [2026-10-04 19:54:50] iteration 1120/ 6855 | consumed samples: 143360 | elapsed time per iteration (ms): 111307.1 | throughput per GPU (TFLOP/s/GPU): 32.7 | MFU 10.48% | FLOPs estimator: workload-aware | logical tokens/s: 150728.5 | valid tokens/s: 150728.5 | physical tokens/s: 151183.9 | loss tokens/s: 112225.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.900523E-01 | lm-global-supervised-tokens: 1.249147E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.166628E-02 | lm-token-buffer-mib: 5.814433E+01 | global_load_balancing_loss: 1.001047E+00 | loss scale: 1.0 | grad norm: 0.040 | params norm: 11580.920 | params rms: 0.062707 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 7 days, 9:19:06.098505 | finish at 2026-10-12 05:13:56 +times across ranks (ms): + save-checkpoint: + rank 0: 24129.56 + rank 1: 24129.52 + rank 2: 24129.57 + rank 3: 24129.55 + rank 4: 24129.52 + rank 5: 24129.55 + rank 6: 24129.51 + rank 7: 24129.54 + rank 8: 24127.51 + rank 9: 24127.46 + rank 10: 24127.44 + rank 11: 24127.45 + rank 12: 24127.43 + rank 13: 24127.46 + rank 14: 24127.48 + rank 15: 24127.46 + rank 16: 24130.56 + rank 17: 24130.56 + rank 18: 24130.57 + rank 19: 24130.56 + rank 20: 24130.56 + rank 21: 24130.61 + rank 22: 24130.56 + rank 23: 24130.54 + rank 24: 24129.33 + rank 25: 24129.48 + rank 26: 24129.56 + rank 27: 24129.53 + rank 28: 24129.25 + rank 29: 24129.42 + rank 30: 24129.40 + rank 31: 24129.35 + rank 32: 24132.23 + rank 33: 24132.09 + rank 34: 24132.09 + rank 35: 24132.07 + rank 36: 24132.15 + rank 37: 24132.06 + rank 38: 24132.12 + rank 39: 24132.07 + rank 40: 24129.60 + rank 41: 24129.57 + rank 42: 24129.57 + rank 43: 24129.59 + rank 44: 24129.54 + rank 45: 24129.56 + rank 46: 24129.57 + rank 47: 24129.57 + rank 48: 24128.68 + rank 49: 24128.84 + rank 50: 24128.77 + rank 51: 24128.65 + rank 52: 24128.82 + rank 53: 24128.80 + rank 54: 24128.68 + rank 55: 24128.83 + rank 56: 24129.62 + rank 57: 24129.58 + rank 58: 24129.58 + rank 59: 24129.59 + rank 60: 24129.62 + rank 61: 24129.58 + rank 62: 24129.59 + rank 63: 24129.58 + [2026-10-04 19:56:53] iteration 1121/ 6855 | consumed samples: 143488 | elapsed time per iteration (ms): 99234.6 | throughput per GPU (TFLOP/s/GPU): 37.9 | MFU 12.15% | FLOPs estimator: workload-aware | logical tokens/s: 169065.4 | valid tokens/s: 169065.4 | physical tokens/s: 169390.3 | loss tokens/s: 122252.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.234642E-01 | lm-global-supervised-tokens: 1.213164E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.218533E-01 | lm-token-buffer-mib: 5.809859E+01 | global_load_balancing_loss: 1.001666E+00 | loss scale: 1.0 | grad norm: 0.100 | params norm: 11580.840 | params rms: 0.062706 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 14:03:31.159873 | finish at 2026-10-11 10:00:24 + [2026-10-04 19:58:25] iteration 1122/ 6855 | consumed samples: 143616 | elapsed time per iteration (ms): 92035.9 | throughput per GPU (TFLOP/s/GPU): 40.2 | MFU 12.88% | FLOPs estimator: workload-aware | logical tokens/s: 182289.6 | valid tokens/s: 182289.6 | physical tokens/s: 182588.7 | loss tokens/s: 129057.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.137484E-01 | lm-global-supervised-tokens: 1.187796E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 4.181692E-02 | lm-token-buffer-mib: 5.810025E+01 | global_load_balancing_loss: 1.001229E+00 | loss scale: 1.0 | grad norm: 0.061 | params norm: 11580.762 | params rms: 0.062706 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 2:34:02.099670 | finish at 2026-10-10 22:32:27 + [2026-10-04 19:59:54] iteration 1123/ 6855 | consumed samples: 143744 | elapsed time per iteration (ms): 88374.9 | throughput per GPU (TFLOP/s/GPU): 41.6 | MFU 13.33% | FLOPs estimator: workload-aware | logical tokens/s: 189841.3 | valid tokens/s: 189841.3 | physical tokens/s: 190174.1 | loss tokens/s: 137805.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.507486E-01 | lm-global-supervised-tokens: 1.217853E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.320326E-02 | lm-token-buffer-mib: 5.807893E+01 | global_load_balancing_loss: 1.000700E+00 | loss scale: 1.0 | grad norm: 0.065 | params norm: 11580.685 | params rms: 0.062706 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 20:42:45.133747 | finish at 2026-10-10 16:42:39 + [2026-10-04 20:01:21] iteration 1124/ 6855 | consumed samples: 143872 | elapsed time per iteration (ms): 87732.8 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.41% | FLOPs estimator: workload-aware | logical tokens/s: 191230.2 | valid tokens/s: 191230.2 | physical tokens/s: 191669.4 | loss tokens/s: 145135.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.601751E-01 | lm-global-supervised-tokens: 1.273314E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.369832E-02 | lm-token-buffer-mib: 5.819562E+01 | global_load_balancing_loss: 9.989722E-01 | loss scale: 1.0 | grad norm: 0.113 | params norm: 11580.607 | params rms: 0.062705 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 19:39:56.674107 | finish at 2026-10-10 15:41:18 + [2026-10-04 20:02:49] iteration 1125/ 6855 | consumed samples: 144000 | elapsed time per iteration (ms): 87984.4 | throughput per GPU (TFLOP/s/GPU): 41.2 | MFU 13.22% | FLOPs estimator: workload-aware | logical tokens/s: 190683.5 | valid tokens/s: 190683.5 | physical tokens/s: 191199.3 | loss tokens/s: 144427.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.636614E-01 | lm-global-supervised-tokens: 1.270741E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.167215E-02 | lm-token-buffer-mib: 5.816798E+01 | global_load_balancing_loss: 9.983385E-01 | loss scale: 1.0 | grad norm: 0.076 | params norm: 11580.530 | params rms: 0.062705 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 20:02:30.728316 | finish at 2026-10-10 16:05:20 + [2026-10-04 20:04:15] iteration 1126/ 6855 | consumed samples: 144128 | elapsed time per iteration (ms): 85268.1 | throughput per GPU (TFLOP/s/GPU): 43.2 | MFU 13.84% | FLOPs estimator: workload-aware | logical tokens/s: 196757.5 | valid tokens/s: 196757.4 | physical tokens/s: 197299.0 | loss tokens/s: 150423.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.360187E-01 | lm-global-supervised-tokens: 1.282630E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.158193E-02 | lm-token-buffer-mib: 5.832502E+01 | global_load_balancing_loss: 9.988148E-01 | loss scale: 1.0 | grad norm: 0.078 | params norm: 11580.453 | params rms: 0.062704 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 15:41:40.916349 | finish at 2026-10-10 11:45:56 + [2026-10-04 20:05:42] iteration 1127/ 6855 | consumed samples: 144256 | elapsed time per iteration (ms): 87202.3 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.64% | FLOPs estimator: workload-aware | logical tokens/s: 192393.7 | valid tokens/s: 192393.7 | physical tokens/s: 192820.2 | loss tokens/s: 142219.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.863221E-01 | lm-global-supervised-tokens: 1.240183E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.147342E-02 | lm-token-buffer-mib: 5.825621E+01 | global_load_balancing_loss: 1.000289E+00 | loss scale: 1.0 | grad norm: 0.029 | params norm: 11580.376 | params rms: 0.062704 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 18:44:54.918205 | finish at 2026-10-10 14:50:37 + [2026-10-04 20:07:07] iteration 1128/ 6855 | consumed samples: 144384 | elapsed time per iteration (ms): 85377.3 | throughput per GPU (TFLOP/s/GPU): 43.9 | MFU 14.07% | FLOPs estimator: workload-aware | logical tokens/s: 196506.3 | valid tokens/s: 196506.3 | physical tokens/s: 196905.1 | loss tokens/s: 141805.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.071458E-01 | lm-global-supervised-tokens: 1.210700E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.508719E-02 | lm-token-buffer-mib: 5.820495E+01 | global_load_balancing_loss: 1.001130E+00 | loss scale: 1.0 | grad norm: 0.040 | params norm: 11580.297 | params rms: 0.062704 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 15:49:15.967916 | finish at 2026-10-10 11:56:23 + [2026-10-04 20:08:37] iteration 1129/ 6855 | consumed samples: 144512 | elapsed time per iteration (ms): 89388.8 | throughput per GPU (TFLOP/s/GPU): 41.3 | MFU 13.22% | FLOPs estimator: workload-aware | logical tokens/s: 187687.8 | valid tokens/s: 187687.8 | physical tokens/s: 188141.9 | loss tokens/s: 140137.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.592194E-01 | lm-global-supervised-tokens: 1.252668E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.514764E-02 | lm-token-buffer-mib: 5.835797E+01 | global_load_balancing_loss: 1.001018E+00 | loss scale: 1.0 | grad norm: 0.041 | params norm: 11580.217 | params rms: 0.062703 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 22:10:40.066213 | finish at 2026-10-10 18:19:17 + [2026-10-04 20:10:26] iteration 1130/ 6855 | consumed samples: 144640 | elapsed time per iteration (ms): 109674.8 | throughput per GPU (TFLOP/s/GPU): 33.7 | MFU 10.80% | FLOPs estimator: workload-aware | logical tokens/s: 152972.2 | valid tokens/s: 152972.2 | physical tokens/s: 153257.1 | loss tokens/s: 112748.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.434123E-01 | lm-global-supervised-tokens: 1.236564E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.166905E-02 | lm-token-buffer-mib: 5.809117E+01 | global_load_balancing_loss: 1.001514E+00 | loss scale: 1.0 | grad norm: 0.068 | params norm: 11580.139 | params rms: 0.062703 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 7 days, 6:24:48.275945 | finish at 2026-10-12 02:35:15 + [2026-10-04 20:11:53] iteration 1131/ 6855 | consumed samples: 144768 | elapsed time per iteration (ms): 86606.4 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.77% | FLOPs estimator: workload-aware | logical tokens/s: 193717.6 | valid tokens/s: 193717.6 | physical tokens/s: 194109.7 | loss tokens/s: 141776.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.890364E-01 | lm-global-supervised-tokens: 1.227873E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.406160E-02 | lm-token-buffer-mib: 5.807409E+01 | global_load_balancing_loss: 1.000761E+00 | loss scale: 1.0 | grad norm: 0.092 | params norm: 11580.061 | params rms: 0.062702 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 17:42:14.878098 | finish at 2026-10-10 13:54:08 + [2026-10-04 20:13:17] iteration 1132/ 6855 | consumed samples: 144896 | elapsed time per iteration (ms): 83561.0 | throughput per GPU (TFLOP/s/GPU): 43.3 | MFU 13.87% | FLOPs estimator: workload-aware | logical tokens/s: 200777.9 | valid tokens/s: 200777.8 | physical tokens/s: 201197.4 | loss tokens/s: 151364.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.732619E-01 | lm-global-supervised-tokens: 1.264821E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.254649E-02 | lm-token-buffer-mib: 5.803064E+01 | global_load_balancing_loss: 9.993111E-01 | loss scale: 1.0 | grad norm: 0.061 | params norm: 11579.985 | params rms: 0.062702 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 12:50:19.675939 | finish at 2026-10-10 09:03:36 + [2026-10-04 20:14:43] iteration 1133/ 6855 | consumed samples: 145024 | elapsed time per iteration (ms): 86267.9 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.59% | FLOPs estimator: workload-aware | logical tokens/s: 194478.0 | valid tokens/s: 194478.0 | physical tokens/s: 194900.4 | loss tokens/s: 150848.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.333289E-01 | lm-global-supervised-tokens: 1.301336E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.153962E-02 | lm-token-buffer-mib: 5.815909E+01 | global_load_balancing_loss: 9.987473E-01 | loss scale: 1.0 | grad norm: 0.033 | params norm: 11579.906 | params rms: 0.062701 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 17:07:05.012418 | finish at 2026-10-10 13:21:48 + [2026-10-04 20:16:13] iteration 1134/ 6855 | consumed samples: 145152 | elapsed time per iteration (ms): 90173.3 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.53% | FLOPs estimator: workload-aware | logical tokens/s: 186055.1 | valid tokens/s: 186055.1 | physical tokens/s: 186511.9 | loss tokens/s: 136474.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.335163E-01 | lm-global-supervised-tokens: 1.230638E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.983955E-02 | lm-token-buffer-mib: 5.804311E+01 | global_load_balancing_loss: 9.992443E-01 | loss scale: 1.0 | grad norm: 0.149 | params norm: 11579.827 | params rms: 0.062701 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 23:18:01.182117 | finish at 2026-10-10 19:34:14 + [2026-10-04 20:17:44] iteration 1135/ 6855 | consumed samples: 145280 | elapsed time per iteration (ms): 90812.5 | throughput per GPU (TFLOP/s/GPU): 41.3 | MFU 13.24% | FLOPs estimator: workload-aware | logical tokens/s: 184745.2 | valid tokens/s: 184745.2 | physical tokens/s: 185208.8 | loss tokens/s: 137004.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.009827E-01 | lm-global-supervised-tokens: 1.244170E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.444497E-02 | lm-token-buffer-mib: 5.815224E+01 | global_load_balancing_loss: 1.000293E+00 | loss scale: 1.0 | grad norm: 0.273 | params norm: 11579.748 | params rms: 0.062701 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 0:17:27.640467 | finish at 2026-10-10 20:35:11 + [2026-10-04 20:19:09] iteration 1136/ 6855 | consumed samples: 145408 | elapsed time per iteration (ms): 85673.1 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.46% | FLOPs estimator: workload-aware | logical tokens/s: 195828.2 | valid tokens/s: 195828.2 | physical tokens/s: 196343.0 | loss tokens/s: 151254.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.646977E-01 | lm-global-supervised-tokens: 1.295841E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.192918E-02 | lm-token-buffer-mib: 5.803548E+01 | global_load_balancing_loss: 1.000620E+00 | loss scale: 1.0 | grad norm: 0.199 | params norm: 11579.667 | params rms: 0.062700 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 16:06:04.506593 | finish at 2026-10-10 12:25:14 + [2026-10-04 20:20:36] iteration 1137/ 6855 | consumed samples: 145536 | elapsed time per iteration (ms): 86191.2 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.56% | FLOPs estimator: workload-aware | logical tokens/s: 194650.9 | valid tokens/s: 194650.9 | physical tokens/s: 195289.8 | loss tokens/s: 145070.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.656546E-01 | lm-global-supervised-tokens: 1.250376E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.470712E-02 | lm-token-buffer-mib: 5.813329E+01 | global_load_balancing_loss: 1.000771E+00 | loss scale: 1.0 | grad norm: 0.034 | params norm: 11579.587 | params rms: 0.062700 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 16:54:01.512096 | finish at 2026-10-10 13:14:37 + [2026-10-04 20:22:01] iteration 1138/ 6855 | consumed samples: 145664 | elapsed time per iteration (ms): 85610.0 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.59% | FLOPs estimator: workload-aware | logical tokens/s: 195972.1 | valid tokens/s: 195972.1 | physical tokens/s: 196457.7 | loss tokens/s: 153239.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.118769E-01 | lm-global-supervised-tokens: 1.311880E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.110337E-02 | lm-token-buffer-mib: 5.819799E+01 | global_load_balancing_loss: 1.000152E+00 | loss scale: 1.0 | grad norm: 0.132 | params norm: 11579.508 | params rms: 0.062699 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 15:57:12.598391 | finish at 2026-10-10 12:19:14 + [2026-10-04 20:23:31] iteration 1139/ 6855 | consumed samples: 145792 | elapsed time per iteration (ms): 89515.9 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.46% | FLOPs estimator: workload-aware | logical tokens/s: 187421.6 | valid tokens/s: 187421.6 | physical tokens/s: 187721.9 | loss tokens/s: 135539.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.743503E-01 | lm-global-supervised-tokens: 1.213293E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.450780E-02 | lm-token-buffer-mib: 5.810500E+01 | global_load_balancing_loss: 9.998367E-01 | loss scale: 1.0 | grad norm: 0.044 | params norm: 11579.430 | params rms: 0.062699 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 22:07:52.976480 | finish at 2026-10-10 18:31:24 + [2026-10-04 20:25:18] iteration 1140/ 6855 | consumed samples: 145920 | elapsed time per iteration (ms): 107504.2 | throughput per GPU (TFLOP/s/GPU): 34.3 | MFU 11.01% | FLOPs estimator: workload-aware | logical tokens/s: 156060.8 | valid tokens/s: 156060.8 | physical tokens/s: 156377.3 | loss tokens/s: 113169.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.114708E-01 | lm-global-supervised-tokens: 1.216620E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.204764E-02 | lm-token-buffer-mib: 5.805035E+01 | global_load_balancing_loss: 1.001574E+00 | loss scale: 1.0 | grad norm: 0.144 | params norm: 11579.351 | params rms: 0.062698 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 7 days, 2:39:46.336663 | finish at 2026-10-11 23:05:05 + [2026-10-04 20:26:44] iteration 1141/ 6855 | consumed samples: 146048 | elapsed time per iteration (ms): 85310.5 | throughput per GPU (TFLOP/s/GPU): 43.6 | MFU 13.99% | FLOPs estimator: workload-aware | logical tokens/s: 196659.6 | valid tokens/s: 196659.5 | physical tokens/s: 197385.6 | loss tokens/s: 153233.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.499478E-01 | lm-global-supervised-tokens: 1.307241E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.855921E-02 | lm-token-buffer-mib: 5.813711E+01 | global_load_balancing_loss: 1.000604E+00 | loss scale: 1.0 | grad norm: 0.126 | params norm: 11579.271 | params rms: 0.062698 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 15:24:24.413075 | finish at 2026-10-10 11:51:08 + [2026-10-04 20:28:09] iteration 1142/ 6855 | consumed samples: 146176 | elapsed time per iteration (ms): 85186.0 | throughput per GPU (TFLOP/s/GPU): 43.3 | MFU 13.86% | FLOPs estimator: workload-aware | logical tokens/s: 196947.8 | valid tokens/s: 196947.8 | physical tokens/s: 197415.0 | loss tokens/s: 150786.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.361518E-01 | lm-global-supervised-tokens: 1.284489E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.191136E-02 | lm-token-buffer-mib: 5.810274E+01 | global_load_balancing_loss: 9.990833E-01 | loss scale: 1.0 | grad norm: 0.121 | params norm: 11579.190 | params rms: 0.062698 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 15:11:07.467430 | finish at 2026-10-10 11:39:16 + [2026-10-04 20:29:35] iteration 1143/ 6855 | consumed samples: 146304 | elapsed time per iteration (ms): 85708.5 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.56% | FLOPs estimator: workload-aware | logical tokens/s: 195747.2 | valid tokens/s: 195747.2 | physical tokens/s: 196342.1 | loss tokens/s: 143239.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.388510E-01 | lm-global-supervised-tokens: 1.227686E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.165886E-02 | lm-token-buffer-mib: 5.823329E+01 | global_load_balancing_loss: 1.001796E+00 | loss scale: 1.0 | grad norm: 0.123 | params norm: 11579.111 | params rms: 0.062697 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 15:59:27.148945 | finish at 2026-10-10 12:29:02 + [2026-10-04 20:31:03] iteration 1144/ 6855 | consumed samples: 146432 | elapsed time per iteration (ms): 88064.2 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.77% | FLOPs estimator: workload-aware | logical tokens/s: 190511.1 | valid tokens/s: 190511.1 | physical tokens/s: 190853.1 | loss tokens/s: 138131.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.647277E-01 | lm-global-supervised-tokens: 1.216443E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.301234E-02 | lm-token-buffer-mib: 5.801667E+01 | global_load_balancing_loss: 9.993018E-01 | loss scale: 1.0 | grad norm: 0.162 | params norm: 11579.032 | params rms: 0.062697 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 19:42:14.780568 | finish at 2026-10-10 16:13:17 + [2026-10-04 20:32:29] iteration 1145/ 6855 | consumed samples: 146560 | elapsed time per iteration (ms): 86196.6 | throughput per GPU (TFLOP/s/GPU): 41.7 | MFU 13.38% | FLOPs estimator: workload-aware | logical tokens/s: 194638.6 | valid tokens/s: 194638.6 | physical tokens/s: 195018.6 | loss tokens/s: 143500.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.356675E-01 | lm-global-supervised-tokens: 1.236924E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.159940E-02 | lm-token-buffer-mib: 5.806342E+01 | global_load_balancing_loss: 9.991272E-01 | loss scale: 1.0 | grad norm: 0.193 | params norm: 11578.953 | params rms: 0.062696 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 16:43:02.340727 | finish at 2026-10-10 13:15:31 + [2026-10-04 20:33:55] iteration 1146/ 6855 | consumed samples: 146688 | elapsed time per iteration (ms): 86261.0 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.67% | FLOPs estimator: workload-aware | logical tokens/s: 194493.7 | valid tokens/s: 194493.7 | physical tokens/s: 194951.6 | loss tokens/s: 143637.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.049335E-01 | lm-global-supervised-tokens: 1.239030E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.193091E-02 | lm-token-buffer-mib: 5.807115E+01 | global_load_balancing_loss: 9.997513E-01 | loss scale: 1.0 | grad norm: 0.087 | params norm: 11578.875 | params rms: 0.062696 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 16:47:43.965503 | finish at 2026-10-10 13:21:39 + [2026-10-04 20:35:24] iteration 1147/ 6855 | consumed samples: 146816 | elapsed time per iteration (ms): 89350.5 | throughput per GPU (TFLOP/s/GPU): 41.5 | MFU 13.31% | FLOPs estimator: workload-aware | logical tokens/s: 187768.2 | valid tokens/s: 187768.2 | physical tokens/s: 188208.9 | loss tokens/s: 134049.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.780120E-01 | lm-global-supervised-tokens: 1.197742E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.334018E-02 | lm-token-buffer-mib: 5.815098E+01 | global_load_balancing_loss: 1.000061E+00 | loss scale: 1.0 | grad norm: 0.449 | params norm: 11578.795 | params rms: 0.062695 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 21:40:12.651888 | finish at 2026-10-10 18:15:37 + [2026-10-04 20:36:56] iteration 1148/ 6855 | consumed samples: 146944 | elapsed time per iteration (ms): 91365.3 | throughput per GPU (TFLOP/s/GPU): 41.4 | MFU 13.27% | FLOPs estimator: workload-aware | logical tokens/s: 183627.6 | valid tokens/s: 183627.6 | physical tokens/s: 183957.6 | loss tokens/s: 133736.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.175798E-01 | lm-global-supervised-tokens: 1.221891E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.900474E-02 | lm-token-buffer-mib: 5.805260E+01 | global_load_balancing_loss: 1.000148E+00 | loss scale: 1.0 | grad norm: 0.244 | params norm: 11578.713 | params rms: 0.062695 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 0:50:21.721857 | finish at 2026-10-10 21:27:17 + [2026-10-04 20:38:26] iteration 1149/ 6855 | consumed samples: 147072 | elapsed time per iteration (ms): 90615.9 | throughput per GPU (TFLOP/s/GPU): 41.3 | MFU 13.25% | FLOPs estimator: workload-aware | logical tokens/s: 185145.6 | valid tokens/s: 185145.6 | physical tokens/s: 185697.2 | loss tokens/s: 137517.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.500736E-01 | lm-global-supervised-tokens: 1.246129E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.191194E-02 | lm-token-buffer-mib: 5.810020E+01 | global_load_balancing_loss: 1.000527E+00 | loss scale: 1.0 | grad norm: 0.401 | params norm: 11578.634 | params rms: 0.062695 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 23:37:34.356916 | finish at 2026-10-10 20:16:01 + [2026-10-04 20:40:15] iteration 1150/ 6855 | consumed samples: 147200 | elapsed time per iteration (ms): 108760.4 | throughput per GPU (TFLOP/s/GPU): 34.2 | MFU 10.97% | FLOPs estimator: workload-aware | logical tokens/s: 154257.9 | valid tokens/s: 154257.9 | physical tokens/s: 154559.6 | loss tokens/s: 112869.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.927964E-01 | lm-global-supervised-tokens: 1.227568E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.170876E-02 | lm-token-buffer-mib: 5.805577E+01 | global_load_balancing_loss: 9.994155E-01 | loss scale: 1.0 | grad norm: 0.271 | params norm: 11578.554 | params rms: 0.062694 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 7 days, 4:21:18.113608 | finish at 2026-10-12 01:01:33 + [2026-10-04 20:41:45] iteration 1151/ 6855 | consumed samples: 147328 | elapsed time per iteration (ms): 89443.3 | throughput per GPU (TFLOP/s/GPU): 41.2 | MFU 13.20% | FLOPs estimator: workload-aware | logical tokens/s: 187573.0 | valid tokens/s: 187573.0 | physical tokens/s: 187941.9 | loss tokens/s: 137526.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.738178E-01 | lm-global-supervised-tokens: 1.230078E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.356866E-02 | lm-token-buffer-mib: 5.813956E+01 | global_load_balancing_loss: 1.001755E+00 | loss scale: 1.0 | grad norm: 0.162 | params norm: 11578.475 | params rms: 0.062694 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 21:43:04.624048 | finish at 2026-10-10 18:24:49 + [2026-10-04 20:43:12] iteration 1152/ 6855 | consumed samples: 147456 | elapsed time per iteration (ms): 87536.6 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.61% | FLOPs estimator: workload-aware | logical tokens/s: 191659.4 | valid tokens/s: 191659.4 | physical tokens/s: 192046.1 | loss tokens/s: 142495.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.575424E-01 | lm-global-supervised-tokens: 1.247356E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.574573E-02 | lm-token-buffer-mib: 5.815437E+01 | global_load_balancing_loss: 1.001020E+00 | loss scale: 1.0 | grad norm: 0.078 | params norm: 11578.395 | params rms: 0.062693 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 18:40:21.064560 | finish at 2026-10-10 15:23:33 + [2026-10-04 20:44:38] iteration 1153/ 6855 | consumed samples: 147584 | elapsed time per iteration (ms): 86176.3 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.45% | FLOPs estimator: workload-aware | logical tokens/s: 194684.4 | valid tokens/s: 194684.4 | physical tokens/s: 195057.0 | loss tokens/s: 145778.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.864607E-01 | lm-global-supervised-tokens: 1.256263E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.190358E-02 | lm-token-buffer-mib: 5.826573E+01 | global_load_balancing_loss: 1.000258E+00 | loss scale: 1.0 | grad norm: 0.125 | params norm: 11578.318 | params rms: 0.062693 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 16:29:37.378433 | finish at 2026-10-10 13:14:16 + [2026-10-04 20:46:03] iteration 1154/ 6855 | consumed samples: 147712 | elapsed time per iteration (ms): 85142.6 | throughput per GPU (TFLOP/s/GPU): 43.1 | MFU 13.83% | FLOPs estimator: workload-aware | logical tokens/s: 197047.9 | valid tokens/s: 197047.8 | physical tokens/s: 197519.3 | loss tokens/s: 147597.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.594731E-01 | lm-global-supervised-tokens: 1.256685E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.352700E-02 | lm-token-buffer-mib: 5.807635E+01 | global_load_balancing_loss: 1.001210E+00 | loss scale: 1.0 | grad norm: 0.168 | params norm: 11578.242 | params rms: 0.062692 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 14:49:58.173619 | finish at 2026-10-10 11:36:02 + [2026-10-04 20:47:31] iteration 1155/ 6855 | consumed samples: 147840 | elapsed time per iteration (ms): 87301.2 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.50% | FLOPs estimator: workload-aware | logical tokens/s: 192176.2 | valid tokens/s: 192176.2 | physical tokens/s: 192609.1 | loss tokens/s: 139923.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.902788E-01 | lm-global-supervised-tokens: 1.221553E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.157109E-02 | lm-token-buffer-mib: 5.809278E+01 | global_load_balancing_loss: 1.000982E+00 | loss scale: 1.0 | grad norm: 0.047 | params norm: 11578.164 | params rms: 0.062692 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 18:13:36.948223 | finish at 2026-10-10 15:01:08 + [2026-10-04 20:48:58] iteration 1156/ 6855 | consumed samples: 147968 | elapsed time per iteration (ms): 86882.5 | throughput per GPU (TFLOP/s/GPU): 44.0 | MFU 14.10% | FLOPs estimator: workload-aware | logical tokens/s: 193101.7 | valid tokens/s: 193101.7 | physical tokens/s: 193471.3 | loss tokens/s: 137018.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.826939E-01 | lm-global-supervised-tokens: 1.190446E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.233939E-02 | lm-token-buffer-mib: 5.814004E+01 | global_load_balancing_loss: 1.001209E+00 | loss scale: 1.0 | grad norm: 0.171 | params norm: 11578.086 | params rms: 0.062692 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 17:32:23.189124 | finish at 2026-10-10 14:21:21 + [2026-10-04 20:50:26] iteration 1157/ 6855 | consumed samples: 148096 | elapsed time per iteration (ms): 88563.0 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.44% | FLOPs estimator: workload-aware | logical tokens/s: 189437.6 | valid tokens/s: 189437.6 | physical tokens/s: 189997.1 | loss tokens/s: 140314.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.199255E-01 | lm-global-supervised-tokens: 1.242668E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.170883E-02 | lm-token-buffer-mib: 5.834608E+01 | global_load_balancing_loss: 9.993027E-01 | loss scale: 1.0 | grad norm: 0.084 | params norm: 11578.009 | params rms: 0.062691 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 20:10:31.768659 | finish at 2026-10-10 17:00:58 + [2026-10-04 20:51:54] iteration 1158/ 6855 | consumed samples: 148224 | elapsed time per iteration (ms): 87587.4 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.76% | FLOPs estimator: workload-aware | logical tokens/s: 191547.5 | valid tokens/s: 191547.5 | physical tokens/s: 191891.2 | loss tokens/s: 145936.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.760282E-01 | lm-global-supervised-tokens: 1.278218E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.732306E-02 | lm-token-buffer-mib: 5.808579E+01 | global_load_balancing_loss: 9.980822E-01 | loss scale: 1.0 | grad norm: 0.055 | params norm: 11577.932 | params rms: 0.062691 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 18:36:25.406703 | finish at 2026-10-10 15:28:19 + [2026-10-04 20:53:24] iteration 1159/ 6855 | consumed samples: 148352 | elapsed time per iteration (ms): 90093.4 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.68% | FLOPs estimator: workload-aware | logical tokens/s: 186219.6 | valid tokens/s: 186219.6 | physical tokens/s: 186634.5 | loss tokens/s: 133512.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.227293E-01 | lm-global-supervised-tokens: 1.202857E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.815666E-02 | lm-token-buffer-mib: 5.819791E+01 | global_load_balancing_loss: 9.992581E-01 | loss scale: 1.0 | grad norm: 0.147 | params norm: 11577.854 | params rms: 0.062690 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 22:32:52.248138 | finish at 2026-10-10 19:26:16 + [2026-10-04 20:55:13] iteration 1160/ 6855 | consumed samples: 148480 | elapsed time per iteration (ms): 109340.2 | throughput per GPU (TFLOP/s/GPU): 34.2 | MFU 10.96% | FLOPs estimator: workload-aware | logical tokens/s: 153439.8 | valid tokens/s: 153439.8 | physical tokens/s: 153690.7 | loss tokens/s: 113468.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.056080E-01 | lm-global-supervised-tokens: 1.240670E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.137532E-02 | lm-token-buffer-mib: 5.812251E+01 | global_load_balancing_loss: 9.989096E-01 | loss scale: 1.0 | grad norm: 0.054 | params norm: 11577.778 | params rms: 0.062690 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 7 days, 4:58:12.535104 | finish at 2026-10-12 01:53:26 +times across ranks (ms): + save-checkpoint: + rank 0: 24583.97 + rank 1: 24583.96 + rank 2: 24584.02 + rank 3: 24583.99 + rank 4: 24584.00 + rank 5: 24584.00 + rank 6: 24583.95 + rank 7: 24584.01 + rank 8: 24583.49 + rank 9: 24583.45 + rank 10: 24583.47 + rank 11: 24583.44 + rank 12: 24583.46 + rank 13: 24583.46 + rank 14: 24583.48 + rank 15: 24583.45 + rank 16: 24583.94 + rank 17: 24583.86 + rank 18: 24583.85 + rank 19: 24583.89 + rank 20: 24583.83 + rank 21: 24583.72 + rank 22: 24583.84 + rank 23: 24583.80 + rank 24: 24583.88 + rank 25: 24583.88 + rank 26: 24583.85 + rank 27: 24583.93 + rank 28: 24583.83 + rank 29: 24583.87 + rank 30: 24583.91 + rank 31: 24583.90 + rank 32: 24583.64 + rank 33: 24583.62 + rank 34: 24583.75 + rank 35: 24583.56 + rank 36: 24583.57 + rank 37: 24583.55 + rank 38: 24583.74 + rank 39: 24583.64 + rank 40: 24582.78 + rank 41: 24582.79 + rank 42: 24582.77 + rank 43: 24582.84 + rank 44: 24582.78 + rank 45: 24582.77 + rank 46: 24582.77 + rank 47: 24582.82 + rank 48: 24583.96 + rank 49: 24583.80 + rank 50: 24584.16 + rank 51: 24583.84 + rank 52: 24583.92 + rank 53: 24583.94 + rank 54: 24583.83 + rank 55: 24583.92 + rank 56: 24583.92 + rank 57: 24583.88 + rank 58: 24583.86 + rank 59: 24583.86 + rank 60: 24583.91 + rank 61: 24583.90 + rank 62: 24583.84 + rank 63: 24583.89 + [2026-10-04 20:57:37] iteration 1161/ 6855 | consumed samples: 148608 | elapsed time per iteration (ms): 118938.4 | throughput per GPU (TFLOP/s/GPU): 32.0 | MFU 10.26% | FLOPs estimator: workload-aware | logical tokens/s: 141058.0 | valid tokens/s: 141058.0 | physical tokens/s: 141325.9 | loss tokens/s: 103255.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.296571E-01 | lm-global-supervised-tokens: 1.228100E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.025346E+00 | lm-token-buffer-mib: 5.814601E+01 | global_load_balancing_loss: 1.000139E+00 | loss scale: 1.0 | grad norm: 0.088 | params norm: 11577.701 | params rms: 0.062689 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 7 days, 20:07:15.149312 | finish at 2026-10-12 17:04:52 + [2026-10-04 20:59:08] iteration 1162/ 6855 | consumed samples: 148736 | elapsed time per iteration (ms): 91620.1 | throughput per GPU (TFLOP/s/GPU): 40.6 | MFU 13.00% | FLOPs estimator: workload-aware | logical tokens/s: 183116.6 | valid tokens/s: 183116.6 | physical tokens/s: 183562.6 | loss tokens/s: 136704.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.845029E-01 | lm-global-supervised-tokens: 1.252489E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.300433E-02 | lm-token-buffer-mib: 5.807060E+01 | global_load_balancing_loss: 1.001641E+00 | loss scale: 1.0 | grad norm: 0.077 | params norm: 11577.621 | params rms: 0.062689 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 0:53:12.948457 | finish at 2026-10-10 21:52:21 + [2026-10-04 21:00:35] iteration 1163/ 6855 | consumed samples: 148864 | elapsed time per iteration (ms): 86125.6 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.67% | FLOPs estimator: workload-aware | logical tokens/s: 194799.2 | valid tokens/s: 194799.2 | physical tokens/s: 195523.1 | loss tokens/s: 147361.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.673994E-01 | lm-global-supervised-tokens: 1.269164E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.248908E-02 | lm-token-buffer-mib: 5.821603E+01 | global_load_balancing_loss: 1.001171E+00 | loss scale: 1.0 | grad norm: 0.063 | params norm: 11577.542 | params rms: 0.062689 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 16:10:27.069117 | finish at 2026-10-10 13:11:02 + [2026-10-04 21:02:02] iteration 1164/ 6855 | consumed samples: 148992 | elapsed time per iteration (ms): 87624.2 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.68% | FLOPs estimator: workload-aware | logical tokens/s: 191467.5 | valid tokens/s: 191467.5 | physical tokens/s: 191948.4 | loss tokens/s: 138790.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.733823E-01 | lm-global-supervised-tokens: 1.216142E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.663430E-02 | lm-token-buffer-mib: 5.813200E+01 | global_load_balancing_loss: 1.001292E+00 | loss scale: 1.0 | grad norm: 0.088 | params norm: 11577.465 | params rms: 0.062688 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 18:31:09.455772 | finish at 2026-10-10 15:33:12 + [2026-10-04 21:03:26] iteration 1165/ 6855 | consumed samples: 149120 | elapsed time per iteration (ms): 83339.1 | throughput per GPU (TFLOP/s/GPU): 43.2 | MFU 13.85% | FLOPs estimator: workload-aware | logical tokens/s: 201312.6 | valid tokens/s: 201312.6 | physical tokens/s: 201806.4 | loss tokens/s: 155447.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.719141E-01 | lm-global-supervised-tokens: 1.295483E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.151306E-02 | lm-token-buffer-mib: 5.808300E+01 | global_load_balancing_loss: 1.000110E+00 | loss scale: 1.0 | grad norm: 0.053 | params norm: 11577.389 | params rms: 0.062688 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 11:43:19.550240 | finish at 2026-10-10 08:46:45 + [2026-10-04 21:04:50] iteration 1166/ 6855 | consumed samples: 149248 | elapsed time per iteration (ms): 84327.8 | throughput per GPU (TFLOP/s/GPU): 42.8 | MFU 13.72% | FLOPs estimator: workload-aware | logical tokens/s: 198952.3 | valid tokens/s: 198952.3 | physical tokens/s: 199402.8 | loss tokens/s: 146237.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.369274E-01 | lm-global-supervised-tokens: 1.233186E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.524196E-02 | lm-token-buffer-mib: 5.805857E+01 | global_load_balancing_loss: 1.000309E+00 | loss scale: 1.0 | grad norm: 0.049 | params norm: 11577.312 | params rms: 0.062687 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 13:15:40.914082 | finish at 2026-10-10 10:20:31 + [2026-10-04 21:06:17] iteration 1167/ 6855 | consumed samples: 149376 | elapsed time per iteration (ms): 86873.6 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.45% | FLOPs estimator: workload-aware | logical tokens/s: 193121.3 | valid tokens/s: 193121.2 | physical tokens/s: 193581.9 | loss tokens/s: 143397.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.653355E-01 | lm-global-supervised-tokens: 1.245742E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.314410E-02 | lm-token-buffer-mib: 5.817542E+01 | global_load_balancing_loss: 1.000890E+00 | loss scale: 1.0 | grad norm: 0.070 | params norm: 11577.234 | params rms: 0.062687 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 17:15:37.312128 | finish at 2026-10-10 14:21:54 + [2026-10-04 21:07:45] iteration 1168/ 6855 | consumed samples: 149504 | elapsed time per iteration (ms): 88375.7 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.41% | FLOPs estimator: workload-aware | logical tokens/s: 189838.9 | valid tokens/s: 189838.9 | physical tokens/s: 190218.4 | loss tokens/s: 137359.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.427752E-01 | lm-global-supervised-tokens: 1.213927E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.732412E-02 | lm-token-buffer-mib: 5.808324E+01 | global_load_balancing_loss: 1.000085E+00 | loss scale: 1.0 | grad norm: 0.068 | params norm: 11577.155 | params rms: 0.062687 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 19:36:32.604527 | finish at 2026-10-10 16:44:18 + [2026-10-04 21:09:12] iteration 1169/ 6855 | consumed samples: 149632 | elapsed time per iteration (ms): 86848.1 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.46% | FLOPs estimator: workload-aware | logical tokens/s: 193178.1 | valid tokens/s: 193178.1 | physical tokens/s: 193732.4 | loss tokens/s: 147708.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.927140E-01 | lm-global-supervised-tokens: 1.282817E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.131748E-02 | lm-token-buffer-mib: 5.807504E+01 | global_load_balancing_loss: 9.992669E-01 | loss scale: 1.0 | grad norm: 0.092 | params norm: 11577.075 | params rms: 0.062686 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 17:10:18.323411 | finish at 2026-10-10 14:19:30 + [2026-10-04 21:11:03] iteration 1170/ 6855 | consumed samples: 149760 | elapsed time per iteration (ms): 110875.5 | throughput per GPU (TFLOP/s/GPU): 33.3 | MFU 10.67% | FLOPs estimator: workload-aware | logical tokens/s: 151314.8 | valid tokens/s: 151314.8 | physical tokens/s: 151574.2 | loss tokens/s: 112046.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.102982E-01 | lm-global-supervised-tokens: 1.242324E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.421071E-02 | lm-token-buffer-mib: 5.806781E+01 | global_load_balancing_loss: 9.994794E-01 | loss scale: 1.0 | grad norm: 0.036 | params norm: 11576.993 | params rms: 0.062686 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 7 days, 7:05:27.314883 | finish at 2026-10-12 04:16:30 + [2026-10-04 21:12:35] iteration 1171/ 6855 | consumed samples: 149888 | elapsed time per iteration (ms): 91873.9 | throughput per GPU (TFLOP/s/GPU): 40.6 | MFU 13.01% | FLOPs estimator: workload-aware | logical tokens/s: 182611.2 | valid tokens/s: 182611.2 | physical tokens/s: 183048.6 | loss tokens/s: 132762.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.968040E-01 | lm-global-supervised-tokens: 1.219740E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.467969E-02 | lm-token-buffer-mib: 5.807590E+01 | global_load_balancing_loss: 1.000122E+00 | loss scale: 1.0 | grad norm: 0.107 | params norm: 11576.908 | params rms: 0.062685 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 1:03:31.262147 | finish at 2026-10-10 22:16:06 + [2026-10-04 21:14:00] iteration 1172/ 6855 | consumed samples: 150016 | elapsed time per iteration (ms): 85340.2 | throughput per GPU (TFLOP/s/GPU): 43.3 | MFU 13.87% | FLOPs estimator: workload-aware | logical tokens/s: 196592.0 | valid tokens/s: 196592.0 | physical tokens/s: 197011.0 | loss tokens/s: 145992.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.871777E-01 | lm-global-supervised-tokens: 1.245905E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.558162E-02 | lm-token-buffer-mib: 5.811140E+01 | global_load_balancing_loss: 9.996330E-01 | loss scale: 1.0 | grad norm: 0.187 | params norm: 11576.825 | params rms: 0.062685 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 14:43:08.524312 | finish at 2026-10-10 11:57:09 + [2026-10-04 21:15:26] iteration 1173/ 6855 | consumed samples: 150144 | elapsed time per iteration (ms): 86016.4 | throughput per GPU (TFLOP/s/GPU): 43.4 | MFU 13.90% | FLOPs estimator: workload-aware | logical tokens/s: 195046.2 | valid tokens/s: 195046.2 | physical tokens/s: 195440.5 | loss tokens/s: 145362.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.827447E-01 | lm-global-supervised-tokens: 1.250357E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.170171E-02 | lm-token-buffer-mib: 5.805753E+01 | global_load_balancing_loss: 9.996775E-01 | loss scale: 1.0 | grad norm: 0.043 | params norm: 11576.743 | params rms: 0.062684 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 15:45:45.262579 | finish at 2026-10-10 13:01:11 + [2026-10-04 21:16:56] iteration 1174/ 6855 | consumed samples: 150272 | elapsed time per iteration (ms): 90350.9 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.52% | FLOPs estimator: workload-aware | logical tokens/s: 185689.3 | valid tokens/s: 185689.3 | physical tokens/s: 186107.4 | loss tokens/s: 140133.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.958444E-01 | lm-global-supervised-tokens: 1.266121E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.514740E-02 | lm-token-buffer-mib: 5.808890E+01 | global_load_balancing_loss: 1.000507E+00 | loss scale: 1.0 | grad norm: 0.170 | params norm: 11576.660 | params rms: 0.062684 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 22:34:43.530252 | finish at 2026-10-10 19:51:40 + [2026-10-04 21:18:26] iteration 1175/ 6855 | consumed samples: 150400 | elapsed time per iteration (ms): 89915.1 | throughput per GPU (TFLOP/s/GPU): 41.2 | MFU 13.19% | FLOPs estimator: workload-aware | logical tokens/s: 186589.5 | valid tokens/s: 186589.5 | physical tokens/s: 187021.2 | loss tokens/s: 136979.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.698622E-01 | lm-global-supervised-tokens: 1.231649E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.360884E-02 | lm-token-buffer-mib: 5.821249E+01 | global_load_balancing_loss: 1.000658E+00 | loss scale: 1.0 | grad norm: 0.119 | params norm: 11576.577 | params rms: 0.062683 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 21:51:57.824078 | finish at 2026-10-10 19:10:24 + [2026-10-04 21:19:54] iteration 1176/ 6855 | consumed samples: 150528 | elapsed time per iteration (ms): 87340.4 | throughput per GPU (TFLOP/s/GPU): 43.1 | MFU 13.80% | FLOPs estimator: workload-aware | logical tokens/s: 192089.6 | valid tokens/s: 192089.6 | physical tokens/s: 192537.4 | loss tokens/s: 138920.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.527774E-01 | lm-global-supervised-tokens: 1.213338E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.162563E-02 | lm-token-buffer-mib: 5.804378E+01 | global_load_balancing_loss: 1.000868E+00 | loss scale: 1.0 | grad norm: 0.050 | params norm: 11576.494 | params rms: 0.062683 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 17:46:46.342710 | finish at 2026-10-10 15:06:40 + [2026-10-04 21:21:19] iteration 1177/ 6855 | consumed samples: 150656 | elapsed time per iteration (ms): 85198.5 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.78% | FLOPs estimator: workload-aware | logical tokens/s: 196918.9 | valid tokens/s: 196918.9 | physical tokens/s: 197327.6 | loss tokens/s: 145963.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.403884E-01 | lm-global-supervised-tokens: 1.243590E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.167026E-02 | lm-token-buffer-mib: 5.813966E+01 | global_load_balancing_loss: 1.000212E+00 | loss scale: 1.0 | grad norm: 0.122 | params norm: 11576.410 | params rms: 0.062682 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 14:22:36.999956 | finish at 2026-10-10 11:43:56 + [2026-10-04 21:22:47] iteration 1178/ 6855 | consumed samples: 150784 | elapsed time per iteration (ms): 87780.4 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.48% | FLOPs estimator: workload-aware | logical tokens/s: 191126.9 | valid tokens/s: 191126.9 | physical tokens/s: 191641.6 | loss tokens/s: 141319.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.992127E-01 | lm-global-supervised-tokens: 1.240505E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.954393E-02 | lm-token-buffer-mib: 5.806574E+01 | global_load_balancing_loss: 1.001179E+00 | loss scale: 1.0 | grad norm: 0.055 | params norm: 11576.327 | params rms: 0.062682 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 18:25:29.521858 | finish at 2026-10-10 15:48:16 + [2026-10-04 21:24:14] iteration 1179/ 6855 | consumed samples: 150912 | elapsed time per iteration (ms): 86891.0 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.67% | FLOPs estimator: workload-aware | logical tokens/s: 193082.6 | valid tokens/s: 193082.5 | physical tokens/s: 193562.1 | loss tokens/s: 136436.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.766978E-01 | lm-global-supervised-tokens: 1.185510E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.179574E-02 | lm-token-buffer-mib: 5.809309E+01 | global_load_balancing_loss: 1.000215E+00 | loss scale: 1.0 | grad norm: 0.242 | params norm: 11576.245 | params rms: 0.062682 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 16:59:53.233635 | finish at 2026-10-10 14:24:07 + [2026-10-04 21:26:03] iteration 1180/ 6855 | consumed samples: 151040 | elapsed time per iteration (ms): 109448.8 | throughput per GPU (TFLOP/s/GPU): 34.5 | MFU 11.06% | FLOPs estimator: workload-aware | logical tokens/s: 153288.2 | valid tokens/s: 153288.2 | physical tokens/s: 153707.6 | loss tokens/s: 113878.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.675764E-01 | lm-global-supervised-tokens: 1.246385E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.175670E-02 | lm-token-buffer-mib: 5.804401E+01 | global_load_balancing_loss: 1.000082E+00 | loss scale: 1.0 | grad norm: 0.065 | params norm: 11576.163 | params rms: 0.062681 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 7 days, 4:32:02.082561 | finish at 2026-10-12 01:58:05 + [2026-10-04 21:27:30] iteration 1181/ 6855 | consumed samples: 151168 | elapsed time per iteration (ms): 86776.0 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.53% | FLOPs estimator: workload-aware | logical tokens/s: 193338.9 | valid tokens/s: 193338.9 | physical tokens/s: 193975.6 | loss tokens/s: 144658.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.671796E-01 | lm-global-supervised-tokens: 1.255291E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.464436E-02 | lm-token-buffer-mib: 5.811533E+01 | global_load_balancing_loss: 1.000152E+00 | loss scale: 1.0 | grad norm: 0.326 | params norm: 11576.082 | params rms: 0.062681 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 16:46:07.122883 | finish at 2026-10-10 14:13:37 + [2026-10-04 21:28:54] iteration 1182/ 6855 | consumed samples: 151296 | elapsed time per iteration (ms): 84348.3 | throughput per GPU (TFLOP/s/GPU): 43.2 | MFU 13.84% | FLOPs estimator: workload-aware | logical tokens/s: 198903.2 | valid tokens/s: 198903.2 | physical tokens/s: 199485.1 | loss tokens/s: 150979.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.539828E-01 | lm-global-supervised-tokens: 1.273486E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.174546E-02 | lm-token-buffer-mib: 5.811025E+01 | global_load_balancing_loss: 9.999012E-01 | loss scale: 1.0 | grad norm: 0.142 | params norm: 11575.999 | params rms: 0.062680 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 12:55:08.168182 | finish at 2026-10-10 10:24:02 + [2026-10-04 21:30:21] iteration 1183/ 6855 | consumed samples: 151424 | elapsed time per iteration (ms): 86615.1 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.78% | FLOPs estimator: workload-aware | logical tokens/s: 193698.1 | valid tokens/s: 193698.0 | physical tokens/s: 194240.8 | loss tokens/s: 136927.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.306232E-01 | lm-global-supervised-tokens: 1.186000E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.204085E-02 | lm-token-buffer-mib: 5.822272E+01 | global_load_balancing_loss: 1.000448E+00 | loss scale: 1.0 | grad norm: 0.287 | params norm: 11575.918 | params rms: 0.062680 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 16:28:01.068451 | finish at 2026-10-10 13:58:22 + [2026-10-04 21:31:49] iteration 1184/ 6855 | consumed samples: 151552 | elapsed time per iteration (ms): 88107.6 | throughput per GPU (TFLOP/s/GPU): 41.3 | MFU 13.25% | FLOPs estimator: workload-aware | logical tokens/s: 190417.2 | valid tokens/s: 190417.2 | physical tokens/s: 190850.7 | loss tokens/s: 146328.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.441831E-01 | lm-global-supervised-tokens: 1.289268E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.162832E-02 | lm-token-buffer-mib: 5.813685E+01 | global_load_balancing_loss: 9.999074E-01 | loss scale: 1.0 | grad norm: 0.182 | params norm: 11575.839 | params rms: 0.062679 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 18:47:38.007457 | finish at 2026-10-10 16:19:27 + [2026-10-04 21:33:16] iteration 1185/ 6855 | consumed samples: 151680 | elapsed time per iteration (ms): 86712.3 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.56% | FLOPs estimator: workload-aware | logical tokens/s: 193481.3 | valid tokens/s: 193481.3 | physical tokens/s: 194038.8 | loss tokens/s: 150941.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.098811E-01 | lm-global-supervised-tokens: 1.308847E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.790213E-02 | lm-token-buffer-mib: 5.821714E+01 | global_load_balancing_loss: 9.994376E-01 | loss scale: 1.0 | grad norm: 0.231 | params norm: 11575.760 | params rms: 0.062679 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 16:34:18.876536 | finish at 2026-10-10 14:07:34 + [2026-10-04 21:34:45] iteration 1186/ 6855 | consumed samples: 151808 | elapsed time per iteration (ms): 88989.5 | throughput per GPU (TFLOP/s/GPU): 40.6 | MFU 13.02% | FLOPs estimator: workload-aware | logical tokens/s: 188530.2 | valid tokens/s: 188530.2 | physical tokens/s: 189153.2 | loss tokens/s: 138689.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.448600E-01 | lm-global-supervised-tokens: 1.234186E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.154221E-02 | lm-token-buffer-mib: 5.828931E+01 | global_load_balancing_loss: 9.997773E-01 | loss scale: 1.0 | grad norm: 0.040 | params norm: 11575.680 | params rms: 0.062679 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 20:08:01.256801 | finish at 2026-10-10 17:42:46 + [2026-10-04 21:36:10] iteration 1187/ 6855 | consumed samples: 151936 | elapsed time per iteration (ms): 85490.9 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.69% | FLOPs estimator: workload-aware | logical tokens/s: 196245.1 | valid tokens/s: 196245.1 | physical tokens/s: 196756.3 | loss tokens/s: 142374.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.618575E-01 | lm-global-supervised-tokens: 1.217176E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.401632E-02 | lm-token-buffer-mib: 5.814439E+01 | global_load_balancing_loss: 1.000264E+00 | loss scale: 1.0 | grad norm: 0.168 | params norm: 11575.599 | params rms: 0.062678 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 14:36:02.240343 | finish at 2026-10-10 12:12:12 + [2026-10-04 21:37:41] iteration 1188/ 6855 | consumed samples: 152064 | elapsed time per iteration (ms): 91015.0 | throughput per GPU (TFLOP/s/GPU): 41.2 | MFU 13.20% | FLOPs estimator: workload-aware | logical tokens/s: 184333.4 | valid tokens/s: 184333.4 | physical tokens/s: 184786.2 | loss tokens/s: 129803.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.197543E-01 | lm-global-supervised-tokens: 1.181411E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.163515E-02 | lm-token-buffer-mib: 5.818008E+01 | global_load_balancing_loss: 1.000636E+00 | loss scale: 1.0 | grad norm: 0.159 | params norm: 11575.519 | params rms: 0.062678 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 23:16:22.232582 | finish at 2026-10-10 20:54:03 + [2026-10-04 21:39:08] iteration 1189/ 6855 | consumed samples: 152192 | elapsed time per iteration (ms): 87336.3 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.59% | FLOPs estimator: workload-aware | logical tokens/s: 192098.5 | valid tokens/s: 192098.5 | physical tokens/s: 192806.5 | loss tokens/s: 141494.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.940596E-01 | lm-global-supervised-tokens: 1.235760E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.402204E-02 | lm-token-buffer-mib: 5.857400E+01 | global_load_balancing_loss: 1.000716E+00 | loss scale: 1.0 | grad norm: 0.041 | params norm: 11575.437 | params rms: 0.062677 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 17:27:27.595440 | finish at 2026-10-10 15:06:36 + [2026-10-04 21:40:58] iteration 1190/ 6855 | consumed samples: 152320 | elapsed time per iteration (ms): 109379.8 | throughput per GPU (TFLOP/s/GPU): 33.8 | MFU 10.82% | FLOPs estimator: workload-aware | logical tokens/s: 153384.5 | valid tokens/s: 153384.5 | physical tokens/s: 153702.6 | loss tokens/s: 110637.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.451168E-01 | lm-global-supervised-tokens: 1.210146E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.408907E-02 | lm-token-buffer-mib: 5.812511E+01 | global_load_balancing_loss: 1.000876E+00 | loss scale: 1.0 | grad norm: 0.104 | params norm: 11575.354 | params rms: 0.062677 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 7 days, 4:07:16.790316 | finish at 2026-10-12 01:48:15 + [2026-10-04 21:42:21] iteration 1191/ 6855 | consumed samples: 152448 | elapsed time per iteration (ms): 83609.5 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.79% | FLOPs estimator: workload-aware | logical tokens/s: 200661.3 | valid tokens/s: 200661.3 | physical tokens/s: 201197.0 | loss tokens/s: 155178.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.807842E-01 | lm-global-supervised-tokens: 1.297436E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.302988E-02 | lm-token-buffer-mib: 5.826242E+01 | global_load_balancing_loss: 1.001001E+00 | loss scale: 1.0 | grad norm: 0.167 | params norm: 11575.272 | params rms: 0.062676 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 11:32:44.348053 | finish at 2026-10-10 09:15:06 + [2026-10-04 21:43:49] iteration 1192/ 6855 | consumed samples: 152576 | elapsed time per iteration (ms): 87735.4 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.79% | FLOPs estimator: workload-aware | logical tokens/s: 191224.8 | valid tokens/s: 191224.8 | physical tokens/s: 191700.0 | loss tokens/s: 134492.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.570132E-01 | lm-global-supervised-tokens: 1.179974E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.554884E-02 | lm-token-buffer-mib: 5.811087E+01 | global_load_balancing_loss: 1.001107E+00 | loss scale: 1.0 | grad norm: 0.034 | params norm: 11575.189 | params rms: 0.062676 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 18:00:45.663143 | finish at 2026-10-10 15:44:35 + [2026-10-04 21:45:14] iteration 1193/ 6855 | consumed samples: 152704 | elapsed time per iteration (ms): 85037.6 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.53% | FLOPs estimator: workload-aware | logical tokens/s: 197290.9 | valid tokens/s: 197290.8 | physical tokens/s: 197781.3 | loss tokens/s: 148863.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.384964E-01 | lm-global-supervised-tokens: 1.265902E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.156234E-02 | lm-token-buffer-mib: 5.811299E+01 | global_load_balancing_loss: 9.990299E-01 | loss scale: 1.0 | grad norm: 0.088 | params norm: 11575.106 | params rms: 0.062675 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 13:44:42.976474 | finish at 2026-10-10 11:29:57 + [2026-10-04 21:46:41] iteration 1194/ 6855 | consumed samples: 152832 | elapsed time per iteration (ms): 86601.5 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.55% | FLOPs estimator: workload-aware | logical tokens/s: 193729.0 | valid tokens/s: 193729.0 | physical tokens/s: 194221.8 | loss tokens/s: 146180.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.259097E-01 | lm-global-supervised-tokens: 1.265945E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.160141E-02 | lm-token-buffer-mib: 5.822518E+01 | global_load_balancing_loss: 9.994228E-01 | loss scale: 1.0 | grad norm: 0.133 | params norm: 11575.024 | params rms: 0.062675 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 16:10:50.882493 | finish at 2026-10-10 13:57:32 + [2026-10-04 21:48:07] iteration 1195/ 6855 | consumed samples: 152960 | elapsed time per iteration (ms): 86222.8 | throughput per GPU (TFLOP/s/GPU): 43.2 | MFU 13.83% | FLOPs estimator: workload-aware | logical tokens/s: 194578.8 | valid tokens/s: 194578.8 | physical tokens/s: 194998.2 | loss tokens/s: 137995.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.967682E-01 | lm-global-supervised-tokens: 1.189831E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.164276E-02 | lm-token-buffer-mib: 5.816000E+01 | global_load_balancing_loss: 9.989157E-01 | loss scale: 1.0 | grad norm: 0.079 | params norm: 11574.942 | params rms: 0.062675 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 15:33:40.844326 | finish at 2026-10-10 13:21:48 + [2026-10-04 21:49:35] iteration 1196/ 6855 | consumed samples: 153088 | elapsed time per iteration (ms): 88221.0 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.57% | FLOPs estimator: workload-aware | logical tokens/s: 190171.7 | valid tokens/s: 190171.7 | physical tokens/s: 190589.9 | loss tokens/s: 145923.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.969412E-01 | lm-global-supervised-tokens: 1.287351E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.417178E-02 | lm-token-buffer-mib: 5.813517E+01 | global_load_balancing_loss: 9.989524E-01 | loss scale: 1.0 | grad norm: 0.071 | params norm: 11574.859 | params rms: 0.062674 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 18:40:42.524069 | finish at 2026-10-10 16:30:18 + [2026-10-04 21:51:08] iteration 1197/ 6855 | consumed samples: 153216 | elapsed time per iteration (ms): 92772.8 | throughput per GPU (TFLOP/s/GPU): 40.6 | MFU 13.01% | FLOPs estimator: workload-aware | logical tokens/s: 180841.7 | valid tokens/s: 180841.7 | physical tokens/s: 181250.4 | loss tokens/s: 131187.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.898309E-01 | lm-global-supervised-tokens: 1.217061E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.333898E-02 | lm-token-buffer-mib: 5.808815E+01 | global_load_balancing_loss: 1.001403E+00 | loss scale: 1.0 | grad norm: 0.120 | params norm: 11574.776 | params rms: 0.062674 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 1:48:28.522458 | finish at 2026-10-10 23:39:36 + [2026-10-04 21:52:33] iteration 1198/ 6855 | consumed samples: 153344 | elapsed time per iteration (ms): 85025.4 | throughput per GPU (TFLOP/s/GPU): 42.8 | MFU 13.71% | FLOPs estimator: workload-aware | logical tokens/s: 197319.2 | valid tokens/s: 197319.2 | physical tokens/s: 197843.2 | loss tokens/s: 149330.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.255256E-01 | lm-global-supervised-tokens: 1.269689E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.274496E-02 | lm-token-buffer-mib: 5.821957E+01 | global_load_balancing_loss: 1.000312E+00 | loss scale: 1.0 | grad norm: 0.047 | params norm: 11574.694 | params rms: 0.062673 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 13:36:28.563283 | finish at 2026-10-10 11:29:02 + [2026-10-04 21:53:59] iteration 1199/ 6855 | consumed samples: 153472 | elapsed time per iteration (ms): 86218.9 | throughput per GPU (TFLOP/s/GPU): 43.2 | MFU 13.83% | FLOPs estimator: workload-aware | logical tokens/s: 194588.4 | valid tokens/s: 194588.4 | physical tokens/s: 195117.9 | loss tokens/s: 142900.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.406796E-01 | lm-global-supervised-tokens: 1.232072E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.487103E-02 | lm-token-buffer-mib: 5.808553E+01 | global_load_balancing_loss: 1.001543E+00 | loss scale: 1.0 | grad norm: 0.079 | params norm: 11574.614 | params rms: 0.062673 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 15:27:34.007854 | finish at 2026-10-10 13:21:33 + [2026-10-04 21:55:48] iteration 1200/ 6855 | consumed samples: 153600 | elapsed time per iteration (ms): 109256.5 | throughput per GPU (TFLOP/s/GPU): 34.0 | MFU 10.89% | FLOPs estimator: workload-aware | logical tokens/s: 153557.3 | valid tokens/s: 153557.3 | physical tokens/s: 153931.4 | loss tokens/s: 108389.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.538320E-01 | lm-global-supervised-tokens: 1.184228E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.143589E-02 | lm-token-buffer-mib: 5.834299E+01 | global_load_balancing_loss: 1.000578E+00 | loss scale: 1.0 | grad norm: 0.073 | params norm: 11574.532 | params rms: 0.062672 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 7 days, 3:37:25.767746 | finish at 2026-10-12 01:33:14 +times across ranks (ms): + save-checkpoint: + rank 0: 25495.41 + rank 1: 25495.49 + rank 2: 25495.53 + rank 3: 25495.51 + rank 4: 25495.52 + rank 5: 25495.50 + rank 6: 25495.50 + rank 7: 25495.54 + rank 8: 25496.66 + rank 9: 25496.60 + rank 10: 25496.62 + rank 11: 25496.62 + rank 12: 25496.62 + rank 13: 25496.61 + rank 14: 25496.63 + rank 15: 25496.61 + rank 16: 25496.46 + rank 17: 25496.49 + rank 18: 25496.42 + rank 19: 25496.47 + rank 20: 25496.42 + rank 21: 25496.46 + rank 22: 25496.43 + rank 23: 25496.42 + rank 24: 25496.45 + rank 25: 25496.37 + rank 26: 25496.45 + rank 27: 25496.44 + rank 28: 25496.39 + rank 29: 25496.37 + rank 30: 25496.37 + rank 31: 25496.43 + rank 32: 25497.12 + rank 33: 25497.08 + rank 34: 25497.03 + rank 35: 25497.00 + rank 36: 25496.98 + rank 37: 25496.93 + rank 38: 25497.00 + rank 39: 25496.94 + rank 40: 25496.81 + rank 41: 25496.76 + rank 42: 25496.79 + rank 43: 25496.78 + rank 44: 25496.76 + rank 45: 25496.78 + rank 46: 25496.76 + rank 47: 25496.76 + rank 48: 25496.57 + rank 49: 25496.67 + rank 50: 25496.60 + rank 51: 25496.49 + rank 52: 25496.62 + rank 53: 25496.56 + rank 54: 25496.61 + rank 55: 25496.58 + rank 56: 25496.82 + rank 57: 25496.81 + rank 58: 25496.71 + rank 59: 25496.77 + rank 60: 25496.80 + rank 61: 25496.84 + rank 62: 25496.79 + rank 63: 25496.81 + [2026-10-04 21:57:50] iteration 1201/ 6855 | consumed samples: 153728 | elapsed time per iteration (ms): 94237.3 | throughput per GPU (TFLOP/s/GPU): 40.1 | MFU 12.85% | FLOPs estimator: workload-aware | logical tokens/s: 178031.1 | valid tokens/s: 178031.1 | physical tokens/s: 178347.3 | loss tokens/s: 125741.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.907908E-01 | lm-global-supervised-tokens: 1.184952E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 8.878815E-01 | lm-token-buffer-mib: 5.808601E+01 | global_load_balancing_loss: 9.995495E-01 | loss scale: 1.0 | grad norm: 0.040 | params norm: 11574.450 | params rms: 0.062672 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 4:00:17.916166 | finish at 2026-10-11 01:58:08 + [2026-10-04 21:59:25] iteration 1202/ 6855 | consumed samples: 153856 | elapsed time per iteration (ms): 95150.6 | throughput per GPU (TFLOP/s/GPU): 40.2 | MFU 12.88% | FLOPs estimator: workload-aware | logical tokens/s: 176322.7 | valid tokens/s: 176322.7 | physical tokens/s: 176691.5 | loss tokens/s: 123794.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.389838E-01 | lm-global-supervised-tokens: 1.177910E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.493072E-02 | lm-token-buffer-mib: 5.820660E+01 | global_load_balancing_loss: 9.995635E-01 | loss scale: 1.0 | grad norm: 0.090 | params norm: 11574.367 | params rms: 0.062671 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 5:24:46.392770 | finish at 2026-10-11 03:24:11 + [2026-10-04 22:00:51] iteration 1203/ 6855 | consumed samples: 153984 | elapsed time per iteration (ms): 85575.5 | throughput per GPU (TFLOP/s/GPU): 42.8 | MFU 13.71% | FLOPs estimator: workload-aware | logical tokens/s: 196050.8 | valid tokens/s: 196050.8 | physical tokens/s: 196394.3 | loss tokens/s: 141859.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.258647E-01 | lm-global-supervised-tokens: 1.213965E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.161754E-02 | lm-token-buffer-mib: 5.803124E+01 | global_load_balancing_loss: 9.998637E-01 | loss scale: 1.0 | grad norm: 0.038 | params norm: 11574.285 | params rms: 0.062671 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 14:21:12.812307 | finish at 2026-10-10 12:22:03 + [2026-10-04 22:02:16] iteration 1204/ 6855 | consumed samples: 154112 | elapsed time per iteration (ms): 85858.4 | throughput per GPU (TFLOP/s/GPU): 43.4 | MFU 13.91% | FLOPs estimator: workload-aware | logical tokens/s: 195405.3 | valid tokens/s: 195405.3 | physical tokens/s: 195815.3 | loss tokens/s: 146588.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.053165E-01 | lm-global-supervised-tokens: 1.258584E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.969527E-02 | lm-token-buffer-mib: 5.804700E+01 | global_load_balancing_loss: 9.994094E-01 | loss scale: 1.0 | grad norm: 0.056 | params norm: 11574.204 | params rms: 0.062671 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 14:46:26.070947 | finish at 2026-10-10 12:48:43 + [2026-10-04 22:03:43] iteration 1205/ 6855 | consumed samples: 154240 | elapsed time per iteration (ms): 86682.6 | throughput per GPU (TFLOP/s/GPU): 43.8 | MFU 14.03% | FLOPs estimator: workload-aware | logical tokens/s: 193547.0 | valid tokens/s: 193546.9 | physical tokens/s: 193969.9 | loss tokens/s: 139987.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.137847E-01 | lm-global-supervised-tokens: 1.213448E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.332460E-02 | lm-token-buffer-mib: 5.806923E+01 | global_load_balancing_loss: 9.998372E-01 | loss scale: 1.0 | grad norm: 0.041 | params norm: 11574.123 | params rms: 0.062670 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 16:02:36.876016 | finish at 2026-10-10 14:06:20 + [2026-10-04 22:05:10] iteration 1206/ 6855 | consumed samples: 154368 | elapsed time per iteration (ms): 87192.4 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.69% | FLOPs estimator: workload-aware | logical tokens/s: 192415.4 | valid tokens/s: 192415.4 | physical tokens/s: 192909.4 | loss tokens/s: 141059.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.965590E-01 | lm-global-supervised-tokens: 1.229932E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.157195E-02 | lm-token-buffer-mib: 5.821941E+01 | global_load_balancing_loss: 9.995209E-01 | loss scale: 1.0 | grad norm: 0.060 | params norm: 11574.042 | params rms: 0.062670 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 16:49:09.669496 | finish at 2026-10-10 14:54:20 + [2026-10-04 22:06:36] iteration 1207/ 6855 | consumed samples: 154496 | elapsed time per iteration (ms): 85264.2 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.77% | FLOPs estimator: workload-aware | logical tokens/s: 196767.2 | valid tokens/s: 196767.2 | physical tokens/s: 197313.9 | loss tokens/s: 148329.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.747426E-01 | lm-global-supervised-tokens: 1.264720E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.541794E-02 | lm-token-buffer-mib: 5.823804E+01 | global_load_balancing_loss: 1.001781E+00 | loss scale: 1.0 | grad norm: 0.057 | params norm: 11573.962 | params rms: 0.062669 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 13:46:12.126034 | finish at 2026-10-10 11:52:48 + [2026-10-04 22:08:01] iteration 1208/ 6855 | consumed samples: 154624 | elapsed time per iteration (ms): 85739.0 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.68% | FLOPs estimator: workload-aware | logical tokens/s: 195677.3 | valid tokens/s: 195677.3 | physical tokens/s: 196186.5 | loss tokens/s: 141618.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.767926E-01 | lm-global-supervised-tokens: 1.214220E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.183156E-02 | lm-token-buffer-mib: 5.826240E+01 | global_load_balancing_loss: 1.001946E+00 | loss scale: 1.0 | grad norm: 0.041 | params norm: 11573.883 | params rms: 0.062669 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 14:29:28.149619 | finish at 2026-10-10 12:37:29 + [2026-10-04 22:09:27] iteration 1209/ 6855 | consumed samples: 154752 | elapsed time per iteration (ms): 85278.8 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.79% | FLOPs estimator: workload-aware | logical tokens/s: 196733.8 | valid tokens/s: 196733.8 | physical tokens/s: 197243.6 | loss tokens/s: 142377.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.453294E-01 | lm-global-supervised-tokens: 1.214177E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.968399E-02 | lm-token-buffer-mib: 5.805965E+01 | global_load_balancing_loss: 1.000862E+00 | loss scale: 1.0 | grad norm: 0.048 | params norm: 11573.804 | params rms: 0.062668 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 13:44:43.877367 | finish at 2026-10-10 11:54:10 + [2026-10-04 22:11:16] iteration 1210/ 6855 | consumed samples: 154880 | elapsed time per iteration (ms): 109473.3 | throughput per GPU (TFLOP/s/GPU): 34.3 | MFU 10.99% | FLOPs estimator: workload-aware | logical tokens/s: 153253.9 | valid tokens/s: 153253.8 | physical tokens/s: 153639.0 | loss tokens/s: 112706.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.857617E-01 | lm-global-supervised-tokens: 1.233830E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.338341E-02 | lm-token-buffer-mib: 5.813391E+01 | global_load_balancing_loss: 9.996426E-01 | loss scale: 1.0 | grad norm: 0.041 | params norm: 11573.723 | params rms: 0.062668 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 7 days, 3:39:36.501138 | finish at 2026-10-12 01:50:53 + [2026-10-04 22:12:44] iteration 1211/ 6855 | consumed samples: 155008 | elapsed time per iteration (ms): 87862.3 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.41% | FLOPs estimator: workload-aware | logical tokens/s: 190947.5 | valid tokens/s: 190947.5 | physical tokens/s: 191320.1 | loss tokens/s: 142026.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.527712E-01 | lm-global-supervised-tokens: 1.247875E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.456447E-02 | lm-token-buffer-mib: 5.811973E+01 | global_load_balancing_loss: 1.000037E+00 | loss scale: 1.0 | grad norm: 0.037 | params norm: 11573.644 | params rms: 0.062668 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 17:44:54.835546 | finish at 2026-10-10 15:57:39 + [2026-10-04 22:14:13] iteration 1212/ 6855 | consumed samples: 155136 | elapsed time per iteration (ms): 89371.2 | throughput per GPU (TFLOP/s/GPU): 40.6 | MFU 13.00% | FLOPs estimator: workload-aware | logical tokens/s: 187724.4 | valid tokens/s: 187724.4 | physical tokens/s: 188211.5 | loss tokens/s: 138266.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.554589E-01 | lm-global-supervised-tokens: 1.235700E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.320534E-02 | lm-token-buffer-mib: 5.810397E+01 | global_load_balancing_loss: 9.989603E-01 | loss scale: 1.0 | grad norm: 0.050 | params norm: 11573.566 | params rms: 0.062667 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 20:05:21.591936 | finish at 2026-10-10 18:19:35 + [2026-10-04 22:15:38] iteration 1213/ 6855 | consumed samples: 155264 | elapsed time per iteration (ms): 84778.2 | throughput per GPU (TFLOP/s/GPU): 43.7 | MFU 14.01% | FLOPs estimator: workload-aware | logical tokens/s: 197895.0 | valid tokens/s: 197895.0 | physical tokens/s: 198169.9 | loss tokens/s: 143177.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.985680E-01 | lm-global-supervised-tokens: 1.213829E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.509672E-02 | lm-token-buffer-mib: 5.820493E+01 | global_load_balancing_loss: 9.992619E-01 | loss scale: 1.0 | grad norm: 0.035 | params norm: 11573.487 | params rms: 0.062667 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 12:51:58.431718 | finish at 2026-10-10 11:07:37 + [2026-10-04 22:17:09] iteration 1214/ 6855 | consumed samples: 155392 | elapsed time per iteration (ms): 91168.7 | throughput per GPU (TFLOP/s/GPU): 41.6 | MFU 13.34% | FLOPs estimator: workload-aware | logical tokens/s: 184023.5 | valid tokens/s: 184023.5 | physical tokens/s: 184376.0 | loss tokens/s: 138450.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.088705E-01 | lm-global-supervised-tokens: 1.262240E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.249578E-02 | lm-token-buffer-mib: 5.811438E+01 | global_load_balancing_loss: 1.000468E+00 | loss scale: 1.0 | grad norm: 0.036 | params norm: 11573.408 | params rms: 0.062666 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 22:51:22.880016 | finish at 2026-10-10 21:08:32 + [2026-10-04 22:18:34] iteration 1215/ 6855 | consumed samples: 155520 | elapsed time per iteration (ms): 84922.6 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.74% | FLOPs estimator: workload-aware | logical tokens/s: 197558.2 | valid tokens/s: 197558.2 | physical tokens/s: 198088.8 | loss tokens/s: 143822.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.616514E-01 | lm-global-supervised-tokens: 1.221373E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.176065E-02 | lm-token-buffer-mib: 5.822236E+01 | global_load_balancing_loss: 1.001012E+00 | loss scale: 1.0 | grad norm: 0.034 | params norm: 11573.329 | params rms: 0.062666 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 13:02:43.269196 | finish at 2026-10-10 11:21:17 + [2026-10-04 22:20:05] iteration 1216/ 6855 | consumed samples: 155648 | elapsed time per iteration (ms): 90840.3 | throughput per GPU (TFLOP/s/GPU): 41.1 | MFU 13.17% | FLOPs estimator: workload-aware | logical tokens/s: 184688.8 | valid tokens/s: 184688.8 | physical tokens/s: 185168.1 | loss tokens/s: 140349.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.136467E-01 | lm-global-supervised-tokens: 1.274942E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.410683E-02 | lm-token-buffer-mib: 5.821274E+01 | global_load_balancing_loss: 1.000323E+00 | loss scale: 1.0 | grad norm: 0.045 | params norm: 11573.250 | params rms: 0.062665 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 22:17:28.379569 | finish at 2026-10-10 20:37:33 + [2026-10-04 22:21:37] iteration 1217/ 6855 | consumed samples: 155776 | elapsed time per iteration (ms): 91601.0 | throughput per GPU (TFLOP/s/GPU): 41.7 | MFU 13.38% | FLOPs estimator: workload-aware | logical tokens/s: 183155.5 | valid tokens/s: 183155.5 | physical tokens/s: 183561.8 | loss tokens/s: 127329.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.371462E-01 | lm-global-supervised-tokens: 1.166350E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.026037E-02 | lm-token-buffer-mib: 5.835123E+01 | global_load_balancing_loss: 1.001557E+00 | loss scale: 1.0 | grad norm: 0.067 | params norm: 11573.170 | params rms: 0.062665 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 23:27:26.193753 | finish at 2026-10-10 21:49:03 + [2026-10-04 22:22:58] iteration 1218/ 6855 | consumed samples: 155904 | elapsed time per iteration (ms): 81803.1 | throughput per GPU (TFLOP/s/GPU): 43.5 | MFU 13.94% | FLOPs estimator: workload-aware | logical tokens/s: 205092.2 | valid tokens/s: 205092.2 | physical tokens/s: 205770.7 | loss tokens/s: 157411.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.490346E-01 | lm-global-supervised-tokens: 1.287678E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.493178E-02 | lm-token-buffer-mib: 5.809808E+01 | global_load_balancing_loss: 1.000219E+00 | loss scale: 1.0 | grad norm: 0.043 | params norm: 11573.086 | params rms: 0.062664 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 8:05:24.297069 | finish at 2026-10-10 06:28:23 + [2026-10-04 22:24:26] iteration 1219/ 6855 | consumed samples: 156032 | elapsed time per iteration (ms): 87280.4 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.68% | FLOPs estimator: workload-aware | logical tokens/s: 192221.5 | valid tokens/s: 192221.5 | physical tokens/s: 192619.5 | loss tokens/s: 152063.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.110834E-01 | lm-global-supervised-tokens: 1.327212E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.792075E-02 | lm-token-buffer-mib: 5.816712E+01 | global_load_balancing_loss: 9.986705E-01 | loss scale: 1.0 | grad norm: 0.035 | params norm: 11573.004 | params rms: 0.062664 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 16:38:32.323863 | finish at 2026-10-10 15:02:58 + [2026-10-04 22:26:11] iteration 1220/ 6855 | consumed samples: 156160 | elapsed time per iteration (ms): 105476.5 | throughput per GPU (TFLOP/s/GPU): 34.4 | MFU 11.01% | FLOPs estimator: workload-aware | logical tokens/s: 159060.6 | valid tokens/s: 159060.6 | physical tokens/s: 159415.6 | loss tokens/s: 121483.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.772469E-01 | lm-global-supervised-tokens: 1.281368E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.383886E-02 | lm-token-buffer-mib: 5.812132E+01 | global_load_balancing_loss: 9.998086E-01 | loss scale: 1.0 | grad norm: 0.036 | params norm: 11572.923 | params rms: 0.062664 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 21:06:00.339674 | finish at 2026-10-11 19:32:12 + [2026-10-04 22:27:42] iteration 1221/ 6855 | consumed samples: 156288 | elapsed time per iteration (ms): 90513.1 | throughput per GPU (TFLOP/s/GPU): 43.1 | MFU 13.81% | FLOPs estimator: workload-aware | logical tokens/s: 185356.6 | valid tokens/s: 185356.6 | physical tokens/s: 185704.7 | loss tokens/s: 133491.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.970208E-01 | lm-global-supervised-tokens: 1.208268E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.333614E-02 | lm-token-buffer-mib: 5.806393E+01 | global_load_balancing_loss: 9.996940E-01 | loss scale: 1.0 | grad norm: 0.047 | params norm: 11572.844 | params rms: 0.062663 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 21:39:10.735037 | finish at 2026-10-10 20:06:52 + [2026-10-04 22:29:08] iteration 1222/ 6855 | consumed samples: 156416 | elapsed time per iteration (ms): 86228.9 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.57% | FLOPs estimator: workload-aware | logical tokens/s: 194565.9 | valid tokens/s: 194565.9 | physical tokens/s: 195209.1 | loss tokens/s: 146023.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.095025E-01 | lm-global-supervised-tokens: 1.259142E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.206923E-02 | lm-token-buffer-mib: 5.806373E+01 | global_load_balancing_loss: 1.000360E+00 | loss scale: 1.0 | grad norm: 0.030 | params norm: 11572.763 | params rms: 0.062663 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 14:55:27.461944 | finish at 2026-10-10 13:24:35 + [2026-10-04 22:30:39] iteration 1223/ 6855 | consumed samples: 156544 | elapsed time per iteration (ms): 90575.3 | throughput per GPU (TFLOP/s/GPU): 41.0 | MFU 13.14% | FLOPs estimator: workload-aware | logical tokens/s: 185229.5 | valid tokens/s: 185229.5 | physical tokens/s: 185732.6 | loss tokens/s: 137972.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.616929E-01 | lm-global-supervised-tokens: 1.249689E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.847241E-02 | lm-token-buffer-mib: 5.812691E+01 | global_load_balancing_loss: 9.998415E-01 | loss scale: 1.0 | grad norm: 0.055 | params norm: 11572.682 | params rms: 0.062662 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 21:41:59.976685 | finish at 2026-10-10 20:12:38 + [2026-10-04 22:32:06] iteration 1224/ 6855 | consumed samples: 156672 | elapsed time per iteration (ms): 87441.7 | throughput per GPU (TFLOP/s/GPU): 41.4 | MFU 13.27% | FLOPs estimator: workload-aware | logical tokens/s: 191867.5 | valid tokens/s: 191867.5 | physical tokens/s: 192445.0 | loss tokens/s: 143855.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.930687E-01 | lm-global-supervised-tokens: 1.257898E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.162617E-02 | lm-token-buffer-mib: 5.802685E+01 | global_load_balancing_loss: 1.000413E+00 | loss scale: 1.0 | grad norm: 0.048 | params norm: 11572.603 | params rms: 0.062662 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 16:46:24.146813 | finish at 2026-10-10 15:18:30 + [2026-10-04 22:33:32] iteration 1225/ 6855 | consumed samples: 156800 | elapsed time per iteration (ms): 85988.1 | throughput per GPU (TFLOP/s/GPU): 42.8 | MFU 13.72% | FLOPs estimator: workload-aware | logical tokens/s: 195110.6 | valid tokens/s: 195110.6 | physical tokens/s: 195485.6 | loss tokens/s: 148557.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.480121E-01 | lm-global-supervised-tokens: 1.277420E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.112733E-02 | lm-token-buffer-mib: 5.803156E+01 | global_load_balancing_loss: 1.000521E+00 | loss scale: 1.0 | grad norm: 0.052 | params norm: 11572.523 | params rms: 0.062661 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 14:28:32.763021 | finish at 2026-10-10 13:02:05 + [2026-10-04 22:34:59] iteration 1226/ 6855 | consumed samples: 156928 | elapsed time per iteration (ms): 87168.1 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.42% | FLOPs estimator: workload-aware | logical tokens/s: 192469.1 | valid tokens/s: 192469.1 | physical tokens/s: 193054.2 | loss tokens/s: 142956.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.886137E-01 | lm-global-supervised-tokens: 1.246123E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.368240E-02 | lm-token-buffer-mib: 5.817216E+01 | global_load_balancing_loss: 1.000965E+00 | loss scale: 1.0 | grad norm: 0.133 | params norm: 11572.443 | params rms: 0.062661 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 16:17:49.055732 | finish at 2026-10-10 14:52:48 + [2026-10-04 22:36:26] iteration 1227/ 6855 | consumed samples: 157056 | elapsed time per iteration (ms): 86700.8 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.58% | FLOPs estimator: workload-aware | logical tokens/s: 193506.8 | valid tokens/s: 193506.8 | physical tokens/s: 193872.5 | loss tokens/s: 147694.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.582336E-01 | lm-global-supervised-tokens: 1.280526E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.195834E-02 | lm-token-buffer-mib: 5.811942E+01 | global_load_balancing_loss: 1.000563E+00 | loss scale: 1.0 | grad norm: 0.109 | params norm: 11572.365 | params rms: 0.062661 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 15:32:31.856521 | finish at 2026-10-10 14:08:58 + [2026-10-04 22:37:55] iteration 1228/ 6855 | consumed samples: 157184 | elapsed time per iteration (ms): 89010.7 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.65% | FLOPs estimator: workload-aware | logical tokens/s: 188485.4 | valid tokens/s: 188485.4 | physical tokens/s: 188886.5 | loss tokens/s: 139260.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.840578E-01 | lm-global-supervised-tokens: 1.239570E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.718576E-02 | lm-token-buffer-mib: 5.806626E+01 | global_load_balancing_loss: 1.001169E+00 | loss scale: 1.0 | grad norm: 0.072 | params norm: 11572.286 | params rms: 0.062660 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 19:07:43.088087 | finish at 2026-10-10 17:45:38 + [2026-10-04 22:39:24] iteration 1229/ 6855 | consumed samples: 157312 | elapsed time per iteration (ms): 89405.2 | throughput per GPU (TFLOP/s/GPU): 41.6 | MFU 13.32% | FLOPs estimator: workload-aware | logical tokens/s: 187653.6 | valid tokens/s: 187653.6 | physical tokens/s: 188137.6 | loss tokens/s: 141124.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.092984E-01 | lm-global-supervised-tokens: 1.261728E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.263802E-02 | lm-token-buffer-mib: 5.807058E+01 | global_load_balancing_loss: 9.993418E-01 | loss scale: 1.0 | grad norm: 0.110 | params norm: 11572.207 | params rms: 0.062660 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 19:43:13.690101 | finish at 2026-10-10 18:22:38 + [2026-10-04 22:41:13] iteration 1230/ 6855 | consumed samples: 157440 | elapsed time per iteration (ms): 108920.0 | throughput per GPU (TFLOP/s/GPU): 34.6 | MFU 11.08% | FLOPs estimator: workload-aware | logical tokens/s: 154032.0 | valid tokens/s: 154032.0 | physical tokens/s: 154406.4 | loss tokens/s: 111018.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.936914E-01 | lm-global-supervised-tokens: 1.209215E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.296470E-02 | lm-token-buffer-mib: 5.825261E+01 | global_load_balancing_loss: 9.998128E-01 | loss scale: 1.0 | grad norm: 0.056 | params norm: 11572.127 | params rms: 0.062659 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 7 days, 2:11:14.776465 | finish at 2026-10-12 00:52:28 + [2026-10-04 22:42:41] iteration 1231/ 6855 | consumed samples: 157568 | elapsed time per iteration (ms): 88158.7 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.46% | FLOPs estimator: workload-aware | logical tokens/s: 190306.9 | valid tokens/s: 190306.9 | physical tokens/s: 190916.9 | loss tokens/s: 142694.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.925620E-01 | lm-global-supervised-tokens: 1.257973E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.486260E-02 | lm-token-buffer-mib: 5.825131E+01 | global_load_balancing_loss: 1.000342E+00 | loss scale: 1.0 | grad norm: 0.103 | params norm: 11572.047 | params rms: 0.062659 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 17:43:24.255203 | finish at 2026-10-10 16:26:06 + [2026-10-04 22:44:09] iteration 1232/ 6855 | consumed samples: 157696 | elapsed time per iteration (ms): 87606.7 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.52% | FLOPs estimator: workload-aware | logical tokens/s: 191505.0 | valid tokens/s: 191505.0 | physical tokens/s: 192103.8 | loss tokens/s: 139703.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.101086E-01 | lm-global-supervised-tokens: 1.223893E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.615786E-02 | lm-token-buffer-mib: 5.820230E+01 | global_load_balancing_loss: 1.000395E+00 | loss scale: 1.0 | grad norm: 0.055 | params norm: 11571.968 | params rms: 0.062658 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 16:50:12.476464 | finish at 2026-10-10 15:34:21 + [2026-10-04 22:45:37] iteration 1233/ 6855 | consumed samples: 157824 | elapsed time per iteration (ms): 88417.6 | throughput per GPU (TFLOP/s/GPU): 41.7 | MFU 13.35% | FLOPs estimator: workload-aware | logical tokens/s: 189749.3 | valid tokens/s: 189749.3 | physical tokens/s: 190236.0 | loss tokens/s: 140156.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.582021E-01 | lm-global-supervised-tokens: 1.239231E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.791006E-02 | lm-token-buffer-mib: 5.815105E+01 | global_load_balancing_loss: 9.997619E-01 | loss scale: 1.0 | grad norm: 0.129 | params norm: 11571.889 | params rms: 0.062658 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 18:04:43.730646 | finish at 2026-10-10 16:50:21 + [2026-10-04 22:47:06] iteration 1234/ 6855 | consumed samples: 157952 | elapsed time per iteration (ms): 88708.0 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.50% | FLOPs estimator: workload-aware | logical tokens/s: 189128.2 | valid tokens/s: 189128.2 | physical tokens/s: 189518.6 | loss tokens/s: 134765.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.863974E-01 | lm-global-supervised-tokens: 1.195481E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.246422E-02 | lm-token-buffer-mib: 5.811354E+01 | global_load_balancing_loss: 9.996684E-01 | loss scale: 1.0 | grad norm: 0.070 | params norm: 11571.811 | params rms: 0.062658 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 18:30:27.882113 | finish at 2026-10-10 17:17:34 + [2026-10-04 22:48:33] iteration 1235/ 6855 | consumed samples: 158080 | elapsed time per iteration (ms): 87140.9 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.55% | FLOPs estimator: workload-aware | logical tokens/s: 192528.2 | valid tokens/s: 192528.2 | physical tokens/s: 193032.3 | loss tokens/s: 137520.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.033294E-01 | lm-global-supervised-tokens: 1.198362E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.342496E-02 | lm-token-buffer-mib: 5.802290E+01 | global_load_balancing_loss: 1.001276E+00 | loss scale: 1.0 | grad norm: 0.085 | params norm: 11571.734 | params rms: 0.062657 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 16:02:11.884217 | finish at 2026-10-10 14:50:45 + [2026-10-04 22:50:02] iteration 1236/ 6855 | consumed samples: 158208 | elapsed time per iteration (ms): 88481.5 | throughput per GPU (TFLOP/s/GPU): 41.3 | MFU 13.24% | FLOPs estimator: workload-aware | logical tokens/s: 189612.1 | valid tokens/s: 189612.1 | physical tokens/s: 190252.2 | loss tokens/s: 135104.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.141954E-01 | lm-global-supervised-tokens: 1.195421E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.506206E-02 | lm-token-buffer-mib: 5.839076E+01 | global_load_balancing_loss: 9.997489E-01 | loss scale: 1.0 | grad norm: 0.071 | params norm: 11571.657 | params rms: 0.062657 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 18:06:17.355083 | finish at 2026-10-10 16:56:19 + [2026-10-04 22:51:31] iteration 1237/ 6855 | consumed samples: 158336 | elapsed time per iteration (ms): 89213.0 | throughput per GPU (TFLOP/s/GPU): 41.7 | MFU 13.37% | FLOPs estimator: workload-aware | logical tokens/s: 188057.9 | valid tokens/s: 188057.9 | physical tokens/s: 188436.5 | loss tokens/s: 141405.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.417777E-01 | lm-global-supervised-tokens: 1.261522E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.191494E-02 | lm-token-buffer-mib: 5.805100E+01 | global_load_balancing_loss: 9.999765E-01 | loss scale: 1.0 | grad norm: 0.066 | params norm: 11571.580 | params rms: 0.062656 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 19:13:18.698625 | finish at 2026-10-10 18:04:50 + [2026-10-04 22:52:59] iteration 1238/ 6855 | consumed samples: 158464 | elapsed time per iteration (ms): 87732.9 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.68% | FLOPs estimator: workload-aware | logical tokens/s: 191230.1 | valid tokens/s: 191230.1 | physical tokens/s: 191667.0 | loss tokens/s: 137217.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.798472E-01 | lm-global-supervised-tokens: 1.203846E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.140988E-02 | lm-token-buffer-mib: 5.819354E+01 | global_load_balancing_loss: 1.000327E+00 | loss scale: 1.0 | grad norm: 0.101 | params norm: 11571.503 | params rms: 0.062656 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 16:53:15.858127 | finish at 2026-10-10 15:46:14 + [2026-10-04 22:54:25] iteration 1239/ 6855 | consumed samples: 158592 | elapsed time per iteration (ms): 86632.3 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.41% | FLOPs estimator: workload-aware | logical tokens/s: 193660.1 | valid tokens/s: 193660.1 | physical tokens/s: 194119.6 | loss tokens/s: 144892.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.619515E-01 | lm-global-supervised-tokens: 1.255236E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.155948E-02 | lm-token-buffer-mib: 5.807820E+01 | global_load_balancing_loss: 9.998465E-01 | loss scale: 1.0 | grad norm: 0.051 | params norm: 11571.426 | params rms: 0.062656 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 15:08:46.874393 | finish at 2026-10-10 14:03:12 + [2026-10-04 22:56:11] iteration 1240/ 6855 | consumed samples: 158720 | elapsed time per iteration (ms): 105857.6 | throughput per GPU (TFLOP/s/GPU): 34.5 | MFU 11.05% | FLOPs estimator: workload-aware | logical tokens/s: 158488.0 | valid tokens/s: 158488.0 | physical tokens/s: 158827.1 | loss tokens/s: 117999.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.728919E-01 | lm-global-supervised-tokens: 1.249110E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.157537E-02 | lm-token-buffer-mib: 5.813770E+01 | global_load_balancing_loss: 1.000808E+00 | loss scale: 1.0 | grad norm: 0.038 | params norm: 11571.349 | params rms: 0.062655 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 21:06:30.603241 | finish at 2026-10-11 20:02:42 +times across ranks (ms): + save-checkpoint: + rank 0: 24996.42 + rank 1: 24996.38 + rank 2: 24996.46 + rank 3: 24996.45 + rank 4: 24996.41 + rank 5: 24996.39 + rank 6: 24996.38 + rank 7: 24996.44 + rank 8: 24997.16 + rank 9: 24997.12 + rank 10: 24997.15 + rank 11: 24997.12 + rank 12: 25028.72 + rank 13: 24997.15 + rank 14: 24997.17 + rank 15: 24997.10 + rank 16: 24996.51 + rank 17: 24996.52 + rank 18: 24996.46 + rank 19: 24996.49 + rank 20: 24996.46 + rank 21: 24996.51 + rank 22: 24996.48 + rank 23: 24996.49 + rank 24: 24996.76 + rank 25: 24996.73 + rank 26: 24996.71 + rank 27: 24996.70 + rank 28: 24996.74 + rank 29: 24996.73 + rank 30: 24996.70 + rank 31: 24996.69 + rank 32: 24996.79 + rank 33: 24996.80 + rank 34: 24996.69 + rank 35: 24996.79 + rank 36: 24996.65 + rank 37: 24996.65 + rank 38: 24996.73 + rank 39: 24996.73 + rank 40: 24996.17 + rank 41: 24996.16 + rank 42: 24996.16 + rank 43: 24996.16 + rank 44: 24996.12 + rank 45: 24996.14 + rank 46: 24996.14 + rank 47: 24996.17 + rank 48: 24996.48 + rank 49: 24996.12 + rank 50: 24996.29 + rank 51: 24996.39 + rank 52: 24996.40 + rank 53: 24996.37 + rank 54: 24996.41 + rank 55: 24996.38 + rank 56: 24996.86 + rank 57: 24996.83 + rank 58: 24996.81 + rank 59: 24996.80 + rank 60: 24996.80 + rank 61: 24996.82 + rank 62: 24996.81 + rank 63: 24996.82 + [2026-10-04 22:58:09] iteration 1241/ 6855 | consumed samples: 158848 | elapsed time per iteration (ms): 92295.2 | throughput per GPU (TFLOP/s/GPU): 40.9 | MFU 13.11% | FLOPs estimator: workload-aware | logical tokens/s: 181777.2 | valid tokens/s: 181777.2 | physical tokens/s: 182331.3 | loss tokens/s: 129079.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.539621E-01 | lm-global-supervised-tokens: 1.191342E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 3.914883E-01 | lm-token-buffer-mib: 5.829286E+01 | global_load_balancing_loss: 1.001398E+00 | loss scale: 1.0 | grad norm: 0.034 | params norm: 11571.273 | params rms: 0.062655 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 23:55:45.194521 | finish at 2026-10-10 22:53:54 + [2026-10-04 22:59:42] iteration 1242/ 6855 | consumed samples: 158976 | elapsed time per iteration (ms): 93010.6 | throughput per GPU (TFLOP/s/GPU): 39.8 | MFU 12.75% | FLOPs estimator: workload-aware | logical tokens/s: 180378.6 | valid tokens/s: 180378.5 | physical tokens/s: 180864.6 | loss tokens/s: 134792.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.646679E-01 | lm-global-supervised-tokens: 1.253713E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.449730E-02 | lm-token-buffer-mib: 5.813275E+01 | global_load_balancing_loss: 1.000333E+00 | loss scale: 1.0 | grad norm: 0.043 | params norm: 11571.195 | params rms: 0.062654 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 1:01:08.265451 | finish at 2026-10-11 00:00:50 + [2026-10-04 23:01:08] iteration 1243/ 6855 | consumed samples: 159104 | elapsed time per iteration (ms): 86136.7 | throughput per GPU (TFLOP/s/GPU): 43.1 | MFU 13.80% | FLOPs estimator: workload-aware | logical tokens/s: 194774.1 | valid tokens/s: 194774.1 | physical tokens/s: 195097.6 | loss tokens/s: 141052.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.961714E-01 | lm-global-supervised-tokens: 1.214976E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.175302E-02 | lm-token-buffer-mib: 5.802552E+01 | global_load_balancing_loss: 1.001086E+00 | loss scale: 1.0 | grad norm: 0.045 | params norm: 11571.117 | params rms: 0.062654 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 14:16:38.894998 | finish at 2026-10-10 13:17:47 + [2026-10-04 23:02:33] iteration 1244/ 6855 | consumed samples: 159232 | elapsed time per iteration (ms): 84652.9 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.74% | FLOPs estimator: workload-aware | logical tokens/s: 198187.5 | valid tokens/s: 198187.5 | physical tokens/s: 198554.4 | loss tokens/s: 150549.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.772591E-01 | lm-global-supervised-tokens: 1.274448E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.579630E-02 | lm-token-buffer-mib: 5.806856E+01 | global_load_balancing_loss: 1.000531E+00 | loss scale: 1.0 | grad norm: 0.047 | params norm: 11571.040 | params rms: 0.062653 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 11:56:27.554230 | finish at 2026-10-10 10:59:00 + [2026-10-04 23:03:58] iteration 1245/ 6855 | consumed samples: 159360 | elapsed time per iteration (ms): 84978.7 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.76% | FLOPs estimator: workload-aware | logical tokens/s: 197427.8 | valid tokens/s: 197427.8 | physical tokens/s: 197873.7 | loss tokens/s: 145422.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.084002E-01 | lm-global-supervised-tokens: 1.235782E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.163286E-02 | lm-token-buffer-mib: 5.826723E+01 | global_load_balancing_loss: 1.000241E+00 | loss scale: 1.0 | grad norm: 0.081 | params norm: 11570.963 | params rms: 0.062653 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 12:25:30.255110 | finish at 2026-10-10 11:29:28 + [2026-10-04 23:05:24] iteration 1246/ 6855 | consumed samples: 159488 | elapsed time per iteration (ms): 86415.6 | throughput per GPU (TFLOP/s/GPU): 42.8 | MFU 13.72% | FLOPs estimator: workload-aware | logical tokens/s: 194145.1 | valid tokens/s: 194145.1 | physical tokens/s: 194645.3 | loss tokens/s: 144503.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.808347E-01 | lm-global-supervised-tokens: 1.248739E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.820775E-02 | lm-token-buffer-mib: 5.823221E+01 | global_load_balancing_loss: 9.978485E-01 | loss scale: 1.0 | grad norm: 0.061 | params norm: 11570.884 | params rms: 0.062653 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 14:38:25.365528 | finish at 2026-10-10 13:43:49 + [2026-10-04 23:06:50] iteration 1247/ 6855 | consumed samples: 159616 | elapsed time per iteration (ms): 86158.6 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.68% | FLOPs estimator: workload-aware | logical tokens/s: 194724.5 | valid tokens/s: 194724.4 | physical tokens/s: 195156.9 | loss tokens/s: 146142.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.774619E-01 | lm-global-supervised-tokens: 1.259146E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.193635E-02 | lm-token-buffer-mib: 5.804245E+01 | global_load_balancing_loss: 9.987603E-01 | loss scale: 1.0 | grad norm: 0.039 | params norm: 11570.805 | params rms: 0.062652 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 14:12:57.568369 | finish at 2026-10-10 13:19:48 + [2026-10-04 23:08:22] iteration 1248/ 6855 | consumed samples: 159744 | elapsed time per iteration (ms): 92044.8 | throughput per GPU (TFLOP/s/GPU): 41.2 | MFU 13.21% | FLOPs estimator: workload-aware | logical tokens/s: 182271.8 | valid tokens/s: 182271.8 | physical tokens/s: 182547.7 | loss tokens/s: 132018.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.042372E-01 | lm-global-supervised-tokens: 1.215161E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.161691E-02 | lm-token-buffer-mib: 5.814132E+01 | global_load_balancing_loss: 1.000465E+00 | loss scale: 1.0 | grad norm: 0.036 | params norm: 11570.725 | params rms: 0.062652 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 23:21:35.082886 | finish at 2026-10-10 22:29:57 + [2026-10-04 23:09:47] iteration 1249/ 6855 | consumed samples: 159872 | elapsed time per iteration (ms): 85284.8 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.62% | FLOPs estimator: workload-aware | logical tokens/s: 196719.6 | valid tokens/s: 196719.6 | physical tokens/s: 197264.7 | loss tokens/s: 139780.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.113220E-01 | lm-global-supervised-tokens: 1.192111E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.140803E-02 | lm-token-buffer-mib: 5.822185E+01 | global_load_balancing_loss: 1.001155E+00 | loss scale: 1.0 | grad norm: 0.089 | params norm: 11570.643 | params rms: 0.062651 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 12:48:26.679982 | finish at 2026-10-10 11:58:14 + [2026-10-04 23:11:39] iteration 1250/ 6855 | consumed samples: 160000 | elapsed time per iteration (ms): 111730.0 | throughput per GPU (TFLOP/s/GPU): 32.9 | MFU 10.55% | FLOPs estimator: workload-aware | logical tokens/s: 150158.4 | valid tokens/s: 150158.4 | physical tokens/s: 150593.5 | loss tokens/s: 110517.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.551540E-01 | lm-global-supervised-tokens: 1.234811E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.151210E-02 | lm-token-buffer-mib: 5.815010E+01 | global_load_balancing_loss: 1.001226E+00 | loss scale: 1.0 | grad norm: 0.261 | params norm: 11570.564 | params rms: 0.062651 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 7 days, 5:57:26.487074 | finish at 2026-10-12 05:09:06 + [2026-10-04 23:13:08] iteration 1251/ 6855 | consumed samples: 160128 | elapsed time per iteration (ms): 88523.0 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.48% | FLOPs estimator: workload-aware | logical tokens/s: 189523.5 | valid tokens/s: 189523.5 | physical tokens/s: 189910.4 | loss tokens/s: 136578.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.442292E-01 | lm-global-supervised-tokens: 1.209030E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.236730E-02 | lm-token-buffer-mib: 5.806861E+01 | global_load_balancing_loss: 1.001716E+00 | loss scale: 1.0 | grad norm: 0.347 | params norm: 11570.483 | params rms: 0.062650 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 17:48:03.046998 | finish at 2026-10-10 17:01:11 + [2026-10-04 23:14:37] iteration 1252/ 6855 | consumed samples: 160256 | elapsed time per iteration (ms): 89111.2 | throughput per GPU (TFLOP/s/GPU): 41.7 | MFU 13.36% | FLOPs estimator: workload-aware | logical tokens/s: 188272.5 | valid tokens/s: 188272.5 | physical tokens/s: 188735.6 | loss tokens/s: 132044.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.134109E-01 | lm-global-supervised-tokens: 1.176662E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.588640E-02 | lm-token-buffer-mib: 5.815788E+01 | global_load_balancing_loss: 1.000885E+00 | loss scale: 1.0 | grad norm: 0.081 | params norm: 11570.404 | params rms: 0.062650 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 18:41:29.837719 | finish at 2026-10-10 17:56:07 + [2026-10-04 23:16:02] iteration 1253/ 6855 | consumed samples: 160384 | elapsed time per iteration (ms): 85035.8 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.55% | FLOPs estimator: workload-aware | logical tokens/s: 197295.9 | valid tokens/s: 197295.9 | physical tokens/s: 197766.8 | loss tokens/s: 152308.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.258635E-01 | lm-global-supervised-tokens: 1.295164E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.199041E-02 | lm-token-buffer-mib: 5.816685E+01 | global_load_balancing_loss: 9.996915E-01 | loss scale: 1.0 | grad norm: 0.168 | params norm: 11570.323 | params rms: 0.062650 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 12:19:30.662345 | finish at 2026-10-10 11:35:33 + [2026-10-04 23:17:25] iteration 1254/ 6855 | consumed samples: 160512 | elapsed time per iteration (ms): 83236.3 | throughput per GPU (TFLOP/s/GPU): 42.8 | MFU 13.70% | FLOPs estimator: workload-aware | logical tokens/s: 201561.3 | valid tokens/s: 201561.3 | physical tokens/s: 202024.4 | loss tokens/s: 153190.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.379395E-01 | lm-global-supervised-tokens: 1.275101E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.159201E-02 | lm-token-buffer-mib: 5.812801E+01 | global_load_balancing_loss: 9.996047E-01 | loss scale: 1.0 | grad norm: 0.061 | params norm: 11570.242 | params rms: 0.062649 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 9:30:06.361349 | finish at 2026-10-10 08:47:31 + [2026-10-04 23:18:52] iteration 1255/ 6855 | consumed samples: 160640 | elapsed time per iteration (ms): 86885.9 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.76% | FLOPs estimator: workload-aware | logical tokens/s: 193093.7 | valid tokens/s: 193093.7 | physical tokens/s: 193454.3 | loss tokens/s: 147672.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.285103E-01 | lm-global-supervised-tokens: 1.283063E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.668868E-02 | lm-token-buffer-mib: 5.822166E+01 | global_load_balancing_loss: 9.982895E-01 | loss scale: 1.0 | grad norm: 0.077 | params norm: 11570.161 | params rms: 0.062649 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 15:09:20.969353 | finish at 2026-10-10 14:28:13 + [2026-10-04 23:20:20] iteration 1256/ 6855 | consumed samples: 160768 | elapsed time per iteration (ms): 87765.4 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.54% | FLOPs estimator: workload-aware | logical tokens/s: 191159.8 | valid tokens/s: 191159.8 | physical tokens/s: 191688.5 | loss tokens/s: 144121.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.712172E-01 | lm-global-supervised-tokens: 1.264891E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.558378E-02 | lm-token-buffer-mib: 5.826952E+01 | global_load_balancing_loss: 1.000116E+00 | loss scale: 1.0 | grad norm: 0.120 | params norm: 11570.079 | params rms: 0.062648 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 16:29:58.338064 | finish at 2026-10-10 15:50:18 + [2026-10-04 23:21:46] iteration 1257/ 6855 | consumed samples: 160896 | elapsed time per iteration (ms): 86607.6 | throughput per GPU (TFLOP/s/GPU): 42.8 | MFU 13.71% | FLOPs estimator: workload-aware | logical tokens/s: 193715.2 | valid tokens/s: 193715.2 | physical tokens/s: 194155.4 | loss tokens/s: 145546.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.375574E-01 | lm-global-supervised-tokens: 1.260545E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.433168E-02 | lm-token-buffer-mib: 5.829418E+01 | global_load_balancing_loss: 9.998758E-01 | loss scale: 1.0 | grad norm: 0.096 | params norm: 11569.997 | params rms: 0.062648 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 14:40:29.300609 | finish at 2026-10-10 14:02:16 + [2026-10-04 23:23:11] iteration 1258/ 6855 | consumed samples: 161024 | elapsed time per iteration (ms): 84651.1 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.68% | FLOPs estimator: workload-aware | logical tokens/s: 198192.5 | valid tokens/s: 198192.5 | physical tokens/s: 198473.1 | loss tokens/s: 147494.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.261533E-01 | lm-global-supervised-tokens: 1.248556E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.100943E-02 | lm-token-buffer-mib: 5.806165E+01 | global_load_balancing_loss: 1.000142E+00 | loss scale: 1.0 | grad norm: 0.072 | params norm: 11569.915 | params rms: 0.062647 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 11:36:32.088824 | finish at 2026-10-10 10:59:43 + [2026-10-04 23:24:37] iteration 1259/ 6855 | consumed samples: 161152 | elapsed time per iteration (ms): 86168.5 | throughput per GPU (TFLOP/s/GPU): 43.9 | MFU 14.08% | FLOPs estimator: workload-aware | logical tokens/s: 194701.8 | valid tokens/s: 194701.8 | physical tokens/s: 195104.5 | loss tokens/s: 137797.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.641453E-01 | lm-global-supervised-tokens: 1.187380E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.516759E-02 | lm-token-buffer-mib: 5.826102E+01 | global_load_balancing_loss: 1.000825E+00 | loss scale: 1.0 | grad norm: 0.071 | params norm: 11569.834 | params rms: 0.062647 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 13:56:38.686881 | finish at 2026-10-10 13:21:16 + [2026-10-04 23:26:21] iteration 1260/ 6855 | consumed samples: 161280 | elapsed time per iteration (ms): 103732.0 | throughput per GPU (TFLOP/s/GPU): 34.7 | MFU 11.11% | FLOPs estimator: workload-aware | logical tokens/s: 161735.9 | valid tokens/s: 161735.9 | physical tokens/s: 162128.9 | loss tokens/s: 121668.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.500151E-01 | lm-global-supervised-tokens: 1.262093E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.115365E-02 | lm-token-buffer-mib: 5.812288E+01 | global_load_balancing_loss: 1.000846E+00 | loss scale: 1.0 | grad norm: 0.099 | params norm: 11569.753 | params rms: 0.062646 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 17:13:00.347207 | finish at 2026-10-11 16:39:21 + [2026-10-04 23:27:50] iteration 1261/ 6855 | consumed samples: 161408 | elapsed time per iteration (ms): 88606.2 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.65% | FLOPs estimator: workload-aware | logical tokens/s: 189345.4 | valid tokens/s: 189345.4 | physical tokens/s: 189709.8 | loss tokens/s: 133319.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.010397E-01 | lm-global-supervised-tokens: 1.181288E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.232015E-02 | lm-token-buffer-mib: 5.818774E+01 | global_load_balancing_loss: 1.001435E+00 | loss scale: 1.0 | grad norm: 0.037 | params norm: 11569.672 | params rms: 0.062646 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 17:41:02.863958 | finish at 2026-10-10 17:08:52 + [2026-10-04 23:29:19] iteration 1262/ 6855 | consumed samples: 161536 | elapsed time per iteration (ms): 89654.3 | throughput per GPU (TFLOP/s/GPU): 41.5 | MFU 13.31% | FLOPs estimator: workload-aware | logical tokens/s: 187132.0 | valid tokens/s: 187132.0 | physical tokens/s: 187506.1 | loss tokens/s: 134490.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.980755E-01 | lm-global-supervised-tokens: 1.205767E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.517389E-02 | lm-token-buffer-mib: 5.809024E+01 | global_load_balancing_loss: 9.995640E-01 | loss scale: 1.0 | grad norm: 0.058 | params norm: 11569.589 | params rms: 0.062646 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 19:17:16.322405 | finish at 2026-10-10 18:46:35 + [2026-10-04 23:30:44] iteration 1263/ 6855 | consumed samples: 161664 | elapsed time per iteration (ms): 84339.9 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.69% | FLOPs estimator: workload-aware | logical tokens/s: 198923.2 | valid tokens/s: 198923.2 | physical tokens/s: 199802.6 | loss tokens/s: 143028.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.362754E-01 | lm-global-supervised-tokens: 1.206298E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.541782E-02 | lm-token-buffer-mib: 5.813007E+01 | global_load_balancing_loss: 9.990852E-01 | loss scale: 1.0 | grad norm: 0.074 | params norm: 11569.507 | params rms: 0.062645 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 11:00:28.622234 | finish at 2026-10-10 10:31:12 + [2026-10-04 23:32:12] iteration 1264/ 6855 | consumed samples: 161792 | elapsed time per iteration (ms): 88159.1 | throughput per GPU (TFLOP/s/GPU): 41.5 | MFU 13.29% | FLOPs estimator: workload-aware | logical tokens/s: 190305.5 | valid tokens/s: 190305.5 | physical tokens/s: 190776.1 | loss tokens/s: 138537.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.590180E-01 | lm-global-supervised-tokens: 1.221338E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.738560E-02 | lm-token-buffer-mib: 5.817794E+01 | global_load_balancing_loss: 9.998943E-01 | loss scale: 1.0 | grad norm: 0.052 | params norm: 11569.427 | params rms: 0.062645 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 16:54:57.717697 | finish at 2026-10-10 16:27:09 + [2026-10-04 23:33:41] iteration 1265/ 6855 | consumed samples: 161920 | elapsed time per iteration (ms): 88841.5 | throughput per GPU (TFLOP/s/GPU): 41.7 | MFU 13.37% | FLOPs estimator: workload-aware | logical tokens/s: 188844.0 | valid tokens/s: 188844.0 | physical tokens/s: 189236.6 | loss tokens/s: 141950.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.837350E-01 | lm-global-supervised-tokens: 1.261105E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.232984E-02 | lm-token-buffer-mib: 5.802885E+01 | global_load_balancing_loss: 9.995822E-01 | loss scale: 1.0 | grad norm: 0.060 | params norm: 11569.345 | params rms: 0.062644 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 17:57:04.002571 | finish at 2026-10-10 17:30:45 + [2026-10-04 23:35:11] iteration 1266/ 6855 | consumed samples: 162048 | elapsed time per iteration (ms): 90019.0 | throughput per GPU (TFLOP/s/GPU): 41.2 | MFU 13.22% | FLOPs estimator: workload-aware | logical tokens/s: 186374.0 | valid tokens/s: 186374.0 | physical tokens/s: 186806.0 | loss tokens/s: 137845.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.389119E-01 | lm-global-supervised-tokens: 1.240873E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.144508E-02 | lm-token-buffer-mib: 5.819385E+01 | global_load_balancing_loss: 1.001437E+00 | loss scale: 1.0 | grad norm: 0.113 | params norm: 11569.263 | params rms: 0.062644 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 19:45:16.349869 | finish at 2026-10-10 19:20:27 + [2026-10-04 23:36:38] iteration 1267/ 6855 | consumed samples: 162176 | elapsed time per iteration (ms): 87012.3 | throughput per GPU (TFLOP/s/GPU): 41.7 | MFU 13.37% | FLOPs estimator: workload-aware | logical tokens/s: 192813.4 | valid tokens/s: 192813.3 | physical tokens/s: 193258.1 | loss tokens/s: 149770.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.644406E-01 | lm-global-supervised-tokens: 1.303188E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.153422E-02 | lm-token-buffer-mib: 5.813777E+01 | global_load_balancing_loss: 1.000934E+00 | loss scale: 1.0 | grad norm: 0.035 | params norm: 11569.182 | params rms: 0.062643 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 15:03:44.593924 | finish at 2026-10-10 14:40:22 + [2026-10-04 23:38:08] iteration 1268/ 6855 | consumed samples: 162304 | elapsed time per iteration (ms): 90239.6 | throughput per GPU (TFLOP/s/GPU): 41.6 | MFU 13.33% | FLOPs estimator: workload-aware | logical tokens/s: 185918.2 | valid tokens/s: 185918.2 | physical tokens/s: 186270.6 | loss tokens/s: 137521.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.234045E-01 | lm-global-supervised-tokens: 1.240992E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.860884E-02 | lm-token-buffer-mib: 5.814815E+01 | global_load_balancing_loss: 1.000362E+00 | loss scale: 1.0 | grad norm: 0.072 | params norm: 11569.102 | params rms: 0.062643 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 20:02:48.386466 | finish at 2026-10-10 19:40:56 + [2026-10-04 23:39:39] iteration 1269/ 6855 | consumed samples: 162432 | elapsed time per iteration (ms): 91131.9 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.49% | FLOPs estimator: workload-aware | logical tokens/s: 184098.1 | valid tokens/s: 184098.1 | physical tokens/s: 184508.2 | loss tokens/s: 138338.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.355753E-01 | lm-global-supervised-tokens: 1.260708E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.634873E-02 | lm-token-buffer-mib: 5.820464E+01 | global_load_balancing_loss: 9.992569E-01 | loss scale: 1.0 | grad norm: 0.100 | params norm: 11569.021 | params rms: 0.062642 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 21:24:23.017546 | finish at 2026-10-10 21:04:02 + [2026-10-04 23:41:27] iteration 1270/ 6855 | consumed samples: 162560 | elapsed time per iteration (ms): 107855.4 | throughput per GPU (TFLOP/s/GPU): 34.5 | MFU 11.05% | FLOPs estimator: workload-aware | logical tokens/s: 155552.3 | valid tokens/s: 155552.2 | physical tokens/s: 155874.8 | loss tokens/s: 112793.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.402069E-01 | lm-global-supervised-tokens: 1.216536E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.151777E-02 | lm-token-buffer-mib: 5.809338E+01 | global_load_balancing_loss: 1.000853E+00 | loss scale: 1.0 | grad norm: 0.040 | params norm: 11568.939 | params rms: 0.062642 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 23:19:32.163137 | finish at 2026-10-11 23:00:59 + [2026-10-04 23:42:54] iteration 1271/ 6855 | consumed samples: 162688 | elapsed time per iteration (ms): 87547.1 | throughput per GPU (TFLOP/s/GPU): 41.4 | MFU 13.27% | FLOPs estimator: workload-aware | logical tokens/s: 191636.4 | valid tokens/s: 191636.3 | physical tokens/s: 192089.4 | loss tokens/s: 147736.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.673853E-01 | lm-global-supervised-tokens: 1.293392E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.147297E-02 | lm-token-buffer-mib: 5.809909E+01 | global_load_balancing_loss: 1.000417E+00 | loss scale: 1.0 | grad norm: 0.066 | params norm: 11568.859 | params rms: 0.062642 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 15:47:42.930889 | finish at 2026-10-10 15:30:37 + [2026-10-04 23:44:23] iteration 1272/ 6855 | consumed samples: 162816 | elapsed time per iteration (ms): 88269.6 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.46% | FLOPs estimator: workload-aware | logical tokens/s: 190067.8 | valid tokens/s: 190067.8 | physical tokens/s: 190557.8 | loss tokens/s: 140996.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.663349E-01 | lm-global-supervised-tokens: 1.244573E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.374336E-02 | lm-token-buffer-mib: 5.813833E+01 | global_load_balancing_loss: 1.000254E+00 | loss scale: 1.0 | grad norm: 0.118 | params norm: 11568.779 | params rms: 0.062641 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 16:53:28.903449 | finish at 2026-10-10 16:37:51 + [2026-10-04 23:45:51] iteration 1273/ 6855 | consumed samples: 162944 | elapsed time per iteration (ms): 87983.0 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.63% | FLOPs estimator: workload-aware | logical tokens/s: 190686.8 | valid tokens/s: 190686.8 | physical tokens/s: 191114.3 | loss tokens/s: 142469.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.493479E-01 | lm-global-supervised-tokens: 1.253492E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.282183E-02 | lm-token-buffer-mib: 5.806895E+01 | global_load_balancing_loss: 1.000095E+00 | loss scale: 1.0 | grad norm: 0.047 | params norm: 11568.700 | params rms: 0.062641 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 16:25:21.265926 | finish at 2026-10-10 16:11:12 + [2026-10-04 23:47:15] iteration 1274/ 6855 | consumed samples: 163072 | elapsed time per iteration (ms): 84451.4 | throughput per GPU (TFLOP/s/GPU): 43.1 | MFU 13.82% | FLOPs estimator: workload-aware | logical tokens/s: 198660.9 | valid tokens/s: 198660.8 | physical tokens/s: 199216.4 | loss tokens/s: 144227.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.932430E-01 | lm-global-supervised-tokens: 1.218022E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.747869E-02 | lm-token-buffer-mib: 5.821564E+01 | global_load_balancing_loss: 1.000185E+00 | loss scale: 1.0 | grad norm: 0.113 | params norm: 11568.619 | params rms: 0.062640 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 10:55:23.384718 | finish at 2026-10-10 10:42:38 + [2026-10-04 23:48:41] iteration 1275/ 6855 | consumed samples: 163200 | elapsed time per iteration (ms): 85965.3 | throughput per GPU (TFLOP/s/GPU): 43.1 | MFU 13.81% | FLOPs estimator: workload-aware | logical tokens/s: 195162.3 | valid tokens/s: 195162.3 | physical tokens/s: 195660.2 | loss tokens/s: 148601.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.338529E-01 | lm-global-supervised-tokens: 1.277458E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.153556E-02 | lm-token-buffer-mib: 5.811039E+01 | global_load_balancing_loss: 9.990890E-01 | loss scale: 1.0 | grad norm: 0.072 | params norm: 11568.539 | params rms: 0.062640 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 13:14:46.306615 | finish at 2026-10-10 13:03:27 + [2026-10-04 23:50:08] iteration 1276/ 6855 | consumed samples: 163328 | elapsed time per iteration (ms): 86923.8 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.66% | FLOPs estimator: workload-aware | logical tokens/s: 193010.5 | valid tokens/s: 193010.5 | physical tokens/s: 193723.9 | loss tokens/s: 137022.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.839473E-01 | lm-global-supervised-tokens: 1.191049E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.330758E-02 | lm-token-buffer-mib: 5.851012E+01 | global_load_balancing_loss: 9.994336E-01 | loss scale: 1.0 | grad norm: 0.160 | params norm: 11568.461 | params rms: 0.062639 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 14:42:27.721867 | finish at 2026-10-10 14:32:36 + [2026-10-04 23:51:36] iteration 1277/ 6855 | consumed samples: 163456 | elapsed time per iteration (ms): 88298.2 | throughput per GPU (TFLOP/s/GPU): 42.8 | MFU 13.73% | FLOPs estimator: workload-aware | logical tokens/s: 190005.7 | valid tokens/s: 190005.7 | physical tokens/s: 190340.1 | loss tokens/s: 137220.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.985886E-01 | lm-global-supervised-tokens: 1.211632E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.396222E-02 | lm-token-buffer-mib: 5.812738E+01 | global_load_balancing_loss: 1.000890E+00 | loss scale: 1.0 | grad norm: 0.068 | params norm: 11568.382 | params rms: 0.062639 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 16:48:47.437462 | finish at 2026-10-10 16:40:24 + [2026-10-04 23:53:03] iteration 1278/ 6855 | consumed samples: 163584 | elapsed time per iteration (ms): 86525.9 | throughput per GPU (TFLOP/s/GPU): 42.8 | MFU 13.72% | FLOPs estimator: workload-aware | logical tokens/s: 193897.6 | valid tokens/s: 193897.6 | physical tokens/s: 194245.6 | loss tokens/s: 137589.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.898606E-01 | lm-global-supervised-tokens: 1.190504E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.457079E-02 | lm-token-buffer-mib: 5.810448E+01 | global_load_balancing_loss: 1.001208E+00 | loss scale: 1.0 | grad norm: 0.233 | params norm: 11568.303 | params rms: 0.062639 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 14:02:35.112537 | finish at 2026-10-10 13:55:38 + [2026-10-04 23:54:30] iteration 1279/ 6855 | consumed samples: 163712 | elapsed time per iteration (ms): 87639.1 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.68% | FLOPs estimator: workload-aware | logical tokens/s: 191435.3 | valid tokens/s: 191435.3 | physical tokens/s: 191687.8 | loss tokens/s: 141334.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.687075E-01 | lm-global-supervised-tokens: 1.238639E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.473232E-02 | lm-token-buffer-mib: 5.804365E+01 | global_load_balancing_loss: 1.001095E+00 | loss scale: 1.0 | grad norm: 0.069 | params norm: 11568.226 | params rms: 0.062638 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 15:44:35.604734 | finish at 2026-10-10 15:39:06 + [2026-10-04 23:56:13] iteration 1280/ 6855 | consumed samples: 163840 | elapsed time per iteration (ms): 103094.7 | throughput per GPU (TFLOP/s/GPU): 35.0 | MFU 11.22% | FLOPs estimator: workload-aware | logical tokens/s: 162736.0 | valid tokens/s: 162736.0 | physical tokens/s: 163325.9 | loss tokens/s: 125961.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.643730E-01 | lm-global-supervised-tokens: 1.298592E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.529879E-02 | lm-token-buffer-mib: 5.816471E+01 | global_load_balancing_loss: 1.001304E+00 | loss scale: 1.0 | grad norm: 0.314 | params norm: 11568.152 | params rms: 0.062638 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 15:39:12.875954 | finish at 2026-10-11 15:35:26 +times across ranks (ms): + save-checkpoint: + rank 0: 24497.16 + rank 1: 24497.17 + rank 2: 24497.21 + rank 3: 24497.16 + rank 4: 24497.16 + rank 5: 24497.19 + rank 6: 24497.16 + rank 7: 24497.20 + rank 8: 24497.00 + rank 9: 24496.98 + rank 10: 24496.97 + rank 11: 24496.97 + rank 12: 24496.98 + rank 13: 24497.00 + rank 14: 24496.97 + rank 15: 24496.99 + rank 16: 24496.89 + rank 17: 24496.84 + rank 18: 24496.82 + rank 19: 24496.91 + rank 20: 24496.88 + rank 21: 24496.94 + rank 22: 24496.83 + rank 23: 24496.82 + rank 24: 24497.22 + rank 25: 24497.29 + rank 26: 24497.19 + rank 27: 24497.23 + rank 28: 24497.18 + rank 29: 24497.19 + rank 30: 24497.19 + rank 31: 24497.19 + rank 32: 24496.19 + rank 33: 24496.18 + rank 34: 24496.16 + rank 35: 24496.11 + rank 36: 24496.22 + rank 37: 24496.15 + rank 38: 24496.12 + rank 39: 24496.10 + rank 40: 24497.13 + rank 41: 24497.08 + rank 42: 24497.15 + rank 43: 24497.11 + rank 44: 24497.08 + rank 45: 24497.11 + rank 46: 24497.14 + rank 47: 24497.11 + rank 48: 24497.95 + rank 49: 24497.96 + rank 50: 24497.94 + rank 51: 24497.97 + rank 52: 24498.00 + rank 53: 24497.92 + rank 54: 24497.96 + rank 55: 24497.94 + rank 56: 24497.76 + rank 57: 24497.76 + rank 58: 24497.75 + rank 59: 24497.75 + rank 60: 24497.76 + rank 61: 24497.74 + rank 62: 24497.75 + rank 63: 24497.78 + [2026-10-04 23:58:05] iteration 1281/ 6855 | consumed samples: 163968 | elapsed time per iteration (ms): 86980.0 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.55% | FLOPs estimator: workload-aware | logical tokens/s: 192884.9 | valid tokens/s: 192884.9 | physical tokens/s: 193343.5 | loss tokens/s: 147595.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.790161E-01 | lm-global-supervised-tokens: 1.283784E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.847024E-02 | lm-token-buffer-mib: 5.821018E+01 | global_load_balancing_loss: 9.996818E-01 | loss scale: 1.0 | grad norm: 0.075 | params norm: 11568.077 | params rms: 0.062637 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 14:40:26.637053 | finish at 2026-10-10 14:38:32 + [2026-10-04 23:59:35] iteration 1282/ 6855 | consumed samples: 164096 | elapsed time per iteration (ms): 90188.9 | throughput per GPU (TFLOP/s/GPU): 40.0 | MFU 12.82% | FLOPs estimator: workload-aware | logical tokens/s: 186022.2 | valid tokens/s: 186022.2 | physical tokens/s: 186514.4 | loss tokens/s: 136752.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.530993E-01 | lm-global-supervised-tokens: 1.233356E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.394883E-02 | lm-token-buffer-mib: 5.808515E+01 | global_load_balancing_loss: 9.997936E-01 | loss scale: 1.0 | grad norm: 0.344 | params norm: 11568.002 | params rms: 0.062637 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 19:37:02.598823 | finish at 2026-10-10 19:36:38 + [2026-10-05 00:00:58] iteration 1283/ 6855 | consumed samples: 164224 | elapsed time per iteration (ms): 83247.4 | throughput per GPU (TFLOP/s/GPU): 43.3 | MFU 13.87% | FLOPs estimator: workload-aware | logical tokens/s: 201533.7 | valid tokens/s: 201533.7 | physical tokens/s: 202035.6 | loss tokens/s: 151026.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.032398E-01 | lm-global-supervised-tokens: 1.257256E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.184395E-02 | lm-token-buffer-mib: 5.805566E+01 | global_load_balancing_loss: 9.996493E-01 | loss scale: 1.0 | grad norm: 0.094 | params norm: 11567.925 | params rms: 0.062637 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 8:50:54.744206 | finish at 2026-10-10 08:51:53 + [2026-10-05 00:02:24] iteration 1284/ 6855 | consumed samples: 164352 | elapsed time per iteration (ms): 85275.4 | throughput per GPU (TFLOP/s/GPU): 43.1 | MFU 13.82% | FLOPs estimator: workload-aware | logical tokens/s: 196740.8 | valid tokens/s: 196740.8 | physical tokens/s: 197208.8 | loss tokens/s: 149726.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.820943E-01 | lm-global-supervised-tokens: 1.276795E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.147343E-02 | lm-token-buffer-mib: 5.803300E+01 | global_load_balancing_loss: 9.998580E-01 | loss scale: 1.0 | grad norm: 0.224 | params norm: 11567.848 | params rms: 0.062636 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 11:57:49.181248 | finish at 2026-10-10 12:00:13 + [2026-10-05 00:03:50] iteration 1285/ 6855 | consumed samples: 164480 | elapsed time per iteration (ms): 85822.2 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.62% | FLOPs estimator: workload-aware | logical tokens/s: 195487.9 | valid tokens/s: 195487.9 | physical tokens/s: 195926.3 | loss tokens/s: 138402.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.261464E-01 | lm-global-supervised-tokens: 1.187803E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.164707E-02 | lm-token-buffer-mib: 5.814580E+01 | global_load_balancing_loss: 1.000962E+00 | loss scale: 1.0 | grad norm: 0.253 | params norm: 11567.770 | params rms: 0.062636 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 12:47:09.914620 | finish at 2026-10-10 12:51:00 + [2026-10-05 00:05:20] iteration 1286/ 6855 | consumed samples: 164608 | elapsed time per iteration (ms): 90025.1 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.40% | FLOPs estimator: workload-aware | logical tokens/s: 186361.4 | valid tokens/s: 186361.4 | physical tokens/s: 186665.3 | loss tokens/s: 135295.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.445422E-01 | lm-global-supervised-tokens: 1.217996E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.389061E-02 | lm-token-buffer-mib: 5.814075E+01 | global_load_balancing_loss: 1.000115E+00 | loss scale: 1.0 | grad norm: 0.077 | params norm: 11567.692 | params rms: 0.062635 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 19:15:49.658379 | finish at 2026-10-10 19:21:09 + [2026-10-05 00:06:48] iteration 1287/ 6855 | consumed samples: 164736 | elapsed time per iteration (ms): 88093.2 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.48% | FLOPs estimator: workload-aware | logical tokens/s: 190448.2 | valid tokens/s: 190448.2 | physical tokens/s: 190898.0 | loss tokens/s: 141955.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.935336E-01 | lm-global-supervised-tokens: 1.250532E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.263980E-02 | lm-token-buffer-mib: 5.825948E+01 | global_load_balancing_loss: 9.995691E-01 | loss scale: 1.0 | grad norm: 0.191 | params norm: 11567.615 | params rms: 0.062635 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 16:15:03.025040 | finish at 2026-10-10 16:21:51 + [2026-10-05 00:08:18] iteration 1288/ 6855 | consumed samples: 164864 | elapsed time per iteration (ms): 89885.1 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.45% | FLOPs estimator: workload-aware | logical tokens/s: 186651.4 | valid tokens/s: 186651.4 | physical tokens/s: 187115.6 | loss tokens/s: 134535.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.943225E-01 | lm-global-supervised-tokens: 1.209274E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.219937E-02 | lm-token-buffer-mib: 5.816758E+01 | global_load_balancing_loss: 9.996685E-01 | loss scale: 1.0 | grad norm: 0.338 | params norm: 11567.536 | params rms: 0.062634 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 18:59:50.113493 | finish at 2026-10-10 19:08:08 + [2026-10-05 00:09:45] iteration 1289/ 6855 | consumed samples: 164992 | elapsed time per iteration (ms): 87465.4 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.53% | FLOPs estimator: workload-aware | logical tokens/s: 191814.6 | valid tokens/s: 191814.6 | physical tokens/s: 192308.8 | loss tokens/s: 142084.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.588948E-01 | lm-global-supervised-tokens: 1.242746E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.775384E-02 | lm-token-buffer-mib: 5.809534E+01 | global_load_balancing_loss: 1.000183E+00 | loss scale: 1.0 | grad norm: 0.171 | params norm: 11567.457 | params rms: 0.062634 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 15:13:52.582171 | finish at 2026-10-10 15:23:38 + [2026-10-05 00:11:35] iteration 1290/ 6855 | consumed samples: 165120 | elapsed time per iteration (ms): 109841.6 | throughput per GPU (TFLOP/s/GPU): 34.0 | MFU 10.91% | FLOPs estimator: workload-aware | logical tokens/s: 152739.6 | valid tokens/s: 152739.6 | physical tokens/s: 153003.0 | loss tokens/s: 114477.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.306484E-01 | lm-global-supervised-tokens: 1.257434E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.401780E-02 | lm-token-buffer-mib: 5.809414E+01 | global_load_balancing_loss: 1.001033E+00 | loss scale: 1.0 | grad norm: 0.124 | params norm: 11567.379 | params rms: 0.062634 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 7 days, 1:47:48.645641 | finish at 2026-10-12 01:59:24 + [2026-10-05 00:13:01] iteration 1291/ 6855 | consumed samples: 165248 | elapsed time per iteration (ms): 85641.2 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.55% | FLOPs estimator: workload-aware | logical tokens/s: 195900.1 | valid tokens/s: 195900.1 | physical tokens/s: 196489.6 | loss tokens/s: 146829.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.418226E-01 | lm-global-supervised-tokens: 1.257466E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.328547E-02 | lm-token-buffer-mib: 5.823392E+01 | global_load_balancing_loss: 1.001052E+00 | loss scale: 1.0 | grad norm: 0.120 | params norm: 11567.301 | params rms: 0.062633 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 12:21:47.824210 | finish at 2026-10-10 12:34:48 + [2026-10-05 00:14:28] iteration 1292/ 6855 | consumed samples: 165376 | elapsed time per iteration (ms): 87410.0 | throughput per GPU (TFLOP/s/GPU): 43.8 | MFU 14.02% | FLOPs estimator: workload-aware | logical tokens/s: 191936.7 | valid tokens/s: 191936.6 | physical tokens/s: 192522.5 | loss tokens/s: 136038.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.877519E-01 | lm-global-supervised-tokens: 1.189109E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.302604E-02 | lm-token-buffer-mib: 5.808600E+01 | global_load_balancing_loss: 1.000476E+00 | loss scale: 1.0 | grad norm: 0.166 | params norm: 11567.223 | params rms: 0.062633 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 15:04:21.711108 | finish at 2026-10-10 15:18:50 + [2026-10-05 00:15:57] iteration 1293/ 6855 | consumed samples: 165504 | elapsed time per iteration (ms): 88741.9 | throughput per GPU (TFLOP/s/GPU): 41.3 | MFU 13.23% | FLOPs estimator: workload-aware | logical tokens/s: 189055.5 | valid tokens/s: 189055.5 | physical tokens/s: 189530.5 | loss tokens/s: 139481.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.323961E-01 | lm-global-supervised-tokens: 1.237782E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.592261E-02 | lm-token-buffer-mib: 5.815122E+01 | global_load_balancing_loss: 1.000724E+00 | loss scale: 1.0 | grad norm: 0.148 | params norm: 11567.147 | params rms: 0.062632 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 17:06:22.462204 | finish at 2026-10-10 17:22:19 + [2026-10-05 00:17:20] iteration 1294/ 6855 | consumed samples: 165632 | elapsed time per iteration (ms): 83691.1 | throughput per GPU (TFLOP/s/GPU): 43.7 | MFU 14.01% | FLOPs estimator: workload-aware | logical tokens/s: 200466.0 | valid tokens/s: 200466.0 | physical tokens/s: 200978.5 | loss tokens/s: 153227.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.809930E-01 | lm-global-supervised-tokens: 1.282379E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.530274E-02 | lm-token-buffer-mib: 5.819569E+01 | global_load_balancing_loss: 9.997676E-01 | loss scale: 1.0 | grad norm: 0.226 | params norm: 11567.071 | params rms: 0.062632 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 9:16:46.118939 | finish at 2026-10-10 09:34:07 + [2026-10-05 00:18:51] iteration 1295/ 6855 | consumed samples: 165760 | elapsed time per iteration (ms): 90502.2 | throughput per GPU (TFLOP/s/GPU): 41.3 | MFU 13.24% | FLOPs estimator: workload-aware | logical tokens/s: 185378.4 | valid tokens/s: 185378.4 | physical tokens/s: 185841.2 | loss tokens/s: 136359.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.486566E-01 | lm-global-supervised-tokens: 1.234080E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.176167E-02 | lm-token-buffer-mib: 5.814593E+01 | global_load_balancing_loss: 9.990357E-01 | loss scale: 1.0 | grad norm: 0.234 | params norm: 11566.996 | params rms: 0.062632 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 19:46:32.373219 | finish at 2026-10-10 20:05:23 + [2026-10-05 00:20:23] iteration 1296/ 6855 | consumed samples: 165888 | elapsed time per iteration (ms): 92432.1 | throughput per GPU (TFLOP/s/GPU): 40.7 | MFU 13.04% | FLOPs estimator: workload-aware | logical tokens/s: 181508.3 | valid tokens/s: 181508.3 | physical tokens/s: 181947.6 | loss tokens/s: 125147.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.733761E-01 | lm-global-supervised-tokens: 1.156763E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.296755E-02 | lm-token-buffer-mib: 5.812652E+01 | global_load_balancing_loss: 9.990409E-01 | loss scale: 1.0 | grad norm: 0.087 | params norm: 11566.919 | params rms: 0.062631 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 22:43:49.875898 | finish at 2026-10-10 23:04:13 + [2026-10-05 00:21:49] iteration 1297/ 6855 | consumed samples: 166016 | elapsed time per iteration (ms): 85557.1 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.56% | FLOPs estimator: workload-aware | logical tokens/s: 196093.0 | valid tokens/s: 196092.9 | physical tokens/s: 196937.6 | loss tokens/s: 144315.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.778585E-01 | lm-global-supervised-tokens: 1.234718E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.177014E-02 | lm-token-buffer-mib: 5.835393E+01 | global_load_balancing_loss: 1.000244E+00 | loss scale: 1.0 | grad norm: 0.316 | params norm: 11566.843 | params rms: 0.062631 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 12:05:26.379347 | finish at 2026-10-10 12:27:15 + [2026-10-05 00:23:16] iteration 1298/ 6855 | consumed samples: 166144 | elapsed time per iteration (ms): 87081.6 | throughput per GPU (TFLOP/s/GPU): 43.3 | MFU 13.86% | FLOPs estimator: workload-aware | logical tokens/s: 192660.6 | valid tokens/s: 192660.6 | physical tokens/s: 192937.2 | loss tokens/s: 132295.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.480291E-01 | lm-global-supervised-tokens: 1.152047E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.140576E-02 | lm-token-buffer-mib: 5.810892E+01 | global_load_balancing_loss: 1.001054E+00 | loss scale: 1.0 | grad norm: 0.175 | params norm: 11566.769 | params rms: 0.062630 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 14:25:12.428403 | finish at 2026-10-10 14:48:28 + [2026-10-05 00:24:42] iteration 1299/ 6855 | consumed samples: 166272 | elapsed time per iteration (ms): 85721.6 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.74% | FLOPs estimator: workload-aware | logical tokens/s: 195717.5 | valid tokens/s: 195717.5 | physical tokens/s: 196340.8 | loss tokens/s: 141861.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.507010E-01 | lm-global-supervised-tokens: 1.216056E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.167621E-02 | lm-token-buffer-mib: 5.810646E+01 | global_load_balancing_loss: 1.000908E+00 | loss scale: 1.0 | grad norm: 0.308 | params norm: 11566.696 | params rms: 0.062630 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 12:17:49.044328 | finish at 2026-10-10 12:42:31 + [2026-10-05 00:26:28] iteration 1300/ 6855 | consumed samples: 166400 | elapsed time per iteration (ms): 105895.9 | throughput per GPU (TFLOP/s/GPU): 34.4 | MFU 11.01% | FLOPs estimator: workload-aware | logical tokens/s: 158430.3 | valid tokens/s: 158430.3 | physical tokens/s: 158753.0 | loss tokens/s: 118360.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.239333E-01 | lm-global-supervised-tokens: 1.253392E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.277203E-02 | lm-token-buffer-mib: 5.802965E+01 | global_load_balancing_loss: 1.001064E+00 | loss scale: 1.0 | grad norm: 0.126 | params norm: 11566.621 | params rms: 0.062629 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 19:24:11.866274 | finish at 2026-10-11 19:50:39 + [2026-10-05 00:27:55] iteration 1301/ 6855 | consumed samples: 166528 | elapsed time per iteration (ms): 87614.7 | throughput per GPU (TFLOP/s/GPU): 43.4 | MFU 13.90% | FLOPs estimator: workload-aware | logical tokens/s: 191488.4 | valid tokens/s: 191488.4 | physical tokens/s: 191740.1 | loss tokens/s: 135137.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.471527E-01 | lm-global-supervised-tokens: 1.184006E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.315471E-02 | lm-token-buffer-mib: 5.808788E+01 | global_load_balancing_loss: 1.000696E+00 | loss scale: 1.0 | grad norm: 0.268 | params norm: 11566.545 | params rms: 0.062629 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 15:10:12.049535 | finish at 2026-10-10 15:38:07 + [2026-10-05 00:29:25] iteration 1302/ 6855 | consumed samples: 166656 | elapsed time per iteration (ms): 89535.2 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.66% | FLOPs estimator: workload-aware | logical tokens/s: 187381.1 | valid tokens/s: 187381.1 | physical tokens/s: 187721.1 | loss tokens/s: 134787.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.975898E-01 | lm-global-supervised-tokens: 1.206826E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.088322E-02 | lm-token-buffer-mib: 5.813213E+01 | global_load_balancing_loss: 1.000091E+00 | loss scale: 1.0 | grad norm: 0.043 | params norm: 11566.471 | params rms: 0.062629 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 18:06:29.003331 | finish at 2026-10-10 18:35:54 + [2026-10-05 00:30:48] iteration 1303/ 6855 | consumed samples: 166784 | elapsed time per iteration (ms): 83653.3 | throughput per GPU (TFLOP/s/GPU): 43.7 | MFU 14.02% | FLOPs estimator: workload-aware | logical tokens/s: 200554.8 | valid tokens/s: 200554.8 | physical tokens/s: 201133.4 | loss tokens/s: 151810.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.650138E-01 | lm-global-supervised-tokens: 1.269946E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.143559E-02 | lm-token-buffer-mib: 5.804905E+01 | global_load_balancing_loss: 1.000168E+00 | loss scale: 1.0 | grad norm: 0.172 | params norm: 11566.394 | params rms: 0.062628 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 9:00:43.063618 | finish at 2026-10-10 09:31:31 + [2026-10-05 00:32:16] iteration 1304/ 6855 | consumed samples: 166912 | elapsed time per iteration (ms): 87811.1 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.50% | FLOPs estimator: workload-aware | logical tokens/s: 191059.9 | valid tokens/s: 191059.9 | physical tokens/s: 191649.2 | loss tokens/s: 142717.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.964918E-01 | lm-global-supervised-tokens: 1.253215E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.258618E-02 | lm-token-buffer-mib: 5.821487E+01 | global_load_balancing_loss: 9.988975E-01 | loss scale: 1.0 | grad norm: 0.046 | params norm: 11566.317 | params rms: 0.062628 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 15:23:59.515393 | finish at 2026-10-10 15:56:16 + [2026-10-05 00:33:46] iteration 1305/ 6855 | consumed samples: 167040 | elapsed time per iteration (ms): 89792.1 | throughput per GPU (TFLOP/s/GPU): 41.4 | MFU 13.28% | FLOPs estimator: workload-aware | logical tokens/s: 186845.1 | valid tokens/s: 186845.1 | physical tokens/s: 187301.8 | loss tokens/s: 138371.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.764937E-01 | lm-global-supervised-tokens: 1.242464E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.153402E-02 | lm-token-buffer-mib: 5.807870E+01 | global_load_balancing_loss: 9.999078E-01 | loss scale: 1.0 | grad norm: 0.189 | params norm: 11566.241 | params rms: 0.062627 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 18:25:46.248686 | finish at 2026-10-10 18:59:32 + [2026-10-05 00:35:15] iteration 1306/ 6855 | consumed samples: 167168 | elapsed time per iteration (ms): 89173.0 | throughput per GPU (TFLOP/s/GPU): 41.7 | MFU 13.38% | FLOPs estimator: workload-aware | logical tokens/s: 188142.2 | valid tokens/s: 188142.1 | physical tokens/s: 188569.9 | loss tokens/s: 139279.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.512078E-01 | lm-global-supervised-tokens: 1.242001E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.354438E-02 | lm-token-buffer-mib: 5.805663E+01 | global_load_balancing_loss: 1.001204E+00 | loss scale: 1.0 | grad norm: 0.041 | params norm: 11566.165 | params rms: 0.062627 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 17:27:00.862440 | finish at 2026-10-10 18:02:16 + [2026-10-05 00:36:41] iteration 1307/ 6855 | consumed samples: 167296 | elapsed time per iteration (ms): 85796.9 | throughput per GPU (TFLOP/s/GPU): 43.7 | MFU 14.02% | FLOPs estimator: workload-aware | logical tokens/s: 195545.3 | valid tokens/s: 195545.3 | physical tokens/s: 195930.9 | loss tokens/s: 142499.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.384734E-01 | lm-global-supervised-tokens: 1.222599E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.789343E-02 | lm-token-buffer-mib: 5.805772E+01 | global_load_balancing_loss: 9.995370E-01 | loss scale: 1.0 | grad norm: 0.096 | params norm: 11566.089 | params rms: 0.062627 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 12:13:20.948744 | finish at 2026-10-10 12:50:02 + [2026-10-05 00:38:08] iteration 1308/ 6855 | consumed samples: 167424 | elapsed time per iteration (ms): 86576.9 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.62% | FLOPs estimator: workload-aware | logical tokens/s: 193783.3 | valid tokens/s: 193783.3 | physical tokens/s: 194098.2 | loss tokens/s: 143415.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.792505E-01 | lm-global-supervised-tokens: 1.241644E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.284768E-02 | lm-token-buffer-mib: 5.808293E+01 | global_load_balancing_loss: 1.000500E+00 | loss scale: 1.0 | grad norm: 0.153 | params norm: 11566.012 | params rms: 0.062626 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 13:24:01.951916 | finish at 2026-10-10 14:02:09 + [2026-10-05 00:39:35] iteration 1309/ 6855 | consumed samples: 167552 | elapsed time per iteration (ms): 87354.0 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.58% | FLOPs estimator: workload-aware | logical tokens/s: 192060.2 | valid tokens/s: 192060.2 | physical tokens/s: 192488.8 | loss tokens/s: 140897.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.235799E-01 | lm-global-supervised-tokens: 1.230792E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.124804E-02 | lm-token-buffer-mib: 5.808766E+01 | global_load_balancing_loss: 1.000247E+00 | loss scale: 1.0 | grad norm: 0.032 | params norm: 11565.933 | params rms: 0.062626 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 14:34:25.022698 | finish at 2026-10-10 15:14:00 + [2026-10-05 00:41:21] iteration 1310/ 6855 | consumed samples: 167680 | elapsed time per iteration (ms): 106448.2 | throughput per GPU (TFLOP/s/GPU): 33.3 | MFU 10.67% | FLOPs estimator: workload-aware | logical tokens/s: 157609.2 | valid tokens/s: 157609.2 | physical tokens/s: 158113.7 | loss tokens/s: 123646.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.434330E-01 | lm-global-supervised-tokens: 1.316195E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.158031E-02 | lm-token-buffer-mib: 5.819647E+01 | global_load_balancing_loss: 9.999326E-01 | loss scale: 1.0 | grad norm: 0.178 | params norm: 11565.854 | params rms: 0.062625 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 19:57:35.208117 | finish at 2026-10-11 20:38:57 + [2026-10-05 00:42:52] iteration 1311/ 6855 | consumed samples: 167808 | elapsed time per iteration (ms): 90272.9 | throughput per GPU (TFLOP/s/GPU): 40.3 | MFU 12.92% | FLOPs estimator: workload-aware | logical tokens/s: 185848.9 | valid tokens/s: 185848.9 | physical tokens/s: 186332.9 | loss tokens/s: 142634.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.297144E-01 | lm-global-supervised-tokens: 1.287602E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.194766E-02 | lm-token-buffer-mib: 5.816823E+01 | global_load_balancing_loss: 1.001210E+00 | loss scale: 1.0 | grad norm: 0.077 | params norm: 11565.775 | params rms: 0.062625 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 19:01:13.214607 | finish at 2026-10-10 19:44:05 + [2026-10-05 00:44:18] iteration 1312/ 6855 | consumed samples: 167936 | elapsed time per iteration (ms): 86519.7 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.65% | FLOPs estimator: workload-aware | logical tokens/s: 193911.4 | valid tokens/s: 193911.4 | physical tokens/s: 194386.8 | loss tokens/s: 142257.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.903471E-01 | lm-global-supervised-tokens: 1.230811E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.221654E-02 | lm-token-buffer-mib: 5.804356E+01 | global_load_balancing_loss: 1.000479E+00 | loss scale: 1.0 | grad norm: 0.091 | params norm: 11565.697 | params rms: 0.062624 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 13:12:58.740990 | finish at 2026-10-10 13:57:17 + [2026-10-05 00:45:42] iteration 1313/ 6855 | consumed samples: 168064 | elapsed time per iteration (ms): 83621.3 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.78% | FLOPs estimator: workload-aware | logical tokens/s: 200633.3 | valid tokens/s: 200633.3 | physical tokens/s: 201283.7 | loss tokens/s: 153396.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.579336E-01 | lm-global-supervised-tokens: 1.282718E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.179917E-02 | lm-token-buffer-mib: 5.806454E+01 | global_load_balancing_loss: 1.000390E+00 | loss scale: 1.0 | grad norm: 0.079 | params norm: 11565.617 | params rms: 0.062624 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 8:43:49.064802 | finish at 2026-10-10 09:29:31 + [2026-10-05 00:47:07] iteration 1314/ 6855 | consumed samples: 168192 | elapsed time per iteration (ms): 85132.9 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.57% | FLOPs estimator: workload-aware | logical tokens/s: 197070.1 | valid tokens/s: 197070.1 | physical tokens/s: 197519.6 | loss tokens/s: 144531.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 5.895136E-01 | lm-global-supervised-tokens: 1.230435E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.924569E-02 | lm-token-buffer-mib: 5.812569E+01 | global_load_balancing_loss: 9.997734E-01 | loss scale: 1.0 | grad norm: 0.075 | params norm: 11565.538 | params rms: 0.062624 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 11:02:01.673682 | finish at 2026-10-10 11:49:09 + [2026-10-05 00:48:35] iteration 1315/ 6855 | consumed samples: 168320 | elapsed time per iteration (ms): 88512.9 | throughput per GPU (TFLOP/s/GPU): 40.9 | MFU 13.11% | FLOPs estimator: workload-aware | logical tokens/s: 189543.0 | valid tokens/s: 189543.0 | physical tokens/s: 190140.6 | loss tokens/s: 145341.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.787019E-01 | lm-global-supervised-tokens: 1.286459E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.105461E-02 | lm-token-buffer-mib: 5.819136E+01 | global_load_balancing_loss: 1.000004E+00 | loss scale: 1.0 | grad norm: 0.091 | params norm: 11565.461 | params rms: 0.062623 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 16:12:41.264544 | finish at 2026-10-10 17:01:17 + [2026-10-05 00:50:02] iteration 1316/ 6855 | consumed samples: 168448 | elapsed time per iteration (ms): 86374.8 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.79% | FLOPs estimator: workload-aware | logical tokens/s: 194237.0 | valid tokens/s: 194237.0 | physical tokens/s: 194607.5 | loss tokens/s: 150062.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.801388E-01 | lm-global-supervised-tokens: 1.296162E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.153178E-02 | lm-token-buffer-mib: 5.815853E+01 | global_load_balancing_loss: 9.989033E-01 | loss scale: 1.0 | grad norm: 0.071 | params norm: 11565.383 | params rms: 0.062623 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 12:53:49.987963 | finish at 2026-10-10 13:43:52 + [2026-10-05 00:51:29] iteration 1317/ 6855 | consumed samples: 168576 | elapsed time per iteration (ms): 87457.1 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.50% | FLOPs estimator: workload-aware | logical tokens/s: 191833.4 | valid tokens/s: 191833.4 | physical tokens/s: 192180.5 | loss tokens/s: 147030.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.501962E-01 | lm-global-supervised-tokens: 1.285888E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.411377E-02 | lm-token-buffer-mib: 5.806702E+01 | global_load_balancing_loss: 9.998626E-01 | loss scale: 1.0 | grad norm: 0.074 | params norm: 11565.303 | params rms: 0.062622 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 14:32:17.144691 | finish at 2026-10-10 15:23:46 + [2026-10-05 00:52:56] iteration 1318/ 6855 | consumed samples: 168704 | elapsed time per iteration (ms): 86927.8 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.49% | FLOPs estimator: workload-aware | logical tokens/s: 193001.6 | valid tokens/s: 193001.6 | physical tokens/s: 193401.8 | loss tokens/s: 147843.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.844636E-01 | lm-global-supervised-tokens: 1.285172E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.244495E-02 | lm-token-buffer-mib: 5.810929E+01 | global_load_balancing_loss: 9.990981E-01 | loss scale: 1.0 | grad norm: 0.046 | params norm: 11565.221 | params rms: 0.062622 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 13:41:59.406485 | finish at 2026-10-10 14:34:56 + [2026-10-05 00:54:24] iteration 1319/ 6855 | consumed samples: 168832 | elapsed time per iteration (ms): 87541.3 | throughput per GPU (TFLOP/s/GPU): 43.3 | MFU 13.87% | FLOPs estimator: workload-aware | logical tokens/s: 191648.7 | valid tokens/s: 191648.7 | physical tokens/s: 192018.5 | loss tokens/s: 140924.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.892825E-01 | lm-global-supervised-tokens: 1.233673E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.143156E-02 | lm-token-buffer-mib: 5.802705E+01 | global_load_balancing_loss: 1.000668E+00 | loss scale: 1.0 | grad norm: 0.075 | params norm: 11565.141 | params rms: 0.062621 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 14:37:08.762512 | finish at 2026-10-10 15:31:32 + [2026-10-05 00:56:14] iteration 1320/ 6855 | consumed samples: 168960 | elapsed time per iteration (ms): 109861.0 | throughput per GPU (TFLOP/s/GPU): 33.3 | MFU 10.67% | FLOPs estimator: workload-aware | logical tokens/s: 152713.1 | valid tokens/s: 152713.1 | physical tokens/s: 153029.8 | loss tokens/s: 109251.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.406635E-01 | lm-global-supervised-tokens: 1.200246E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.029303E-02 | lm-token-buffer-mib: 5.807436E+01 | global_load_balancing_loss: 1.002816E+00 | loss scale: 1.0 | grad norm: 0.054 | params norm: 11565.060 | params rms: 0.062621 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 7 days, 0:54:40.831965 | finish at 2026-10-12 01:50:54 +times across ranks (ms): + save-checkpoint: + rank 0: 23565.46 + rank 1: 23565.44 + rank 2: 23565.48 + rank 3: 23565.45 + rank 4: 23565.46 + rank 5: 23565.45 + rank 6: 23565.46 + rank 7: 23565.50 + rank 8: 23565.00 + rank 9: 23564.98 + rank 10: 23564.98 + rank 11: 23564.98 + rank 12: 23564.98 + rank 13: 23565.01 + rank 14: 23564.99 + rank 15: 23564.97 + rank 16: 23564.76 + rank 17: 23564.75 + rank 18: 23564.75 + rank 19: 23564.77 + rank 20: 23564.75 + rank 21: 23564.75 + rank 22: 23564.76 + rank 23: 23564.75 + rank 24: 23565.47 + rank 25: 23565.46 + rank 26: 23565.50 + rank 27: 23565.53 + rank 28: 23565.50 + rank 29: 23565.46 + rank 30: 23565.49 + rank 31: 23565.47 + rank 32: 23565.33 + rank 33: 23565.36 + rank 34: 23565.28 + rank 35: 23565.28 + rank 36: 23565.31 + rank 37: 23565.28 + rank 38: 23565.28 + rank 39: 23565.28 + rank 40: 23564.63 + rank 41: 23564.64 + rank 42: 23564.60 + rank 43: 23564.61 + rank 44: 23564.65 + rank 45: 23564.61 + rank 46: 23564.62 + rank 47: 23564.61 + rank 48: 23565.15 + rank 49: 23565.14 + rank 50: 23565.14 + rank 51: 23565.13 + rank 52: 23565.12 + rank 53: 23565.12 + rank 54: 23565.12 + rank 55: 23565.12 + rank 56: 23565.04 + rank 57: 23565.05 + rank 58: 23565.01 + rank 59: 23565.04 + rank 60: 23565.02 + rank 61: 23565.04 + rank 62: 23565.02 + rank 63: 23565.02 + [2026-10-05 00:58:03] iteration 1321/ 6855 | consumed samples: 169088 | elapsed time per iteration (ms): 85976.1 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.47% | FLOPs estimator: workload-aware | logical tokens/s: 195138.2 | valid tokens/s: 195138.2 | physical tokens/s: 195604.2 | loss tokens/s: 142700.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.541433E-01 | lm-global-supervised-tokens: 1.226880E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 8.457218E-01 | lm-token-buffer-mib: 5.802354E+01 | global_load_balancing_loss: 1.001529E+00 | loss scale: 1.0 | grad norm: 0.058 | params norm: 11564.979 | params rms: 0.062621 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 12:09:51.518201 | finish at 2026-10-10 13:07:55 + [2026-10-05 00:59:38] iteration 1322/ 6855 | consumed samples: 169216 | elapsed time per iteration (ms): 94468.5 | throughput per GPU (TFLOP/s/GPU): 39.3 | MFU 12.60% | FLOPs estimator: workload-aware | logical tokens/s: 177595.2 | valid tokens/s: 177595.2 | physical tokens/s: 177990.9 | loss tokens/s: 127565.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.992612E-01 | lm-global-supervised-tokens: 1.205088E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.407701E-02 | lm-token-buffer-mib: 5.815352E+01 | global_load_balancing_loss: 1.000519E+00 | loss scale: 1.0 | grad norm: 0.105 | params norm: 11564.899 | params rms: 0.062620 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 1:11:34.154536 | finish at 2026-10-11 02:11:12 + [2026-10-05 01:01:08] iteration 1323/ 6855 | consumed samples: 169344 | elapsed time per iteration (ms): 90012.7 | throughput per GPU (TFLOP/s/GPU): 40.6 | MFU 13.01% | FLOPs estimator: workload-aware | logical tokens/s: 186386.4 | valid tokens/s: 186386.4 | physical tokens/s: 186795.2 | loss tokens/s: 138554.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.535773E-01 | lm-global-supervised-tokens: 1.247164E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.156816E-02 | lm-token-buffer-mib: 5.821266E+01 | global_load_balancing_loss: 1.000812E+00 | loss scale: 1.0 | grad norm: 0.108 | params norm: 11564.819 | params rms: 0.062620 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 18:19:10.040543 | finish at 2026-10-10 19:20:18 + [2026-10-05 01:02:37] iteration 1324/ 6855 | consumed samples: 169472 | elapsed time per iteration (ms): 88828.5 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.59% | FLOPs estimator: workload-aware | logical tokens/s: 188871.5 | valid tokens/s: 188871.5 | physical tokens/s: 189360.6 | loss tokens/s: 135435.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.844051E-01 | lm-global-supervised-tokens: 1.203050E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.167713E-02 | lm-token-buffer-mib: 5.813991E+01 | global_load_balancing_loss: 9.988374E-01 | loss scale: 1.0 | grad norm: 0.173 | params norm: 11564.739 | params rms: 0.062619 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 16:28:30.615659 | finish at 2026-10-10 17:31:07 + [2026-10-05 01:04:02] iteration 1325/ 6855 | consumed samples: 169600 | elapsed time per iteration (ms): 85875.1 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.47% | FLOPs estimator: workload-aware | logical tokens/s: 195367.3 | valid tokens/s: 195367.3 | physical tokens/s: 195905.5 | loss tokens/s: 140904.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.196776E-01 | lm-global-supervised-tokens: 1.210019E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.180550E-02 | lm-token-buffer-mib: 5.821962E+01 | global_load_balancing_loss: 9.984730E-01 | loss scale: 1.0 | grad norm: 0.184 | params norm: 11564.660 | params rms: 0.062619 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 11:54:49.285293 | finish at 2026-10-10 12:58:52 + [2026-10-05 01:05:29] iteration 1326/ 6855 | consumed samples: 169728 | elapsed time per iteration (ms): 86334.0 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.65% | FLOPs estimator: workload-aware | logical tokens/s: 194327.9 | valid tokens/s: 194327.9 | physical tokens/s: 194796.5 | loss tokens/s: 145103.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.710684E-01 | lm-global-supervised-tokens: 1.252733E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.199069E-02 | lm-token-buffer-mib: 5.823727E+01 | global_load_balancing_loss: 9.988373E-01 | loss scale: 1.0 | grad norm: 0.173 | params norm: 11564.584 | params rms: 0.062618 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 12:35:40.462515 | finish at 2026-10-10 13:41:09 + [2026-10-05 01:06:57] iteration 1327/ 6855 | consumed samples: 169856 | elapsed time per iteration (ms): 87830.4 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.39% | FLOPs estimator: workload-aware | logical tokens/s: 191017.9 | valid tokens/s: 191017.9 | physical tokens/s: 191449.4 | loss tokens/s: 139372.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.351727E-01 | lm-global-supervised-tokens: 1.224113E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.615838E-02 | lm-token-buffer-mib: 5.823872E+01 | global_load_balancing_loss: 9.996840E-01 | loss scale: 1.0 | grad norm: 0.073 | params norm: 11564.508 | params rms: 0.062618 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 14:52:06.231043 | finish at 2026-10-10 15:59:03 + [2026-10-05 01:08:21] iteration 1328/ 6855 | consumed samples: 169984 | elapsed time per iteration (ms): 84299.1 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.60% | FLOPs estimator: workload-aware | logical tokens/s: 199019.1 | valid tokens/s: 199019.1 | physical tokens/s: 199692.8 | loss tokens/s: 154713.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.520092E-01 | lm-global-supervised-tokens: 1.304218E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.554051E-02 | lm-token-buffer-mib: 5.820205E+01 | global_load_balancing_loss: 1.000358E+00 | loss scale: 1.0 | grad norm: 0.145 | params norm: 11564.431 | params rms: 0.062618 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 9:25:21.386182 | finish at 2026-10-10 10:33:42 + [2026-10-05 01:09:46] iteration 1329/ 6855 | consumed samples: 170112 | elapsed time per iteration (ms): 85503.6 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.58% | FLOPs estimator: workload-aware | logical tokens/s: 196216.1 | valid tokens/s: 196216.1 | physical tokens/s: 196726.4 | loss tokens/s: 148675.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.725779E-01 | lm-global-supervised-tokens: 1.271226E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.477746E-02 | lm-token-buffer-mib: 5.813336E+01 | global_load_balancing_loss: 1.001278E+00 | loss scale: 1.0 | grad norm: 0.102 | params norm: 11564.354 | params rms: 0.062617 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 11:14:52.904806 | finish at 2026-10-10 12:24:39 + [2026-10-05 01:11:34] iteration 1330/ 6855 | consumed samples: 170240 | elapsed time per iteration (ms): 107227.9 | throughput per GPU (TFLOP/s/GPU): 34.1 | MFU 10.93% | FLOPs estimator: workload-aware | logical tokens/s: 156463.0 | valid tokens/s: 156463.0 | physical tokens/s: 156802.0 | loss tokens/s: 115841.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.881936E-01 | lm-global-supervised-tokens: 1.242148E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.203265E-02 | lm-token-buffer-mib: 5.827528E+01 | global_load_balancing_loss: 1.001440E+00 | loss scale: 1.0 | grad norm: 0.095 | params norm: 11564.278 | params rms: 0.062617 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 20:33:54.312314 | finish at 2026-10-11 21:45:28 + [2026-10-05 01:13:03] iteration 1331/ 6855 | consumed samples: 170368 | elapsed time per iteration (ms): 89164.0 | throughput per GPU (TFLOP/s/GPU): 41.3 | MFU 13.23% | FLOPs estimator: workload-aware | logical tokens/s: 188160.9 | valid tokens/s: 188160.9 | physical tokens/s: 188909.4 | loss tokens/s: 142003.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.825443E-01 | lm-global-supervised-tokens: 1.266160E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.266609E-02 | lm-token-buffer-mib: 5.885191E+01 | global_load_balancing_loss: 1.001757E+00 | loss scale: 1.0 | grad norm: 0.044 | params norm: 11564.202 | params rms: 0.062616 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 16:49:01.660932 | finish at 2026-10-10 18:02:04 + [2026-10-05 01:14:33] iteration 1332/ 6855 | consumed samples: 170496 | elapsed time per iteration (ms): 90483.1 | throughput per GPU (TFLOP/s/GPU): 40.4 | MFU 12.96% | FLOPs estimator: workload-aware | logical tokens/s: 185418.2 | valid tokens/s: 185418.2 | physical tokens/s: 185813.6 | loss tokens/s: 135942.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.698579E-01 | lm-global-supervised-tokens: 1.230047E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.146424E-02 | lm-token-buffer-mib: 5.806411E+01 | global_load_balancing_loss: 1.001589E+00 | loss scale: 1.0 | grad norm: 0.077 | params norm: 11564.124 | params rms: 0.062616 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 18:48:58.275515 | finish at 2026-10-10 20:03:32 + [2026-10-05 01:16:02] iteration 1333/ 6855 | consumed samples: 170624 | elapsed time per iteration (ms): 88964.1 | throughput per GPU (TFLOP/s/GPU): 41.0 | MFU 13.15% | FLOPs estimator: workload-aware | logical tokens/s: 188583.9 | valid tokens/s: 188583.9 | physical tokens/s: 189141.4 | loss tokens/s: 134236.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.172534E-01 | lm-global-supervised-tokens: 1.194224E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.453492E-02 | lm-token-buffer-mib: 5.816028E+01 | global_load_balancing_loss: 1.000559E+00 | loss scale: 1.0 | grad norm: 0.035 | params norm: 11564.046 | params rms: 0.062616 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 16:27:40.000488 | finish at 2026-10-10 17:43:42 + [2026-10-05 01:17:30] iteration 1334/ 6855 | consumed samples: 170752 | elapsed time per iteration (ms): 87400.8 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.63% | FLOPs estimator: workload-aware | logical tokens/s: 191956.8 | valid tokens/s: 191956.7 | physical tokens/s: 192271.8 | loss tokens/s: 139836.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.560531E-01 | lm-global-supervised-tokens: 1.222186E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.542943E-02 | lm-token-buffer-mib: 5.812380E+01 | global_load_balancing_loss: 9.999470E-01 | loss scale: 1.0 | grad norm: 0.052 | params norm: 11563.968 | params rms: 0.062615 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 14:02:19.603501 | finish at 2026-10-10 15:19:49 + [2026-10-05 01:18:57] iteration 1335/ 6855 | consumed samples: 170880 | elapsed time per iteration (ms): 87204.7 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.66% | FLOPs estimator: workload-aware | logical tokens/s: 192388.7 | valid tokens/s: 192388.6 | physical tokens/s: 192830.3 | loss tokens/s: 145665.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.805862E-01 | lm-global-supervised-tokens: 1.270268E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.224920E-02 | lm-token-buffer-mib: 5.806637E+01 | global_load_balancing_loss: 9.981318E-01 | loss scale: 1.0 | grad norm: 0.031 | params norm: 11563.890 | params rms: 0.062615 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 13:42:50.079517 | finish at 2026-10-10 15:01:47 + [2026-10-05 01:20:25] iteration 1336/ 6855 | consumed samples: 171008 | elapsed time per iteration (ms): 88398.6 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.55% | FLOPs estimator: workload-aware | logical tokens/s: 189790.5 | valid tokens/s: 189790.5 | physical tokens/s: 190183.1 | loss tokens/s: 130854.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.818254E-01 | lm-global-supervised-tokens: 1.156732E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.395283E-02 | lm-token-buffer-mib: 5.805251E+01 | global_load_balancing_loss: 1.000068E+00 | loss scale: 1.0 | grad norm: 0.035 | params norm: 11563.810 | params rms: 0.062614 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 15:31:11.955542 | finish at 2026-10-10 16:51:37 + [2026-10-05 01:21:50] iteration 1337/ 6855 | consumed samples: 171136 | elapsed time per iteration (ms): 84561.7 | throughput per GPU (TFLOP/s/GPU): 43.3 | MFU 13.86% | FLOPs estimator: workload-aware | logical tokens/s: 198401.9 | valid tokens/s: 198401.9 | physical tokens/s: 198968.3 | loss tokens/s: 144609.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.405329E-01 | lm-global-supervised-tokens: 1.222843E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.470421E-02 | lm-token-buffer-mib: 5.824861E+01 | global_load_balancing_loss: 9.998560E-01 | loss scale: 1.0 | grad norm: 0.045 | params norm: 11563.730 | params rms: 0.062614 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 9:36:51.704568 | finish at 2026-10-10 10:58:41 + [2026-10-05 01:23:18] iteration 1338/ 6855 | consumed samples: 171264 | elapsed time per iteration (ms): 88137.3 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.66% | FLOPs estimator: workload-aware | logical tokens/s: 190352.6 | valid tokens/s: 190352.6 | physical tokens/s: 190693.3 | loss tokens/s: 137544.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.906225E-01 | lm-global-supervised-tokens: 1.212282E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.127094E-02 | lm-token-buffer-mib: 5.809613E+01 | global_load_balancing_loss: 9.998427E-01 | loss scale: 1.0 | grad norm: 0.042 | params norm: 11563.650 | params rms: 0.062613 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 15:04:13.282931 | finish at 2026-10-10 16:27:31 + [2026-10-05 01:24:45] iteration 1339/ 6855 | consumed samples: 171392 | elapsed time per iteration (ms): 87399.8 | throughput per GPU (TFLOP/s/GPU): 43.1 | MFU 13.82% | FLOPs estimator: workload-aware | logical tokens/s: 191958.6 | valid tokens/s: 191958.6 | physical tokens/s: 192286.4 | loss tokens/s: 145334.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.164633E-01 | lm-global-supervised-tokens: 1.270223E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.161275E-02 | lm-token-buffer-mib: 5.812328E+01 | global_load_balancing_loss: 9.996395E-01 | loss scale: 1.0 | grad norm: 0.035 | params norm: 11563.571 | params rms: 0.062613 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 13:54:57.390516 | finish at 2026-10-10 15:19:43 + [2026-10-05 01:26:36] iteration 1340/ 6855 | consumed samples: 171520 | elapsed time per iteration (ms): 110838.0 | throughput per GPU (TFLOP/s/GPU): 33.8 | MFU 10.82% | FLOPs estimator: workload-aware | logical tokens/s: 151366.4 | valid tokens/s: 151366.4 | physical tokens/s: 151712.0 | loss tokens/s: 112433.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.770974E-01 | lm-global-supervised-tokens: 1.246187E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.365039E-02 | lm-token-buffer-mib: 5.828730E+01 | global_load_balancing_loss: 1.000744E+00 | loss scale: 1.0 | grad norm: 0.069 | params norm: 11563.493 | params rms: 0.062613 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 7 days, 1:47:51.712333 | finish at 2026-10-12 03:14:28 + [2026-10-05 01:28:05] iteration 1341/ 6855 | consumed samples: 171648 | elapsed time per iteration (ms): 88570.2 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.40% | FLOPs estimator: workload-aware | logical tokens/s: 189422.7 | valid tokens/s: 189422.7 | physical tokens/s: 189777.0 | loss tokens/s: 140230.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.335520E-01 | lm-global-supervised-tokens: 1.242021E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.290825E-02 | lm-token-buffer-mib: 5.818557E+01 | global_load_balancing_loss: 1.000082E+00 | loss scale: 1.0 | grad norm: 0.035 | params norm: 11563.414 | params rms: 0.062612 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 15:39:36.296901 | finish at 2026-10-10 17:07:41 + [2026-10-05 01:29:31] iteration 1342/ 6855 | consumed samples: 171776 | elapsed time per iteration (ms): 85876.5 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.68% | FLOPs estimator: workload-aware | logical tokens/s: 195364.1 | valid tokens/s: 195364.1 | physical tokens/s: 195843.6 | loss tokens/s: 139996.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.529396E-01 | lm-global-supervised-tokens: 1.202243E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.457756E-02 | lm-token-buffer-mib: 5.807639E+01 | global_load_balancing_loss: 1.000769E+00 | loss scale: 1.0 | grad norm: 0.079 | params norm: 11563.335 | params rms: 0.062612 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 11:30:36.934911 | finish at 2026-10-10 13:00:08 + [2026-10-05 01:31:00] iteration 1343/ 6855 | consumed samples: 171904 | elapsed time per iteration (ms): 89099.2 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.66% | FLOPs estimator: workload-aware | logical tokens/s: 188297.8 | valid tokens/s: 188297.8 | physical tokens/s: 188722.7 | loss tokens/s: 132869.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.838285E-01 | lm-global-supervised-tokens: 1.183855E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.158694E-02 | lm-token-buffer-mib: 5.825381E+01 | global_load_balancing_loss: 1.001854E+00 | loss scale: 1.0 | grad norm: 0.033 | params norm: 11563.255 | params rms: 0.062611 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 16:25:14.843023 | finish at 2026-10-10 17:56:15 + [2026-10-05 01:32:27] iteration 1344/ 6855 | consumed samples: 172032 | elapsed time per iteration (ms): 87067.1 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.47% | FLOPs estimator: workload-aware | logical tokens/s: 192692.8 | valid tokens/s: 192692.8 | physical tokens/s: 193047.8 | loss tokens/s: 152339.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.158219E-01 | lm-global-supervised-tokens: 1.326376E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.322121E-02 | lm-token-buffer-mib: 5.809435E+01 | global_load_balancing_loss: 9.991599E-01 | loss scale: 1.0 | grad norm: 0.054 | params norm: 11563.175 | params rms: 0.062611 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 13:17:06.663542 | finish at 2026-10-10 14:49:33 + [2026-10-05 01:33:53] iteration 1345/ 6855 | consumed samples: 172160 | elapsed time per iteration (ms): 86338.1 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.57% | FLOPs estimator: workload-aware | logical tokens/s: 194319.6 | valid tokens/s: 194319.6 | physical tokens/s: 194782.6 | loss tokens/s: 148148.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.704626E-01 | lm-global-supervised-tokens: 1.279085E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.221970E-02 | lm-token-buffer-mib: 5.814469E+01 | global_load_balancing_loss: 9.990414E-01 | loss scale: 1.0 | grad norm: 0.045 | params norm: 11563.096 | params rms: 0.062610 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 12:08:42.721884 | finish at 2026-10-10 13:42:36 + [2026-10-05 01:35:21] iteration 1346/ 6855 | consumed samples: 172288 | elapsed time per iteration (ms): 87894.8 | throughput per GPU (TFLOP/s/GPU): 41.6 | MFU 13.33% | FLOPs estimator: workload-aware | logical tokens/s: 190878.2 | valid tokens/s: 190878.2 | physical tokens/s: 191461.5 | loss tokens/s: 144680.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.787173E-01 | lm-global-supervised-tokens: 1.271663E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.131849E-02 | lm-token-buffer-mib: 5.813365E+01 | global_load_balancing_loss: 9.996026E-01 | loss scale: 1.0 | grad norm: 0.092 | params norm: 11563.016 | params rms: 0.062610 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 14:30:12.726109 | finish at 2026-10-10 16:05:34 + [2026-10-05 01:36:48] iteration 1347/ 6855 | consumed samples: 172416 | elapsed time per iteration (ms): 86823.6 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.44% | FLOPs estimator: workload-aware | logical tokens/s: 193232.7 | valid tokens/s: 193232.7 | physical tokens/s: 193921.5 | loss tokens/s: 144614.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.813672E-01 | lm-global-supervised-tokens: 1.255595E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.158079E-02 | lm-token-buffer-mib: 5.802528E+01 | global_load_balancing_loss: 1.000273E+00 | loss scale: 1.0 | grad norm: 0.344 | params norm: 11562.938 | params rms: 0.062610 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 12:50:24.647799 | finish at 2026-10-10 14:27:12 + [2026-10-05 01:38:17] iteration 1348/ 6855 | consumed samples: 172544 | elapsed time per iteration (ms): 89591.6 | throughput per GPU (TFLOP/s/GPU): 41.6 | MFU 13.34% | FLOPs estimator: workload-aware | logical tokens/s: 187263.0 | valid tokens/s: 187263.0 | physical tokens/s: 187727.6 | loss tokens/s: 139542.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.324610E-01 | lm-global-supervised-tokens: 1.250186E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.157671E-02 | lm-token-buffer-mib: 5.813097E+01 | global_load_balancing_loss: 1.002492E+00 | loss scale: 1.0 | grad norm: 0.338 | params norm: 11562.859 | params rms: 0.062609 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 17:03:00.887045 | finish at 2026-10-10 18:41:18 + [2026-10-05 01:39:43] iteration 1349/ 6855 | consumed samples: 172672 | elapsed time per iteration (ms): 85299.8 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.78% | FLOPs estimator: workload-aware | logical tokens/s: 196685.3 | valid tokens/s: 196685.3 | physical tokens/s: 197161.4 | loss tokens/s: 144674.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.960561E-01 | lm-global-supervised-tokens: 1.234068E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.681624E-02 | lm-token-buffer-mib: 5.829839E+01 | global_load_balancing_loss: 1.000309E+00 | loss scale: 1.0 | grad norm: 0.209 | params norm: 11562.781 | params rms: 0.062609 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 10:27:40.623529 | finish at 2026-10-10 12:07:23 + [2026-10-05 01:41:30] iteration 1350/ 6855 | consumed samples: 172800 | elapsed time per iteration (ms): 106985.0 | throughput per GPU (TFLOP/s/GPU): 35.2 | MFU 11.28% | FLOPs estimator: workload-aware | logical tokens/s: 156817.9 | valid tokens/s: 156817.9 | physical tokens/s: 157147.2 | loss tokens/s: 113728.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.622515E-01 | lm-global-supervised-tokens: 1.216720E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.359637E-02 | lm-token-buffer-mib: 5.839819E+01 | global_load_balancing_loss: 9.996491E-01 | loss scale: 1.0 | grad norm: 0.274 | params norm: 11562.702 | params rms: 0.062608 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 19:35:52.669859 | finish at 2026-10-11 21:17:22 + [2026-10-05 01:42:55] iteration 1351/ 6855 | consumed samples: 172928 | elapsed time per iteration (ms): 84945.8 | throughput per GPU (TFLOP/s/GPU): 43.3 | MFU 13.89% | FLOPs estimator: workload-aware | logical tokens/s: 197504.6 | valid tokens/s: 197504.6 | physical tokens/s: 197984.3 | loss tokens/s: 142910.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.357614E-01 | lm-global-supervised-tokens: 1.213963E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.534726E-02 | lm-token-buffer-mib: 5.813478E+01 | global_load_balancing_loss: 1.000778E+00 | loss scale: 1.0 | grad norm: 0.204 | params norm: 11562.622 | params rms: 0.062608 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 9:52:21.675690 | finish at 2026-10-10 11:35:16 + [2026-10-05 01:44:18] iteration 1352/ 6855 | consumed samples: 173056 | elapsed time per iteration (ms): 83229.8 | throughput per GPU (TFLOP/s/GPU): 43.6 | MFU 13.98% | FLOPs estimator: workload-aware | logical tokens/s: 201576.3 | valid tokens/s: 201576.3 | physical tokens/s: 201961.3 | loss tokens/s: 155953.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.673198E-01 | lm-global-supervised-tokens: 1.297999E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.624569E-02 | lm-token-buffer-mib: 5.803900E+01 | global_load_balancing_loss: 9.993661E-01 | loss scale: 1.0 | grad norm: 0.081 | params norm: 11562.543 | params rms: 0.062607 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 7:13:33.644427 | finish at 2026-10-10 08:57:52 + [2026-10-05 01:45:45] iteration 1353/ 6855 | consumed samples: 173184 | elapsed time per iteration (ms): 86733.0 | throughput per GPU (TFLOP/s/GPU): 43.2 | MFU 13.86% | FLOPs estimator: workload-aware | logical tokens/s: 193434.5 | valid tokens/s: 193434.5 | physical tokens/s: 193868.1 | loss tokens/s: 147779.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.445080E-01 | lm-global-supervised-tokens: 1.281734E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.255925E-02 | lm-token-buffer-mib: 5.810397E+01 | global_load_balancing_loss: 9.991977E-01 | loss scale: 1.0 | grad norm: 0.319 | params norm: 11562.466 | params rms: 0.062607 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 12:33:24.770765 | finish at 2026-10-10 14:19:09 + [2026-10-05 01:47:12] iteration 1354/ 6855 | consumed samples: 173312 | elapsed time per iteration (ms): 87255.9 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.46% | FLOPs estimator: workload-aware | logical tokens/s: 192276.1 | valid tokens/s: 192276.1 | physical tokens/s: 192783.1 | loss tokens/s: 145448.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.582183E-01 | lm-global-supervised-tokens: 1.269126E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.582899E-02 | lm-token-buffer-mib: 5.816609E+01 | global_load_balancing_loss: 9.990721E-01 | loss scale: 1.0 | grad norm: 0.173 | params norm: 11562.387 | params rms: 0.062607 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 13:19:54.509964 | finish at 2026-10-10 15:07:06 + [2026-10-05 01:48:37] iteration 1355/ 6855 | consumed samples: 173440 | elapsed time per iteration (ms): 85217.6 | throughput per GPU (TFLOP/s/GPU): 43.3 | MFU 13.88% | FLOPs estimator: workload-aware | logical tokens/s: 196874.6 | valid tokens/s: 196874.6 | physical tokens/s: 197649.1 | loss tokens/s: 144846.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.837319E-01 | lm-global-supervised-tokens: 1.234349E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.173242E-02 | lm-token-buffer-mib: 5.811327E+01 | global_load_balancing_loss: 1.000068E+00 | loss scale: 1.0 | grad norm: 0.126 | params norm: 11562.306 | params rms: 0.062606 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 10:11:36.780920 | finish at 2026-10-10 12:00:14 + [2026-10-05 01:50:05] iteration 1356/ 6855 | consumed samples: 173568 | elapsed time per iteration (ms): 87486.5 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.77% | FLOPs estimator: workload-aware | logical tokens/s: 191769.2 | valid tokens/s: 191769.2 | physical tokens/s: 192183.8 | loss tokens/s: 142462.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.757764E-01 | lm-global-supervised-tokens: 1.246357E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.147321E-02 | lm-token-buffer-mib: 5.814311E+01 | global_load_balancing_loss: 1.000152E+00 | loss scale: 1.0 | grad norm: 0.122 | params norm: 11562.223 | params rms: 0.062606 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 13:38:08.355411 | finish at 2026-10-10 15:28:13 + [2026-10-05 01:51:33] iteration 1357/ 6855 | consumed samples: 173696 | elapsed time per iteration (ms): 88787.4 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.52% | FLOPs estimator: workload-aware | logical tokens/s: 188959.2 | valid tokens/s: 188959.1 | physical tokens/s: 189404.9 | loss tokens/s: 140961.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.074002E-01 | lm-global-supervised-tokens: 1.251559E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.410903E-02 | lm-token-buffer-mib: 5.816200E+01 | global_load_balancing_loss: 9.999865E-01 | loss scale: 1.0 | grad norm: 0.146 | params norm: 11562.143 | params rms: 0.062605 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 15:35:53.041347 | finish at 2026-10-10 17:27:26 + [2026-10-05 01:53:01] iteration 1358/ 6855 | consumed samples: 173824 | elapsed time per iteration (ms): 88081.7 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.58% | FLOPs estimator: workload-aware | logical tokens/s: 190473.4 | valid tokens/s: 190473.4 | physical tokens/s: 190911.0 | loss tokens/s: 135751.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.380892E-01 | lm-global-supervised-tokens: 1.195721E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.306903E-02 | lm-token-buffer-mib: 5.810049E+01 | global_load_balancing_loss: 1.001500E+00 | loss scale: 1.0 | grad norm: 0.109 | params norm: 11562.063 | params rms: 0.062605 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 14:29:44.912720 | finish at 2026-10-10 16:22:46 + [2026-10-05 01:54:33] iteration 1359/ 6855 | consumed samples: 173952 | elapsed time per iteration (ms): 91437.5 | throughput per GPU (TFLOP/s/GPU): 40.5 | MFU 12.99% | FLOPs estimator: workload-aware | logical tokens/s: 183482.8 | valid tokens/s: 183482.8 | physical tokens/s: 183978.0 | loss tokens/s: 135052.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.912882E-01 | lm-global-supervised-tokens: 1.234885E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.169532E-02 | lm-token-buffer-mib: 5.803469E+01 | global_load_balancing_loss: 1.001714E+00 | loss scale: 1.0 | grad norm: 0.081 | params norm: 11561.984 | params rms: 0.062604 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 19:35:40.594345 | finish at 2026-10-10 21:30:13 + [2026-10-05 01:56:23] iteration 1360/ 6855 | consumed samples: 174080 | elapsed time per iteration (ms): 109948.4 | throughput per GPU (TFLOP/s/GPU): 33.8 | MFU 10.82% | FLOPs estimator: workload-aware | logical tokens/s: 152590.9 | valid tokens/s: 152590.9 | physical tokens/s: 152989.4 | loss tokens/s: 114744.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.055458E-01 | lm-global-supervised-tokens: 1.261592E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.126994E-02 | lm-token-buffer-mib: 5.819421E+01 | global_load_balancing_loss: 1.000241E+00 | loss scale: 1.0 | grad norm: 0.081 | params norm: 11561.905 | params rms: 0.062604 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 23:49:26.548511 | finish at 2026-10-12 01:45:49 +times across ranks (ms): + save-checkpoint: + rank 0: 23892.54 + rank 1: 23892.55 + rank 2: 23892.55 + rank 3: 23892.55 + rank 4: 23892.56 + rank 5: 23892.55 + rank 6: 23892.60 + rank 7: 23892.54 + rank 8: 23893.95 + rank 9: 23893.91 + rank 10: 23893.91 + rank 11: 23893.90 + rank 12: 23893.91 + rank 13: 23893.90 + rank 14: 23893.90 + rank 15: 23893.93 + rank 16: 23893.51 + rank 17: 23893.51 + rank 18: 23893.53 + rank 19: 23893.53 + rank 20: 23893.53 + rank 21: 23893.54 + rank 22: 23893.49 + rank 23: 23893.49 + rank 24: 23894.57 + rank 25: 23894.44 + rank 26: 23894.35 + rank 27: 23894.38 + rank 28: 23894.31 + rank 29: 23894.48 + rank 30: 23894.54 + rank 31: 23894.55 + rank 32: 23893.86 + rank 33: 23893.86 + rank 34: 23893.85 + rank 35: 23893.83 + rank 36: 23893.83 + rank 37: 23893.87 + rank 38: 23893.85 + rank 39: 23893.81 + rank 40: 23893.66 + rank 41: 23893.67 + rank 42: 23893.66 + rank 43: 23893.66 + rank 44: 23893.67 + rank 45: 23893.68 + rank 46: 23893.65 + rank 47: 23893.67 + rank 48: 23893.16 + rank 49: 23893.16 + rank 50: 23893.14 + rank 51: 23893.13 + rank 52: 23893.14 + rank 53: 23893.22 + rank 54: 23893.22 + rank 55: 23893.31 + rank 56: 23893.68 + rank 57: 23893.70 + rank 58: 23893.64 + rank 59: 23893.65 + rank 60: 23893.64 + rank 61: 23893.68 + rank 62: 23893.66 + rank 63: 23893.67 + [2026-10-05 01:58:25] iteration 1361/ 6855 | consumed samples: 174208 | elapsed time per iteration (ms): 98023.3 | throughput per GPU (TFLOP/s/GPU): 37.4 | MFU 11.98% | FLOPs estimator: workload-aware | logical tokens/s: 171154.7 | valid tokens/s: 171154.7 | physical tokens/s: 171737.3 | loss tokens/s: 129890.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.992592E-01 | lm-global-supervised-tokens: 1.273228E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 4.418378E-01 | lm-token-buffer-mib: 5.822720E+01 | global_load_balancing_loss: 9.999991E-01 | loss scale: 1.0 | grad norm: 0.154 | params norm: 11561.825 | params rms: 0.062604 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 5:35:39.935166 | finish at 2026-10-11 07:34:05 + [2026-10-05 01:59:59] iteration 1362/ 6855 | consumed samples: 174336 | elapsed time per iteration (ms): 94339.0 | throughput per GPU (TFLOP/s/GPU): 39.6 | MFU 12.70% | FLOPs estimator: workload-aware | logical tokens/s: 177838.4 | valid tokens/s: 177838.4 | physical tokens/s: 178199.2 | loss tokens/s: 132704.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.002813E-01 | lm-global-supervised-tokens: 1.251923E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.728050E-02 | lm-token-buffer-mib: 5.806937E+01 | global_load_balancing_loss: 1.000212E+00 | loss scale: 1.0 | grad norm: 0.070 | params norm: 11561.745 | params rms: 0.062603 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 23:56:44.085018 | finish at 2026-10-11 01:56:43 + [2026-10-05 02:01:29] iteration 1363/ 6855 | consumed samples: 174464 | elapsed time per iteration (ms): 90111.0 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.47% | FLOPs estimator: workload-aware | logical tokens/s: 186183.9 | valid tokens/s: 186183.9 | physical tokens/s: 186608.9 | loss tokens/s: 132938.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.761184E-01 | lm-global-supervised-tokens: 1.197922E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.057737E-02 | lm-token-buffer-mib: 5.836835E+01 | global_load_balancing_loss: 9.995388E-01 | loss scale: 1.0 | grad norm: 0.142 | params norm: 11561.666 | params rms: 0.062603 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 17:28:09.642451 | finish at 2026-10-10 19:29:39 + [2026-10-05 02:02:54] iteration 1364/ 6855 | consumed samples: 174592 | elapsed time per iteration (ms): 85102.3 | throughput per GPU (TFLOP/s/GPU): 43.2 | MFU 13.84% | FLOPs estimator: workload-aware | logical tokens/s: 197140.9 | valid tokens/s: 197140.8 | physical tokens/s: 197632.4 | loss tokens/s: 145126.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.532311E-01 | lm-global-supervised-tokens: 1.235059E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.178821E-02 | lm-token-buffer-mib: 5.814037E+01 | global_load_balancing_loss: 1.000617E+00 | loss scale: 1.0 | grad norm: 0.056 | params norm: 11561.586 | params rms: 0.062602 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 9:48:16.912190 | finish at 2026-10-10 11:51:11 + [2026-10-05 02:04:20] iteration 1365/ 6855 | consumed samples: 174720 | elapsed time per iteration (ms): 85712.2 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.76% | FLOPs estimator: workload-aware | logical tokens/s: 195738.5 | valid tokens/s: 195738.5 | physical tokens/s: 196149.2 | loss tokens/s: 148190.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.492773E-01 | lm-global-supervised-tokens: 1.270171E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.350117E-02 | lm-token-buffer-mib: 5.818501E+01 | global_load_balancing_loss: 9.998853E-01 | loss scale: 1.0 | grad norm: 0.060 | params norm: 11561.506 | params rms: 0.062602 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 10:42:40.139902 | finish at 2026-10-10 12:47:00 + [2026-10-05 02:05:49] iteration 1366/ 6855 | consumed samples: 174848 | elapsed time per iteration (ms): 88928.6 | throughput per GPU (TFLOP/s/GPU): 41.2 | MFU 13.19% | FLOPs estimator: workload-aware | logical tokens/s: 188658.6 | valid tokens/s: 188658.6 | physical tokens/s: 189206.5 | loss tokens/s: 139311.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.086861E-01 | lm-global-supervised-tokens: 1.238875E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.530125E-02 | lm-token-buffer-mib: 5.804922E+01 | global_load_balancing_loss: 9.997867E-01 | loss scale: 1.0 | grad norm: 0.126 | params norm: 11561.427 | params rms: 0.062601 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 15:35:29.101504 | finish at 2026-10-10 17:41:18 + [2026-10-05 02:07:14] iteration 1367/ 6855 | consumed samples: 174976 | elapsed time per iteration (ms): 84669.7 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.78% | FLOPs estimator: workload-aware | logical tokens/s: 198148.0 | valid tokens/s: 198148.0 | physical tokens/s: 198674.6 | loss tokens/s: 147138.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.478037E-01 | lm-global-supervised-tokens: 1.245821E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.366570E-02 | lm-token-buffer-mib: 5.813048E+01 | global_load_balancing_loss: 1.000205E+00 | loss scale: 1.0 | grad norm: 0.056 | params norm: 11561.350 | params rms: 0.062601 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 9:04:27.565620 | finish at 2026-10-10 11:11:41 + [2026-10-05 02:08:44] iteration 1368/ 6855 | consumed samples: 175104 | elapsed time per iteration (ms): 90107.0 | throughput per GPU (TFLOP/s/GPU): 41.6 | MFU 13.34% | FLOPs estimator: workload-aware | logical tokens/s: 186191.8 | valid tokens/s: 186191.8 | physical tokens/s: 186543.3 | loss tokens/s: 136818.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.933471E-01 | lm-global-supervised-tokens: 1.232830E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.330850E-02 | lm-token-buffer-mib: 5.809244E+01 | global_load_balancing_loss: 1.000571E+00 | loss scale: 1.0 | grad norm: 0.059 | params norm: 11561.273 | params rms: 0.062601 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 17:20:17.228664 | finish at 2026-10-10 19:29:01 + [2026-10-05 02:10:14] iteration 1369/ 6855 | consumed samples: 175232 | elapsed time per iteration (ms): 90355.3 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.48% | FLOPs estimator: workload-aware | logical tokens/s: 185680.1 | valid tokens/s: 185680.1 | physical tokens/s: 186052.4 | loss tokens/s: 138572.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.187172E-01 | lm-global-supervised-tokens: 1.252077E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.590683E-02 | lm-token-buffer-mib: 5.804964E+01 | global_load_balancing_loss: 1.000018E+00 | loss scale: 1.0 | grad norm: 0.073 | params norm: 11561.193 | params rms: 0.062600 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 17:41:29.399186 | finish at 2026-10-10 19:51:44 + [2026-10-05 02:12:00] iteration 1370/ 6855 | consumed samples: 175360 | elapsed time per iteration (ms): 106276.1 | throughput per GPU (TFLOP/s/GPU): 33.9 | MFU 10.87% | FLOPs estimator: workload-aware | logical tokens/s: 157864.0 | valid tokens/s: 157864.0 | physical tokens/s: 158221.8 | loss tokens/s: 115162.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.416335E-01 | lm-global-supervised-tokens: 1.223898E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.179101E-02 | lm-token-buffer-mib: 5.806287E+01 | global_load_balancing_loss: 1.000005E+00 | loss scale: 1.0 | grad norm: 0.069 | params norm: 11561.113 | params rms: 0.062600 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 17:55:24.465603 | finish at 2026-10-11 20:07:25 + [2026-10-05 02:13:29] iteration 1371/ 6855 | consumed samples: 175488 | elapsed time per iteration (ms): 88205.9 | throughput per GPU (TFLOP/s/GPU): 41.7 | MFU 13.37% | FLOPs estimator: workload-aware | logical tokens/s: 190204.7 | valid tokens/s: 190204.7 | physical tokens/s: 190583.3 | loss tokens/s: 148146.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.309836E-01 | lm-global-supervised-tokens: 1.306740E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.173745E-02 | lm-token-buffer-mib: 5.811024E+01 | global_load_balancing_loss: 1.000317E+00 | loss scale: 1.0 | grad norm: 0.039 | params norm: 11561.034 | params rms: 0.062599 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 14:22:01.373697 | finish at 2026-10-10 16:35:30 + [2026-10-05 02:14:57] iteration 1372/ 6855 | consumed samples: 175616 | elapsed time per iteration (ms): 88333.5 | throughput per GPU (TFLOP/s/GPU): 41.1 | MFU 13.19% | FLOPs estimator: workload-aware | logical tokens/s: 189930.2 | valid tokens/s: 189930.2 | physical tokens/s: 190378.3 | loss tokens/s: 145000.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.801919E-01 | lm-global-supervised-tokens: 1.280842E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.479180E-02 | lm-token-buffer-mib: 5.811028E+01 | global_load_balancing_loss: 1.000099E+00 | loss scale: 1.0 | grad norm: 0.058 | params norm: 11560.955 | params rms: 0.062599 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 14:32:12.620523 | finish at 2026-10-10 16:47:10 + [2026-10-05 02:16:25] iteration 1373/ 6855 | consumed samples: 175744 | elapsed time per iteration (ms): 88166.9 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.75% | FLOPs estimator: workload-aware | logical tokens/s: 190288.8 | valid tokens/s: 190288.8 | physical tokens/s: 190763.5 | loss tokens/s: 134506.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.278414E-01 | lm-global-supervised-tokens: 1.185902E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.364203E-02 | lm-token-buffer-mib: 5.809533E+01 | global_load_balancing_loss: 1.000145E+00 | loss scale: 1.0 | grad norm: 0.040 | params norm: 11560.874 | params rms: 0.062598 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 14:15:30.968885 | finish at 2026-10-10 16:31:56 + [2026-10-05 02:17:53] iteration 1374/ 6855 | consumed samples: 175872 | elapsed time per iteration (ms): 87492.7 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.64% | FLOPs estimator: workload-aware | logical tokens/s: 191755.5 | valid tokens/s: 191755.5 | physical tokens/s: 192125.3 | loss tokens/s: 141065.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.229427E-01 | lm-global-supervised-tokens: 1.234223E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.272660E-02 | lm-token-buffer-mib: 5.812399E+01 | global_load_balancing_loss: 1.001534E+00 | loss scale: 1.0 | grad norm: 0.082 | params norm: 11560.793 | params rms: 0.062598 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 13:12:27.700945 | finish at 2026-10-10 15:30:20 + [2026-10-05 02:19:21] iteration 1375/ 6855 | consumed samples: 176000 | elapsed time per iteration (ms): 88066.5 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.77% | FLOPs estimator: workload-aware | logical tokens/s: 190506.1 | valid tokens/s: 190506.1 | physical tokens/s: 190979.2 | loss tokens/s: 137981.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.642052E-01 | lm-global-supervised-tokens: 1.215152E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.839784E-02 | lm-token-buffer-mib: 5.830259E+01 | global_load_balancing_loss: 9.995727E-01 | loss scale: 1.0 | grad norm: 0.082 | params norm: 11560.713 | params rms: 0.062597 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 14:03:24.191074 | finish at 2026-10-10 16:22:45 + [2026-10-05 02:20:47] iteration 1376/ 6855 | consumed samples: 176128 | elapsed time per iteration (ms): 86496.7 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.77% | FLOPs estimator: workload-aware | logical tokens/s: 193963.5 | valid tokens/s: 193963.4 | physical tokens/s: 194486.1 | loss tokens/s: 145493.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.129720E-01 | lm-global-supervised-tokens: 1.258472E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.151142E-02 | lm-token-buffer-mib: 5.811011E+01 | global_load_balancing_loss: 9.997610E-01 | loss scale: 1.0 | grad norm: 0.347 | params norm: 11560.631 | params rms: 0.062597 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 11:38:35.354251 | finish at 2026-10-10 13:59:23 + [2026-10-05 02:22:16] iteration 1377/ 6855 | consumed samples: 176256 | elapsed time per iteration (ms): 88901.4 | throughput per GPU (TFLOP/s/GPU): 40.8 | MFU 13.07% | FLOPs estimator: workload-aware | logical tokens/s: 188717.0 | valid tokens/s: 188717.0 | physical tokens/s: 189116.4 | loss tokens/s: 142250.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.269825E-01 | lm-global-supervised-tokens: 1.264628E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.183894E-02 | lm-token-buffer-mib: 5.810025E+01 | global_load_balancing_loss: 1.000123E+00 | loss scale: 1.0 | grad norm: 0.283 | params norm: 11560.548 | params rms: 0.062597 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 15:16:41.656802 | finish at 2026-10-10 17:38:58 + [2026-10-05 02:23:40] iteration 1378/ 6855 | consumed samples: 176384 | elapsed time per iteration (ms): 84352.4 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.58% | FLOPs estimator: workload-aware | logical tokens/s: 198894.0 | valid tokens/s: 198893.9 | physical tokens/s: 199647.8 | loss tokens/s: 148489.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.344259E-01 | lm-global-supervised-tokens: 1.252545E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.250448E-02 | lm-token-buffer-mib: 5.826136E+01 | global_load_balancing_loss: 1.000471E+00 | loss scale: 1.0 | grad norm: 0.143 | params norm: 11560.467 | params rms: 0.062596 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 8:19:58.054673 | finish at 2026-10-10 10:43:39 + [2026-10-05 02:25:07] iteration 1379/ 6855 | consumed samples: 176512 | elapsed time per iteration (ms): 86875.7 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.80% | FLOPs estimator: workload-aware | logical tokens/s: 193116.5 | valid tokens/s: 193116.5 | physical tokens/s: 193609.5 | loss tokens/s: 139635.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.895563E-01 | lm-global-supervised-tokens: 1.213091E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.222475E-02 | lm-token-buffer-mib: 5.816948E+01 | global_load_balancing_loss: 9.994459E-01 | loss scale: 1.0 | grad norm: 0.209 | params norm: 11560.388 | params rms: 0.062596 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 12:08:51.517270 | finish at 2026-10-10 14:33:59 + [2026-10-05 02:26:56] iteration 1380/ 6855 | consumed samples: 176640 | elapsed time per iteration (ms): 108490.8 | throughput per GPU (TFLOP/s/GPU): 34.4 | MFU 11.02% | FLOPs estimator: workload-aware | logical tokens/s: 154641.6 | valid tokens/s: 154641.6 | physical tokens/s: 155094.2 | loss tokens/s: 111708.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.766067E-01 | lm-global-supervised-tokens: 1.211929E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.144628E-02 | lm-token-buffer-mib: 5.807566E+01 | global_load_balancing_loss: 1.000258E+00 | loss scale: 1.0 | grad norm: 0.147 | params norm: 11560.308 | params rms: 0.062595 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 20:59:47.032878 | finish at 2026-10-11 23:26:43 + [2026-10-05 02:28:28] iteration 1381/ 6855 | consumed samples: 176768 | elapsed time per iteration (ms): 92013.3 | throughput per GPU (TFLOP/s/GPU): 41.1 | MFU 13.17% | FLOPs estimator: workload-aware | logical tokens/s: 182334.5 | valid tokens/s: 182334.5 | physical tokens/s: 182725.2 | loss tokens/s: 136283.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.497149E-01 | lm-global-supervised-tokens: 1.253987E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.334121E-02 | lm-token-buffer-mib: 5.806848E+01 | global_load_balancing_loss: 9.995024E-01 | loss scale: 1.0 | grad norm: 0.128 | params norm: 11560.228 | params rms: 0.062595 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 19:54:40.903070 | finish at 2026-10-10 22:23:09 + [2026-10-05 02:29:55] iteration 1382/ 6855 | consumed samples: 176896 | elapsed time per iteration (ms): 86802.3 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.75% | FLOPs estimator: workload-aware | logical tokens/s: 193279.4 | valid tokens/s: 193279.4 | physical tokens/s: 193735.0 | loss tokens/s: 140570.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.380885E-01 | lm-global-supervised-tokens: 1.220181E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.506304E-02 | lm-token-buffer-mib: 5.818235E+01 | global_load_balancing_loss: 1.000925E+00 | loss scale: 1.0 | grad norm: 0.159 | params norm: 11560.149 | params rms: 0.062594 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 11:57:49.244829 | finish at 2026-10-10 14:27:44 + [2026-10-05 02:31:27] iteration 1383/ 6855 | consumed samples: 177024 | elapsed time per iteration (ms): 92722.6 | throughput per GPU (TFLOP/s/GPU): 41.5 | MFU 13.31% | FLOPs estimator: workload-aware | logical tokens/s: 180939.0 | valid tokens/s: 180939.0 | physical tokens/s: 181257.5 | loss tokens/s: 137317.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.097582E-01 | lm-global-supervised-tokens: 1.273243E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.165047E-02 | lm-token-buffer-mib: 5.809531E+01 | global_load_balancing_loss: 1.001305E+00 | loss scale: 1.0 | grad norm: 0.054 | params norm: 11560.072 | params rms: 0.062594 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 20:56:18.247147 | finish at 2026-10-10 23:27:46 + [2026-10-05 02:32:57] iteration 1384/ 6855 | consumed samples: 177152 | elapsed time per iteration (ms): 89368.4 | throughput per GPU (TFLOP/s/GPU): 41.5 | MFU 13.29% | FLOPs estimator: workload-aware | logical tokens/s: 187730.7 | valid tokens/s: 187730.7 | physical tokens/s: 188200.7 | loss tokens/s: 143417.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.721313E-01 | lm-global-supervised-tokens: 1.281696E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.330973E-02 | lm-token-buffer-mib: 5.806780E+01 | global_load_balancing_loss: 1.001353E+00 | loss scale: 1.0 | grad norm: 0.161 | params norm: 11559.995 | params rms: 0.062594 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 15:48:54.707371 | finish at 2026-10-10 18:21:51 + [2026-10-05 02:34:22] iteration 1385/ 6855 | consumed samples: 177280 | elapsed time per iteration (ms): 85554.1 | throughput per GPU (TFLOP/s/GPU): 43.1 | MFU 13.80% | FLOPs estimator: workload-aware | logical tokens/s: 196100.3 | valid tokens/s: 196100.3 | physical tokens/s: 196564.9 | loss tokens/s: 147171.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.652044E-01 | lm-global-supervised-tokens: 1.259114E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.145668E-02 | lm-token-buffer-mib: 5.831635E+01 | global_load_balancing_loss: 1.000219E+00 | loss scale: 1.0 | grad norm: 0.038 | params norm: 11559.917 | params rms: 0.062593 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 9:59:40.798090 | finish at 2026-10-10 12:34:03 + [2026-10-05 02:35:50] iteration 1386/ 6855 | consumed samples: 177408 | elapsed time per iteration (ms): 87612.3 | throughput per GPU (TFLOP/s/GPU): 41.3 | MFU 13.24% | FLOPs estimator: workload-aware | logical tokens/s: 191493.5 | valid tokens/s: 191493.4 | physical tokens/s: 192171.9 | loss tokens/s: 140003.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.008565E-01 | lm-global-supervised-tokens: 1.226604E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.133294E-02 | lm-token-buffer-mib: 5.838161E+01 | global_load_balancing_loss: 1.000705E+00 | loss scale: 1.0 | grad norm: 0.090 | params norm: 11559.839 | params rms: 0.062593 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 13:05:51.663042 | finish at 2026-10-10 15:41:42 + [2026-10-05 02:37:19] iteration 1387/ 6855 | consumed samples: 177536 | elapsed time per iteration (ms): 89556.6 | throughput per GPU (TFLOP/s/GPU): 41.3 | MFU 13.25% | FLOPs estimator: workload-aware | logical tokens/s: 187335.8 | valid tokens/s: 187335.7 | physical tokens/s: 187764.8 | loss tokens/s: 140003.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.466281E-01 | lm-global-supervised-tokens: 1.253822E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.547080E-02 | lm-token-buffer-mib: 5.809751E+01 | global_load_balancing_loss: 1.000077E+00 | loss scale: 1.0 | grad norm: 0.062 | params norm: 11559.763 | params rms: 0.062592 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 16:01:35.278118 | finish at 2026-10-10 18:38:55 + [2026-10-05 02:38:47] iteration 1388/ 6855 | consumed samples: 177664 | elapsed time per iteration (ms): 87617.1 | throughput per GPU (TFLOP/s/GPU): 41.7 | MFU 13.35% | FLOPs estimator: workload-aware | logical tokens/s: 191483.2 | valid tokens/s: 191483.2 | physical tokens/s: 191936.2 | loss tokens/s: 145021.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.734692E-01 | lm-global-supervised-tokens: 1.270640E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.646313E-02 | lm-token-buffer-mib: 5.804407E+01 | global_load_balancing_loss: 9.986055E-01 | loss scale: 1.0 | grad norm: 0.061 | params norm: 11559.687 | params rms: 0.062592 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 13:03:22.758694 | finish at 2026-10-10 15:42:10 + [2026-10-05 02:40:14] iteration 1389/ 6855 | consumed samples: 177792 | elapsed time per iteration (ms): 87388.3 | throughput per GPU (TFLOP/s/GPU): 43.4 | MFU 13.92% | FLOPs estimator: workload-aware | logical tokens/s: 191984.7 | valid tokens/s: 191984.7 | physical tokens/s: 192381.4 | loss tokens/s: 138757.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.879473E-01 | lm-global-supervised-tokens: 1.212580E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.262900E-02 | lm-token-buffer-mib: 5.817070E+01 | global_load_balancing_loss: 9.985018E-01 | loss scale: 1.0 | grad norm: 0.059 | params norm: 11559.611 | params rms: 0.062592 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 12:41:04.184237 | finish at 2026-10-10 15:21:19 + [2026-10-05 02:41:56] iteration 1390/ 6855 | consumed samples: 177920 | elapsed time per iteration (ms): 101153.4 | throughput per GPU (TFLOP/s/GPU): 35.2 | MFU 11.29% | FLOPs estimator: workload-aware | logical tokens/s: 165859.2 | valid tokens/s: 165859.2 | physical tokens/s: 166242.6 | loss tokens/s: 123968.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.744510E-01 | lm-global-supervised-tokens: 1.253983E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.369171E-02 | lm-token-buffer-mib: 5.824383E+01 | global_load_balancing_loss: 1.000555E+00 | loss scale: 1.0 | grad norm: 0.049 | params norm: 11559.532 | params rms: 0.062591 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 9:33:23.246003 | finish at 2026-10-11 12:15:19 + [2026-10-05 02:43:23] iteration 1391/ 6855 | consumed samples: 178048 | elapsed time per iteration (ms): 87193.2 | throughput per GPU (TFLOP/s/GPU): 41.5 | MFU 13.29% | FLOPs estimator: workload-aware | logical tokens/s: 192413.6 | valid tokens/s: 192413.6 | physical tokens/s: 193094.0 | loss tokens/s: 141150.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.224387E-01 | lm-global-supervised-tokens: 1.230738E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.145138E-02 | lm-token-buffer-mib: 5.809244E+01 | global_load_balancing_loss: 9.992413E-01 | loss scale: 1.0 | grad norm: 0.051 | params norm: 11559.453 | params rms: 0.062591 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 12:20:23.891623 | finish at 2026-10-10 15:03:47 + [2026-10-05 02:44:50] iteration 1392/ 6855 | consumed samples: 178176 | elapsed time per iteration (ms): 87226.4 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.67% | FLOPs estimator: workload-aware | logical tokens/s: 192340.8 | valid tokens/s: 192340.8 | physical tokens/s: 192715.3 | loss tokens/s: 134029.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.260611E-01 | lm-global-supervised-tokens: 1.169091E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.162199E-02 | lm-token-buffer-mib: 5.824058E+01 | global_load_balancing_loss: 1.000367E+00 | loss scale: 1.0 | grad norm: 0.047 | params norm: 11559.374 | params rms: 0.062590 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 12:21:57.754917 | finish at 2026-10-10 15:06:48 + [2026-10-05 02:46:17] iteration 1393/ 6855 | consumed samples: 178304 | elapsed time per iteration (ms): 86433.9 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.76% | FLOPs estimator: workload-aware | logical tokens/s: 194104.5 | valid tokens/s: 194104.5 | physical tokens/s: 194732.7 | loss tokens/s: 145801.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.645331E-01 | lm-global-supervised-tokens: 1.260223E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.583616E-02 | lm-token-buffer-mib: 5.809737E+01 | global_load_balancing_loss: 1.000771E+00 | loss scale: 1.0 | grad norm: 0.055 | params norm: 11559.293 | params rms: 0.062590 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 11:08:21.846544 | finish at 2026-10-10 13:54:38 + [2026-10-05 02:47:41] iteration 1394/ 6855 | consumed samples: 178432 | elapsed time per iteration (ms): 84396.9 | throughput per GPU (TFLOP/s/GPU): 43.3 | MFU 13.89% | FLOPs estimator: workload-aware | logical tokens/s: 198788.9 | valid tokens/s: 198788.9 | physical tokens/s: 199237.7 | loss tokens/s: 153601.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.546559E-01 | lm-global-supervised-tokens: 1.296346E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.158584E-02 | lm-token-buffer-mib: 5.811990E+01 | global_load_balancing_loss: 1.000607E+00 | loss scale: 1.0 | grad norm: 0.049 | params norm: 11559.212 | params rms: 0.062589 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 8:01:31.255734 | finish at 2026-10-10 10:49:12 + [2026-10-05 02:49:06] iteration 1395/ 6855 | consumed samples: 178560 | elapsed time per iteration (ms): 85343.6 | throughput per GPU (TFLOP/s/GPU): 42.8 | MFU 13.73% | FLOPs estimator: workload-aware | logical tokens/s: 196584.4 | valid tokens/s: 196584.3 | physical tokens/s: 197053.6 | loss tokens/s: 149250.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.009128E-01 | lm-global-supervised-tokens: 1.273758E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.285713E-02 | lm-token-buffer-mib: 5.819749E+01 | global_load_balancing_loss: 1.000080E+00 | loss scale: 1.0 | grad norm: 0.043 | params norm: 11559.131 | params rms: 0.062589 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 9:26:15.975480 | finish at 2026-10-10 12:15:22 + [2026-10-05 02:50:35] iteration 1396/ 6855 | consumed samples: 178688 | elapsed time per iteration (ms): 88474.0 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.67% | FLOPs estimator: workload-aware | logical tokens/s: 189628.5 | valid tokens/s: 189628.5 | physical tokens/s: 190000.5 | loss tokens/s: 138319.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.892381E-01 | lm-global-supervised-tokens: 1.223767E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.119032E-02 | lm-token-buffer-mib: 5.810630E+01 | global_load_balancing_loss: 1.000817E+00 | loss scale: 1.0 | grad norm: 0.042 | params norm: 11559.052 | params rms: 0.062589 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 14:09:39.755898 | finish at 2026-10-10 17:00:14 + [2026-10-05 02:52:00] iteration 1397/ 6855 | consumed samples: 178816 | elapsed time per iteration (ms): 85163.6 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.54% | FLOPs estimator: workload-aware | logical tokens/s: 196999.6 | valid tokens/s: 196999.6 | physical tokens/s: 197632.3 | loss tokens/s: 145311.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.314710E-01 | lm-global-supervised-tokens: 1.237527E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.478837E-02 | lm-token-buffer-mib: 5.813624E+01 | global_load_balancing_loss: 1.001355E+00 | loss scale: 1.0 | grad norm: 0.085 | params norm: 11558.973 | params rms: 0.062588 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 9:07:02.803701 | finish at 2026-10-10 11:59:03 + [2026-10-05 02:53:27] iteration 1398/ 6855 | consumed samples: 178944 | elapsed time per iteration (ms): 87192.0 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.57% | FLOPs estimator: workload-aware | logical tokens/s: 192416.6 | valid tokens/s: 192416.6 | physical tokens/s: 192807.9 | loss tokens/s: 136316.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.481802E-01 | lm-global-supervised-tokens: 1.188569E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.135005E-02 | lm-token-buffer-mib: 5.806690E+01 | global_load_balancing_loss: 1.000481E+00 | loss scale: 1.0 | grad norm: 0.041 | params norm: 11558.895 | params rms: 0.062588 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 12:10:06.997226 | finish at 2026-10-10 15:03:34 + [2026-10-05 02:54:59] iteration 1399/ 6855 | consumed samples: 179072 | elapsed time per iteration (ms): 92204.0 | throughput per GPU (TFLOP/s/GPU): 40.5 | MFU 12.99% | FLOPs estimator: workload-aware | logical tokens/s: 181957.2 | valid tokens/s: 181957.2 | physical tokens/s: 182458.0 | loss tokens/s: 129257.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.381968E-01 | lm-global-supervised-tokens: 1.191810E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.145070E-02 | lm-token-buffer-mib: 5.818855E+01 | global_load_balancing_loss: 1.000529E+00 | loss scale: 1.0 | grad norm: 0.072 | params norm: 11558.816 | params rms: 0.062587 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 19:44:25.052597 | finish at 2026-10-10 22:39:24 + [2026-10-05 02:56:46] iteration 1400/ 6855 | consumed samples: 179200 | elapsed time per iteration (ms): 106618.4 | throughput per GPU (TFLOP/s/GPU): 35.1 | MFU 11.24% | FLOPs estimator: workload-aware | logical tokens/s: 157357.6 | valid tokens/s: 157357.6 | physical tokens/s: 157613.8 | loss tokens/s: 118714.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.886950E-01 | lm-global-supervised-tokens: 1.265716E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.403408E-02 | lm-token-buffer-mib: 5.819514E+01 | global_load_balancing_loss: 9.993590E-01 | loss scale: 1.0 | grad norm: 0.051 | params norm: 11558.741 | params rms: 0.062587 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 17:33:23.428453 | finish at 2026-10-11 20:30:09 +times across ranks (ms): + save-checkpoint: + rank 0: 27116.12 + rank 1: 27116.11 + rank 2: 27116.13 + rank 3: 27116.16 + rank 4: 27116.15 + rank 5: 27116.17 + rank 6: 27116.09 + rank 7: 27116.10 + rank 8: 26538.81 + rank 9: 26538.77 + rank 10: 26538.77 + rank 11: 26538.78 + rank 12: 26538.79 + rank 13: 26538.76 + rank 14: 26538.79 + rank 15: 26538.77 + rank 16: 26539.61 + rank 17: 26539.61 + rank 18: 26539.61 + rank 19: 26539.62 + rank 20: 26539.59 + rank 21: 26539.62 + rank 22: 26539.61 + rank 23: 26539.60 + rank 24: 26539.51 + rank 25: 26539.29 + rank 26: 26539.56 + rank 27: 26539.54 + rank 28: 26539.48 + rank 29: 26539.58 + rank 30: 26539.40 + rank 31: 26539.52 + rank 32: 26539.84 + rank 33: 26539.85 + rank 34: 26539.72 + rank 35: 26539.75 + rank 36: 26539.74 + rank 37: 26539.79 + rank 38: 26539.78 + rank 39: 26539.78 + rank 40: 26539.56 + rank 41: 26539.54 + rank 42: 26539.54 + rank 43: 26539.57 + rank 44: 26539.54 + rank 45: 26539.54 + rank 46: 26539.54 + rank 47: 26539.54 + rank 48: 26539.27 + rank 49: 26539.25 + rank 50: 26539.38 + rank 51: 26539.23 + rank 52: 26539.37 + rank 53: 26539.32 + rank 54: 26539.19 + rank 55: 26539.30 + rank 56: 26538.95 + rank 57: 26538.94 + rank 58: 26538.92 + rank 59: 26538.77 + rank 60: 26538.90 + rank 61: 26538.94 + rank 62: 26538.90 + rank 63: 26538.94 + [2026-10-05 02:58:50] iteration 1401/ 6855 | consumed samples: 179328 | elapsed time per iteration (ms): 96545.4 | throughput per GPU (TFLOP/s/GPU): 39.0 | MFU 12.51% | FLOPs estimator: workload-aware | logical tokens/s: 173774.9 | valid tokens/s: 173774.8 | physical tokens/s: 174114.8 | loss tokens/s: 131247.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.622080E-01 | lm-global-supervised-tokens: 1.267137E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.303412E+00 | lm-token-buffer-mib: 5.807363E+01 | global_load_balancing_loss: 9.989889E-01 | loss scale: 1.0 | grad norm: 0.057 | params norm: 11558.663 | params rms: 0.062586 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 2:15:58.579870 | finish at 2026-10-11 05:14:49 + [2026-10-05 03:00:21] iteration 1402/ 6855 | consumed samples: 179456 | elapsed time per iteration (ms): 90398.0 | throughput per GPU (TFLOP/s/GPU): 40.8 | MFU 13.07% | FLOPs estimator: workload-aware | logical tokens/s: 185592.2 | valid tokens/s: 185592.2 | physical tokens/s: 185990.6 | loss tokens/s: 132057.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.633769E-01 | lm-global-supervised-tokens: 1.193776E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.691433E-02 | lm-token-buffer-mib: 5.812374E+01 | global_load_balancing_loss: 1.000135E+00 | loss scale: 1.0 | grad norm: 0.041 | params norm: 11558.586 | params rms: 0.062586 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 16:55:40.080795 | finish at 2026-10-10 19:56:01 + [2026-10-05 03:01:47] iteration 1403/ 6855 | consumed samples: 179584 | elapsed time per iteration (ms): 86232.2 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.75% | FLOPs estimator: workload-aware | logical tokens/s: 194558.5 | valid tokens/s: 194558.5 | physical tokens/s: 195021.1 | loss tokens/s: 140308.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.734619E-01 | lm-global-supervised-tokens: 1.209910E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.142451E-02 | lm-token-buffer-mib: 5.811875E+01 | global_load_balancing_loss: 1.000362E+00 | loss scale: 1.0 | grad norm: 0.065 | params norm: 11558.510 | params rms: 0.062586 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 10:35:38.133275 | finish at 2026-10-10 13:37:25 + [2026-10-05 03:03:14] iteration 1404/ 6855 | consumed samples: 179712 | elapsed time per iteration (ms): 87144.7 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.59% | FLOPs estimator: workload-aware | logical tokens/s: 192520.9 | valid tokens/s: 192520.9 | physical tokens/s: 193031.6 | loss tokens/s: 143609.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.659083E-01 | lm-global-supervised-tokens: 1.251482E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.512576E-02 | lm-token-buffer-mib: 5.806330E+01 | global_load_balancing_loss: 1.000037E+00 | loss scale: 1.0 | grad norm: 0.062 | params norm: 11558.434 | params rms: 0.062585 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 11:57:05.643009 | finish at 2026-10-10 15:00:20 + [2026-10-05 03:04:40] iteration 1405/ 6855 | consumed samples: 179840 | elapsed time per iteration (ms): 86109.5 | throughput per GPU (TFLOP/s/GPU): 43.2 | MFU 13.86% | FLOPs estimator: workload-aware | logical tokens/s: 194835.4 | valid tokens/s: 194835.4 | physical tokens/s: 195299.0 | loss tokens/s: 144182.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.660354E-01 | lm-global-supervised-tokens: 1.241546E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.155249E-02 | lm-token-buffer-mib: 5.809677E+01 | global_load_balancing_loss: 9.996010E-01 | loss scale: 1.0 | grad norm: 0.042 | params norm: 11558.358 | params rms: 0.062585 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 10:21:36.942246 | finish at 2026-10-10 13:26:17 + [2026-10-05 03:06:06] iteration 1406/ 6855 | consumed samples: 179968 | elapsed time per iteration (ms): 85375.6 | throughput per GPU (TFLOP/s/GPU): 43.6 | MFU 13.98% | FLOPs estimator: workload-aware | logical tokens/s: 196510.2 | valid tokens/s: 196510.2 | physical tokens/s: 196861.3 | loss tokens/s: 143287.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.704645E-01 | lm-global-supervised-tokens: 1.223321E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.169633E-02 | lm-token-buffer-mib: 5.810325E+01 | global_load_balancing_loss: 1.000771E+00 | loss scale: 1.0 | grad norm: 0.051 | params norm: 11558.280 | params rms: 0.062584 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 9:13:31.664430 | finish at 2026-10-10 12:19:37 + [2026-10-05 03:07:33] iteration 1407/ 6855 | consumed samples: 180096 | elapsed time per iteration (ms): 87514.7 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.74% | FLOPs estimator: workload-aware | logical tokens/s: 191706.9 | valid tokens/s: 191706.9 | physical tokens/s: 192060.6 | loss tokens/s: 141168.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.637819E-01 | lm-global-supervised-tokens: 1.235430E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.763402E-02 | lm-token-buffer-mib: 5.807154E+01 | global_load_balancing_loss: 1.001167E+00 | loss scale: 1.0 | grad norm: 0.037 | params norm: 11558.204 | params rms: 0.062584 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 12:26:20.116425 | finish at 2026-10-10 15:33:53 + [2026-10-05 03:09:01] iteration 1408/ 6855 | consumed samples: 180224 | elapsed time per iteration (ms): 88185.6 | throughput per GPU (TFLOP/s/GPU): 41.6 | MFU 13.33% | FLOPs estimator: workload-aware | logical tokens/s: 190248.7 | valid tokens/s: 190248.7 | physical tokens/s: 190788.6 | loss tokens/s: 139719.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.867718E-01 | lm-global-supervised-tokens: 1.232123E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.678614E-02 | lm-token-buffer-mib: 5.808578E+01 | global_load_balancing_loss: 1.000359E+00 | loss scale: 1.0 | grad norm: 0.043 | params norm: 11558.129 | params rms: 0.062584 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 13:25:46.708892 | finish at 2026-10-10 16:34:48 + [2026-10-05 03:10:29] iteration 1409/ 6855 | consumed samples: 180352 | elapsed time per iteration (ms): 88078.7 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.54% | FLOPs estimator: workload-aware | logical tokens/s: 190479.9 | valid tokens/s: 190479.9 | physical tokens/s: 190857.6 | loss tokens/s: 138249.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.611035E-01 | lm-global-supervised-tokens: 1.217685E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.148795E-02 | lm-token-buffer-mib: 5.803217E+01 | global_load_balancing_loss: 1.000818E+00 | loss scale: 1.0 | grad norm: 0.077 | params norm: 11558.053 | params rms: 0.062583 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 13:14:36.412285 | finish at 2026-10-10 16:25:06 + [2026-10-05 03:12:20] iteration 1410/ 6855 | consumed samples: 180480 | elapsed time per iteration (ms): 110302.0 | throughput per GPU (TFLOP/s/GPU): 33.6 | MFU 10.77% | FLOPs estimator: workload-aware | logical tokens/s: 152102.4 | valid tokens/s: 152102.4 | physical tokens/s: 152508.5 | loss tokens/s: 112356.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.817450E-01 | lm-global-supervised-tokens: 1.239309E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.260627E-02 | lm-token-buffer-mib: 5.808329E+01 | global_load_balancing_loss: 9.999107E-01 | loss scale: 1.0 | grad norm: 0.081 | params norm: 11557.974 | params rms: 0.062583 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 22:49:54.277307 | finish at 2026-10-12 02:02:14 + [2026-10-05 03:13:48] iteration 1411/ 6855 | consumed samples: 180608 | elapsed time per iteration (ms): 87876.8 | throughput per GPU (TFLOP/s/GPU): 41.6 | MFU 13.32% | FLOPs estimator: workload-aware | logical tokens/s: 190916.7 | valid tokens/s: 190916.7 | physical tokens/s: 191414.8 | loss tokens/s: 144867.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.773473E-01 | lm-global-supervised-tokens: 1.273051E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.133794E-02 | lm-token-buffer-mib: 5.826027E+01 | global_load_balancing_loss: 9.993304E-01 | loss scale: 1.0 | grad norm: 0.042 | params norm: 11557.898 | params rms: 0.062582 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 12:53:21.486433 | finish at 2026-10-10 16:07:09 + [2026-10-05 03:15:14] iteration 1412/ 6855 | consumed samples: 180736 | elapsed time per iteration (ms): 86868.5 | throughput per GPU (TFLOP/s/GPU): 43.2 | MFU 13.84% | FLOPs estimator: workload-aware | logical tokens/s: 193133.3 | valid tokens/s: 193133.3 | physical tokens/s: 193526.4 | loss tokens/s: 144836.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.838477E-01 | lm-global-supervised-tokens: 1.258176E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.551877E-02 | lm-token-buffer-mib: 5.813546E+01 | global_load_balancing_loss: 9.995362E-01 | loss scale: 1.0 | grad norm: 0.040 | params norm: 11557.823 | params rms: 0.062582 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 11:20:25.495927 | finish at 2026-10-10 14:35:40 + [2026-10-05 03:16:42] iteration 1413/ 6855 | consumed samples: 180864 | elapsed time per iteration (ms): 87163.8 | throughput per GPU (TFLOP/s/GPU): 43.2 | MFU 13.84% | FLOPs estimator: workload-aware | logical tokens/s: 192478.8 | valid tokens/s: 192478.8 | physical tokens/s: 192961.7 | loss tokens/s: 139612.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.346348E-01 | lm-global-supervised-tokens: 1.216913E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.304066E-02 | lm-token-buffer-mib: 5.807465E+01 | global_load_balancing_loss: 1.000416E+00 | loss scale: 1.0 | grad norm: 0.042 | params norm: 11557.748 | params rms: 0.062581 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 11:45:45.509892 | finish at 2026-10-10 15:02:27 + [2026-10-05 03:18:08] iteration 1414/ 6855 | consumed samples: 180992 | elapsed time per iteration (ms): 86379.1 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.67% | FLOPs estimator: workload-aware | logical tokens/s: 194227.3 | valid tokens/s: 194227.3 | physical tokens/s: 194590.7 | loss tokens/s: 147065.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.550685E-01 | lm-global-supervised-tokens: 1.270339E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.500081E-02 | lm-token-buffer-mib: 5.816666E+01 | global_load_balancing_loss: 1.001245E+00 | loss scale: 1.0 | grad norm: 0.084 | params norm: 11557.672 | params rms: 0.062581 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 10:33:08.928736 | finish at 2026-10-10 13:51:17 + [2026-10-05 03:19:37] iteration 1415/ 6855 | consumed samples: 181120 | elapsed time per iteration (ms): 88502.1 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.55% | FLOPs estimator: workload-aware | logical tokens/s: 189567.7 | valid tokens/s: 189567.7 | physical tokens/s: 189985.6 | loss tokens/s: 136690.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.820881E-01 | lm-global-supervised-tokens: 1.209741E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.814683E-02 | lm-token-buffer-mib: 5.809576E+01 | global_load_balancing_loss: 1.000037E+00 | loss scale: 1.0 | grad norm: 0.155 | params norm: 11557.595 | params rms: 0.062581 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 13:44:11.462860 | finish at 2026-10-10 17:03:48 + [2026-10-05 03:21:03] iteration 1416/ 6855 | consumed samples: 181248 | elapsed time per iteration (ms): 86319.8 | throughput per GPU (TFLOP/s/GPU): 42.8 | MFU 13.70% | FLOPs estimator: workload-aware | logical tokens/s: 194360.8 | valid tokens/s: 194360.8 | physical tokens/s: 194721.8 | loss tokens/s: 147661.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.772639E-01 | lm-global-supervised-tokens: 1.274611E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.148070E-02 | lm-token-buffer-mib: 5.826645E+01 | global_load_balancing_loss: 1.000865E+00 | loss scale: 1.0 | grad norm: 0.141 | params norm: 11557.520 | params rms: 0.062580 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 10:24:53.621183 | finish at 2026-10-10 13:45:56 + [2026-10-05 03:22:31] iteration 1417/ 6855 | consumed samples: 181376 | elapsed time per iteration (ms): 88068.8 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.38% | FLOPs estimator: workload-aware | logical tokens/s: 190501.1 | valid tokens/s: 190501.0 | physical tokens/s: 190943.7 | loss tokens/s: 138477.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.776608E-01 | lm-global-supervised-tokens: 1.219554E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.809059E-02 | lm-token-buffer-mib: 5.822414E+01 | global_load_balancing_loss: 1.000371E+00 | loss scale: 1.0 | grad norm: 0.075 | params norm: 11557.445 | params rms: 0.062580 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 13:01:58.052570 | finish at 2026-10-10 16:24:29 + [2026-10-05 03:24:01] iteration 1418/ 6855 | consumed samples: 181504 | elapsed time per iteration (ms): 90135.8 | throughput per GPU (TFLOP/s/GPU): 41.6 | MFU 13.35% | FLOPs estimator: workload-aware | logical tokens/s: 186132.3 | valid tokens/s: 186132.3 | physical tokens/s: 186469.3 | loss tokens/s: 134887.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.223695E-01 | lm-global-supervised-tokens: 1.215821E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.407222E-02 | lm-token-buffer-mib: 5.807847E+01 | global_load_balancing_loss: 9.998302E-01 | loss scale: 1.0 | grad norm: 0.221 | params norm: 11557.370 | params rms: 0.062579 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 16:07:48.130087 | finish at 2026-10-10 19:31:49 + [2026-10-05 03:25:27] iteration 1419/ 6855 | consumed samples: 181632 | elapsed time per iteration (ms): 85947.8 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.64% | FLOPs estimator: workload-aware | logical tokens/s: 195202.1 | valid tokens/s: 195202.1 | physical tokens/s: 195833.9 | loss tokens/s: 143227.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.660514E-01 | lm-global-supervised-tokens: 1.231011E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.508874E-02 | lm-token-buffer-mib: 5.811379E+01 | global_load_balancing_loss: 9.989901E-01 | loss scale: 1.0 | grad norm: 0.055 | params norm: 11557.294 | params rms: 0.062579 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 9:46:52.104286 | finish at 2026-10-10 13:12:19 + [2026-10-05 03:27:13] iteration 1420/ 6855 | consumed samples: 181760 | elapsed time per iteration (ms): 106314.8 | throughput per GPU (TFLOP/s/GPU): 34.2 | MFU 10.96% | FLOPs estimator: workload-aware | logical tokens/s: 157806.4 | valid tokens/s: 157806.4 | physical tokens/s: 158101.0 | loss tokens/s: 118702.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.710214E-01 | lm-global-supervised-tokens: 1.261984E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.214193E-02 | lm-token-buffer-mib: 5.805365E+01 | global_load_balancing_loss: 1.000305E+00 | loss scale: 1.0 | grad norm: 0.175 | params norm: 11557.219 | params rms: 0.062579 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 16:30:20.691928 | finish at 2026-10-11 19:57:34 + [2026-10-05 03:28:39] iteration 1421/ 6855 | consumed samples: 181888 | elapsed time per iteration (ms): 85725.9 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.78% | FLOPs estimator: workload-aware | logical tokens/s: 195707.0 | valid tokens/s: 195707.0 | physical tokens/s: 196255.3 | loss tokens/s: 147323.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.983646E-01 | lm-global-supervised-tokens: 1.262942E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.289895E-02 | lm-token-buffer-mib: 5.830714E+01 | global_load_balancing_loss: 9.991971E-01 | loss scale: 1.0 | grad norm: 0.034 | params norm: 11557.146 | params rms: 0.062578 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 9:23:54.501379 | finish at 2026-10-10 12:52:34 + [2026-10-05 03:30:07] iteration 1422/ 6855 | consumed samples: 182016 | elapsed time per iteration (ms): 87901.6 | throughput per GPU (TFLOP/s/GPU): 41.5 | MFU 13.30% | FLOPs estimator: workload-aware | logical tokens/s: 190863.0 | valid tokens/s: 190863.0 | physical tokens/s: 191422.6 | loss tokens/s: 147583.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.986335E-01 | lm-global-supervised-tokens: 1.297284E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.255106E-02 | lm-token-buffer-mib: 5.804351E+01 | global_load_balancing_loss: 9.997437E-01 | loss scale: 1.0 | grad norm: 0.218 | params norm: 11557.073 | params rms: 0.062578 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 12:39:29.298906 | finish at 2026-10-10 16:09:36 + [2026-10-05 03:31:35] iteration 1423/ 6855 | consumed samples: 182144 | elapsed time per iteration (ms): 87555.5 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.61% | FLOPs estimator: workload-aware | logical tokens/s: 191617.4 | valid tokens/s: 191617.4 | physical tokens/s: 192160.2 | loss tokens/s: 144913.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.159859E-01 | lm-global-supervised-tokens: 1.268800E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.353133E-02 | lm-token-buffer-mib: 5.822084E+01 | global_load_balancing_loss: 1.000543E+00 | loss scale: 1.0 | grad norm: 0.070 | params norm: 11556.999 | params rms: 0.062577 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 12:06:41.652298 | finish at 2026-10-10 15:38:16 + [2026-10-05 03:33:02] iteration 1424/ 6855 | consumed samples: 182272 | elapsed time per iteration (ms): 87257.0 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.67% | FLOPs estimator: workload-aware | logical tokens/s: 192273.2 | valid tokens/s: 192273.2 | physical tokens/s: 192725.2 | loss tokens/s: 141537.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.128351E-01 | lm-global-supervised-tokens: 1.235010E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.284409E-02 | lm-token-buffer-mib: 5.808861E+01 | global_load_balancing_loss: 1.002111E+00 | loss scale: 1.0 | grad norm: 0.143 | params norm: 11556.925 | params rms: 0.062577 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 11:38:12.731873 | finish at 2026-10-10 15:11:14 + [2026-10-05 03:34:31] iteration 1425/ 6855 | consumed samples: 182400 | elapsed time per iteration (ms): 89576.2 | throughput per GPU (TFLOP/s/GPU): 41.1 | MFU 13.17% | FLOPs estimator: workload-aware | logical tokens/s: 187295.3 | valid tokens/s: 187295.3 | physical tokens/s: 187786.4 | loss tokens/s: 136837.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.836929E-01 | lm-global-supervised-tokens: 1.225738E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.268154E-02 | lm-token-buffer-mib: 5.815204E+01 | global_load_balancing_loss: 1.001643E+00 | loss scale: 1.0 | grad norm: 0.071 | params norm: 11556.850 | params rms: 0.062577 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 15:06:38.956354 | finish at 2026-10-10 18:41:10 + [2026-10-05 03:36:00] iteration 1426/ 6855 | consumed samples: 182528 | elapsed time per iteration (ms): 88462.2 | throughput per GPU (TFLOP/s/GPU): 41.4 | MFU 13.28% | FLOPs estimator: workload-aware | logical tokens/s: 189653.4 | valid tokens/s: 189653.4 | physical tokens/s: 190033.6 | loss tokens/s: 143931.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.021655E-01 | lm-global-supervised-tokens: 1.273245E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 3.159728E-02 | lm-token-buffer-mib: 5.807725E+01 | global_load_balancing_loss: 1.000842E+00 | loss scale: 1.0 | grad norm: 0.104 | params norm: 11556.778 | params rms: 0.062576 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 13:24:21.173378 | finish at 2026-10-10 17:00:21 + [2026-10-05 03:37:24] iteration 1427/ 6855 | consumed samples: 182656 | elapsed time per iteration (ms): 83769.4 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.77% | FLOPs estimator: workload-aware | logical tokens/s: 200277.7 | valid tokens/s: 200277.6 | physical tokens/s: 200703.1 | loss tokens/s: 153451.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.894197E-01 | lm-global-supervised-tokens: 1.285453E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.352232E-02 | lm-token-buffer-mib: 5.811864E+01 | global_load_balancing_loss: 9.984463E-01 | loss scale: 1.0 | grad norm: 0.077 | params norm: 11556.706 | params rms: 0.062576 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 6:18:20.395734 | finish at 2026-10-10 09:55:44 + [2026-10-05 03:38:50] iteration 1428/ 6855 | consumed samples: 182784 | elapsed time per iteration (ms): 86629.4 | throughput per GPU (TFLOP/s/GPU): 43.2 | MFU 13.85% | FLOPs estimator: workload-aware | logical tokens/s: 193666.0 | valid tokens/s: 193666.0 | physical tokens/s: 194079.8 | loss tokens/s: 137467.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.869729E-01 | lm-global-supervised-tokens: 1.190875E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.253722E-02 | lm-token-buffer-mib: 5.817689E+01 | global_load_balancing_loss: 9.992515E-01 | loss scale: 1.0 | grad norm: 0.063 | params norm: 11556.635 | params rms: 0.062575 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 10:35:37.884564 | finish at 2026-10-10 14:14:28 + [2026-10-05 03:40:21] iteration 1429/ 6855 | consumed samples: 182912 | elapsed time per iteration (ms): 90993.9 | throughput per GPU (TFLOP/s/GPU): 40.9 | MFU 13.10% | FLOPs estimator: workload-aware | logical tokens/s: 184376.2 | valid tokens/s: 184376.2 | physical tokens/s: 184794.0 | loss tokens/s: 133623.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.825848E-01 | lm-global-supervised-tokens: 1.215888E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.966298E-02 | lm-token-buffer-mib: 5.812112E+01 | global_load_balancing_loss: 1.000654E+00 | loss scale: 1.0 | grad norm: 0.060 | params norm: 11556.564 | params rms: 0.062575 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 17:08:52.697331 | finish at 2026-10-10 20:49:14 + [2026-10-05 03:42:14] iteration 1430/ 6855 | consumed samples: 183040 | elapsed time per iteration (ms): 112883.5 | throughput per GPU (TFLOP/s/GPU): 33.8 | MFU 10.84% | FLOPs estimator: workload-aware | logical tokens/s: 148623.9 | valid tokens/s: 148623.9 | physical tokens/s: 148879.0 | loss tokens/s: 102575.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.232460E-01 | lm-global-supervised-tokens: 1.157902E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.585514E-02 | lm-token-buffer-mib: 5.806326E+01 | global_load_balancing_loss: 9.989746E-01 | loss scale: 1.0 | grad norm: 0.057 | params norm: 11556.493 | params rms: 0.062575 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 7 days, 2:06:32.924660 | finish at 2026-10-12 05:48:47 + [2026-10-05 03:43:42] iteration 1431/ 6855 | consumed samples: 183168 | elapsed time per iteration (ms): 87948.5 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.40% | FLOPs estimator: workload-aware | logical tokens/s: 190761.4 | valid tokens/s: 190761.4 | physical tokens/s: 191334.8 | loss tokens/s: 144016.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.519142E-01 | lm-global-supervised-tokens: 1.266606E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.395403E-02 | lm-token-buffer-mib: 5.816541E+01 | global_load_balancing_loss: 1.000704E+00 | loss scale: 1.0 | grad norm: 0.040 | params norm: 11556.422 | params rms: 0.062574 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 12:30:32.752785 | finish at 2026-10-10 16:14:15 + [2026-10-05 03:45:13] iteration 1432/ 6855 | consumed samples: 183296 | elapsed time per iteration (ms): 90545.8 | throughput per GPU (TFLOP/s/GPU): 41.1 | MFU 13.18% | FLOPs estimator: workload-aware | logical tokens/s: 185289.1 | valid tokens/s: 185289.1 | physical tokens/s: 185826.8 | loss tokens/s: 139679.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.447797E-01 | lm-global-supervised-tokens: 1.264742E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.160075E-02 | lm-token-buffer-mib: 5.820429E+01 | global_load_balancing_loss: 1.000418E+00 | loss scale: 1.0 | grad norm: 0.037 | params norm: 11556.351 | params rms: 0.062574 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 16:23:50.055546 | finish at 2026-10-10 20:09:03 + [2026-10-05 03:46:40] iteration 1433/ 6855 | consumed samples: 183424 | elapsed time per iteration (ms): 87836.9 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.49% | FLOPs estimator: workload-aware | logical tokens/s: 191003.7 | valid tokens/s: 191003.7 | physical tokens/s: 191507.2 | loss tokens/s: 138681.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.293315E-01 | lm-global-supervised-tokens: 1.218134E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.199486E-02 | lm-token-buffer-mib: 5.810182E+01 | global_load_balancing_loss: 1.001359E+00 | loss scale: 1.0 | grad norm: 0.094 | params norm: 11556.280 | params rms: 0.062573 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 12:17:31.941953 | finish at 2026-10-10 16:04:12 + [2026-10-05 03:48:05] iteration 1434/ 6855 | consumed samples: 183552 | elapsed time per iteration (ms): 84706.8 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.74% | FLOPs estimator: workload-aware | logical tokens/s: 198061.5 | valid tokens/s: 198061.5 | physical tokens/s: 198712.3 | loss tokens/s: 147735.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.572135E-01 | lm-global-supervised-tokens: 1.251418E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.160351E-02 | lm-token-buffer-mib: 5.807722E+01 | global_load_balancing_loss: 1.001019E+00 | loss scale: 1.0 | grad norm: 0.076 | params norm: 11556.208 | params rms: 0.062573 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 7:33:15.342994 | finish at 2026-10-10 11:21:20 + [2026-10-05 03:49:33] iteration 1435/ 6855 | consumed samples: 183680 | elapsed time per iteration (ms): 88035.5 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.63% | FLOPs estimator: workload-aware | logical tokens/s: 190572.4 | valid tokens/s: 190572.4 | physical tokens/s: 190971.7 | loss tokens/s: 139105.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.962818E-01 | lm-global-supervised-tokens: 1.224626E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.246170E-02 | lm-token-buffer-mib: 5.815633E+01 | global_load_balancing_loss: 1.000403E+00 | loss scale: 1.0 | grad norm: 0.048 | params norm: 11556.138 | params rms: 0.062573 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 12:32:32.380548 | finish at 2026-10-10 16:22:06 + [2026-10-05 03:51:00] iteration 1436/ 6855 | consumed samples: 183808 | elapsed time per iteration (ms): 87278.1 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.58% | FLOPs estimator: workload-aware | logical tokens/s: 192226.8 | valid tokens/s: 192226.8 | physical tokens/s: 192754.8 | loss tokens/s: 141269.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.820846E-01 | lm-global-supervised-tokens: 1.232976E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.168707E-02 | lm-token-buffer-mib: 5.822602E+01 | global_load_balancing_loss: 9.999854E-01 | loss scale: 1.0 | grad norm: 0.039 | params norm: 11556.065 | params rms: 0.062572 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 11:22:40.186765 | finish at 2026-10-10 15:13:41 + [2026-10-05 03:52:26] iteration 1437/ 6855 | consumed samples: 183936 | elapsed time per iteration (ms): 85091.9 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.78% | FLOPs estimator: workload-aware | logical tokens/s: 197165.6 | valid tokens/s: 197165.5 | physical tokens/s: 197552.6 | loss tokens/s: 147359.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.521188E-01 | lm-global-supervised-tokens: 1.253911E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.883416E-02 | lm-token-buffer-mib: 5.807379E+01 | global_load_balancing_loss: 1.000107E+00 | loss scale: 1.0 | grad norm: 0.069 | params norm: 11555.991 | params rms: 0.062572 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 8:03:48.036222 | finish at 2026-10-10 11:56:14 + [2026-10-05 03:53:53] iteration 1438/ 6855 | consumed samples: 184064 | elapsed time per iteration (ms): 87351.6 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.66% | FLOPs estimator: workload-aware | logical tokens/s: 192065.4 | valid tokens/s: 192065.4 | physical tokens/s: 192433.9 | loss tokens/s: 138487.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.418396E-01 | lm-global-supervised-tokens: 1.209707E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.357542E-02 | lm-token-buffer-mib: 5.806293E+01 | global_load_balancing_loss: 9.995283E-01 | loss scale: 1.0 | grad norm: 0.054 | params norm: 11555.918 | params rms: 0.062572 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 11:26:23.478638 | finish at 2026-10-10 15:20:16 + [2026-10-05 03:55:20] iteration 1439/ 6855 | consumed samples: 184192 | elapsed time per iteration (ms): 87394.1 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.66% | FLOPs estimator: workload-aware | logical tokens/s: 191970.5 | valid tokens/s: 191970.5 | physical tokens/s: 192473.0 | loss tokens/s: 142081.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.606029E-01 | lm-global-supervised-tokens: 1.241707E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.166936E-02 | lm-token-buffer-mib: 5.802517E+01 | global_load_balancing_loss: 9.997751E-01 | loss scale: 1.0 | grad norm: 0.061 | params norm: 11555.846 | params rms: 0.062571 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 11:28:46.637733 | finish at 2026-10-10 15:24:07 + [2026-10-05 03:57:08] iteration 1440/ 6855 | consumed samples: 184320 | elapsed time per iteration (ms): 107817.0 | throughput per GPU (TFLOP/s/GPU): 33.7 | MFU 10.80% | FLOPs estimator: workload-aware | logical tokens/s: 155608.0 | valid tokens/s: 155608.0 | physical tokens/s: 156022.1 | loss tokens/s: 115811.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.830078E-01 | lm-global-supervised-tokens: 1.248641E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.528246E-02 | lm-token-buffer-mib: 5.807875E+01 | global_load_balancing_loss: 9.997470E-01 | loss scale: 1.0 | grad norm: 0.092 | params norm: 11555.773 | params rms: 0.062571 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 18:10:29.160390 | finish at 2026-10-11 22:07:37 +times across ranks (ms): + save-checkpoint: + rank 0: 25584.20 + rank 1: 25584.18 + rank 2: 25584.25 + rank 3: 25584.21 + rank 4: 25584.20 + rank 5: 25584.19 + rank 6: 25584.16 + rank 7: 25584.20 + rank 8: 25583.25 + rank 9: 25583.20 + rank 10: 25583.22 + rank 11: 25583.22 + rank 12: 25583.23 + rank 13: 25583.24 + rank 14: 25583.25 + rank 15: 25583.20 + rank 16: 25583.94 + rank 17: 25583.92 + rank 18: 25583.90 + rank 19: 25583.92 + rank 20: 25583.90 + rank 21: 25583.92 + rank 22: 25583.90 + rank 23: 25583.89 + rank 24: 25584.38 + rank 25: 25584.34 + rank 26: 25584.44 + rank 27: 25584.42 + rank 28: 25584.39 + rank 29: 25584.39 + rank 30: 25584.31 + rank 31: 25584.37 + rank 32: 25584.25 + rank 33: 25584.17 + rank 34: 25584.20 + rank 35: 25584.16 + rank 36: 25584.12 + rank 37: 25584.11 + rank 38: 25584.09 + rank 39: 25584.10 + rank 40: 25584.61 + rank 41: 25584.59 + rank 42: 25584.58 + rank 43: 25584.61 + rank 44: 25584.57 + rank 45: 25584.58 + rank 46: 25584.58 + rank 47: 25584.58 + rank 48: 25583.59 + rank 49: 25583.55 + rank 50: 25583.54 + rank 51: 25583.34 + rank 52: 25583.64 + rank 53: 25583.60 + rank 54: 25583.57 + rank 55: 25583.43 + rank 56: 25584.20 + rank 57: 25584.18 + rank 58: 25584.13 + rank 59: 25584.13 + rank 60: 25584.13 + rank 61: 25584.17 + rank 62: 25584.13 + rank 63: 25584.17 + [2026-10-05 03:59:02] iteration 1441/ 6855 | consumed samples: 184448 | elapsed time per iteration (ms): 88283.1 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.40% | FLOPs estimator: workload-aware | logical tokens/s: 190038.7 | valid tokens/s: 190038.7 | physical tokens/s: 190516.8 | loss tokens/s: 140002.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.660285E-01 | lm-global-supervised-tokens: 1.235986E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 5.954390E-01 | lm-token-buffer-mib: 5.812536E+01 | global_load_balancing_loss: 1.000573E+00 | loss scale: 1.0 | grad norm: 0.089 | params norm: 11555.699 | params rms: 0.062570 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 12:46:04.438190 | finish at 2026-10-10 16:45:07 + [2026-10-05 04:00:36] iteration 1442/ 6855 | consumed samples: 184576 | elapsed time per iteration (ms): 93881.1 | throughput per GPU (TFLOP/s/GPU): 39.3 | MFU 12.59% | FLOPs estimator: workload-aware | logical tokens/s: 178706.6 | valid tokens/s: 178706.6 | physical tokens/s: 179108.7 | loss tokens/s: 131650.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.670405E-01 | lm-global-supervised-tokens: 1.235950E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.421245E-02 | lm-token-buffer-mib: 5.805685E+01 | global_load_balancing_loss: 1.001383E+00 | loss scale: 1.0 | grad norm: 0.089 | params norm: 11555.626 | params rms: 0.062570 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 21:09:38.232652 | finish at 2026-10-11 01:10:15 + [2026-10-05 04:02:03] iteration 1443/ 6855 | consumed samples: 184704 | elapsed time per iteration (ms): 86889.5 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.78% | FLOPs estimator: workload-aware | logical tokens/s: 193086.7 | valid tokens/s: 193086.7 | physical tokens/s: 193464.3 | loss tokens/s: 138482.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.779909E-01 | lm-global-supervised-tokens: 1.203266E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.311651E-02 | lm-token-buffer-mib: 5.820287E+01 | global_load_balancing_loss: 1.000377E+00 | loss scale: 1.0 | grad norm: 0.067 | params norm: 11555.553 | params rms: 0.062570 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 10:37:25.945086 | finish at 2026-10-10 14:39:29 + [2026-10-05 04:03:27] iteration 1444/ 6855 | consumed samples: 184832 | elapsed time per iteration (ms): 84229.9 | throughput per GPU (TFLOP/s/GPU): 43.6 | MFU 13.98% | FLOPs estimator: workload-aware | logical tokens/s: 199183.1 | valid tokens/s: 199183.1 | physical tokens/s: 199656.8 | loss tokens/s: 146378.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.154261E-01 | lm-global-supervised-tokens: 1.232948E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.134121E-02 | lm-token-buffer-mib: 5.805120E+01 | global_load_balancing_loss: 1.000910E+00 | loss scale: 1.0 | grad norm: 0.036 | params norm: 11555.478 | params rms: 0.062569 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 6:36:07.925054 | finish at 2026-10-10 10:39:35 + [2026-10-05 04:04:53] iteration 1445/ 6855 | consumed samples: 184960 | elapsed time per iteration (ms): 85298.9 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.73% | FLOPs estimator: workload-aware | logical tokens/s: 196687.2 | valid tokens/s: 196687.2 | physical tokens/s: 197140.5 | loss tokens/s: 147058.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.194238E-01 | lm-global-supervised-tokens: 1.254392E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.383430E-02 | lm-token-buffer-mib: 5.811313E+01 | global_load_balancing_loss: 9.997044E-01 | loss scale: 1.0 | grad norm: 0.063 | params norm: 11555.401 | params rms: 0.062569 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 8:11:07.083225 | finish at 2026-10-10 12:16:00 + [2026-10-05 04:06:15] iteration 1446/ 6855 | consumed samples: 185088 | elapsed time per iteration (ms): 82238.5 | throughput per GPU (TFLOP/s/GPU): 44.4 | MFU 14.23% | FLOPs estimator: workload-aware | logical tokens/s: 204006.5 | valid tokens/s: 204006.5 | physical tokens/s: 204719.4 | loss tokens/s: 151095.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.743817E-01 | lm-global-supervised-tokens: 1.242587E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.425017E-02 | lm-token-buffer-mib: 5.814545E+01 | global_load_balancing_loss: 1.000349E+00 | loss scale: 1.0 | grad norm: 0.148 | params norm: 11555.325 | params rms: 0.062568 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 3:33:48.221236 | finish at 2026-10-10 07:40:03 + [2026-10-05 04:07:39] iteration 1447/ 6855 | consumed samples: 185216 | elapsed time per iteration (ms): 83664.1 | throughput per GPU (TFLOP/s/GPU): 43.6 | MFU 13.98% | FLOPs estimator: workload-aware | logical tokens/s: 200530.3 | valid tokens/s: 200530.3 | physical tokens/s: 201211.9 | loss tokens/s: 151842.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.743400E-01 | lm-global-supervised-tokens: 1.270375E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.112845E-02 | lm-token-buffer-mib: 5.807169E+01 | global_load_balancing_loss: 1.000196E+00 | loss scale: 1.0 | grad norm: 0.150 | params norm: 11555.250 | params rms: 0.062568 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 5:40:55.429222 | finish at 2026-10-10 09:48:34 + [2026-10-05 04:09:04] iteration 1448/ 6855 | consumed samples: 185344 | elapsed time per iteration (ms): 85400.6 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.63% | FLOPs estimator: workload-aware | logical tokens/s: 196452.1 | valid tokens/s: 196452.1 | physical tokens/s: 197122.4 | loss tokens/s: 152700.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.618201E-01 | lm-global-supervised-tokens: 1.304070E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.155967E-02 | lm-token-buffer-mib: 5.804782E+01 | global_load_balancing_loss: 9.984969E-01 | loss scale: 1.0 | grad norm: 0.054 | params norm: 11555.174 | params rms: 0.062568 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 8:16:01.007869 | finish at 2026-10-10 12:25:05 + [2026-10-05 04:10:28] iteration 1449/ 6855 | consumed samples: 185472 | elapsed time per iteration (ms): 84180.8 | throughput per GPU (TFLOP/s/GPU): 43.7 | MFU 14.02% | FLOPs estimator: workload-aware | logical tokens/s: 199298.9 | valid tokens/s: 199298.9 | physical tokens/s: 199799.0 | loss tokens/s: 145381.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.147061E-01 | lm-global-supervised-tokens: 1.223834E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.151358E-02 | lm-token-buffer-mib: 5.806258E+01 | global_load_balancing_loss: 9.989285E-01 | loss scale: 1.0 | grad norm: 0.186 | params norm: 11555.099 | params rms: 0.062567 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 6:24:41.595345 | finish at 2026-10-10 10:35:10 + [2026-10-05 04:12:14] iteration 1450/ 6855 | consumed samples: 185600 | elapsed time per iteration (ms): 105346.2 | throughput per GPU (TFLOP/s/GPU): 34.7 | MFU 11.11% | FLOPs estimator: workload-aware | logical tokens/s: 159257.9 | valid tokens/s: 159257.9 | physical tokens/s: 159622.4 | loss tokens/s: 120860.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.669303E-01 | lm-global-supervised-tokens: 1.273222E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.329494E-02 | lm-token-buffer-mib: 5.810856E+01 | global_load_balancing_loss: 1.000762E+00 | loss scale: 1.0 | grad norm: 0.048 | params norm: 11555.021 | params rms: 0.062567 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 14:09:56.231561 | finish at 2026-10-11 18:22:10 + [2026-10-05 04:13:40] iteration 1451/ 6855 | consumed samples: 185728 | elapsed time per iteration (ms): 86545.0 | throughput per GPU (TFLOP/s/GPU): 43.1 | MFU 13.81% | FLOPs estimator: workload-aware | logical tokens/s: 193854.8 | valid tokens/s: 193854.8 | physical tokens/s: 194394.2 | loss tokens/s: 139365.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.753163E-01 | lm-global-supervised-tokens: 1.206140E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.159185E-02 | lm-token-buffer-mib: 5.816705E+01 | global_load_balancing_loss: 1.001322E+00 | loss scale: 1.0 | grad norm: 0.194 | params norm: 11554.944 | params rms: 0.062566 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 9:54:49.347906 | finish at 2026-10-10 14:08:29 + [2026-10-05 04:15:09] iteration 1452/ 6855 | consumed samples: 185856 | elapsed time per iteration (ms): 88542.9 | throughput per GPU (TFLOP/s/GPU): 41.2 | MFU 13.21% | FLOPs estimator: workload-aware | logical tokens/s: 189481.1 | valid tokens/s: 189481.1 | physical tokens/s: 189827.5 | loss tokens/s: 141120.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.838553E-01 | lm-global-supervised-tokens: 1.249520E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.378906E-02 | lm-token-buffer-mib: 5.809859E+01 | global_load_balancing_loss: 1.000186E+00 | loss scale: 1.0 | grad norm: 0.090 | params norm: 11554.867 | params rms: 0.062566 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 12:53:17.546797 | finish at 2026-10-10 17:08:26 + [2026-10-05 04:16:34] iteration 1453/ 6855 | consumed samples: 185984 | elapsed time per iteration (ms): 85801.5 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.70% | FLOPs estimator: workload-aware | logical tokens/s: 195535.0 | valid tokens/s: 195535.0 | physical tokens/s: 195994.3 | loss tokens/s: 146008.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.922048E-01 | lm-global-supervised-tokens: 1.252779E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.572108E-02 | lm-token-buffer-mib: 5.812939E+01 | global_load_balancing_loss: 1.001073E+00 | loss scale: 1.0 | grad norm: 0.200 | params norm: 11554.789 | params rms: 0.062565 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 8:44:59.769128 | finish at 2026-10-10 13:01:34 + [2026-10-05 04:17:57] iteration 1454/ 6855 | consumed samples: 186112 | elapsed time per iteration (ms): 82184.3 | throughput per GPU (TFLOP/s/GPU): 43.3 | MFU 13.86% | FLOPs estimator: workload-aware | logical tokens/s: 204141.0 | valid tokens/s: 204141.0 | physical tokens/s: 204602.7 | loss tokens/s: 156260.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.474934E-01 | lm-global-supervised-tokens: 1.284219E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.120348E-02 | lm-token-buffer-mib: 5.820036E+01 | global_load_balancing_loss: 1.001513E+00 | loss scale: 1.0 | grad norm: 0.203 | params norm: 11554.711 | params rms: 0.062565 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 3:17:57.212140 | finish at 2026-10-10 07:35:54 + [2026-10-05 04:19:22] iteration 1455/ 6855 | consumed samples: 186240 | elapsed time per iteration (ms): 85101.8 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.70% | FLOPs estimator: workload-aware | logical tokens/s: 197142.4 | valid tokens/s: 197142.4 | physical tokens/s: 197593.5 | loss tokens/s: 144462.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.783411E-01 | lm-global-supervised-tokens: 1.229401E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.331272E-02 | lm-token-buffer-mib: 5.814619E+01 | global_load_balancing_loss: 1.000240E+00 | loss scale: 1.0 | grad norm: 0.185 | params norm: 11554.633 | params rms: 0.062565 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 7:39:09.937391 | finish at 2026-10-10 11:58:32 + [2026-10-05 04:20:50] iteration 1456/ 6855 | consumed samples: 186368 | elapsed time per iteration (ms): 88792.1 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.47% | FLOPs estimator: workload-aware | logical tokens/s: 188947.7 | valid tokens/s: 188947.7 | physical tokens/s: 189407.8 | loss tokens/s: 141141.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.047299E-01 | lm-global-supervised-tokens: 1.253222E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.175148E-02 | lm-token-buffer-mib: 5.813411E+01 | global_load_balancing_loss: 1.000198E+00 | loss scale: 1.0 | grad norm: 0.063 | params norm: 11554.556 | params rms: 0.062564 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 13:09:48.730430 | finish at 2026-10-10 17:30:39 + [2026-10-05 04:22:16] iteration 1457/ 6855 | consumed samples: 186496 | elapsed time per iteration (ms): 85305.6 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.60% | FLOPs estimator: workload-aware | logical tokens/s: 196671.3 | valid tokens/s: 196671.3 | physical tokens/s: 197241.9 | loss tokens/s: 145616.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.404986E-01 | lm-global-supervised-tokens: 1.242194E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.211523E-02 | lm-token-buffer-mib: 5.808147E+01 | global_load_balancing_loss: 9.991933E-01 | loss scale: 1.0 | grad norm: 0.198 | params norm: 11554.479 | params rms: 0.062564 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 7:54:39.769979 | finish at 2026-10-10 12:16:56 + [2026-10-05 04:23:42] iteration 1458/ 6855 | consumed samples: 186624 | elapsed time per iteration (ms): 86342.1 | throughput per GPU (TFLOP/s/GPU): 43.1 | MFU 13.80% | FLOPs estimator: workload-aware | logical tokens/s: 194310.8 | valid tokens/s: 194310.7 | physical tokens/s: 194760.5 | loss tokens/s: 137501.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.532808E-01 | lm-global-supervised-tokens: 1.187220E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.140219E-02 | lm-token-buffer-mib: 5.820415E+01 | global_load_balancing_loss: 9.987037E-01 | loss scale: 1.0 | grad norm: 0.068 | params norm: 11554.401 | params rms: 0.062563 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 9:26:28.398113 | finish at 2026-10-10 13:50:11 + [2026-10-05 04:25:11] iteration 1459/ 6855 | consumed samples: 186752 | elapsed time per iteration (ms): 89300.2 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.52% | FLOPs estimator: workload-aware | logical tokens/s: 187873.7 | valid tokens/s: 187873.7 | physical tokens/s: 188234.7 | loss tokens/s: 133221.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.652755E-01 | lm-global-supervised-tokens: 1.189670E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.271902E-02 | lm-token-buffer-mib: 5.806796E+01 | global_load_balancing_loss: 9.998513E-01 | loss scale: 1.0 | grad norm: 0.102 | params norm: 11554.322 | params rms: 0.062563 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 13:51:03.750471 | finish at 2026-10-10 18:16:15 + [2026-10-05 04:26:59] iteration 1460/ 6855 | consumed samples: 186880 | elapsed time per iteration (ms): 107577.1 | throughput per GPU (TFLOP/s/GPU): 33.9 | MFU 10.87% | FLOPs estimator: workload-aware | logical tokens/s: 155954.9 | valid tokens/s: 155954.9 | physical tokens/s: 156297.5 | loss tokens/s: 116285.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.645387E-01 | lm-global-supervised-tokens: 1.250966E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.567488E-02 | lm-token-buffer-mib: 5.833572E+01 | global_load_balancing_loss: 1.000145E+00 | loss scale: 1.0 | grad norm: 0.072 | params norm: 11554.242 | params rms: 0.062562 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 17:12:58.313218 | finish at 2026-10-11 21:39:57 + [2026-10-05 04:28:26] iteration 1461/ 6855 | consumed samples: 187008 | elapsed time per iteration (ms): 86569.1 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.59% | FLOPs estimator: workload-aware | logical tokens/s: 193800.8 | valid tokens/s: 193800.8 | physical tokens/s: 194354.6 | loss tokens/s: 145001.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.513249E-01 | lm-global-supervised-tokens: 1.255267E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.206313E-02 | lm-token-buffer-mib: 5.813830E+01 | global_load_balancing_loss: 1.000206E+00 | loss scale: 1.0 | grad norm: 0.200 | params norm: 11554.164 | params rms: 0.062562 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 9:42:33.751884 | finish at 2026-10-10 14:10:59 + [2026-10-05 04:29:53] iteration 1462/ 6855 | consumed samples: 187136 | elapsed time per iteration (ms): 87496.6 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.63% | FLOPs estimator: workload-aware | logical tokens/s: 191745.8 | valid tokens/s: 191745.8 | physical tokens/s: 192027.8 | loss tokens/s: 140759.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.979833E-01 | lm-global-supervised-tokens: 1.231602E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.458232E-02 | lm-token-buffer-mib: 5.802681E+01 | global_load_balancing_loss: 1.001388E+00 | loss scale: 1.0 | grad norm: 0.035 | params norm: 11554.087 | params rms: 0.062562 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 11:04:29.429488 | finish at 2026-10-10 15:34:23 + [2026-10-05 04:31:18] iteration 1463/ 6855 | consumed samples: 187264 | elapsed time per iteration (ms): 85369.3 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.65% | FLOPs estimator: workload-aware | logical tokens/s: 196524.5 | valid tokens/s: 196524.5 | physical tokens/s: 197002.9 | loss tokens/s: 147522.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.653121E-01 | lm-global-supervised-tokens: 1.259387E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.154758E-02 | lm-token-buffer-mib: 5.808050E+01 | global_load_balancing_loss: 1.001111E+00 | loss scale: 1.0 | grad norm: 0.191 | params norm: 11554.010 | params rms: 0.062561 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 7:51:51.266285 | finish at 2026-10-10 12:23:10 + [2026-10-05 04:32:49] iteration 1464/ 6855 | consumed samples: 187392 | elapsed time per iteration (ms): 90609.5 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.68% | FLOPs estimator: workload-aware | logical tokens/s: 185159.2 | valid tokens/s: 185159.2 | physical tokens/s: 185508.5 | loss tokens/s: 131529.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.789325E-01 | lm-global-supervised-tokens: 1.191783E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.166078E-02 | lm-token-buffer-mib: 5.818468E+01 | global_load_balancing_loss: 1.001545E+00 | loss scale: 1.0 | grad norm: 0.065 | params norm: 11553.932 | params rms: 0.062561 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 15:41:15.702307 | finish at 2026-10-10 20:14:05 + [2026-10-05 04:34:17] iteration 1465/ 6855 | consumed samples: 187520 | elapsed time per iteration (ms): 88311.8 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.58% | FLOPs estimator: workload-aware | logical tokens/s: 189976.7 | valid tokens/s: 189976.7 | physical tokens/s: 190421.2 | loss tokens/s: 141781.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.142329E-01 | lm-global-supervised-tokens: 1.252096E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.269246E-02 | lm-token-buffer-mib: 5.816129E+01 | global_load_balancing_loss: 1.000363E+00 | loss scale: 1.0 | grad norm: 0.071 | params norm: 11553.855 | params rms: 0.062560 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 12:13:20.855176 | finish at 2026-10-10 16:47:38 + [2026-10-05 04:35:47] iteration 1466/ 6855 | consumed samples: 187648 | elapsed time per iteration (ms): 89186.7 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.46% | FLOPs estimator: workload-aware | logical tokens/s: 188113.3 | valid tokens/s: 188113.3 | physical tokens/s: 188567.6 | loss tokens/s: 134389.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.933105E-01 | lm-global-supervised-tokens: 1.198576E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.315037E-02 | lm-token-buffer-mib: 5.809702E+01 | global_load_balancing_loss: 9.989090E-01 | loss scale: 1.0 | grad norm: 0.052 | params norm: 11553.777 | params rms: 0.062560 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 13:30:26.890459 | finish at 2026-10-10 18:06:13 + [2026-10-05 04:37:16] iteration 1467/ 6855 | consumed samples: 187776 | elapsed time per iteration (ms): 89757.1 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.42% | FLOPs estimator: workload-aware | logical tokens/s: 186917.1 | valid tokens/s: 186917.1 | physical tokens/s: 187522.2 | loss tokens/s: 139726.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.741434E-01 | lm-global-supervised-tokens: 1.254148E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.343871E-02 | lm-token-buffer-mib: 5.831018E+01 | global_load_balancing_loss: 9.981489E-01 | loss scale: 1.0 | grad norm: 0.098 | params norm: 11553.701 | params rms: 0.062560 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 14:20:11.478888 | finish at 2026-10-10 18:57:28 + [2026-10-05 04:38:45] iteration 1468/ 6855 | consumed samples: 187904 | elapsed time per iteration (ms): 88270.1 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.61% | FLOPs estimator: workload-aware | logical tokens/s: 190066.2 | valid tokens/s: 190066.2 | physical tokens/s: 190548.9 | loss tokens/s: 138091.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.483228E-01 | lm-global-supervised-tokens: 1.218937E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.511548E-02 | lm-token-buffer-mib: 5.815912E+01 | global_load_balancing_loss: 9.995445E-01 | loss scale: 1.0 | grad norm: 0.257 | params norm: 11553.624 | params rms: 0.062559 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 12:05:11.246386 | finish at 2026-10-10 16:43:56 + [2026-10-05 04:40:10] iteration 1469/ 6855 | consumed samples: 188032 | elapsed time per iteration (ms): 85099.9 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.51% | FLOPs estimator: workload-aware | logical tokens/s: 197146.4 | valid tokens/s: 197146.4 | physical tokens/s: 197639.1 | loss tokens/s: 151250.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.217690E-01 | lm-global-supervised-tokens: 1.287142E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.553600E-02 | lm-token-buffer-mib: 5.816422E+01 | global_load_balancing_loss: 1.001116E+00 | loss scale: 1.0 | grad norm: 0.158 | params norm: 11553.547 | params rms: 0.062559 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 7:19:08.150043 | finish at 2026-10-10 11:59:18 + [2026-10-05 04:42:01] iteration 1470/ 6855 | consumed samples: 188160 | elapsed time per iteration (ms): 111671.2 | throughput per GPU (TFLOP/s/GPU): 33.7 | MFU 10.82% | FLOPs estimator: workload-aware | logical tokens/s: 150236.9 | valid tokens/s: 150236.9 | physical tokens/s: 150580.0 | loss tokens/s: 106901.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.893190E-01 | lm-global-supervised-tokens: 1.193780E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.150869E-02 | lm-token-buffer-mib: 5.805172E+01 | global_load_balancing_loss: 1.001409E+00 | loss scale: 1.0 | grad norm: 0.061 | params norm: 11553.471 | params rms: 0.062558 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 23:02:29.536736 | finish at 2026-10-12 03:44:31 + [2026-10-05 04:43:26] iteration 1471/ 6855 | consumed samples: 188288 | elapsed time per iteration (ms): 84388.8 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.54% | FLOPs estimator: workload-aware | logical tokens/s: 198808.0 | valid tokens/s: 198808.0 | physical tokens/s: 199392.1 | loss tokens/s: 149175.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.556960E-01 | lm-global-supervised-tokens: 1.258872E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.608581E-02 | lm-token-buffer-mib: 5.815247E+01 | global_load_balancing_loss: 1.000832E+00 | loss scale: 1.0 | grad norm: 0.150 | params norm: 11553.395 | params rms: 0.062558 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 6:12:29.373144 | finish at 2026-10-10 10:55:55 + [2026-10-05 04:44:53] iteration 1472/ 6855 | consumed samples: 188416 | elapsed time per iteration (ms): 87336.6 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.55% | FLOPs estimator: workload-aware | logical tokens/s: 192098.1 | valid tokens/s: 192098.1 | physical tokens/s: 192482.8 | loss tokens/s: 142765.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.383461E-01 | lm-global-supervised-tokens: 1.246864E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.621776E-02 | lm-token-buffer-mib: 5.812702E+01 | global_load_balancing_loss: 1.001801E+00 | loss scale: 1.0 | grad norm: 0.140 | params norm: 11553.320 | params rms: 0.062557 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 10:35:33.050342 | finish at 2026-10-10 15:20:26 + [2026-10-05 04:46:20] iteration 1473/ 6855 | consumed samples: 188544 | elapsed time per iteration (ms): 86458.2 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.52% | FLOPs estimator: workload-aware | logical tokens/s: 194049.8 | valid tokens/s: 194049.7 | physical tokens/s: 194477.1 | loss tokens/s: 141765.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.693774E-01 | lm-global-supervised-tokens: 1.225681E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.179298E-02 | lm-token-buffer-mib: 5.817936E+01 | global_load_balancing_loss: 9.993201E-01 | loss scale: 1.0 | grad norm: 0.120 | params norm: 11553.243 | params rms: 0.062557 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 9:15:18.149049 | finish at 2026-10-10 14:01:38 + [2026-10-05 04:47:46] iteration 1474/ 6855 | consumed samples: 188672 | elapsed time per iteration (ms): 86079.3 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.69% | FLOPs estimator: workload-aware | logical tokens/s: 194904.2 | valid tokens/s: 194904.1 | physical tokens/s: 195275.8 | loss tokens/s: 146869.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.814212E-01 | lm-global-supervised-tokens: 1.264246E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.543634E-02 | lm-token-buffer-mib: 5.811895E+01 | global_load_balancing_loss: 9.985758E-01 | loss scale: 1.0 | grad norm: 0.097 | params norm: 11553.166 | params rms: 0.062557 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 8:39:52.643628 | finish at 2026-10-10 13:27:38 + [2026-10-05 04:49:10] iteration 1475/ 6855 | consumed samples: 188800 | elapsed time per iteration (ms): 84351.4 | throughput per GPU (TFLOP/s/GPU): 43.9 | MFU 14.08% | FLOPs estimator: workload-aware | logical tokens/s: 198896.7 | valid tokens/s: 198896.7 | physical tokens/s: 199298.4 | loss tokens/s: 149690.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.361652E-01 | lm-global-supervised-tokens: 1.262660E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.595561E-02 | lm-token-buffer-mib: 5.811517E+01 | global_load_balancing_loss: 9.987319E-01 | loss scale: 1.0 | grad norm: 0.187 | params norm: 11553.089 | params rms: 0.062556 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 6:03:30.298004 | finish at 2026-10-10 10:52:40 + [2026-10-05 04:50:38] iteration 1476/ 6855 | consumed samples: 188928 | elapsed time per iteration (ms): 87753.9 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.78% | FLOPs estimator: workload-aware | logical tokens/s: 191184.5 | valid tokens/s: 191184.5 | physical tokens/s: 191643.4 | loss tokens/s: 132946.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.422817E-01 | lm-global-supervised-tokens: 1.166660E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.184896E-02 | lm-token-buffer-mib: 5.817918E+01 | global_load_balancing_loss: 1.000364E+00 | loss scale: 1.0 | grad norm: 0.127 | params norm: 11553.012 | params rms: 0.062556 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 11:07:08.270696 | finish at 2026-10-10 15:57:46 + [2026-10-05 04:52:03] iteration 1477/ 6855 | consumed samples: 189056 | elapsed time per iteration (ms): 85514.0 | throughput per GPU (TFLOP/s/GPU): 43.4 | MFU 13.90% | FLOPs estimator: workload-aware | logical tokens/s: 196192.6 | valid tokens/s: 196192.5 | physical tokens/s: 196742.1 | loss tokens/s: 140066.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.631103E-01 | lm-global-supervised-tokens: 1.197768E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.113318E-02 | lm-token-buffer-mib: 5.810088E+01 | global_load_balancing_loss: 1.001289E+00 | loss scale: 1.0 | grad norm: 0.145 | params norm: 11552.934 | params rms: 0.062555 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 7:44:54.317316 | finish at 2026-10-10 12:36:58 + [2026-10-05 04:53:29] iteration 1478/ 6855 | consumed samples: 189184 | elapsed time per iteration (ms): 85966.1 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.65% | FLOPs estimator: workload-aware | logical tokens/s: 195160.0 | valid tokens/s: 195160.0 | physical tokens/s: 195758.9 | loss tokens/s: 144909.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.888905E-01 | lm-global-supervised-tokens: 1.245725E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.114696E-02 | lm-token-buffer-mib: 5.810141E+01 | global_load_balancing_loss: 1.000137E+00 | loss scale: 1.0 | grad norm: 0.124 | params norm: 11552.857 | params rms: 0.062555 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 8:23:59.506771 | finish at 2026-10-10 13:17:29 + [2026-10-05 04:54:58] iteration 1479/ 6855 | consumed samples: 189312 | elapsed time per iteration (ms): 89275.9 | throughput per GPU (TFLOP/s/GPU): 41.5 | MFU 13.29% | FLOPs estimator: workload-aware | logical tokens/s: 187925.2 | valid tokens/s: 187925.2 | physical tokens/s: 188436.9 | loss tokens/s: 141289.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.030506E-01 | lm-global-supervised-tokens: 1.261373E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.440843E-02 | lm-token-buffer-mib: 5.803983E+01 | global_load_balancing_loss: 1.000193E+00 | loss scale: 1.0 | grad norm: 0.092 | params norm: 11552.779 | params rms: 0.062555 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 13:19:07.013489 | finish at 2026-10-10 18:14:06 + [2026-10-05 04:56:47] iteration 1480/ 6855 | consumed samples: 189440 | elapsed time per iteration (ms): 108724.1 | throughput per GPU (TFLOP/s/GPU): 33.8 | MFU 10.82% | FLOPs estimator: workload-aware | logical tokens/s: 154310.0 | valid tokens/s: 154310.0 | physical tokens/s: 154662.7 | loss tokens/s: 113964.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.712146E-01 | lm-global-supervised-tokens: 1.239072E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.460306E-02 | lm-token-buffer-mib: 5.810866E+01 | global_load_balancing_loss: 1.001208E+00 | loss scale: 1.0 | grad norm: 0.062 | params norm: 11552.701 | params rms: 0.062554 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 18:19:51.821533 | finish at 2026-10-11 23:16:39 +times across ranks (ms): + save-checkpoint: + rank 0: 24880.27 + rank 1: 24880.20 + rank 2: 24880.29 + rank 3: 24880.22 + rank 4: 24880.22 + rank 5: 24880.19 + rank 6: 24880.18 + rank 7: 24880.26 + rank 8: 24880.58 + rank 9: 24880.53 + rank 10: 24880.53 + rank 11: 24880.61 + rank 12: 24880.55 + rank 13: 24880.54 + rank 14: 24880.55 + rank 15: 24880.53 + rank 16: 24880.75 + rank 17: 24880.71 + rank 18: 24880.69 + rank 19: 24880.72 + rank 20: 24880.68 + rank 21: 24880.73 + rank 22: 24880.67 + rank 23: 24880.67 + rank 24: 24880.18 + rank 25: 24880.16 + rank 26: 24880.17 + rank 27: 24880.21 + rank 28: 24880.12 + rank 29: 24880.46 + rank 30: 24880.13 + rank 31: 24880.09 + rank 32: 24880.81 + rank 33: 24880.94 + rank 34: 24880.96 + rank 35: 24880.93 + rank 36: 24880.86 + rank 37: 24880.85 + rank 38: 24880.92 + rank 39: 24880.86 + rank 40: 24880.10 + rank 41: 24880.06 + rank 42: 24880.08 + rank 43: 24880.08 + rank 44: 24880.08 + rank 45: 24880.06 + rank 46: 24880.04 + rank 47: 24880.06 + rank 48: 24880.48 + rank 49: 24880.73 + rank 50: 24880.66 + rank 51: 24880.56 + rank 52: 24881.19 + rank 53: 24880.68 + rank 54: 24880.67 + rank 55: 24880.63 + rank 56: 24880.68 + rank 57: 24880.66 + rank 58: 24880.64 + rank 59: 24880.64 + rank 60: 24880.64 + rank 61: 24880.64 + rank 62: 24880.65 + rank 63: 24880.67 + [2026-10-05 04:58:49] iteration 1481/ 6855 | consumed samples: 189568 | elapsed time per iteration (ms): 96834.4 | throughput per GPU (TFLOP/s/GPU): 37.3 | MFU 11.96% | FLOPs estimator: workload-aware | logical tokens/s: 173256.4 | valid tokens/s: 173256.4 | physical tokens/s: 173733.9 | loss tokens/s: 129677.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.608667E-01 | lm-global-supervised-tokens: 1.255727E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 8.769991E-01 | lm-token-buffer-mib: 5.804433E+01 | global_load_balancing_loss: 1.000278E+00 | loss scale: 1.0 | grad norm: 0.080 | params norm: 11552.623 | params rms: 0.062554 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 0:33:08.166490 | finish at 2026-10-11 05:31:57 + [2026-10-05 05:00:22] iteration 1482/ 6855 | consumed samples: 189696 | elapsed time per iteration (ms): 92276.9 | throughput per GPU (TFLOP/s/GPU): 39.2 | MFU 12.57% | FLOPs estimator: workload-aware | logical tokens/s: 181813.2 | valid tokens/s: 181813.2 | physical tokens/s: 182299.8 | loss tokens/s: 137807.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.780955E-01 | lm-global-supervised-tokens: 1.271640E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.359491E-02 | lm-token-buffer-mib: 5.828107E+01 | global_load_balancing_loss: 1.000168E+00 | loss scale: 1.0 | grad norm: 0.062 | params norm: 11552.547 | params rms: 0.062553 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 17:43:23.908244 | finish at 2026-10-10 22:43:45 + [2026-10-05 05:01:48] iteration 1483/ 6855 | consumed samples: 189824 | elapsed time per iteration (ms): 86158.1 | throughput per GPU (TFLOP/s/GPU): 42.8 | MFU 13.73% | FLOPs estimator: workload-aware | logical tokens/s: 194726.0 | valid tokens/s: 194726.0 | physical tokens/s: 195328.8 | loss tokens/s: 142269.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.475635E-01 | lm-global-supervised-tokens: 1.225769E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.161188E-02 | lm-token-buffer-mib: 5.813858E+01 | global_load_balancing_loss: 1.000255E+00 | loss scale: 1.0 | grad norm: 0.120 | params norm: 11552.471 | params rms: 0.062553 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 8:34:01.059012 | finish at 2026-10-10 13:35:49 + [2026-10-05 05:03:13] iteration 1484/ 6855 | consumed samples: 189952 | elapsed time per iteration (ms): 84989.9 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.60% | FLOPs estimator: workload-aware | logical tokens/s: 197401.8 | valid tokens/s: 197401.8 | physical tokens/s: 198034.4 | loss tokens/s: 143585.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.336659E-01 | lm-global-supervised-tokens: 1.220334E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.167235E-02 | lm-token-buffer-mib: 5.805449E+01 | global_load_balancing_loss: 9.997527E-01 | loss scale: 1.0 | grad norm: 0.052 | params norm: 11552.393 | params rms: 0.062552 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 6:48:00.967354 | finish at 2026-10-10 11:51:14 + [2026-10-05 05:04:42] iteration 1485/ 6855 | consumed samples: 190080 | elapsed time per iteration (ms): 89176.6 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.42% | FLOPs estimator: workload-aware | logical tokens/s: 188133.5 | valid tokens/s: 188133.5 | physical tokens/s: 188448.5 | loss tokens/s: 131304.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.772243E-01 | lm-global-supervised-tokens: 1.170931E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.511308E-02 | lm-token-buffer-mib: 5.823853E+01 | global_load_balancing_loss: 1.000230E+00 | loss scale: 1.0 | grad norm: 0.088 | params norm: 11552.314 | params rms: 0.062552 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 13:01:18.431511 | finish at 2026-10-10 18:06:00 + [2026-10-05 05:06:07] iteration 1486/ 6855 | consumed samples: 190208 | elapsed time per iteration (ms): 84735.3 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.77% | FLOPs estimator: workload-aware | logical tokens/s: 197993.9 | valid tokens/s: 197993.9 | physical tokens/s: 198432.4 | loss tokens/s: 145588.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.434211E-01 | lm-global-supervised-tokens: 1.233650E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.163089E-02 | lm-token-buffer-mib: 5.821242E+01 | global_load_balancing_loss: 9.994661E-01 | loss scale: 1.0 | grad norm: 0.030 | params norm: 11552.237 | params rms: 0.062552 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 6:22:23.637874 | finish at 2026-10-10 11:28:30 + [2026-10-05 05:07:33] iteration 1487/ 6855 | consumed samples: 190336 | elapsed time per iteration (ms): 86022.4 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.48% | FLOPs estimator: workload-aware | logical tokens/s: 195033.0 | valid tokens/s: 195033.0 | physical tokens/s: 195528.4 | loss tokens/s: 148984.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.164897E-01 | lm-global-supervised-tokens: 1.281603E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.904582E-02 | lm-token-buffer-mib: 5.832470E+01 | global_load_balancing_loss: 9.990025E-01 | loss scale: 1.0 | grad norm: 0.037 | params norm: 11552.160 | params rms: 0.062551 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 8:16:08.392416 | finish at 2026-10-10 13:23:41 + [2026-10-05 05:09:01] iteration 1488/ 6855 | consumed samples: 190464 | elapsed time per iteration (ms): 88600.7 | throughput per GPU (TFLOP/s/GPU): 41.2 | MFU 13.22% | FLOPs estimator: workload-aware | logical tokens/s: 189356.9 | valid tokens/s: 189356.9 | physical tokens/s: 189954.5 | loss tokens/s: 133418.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.425235E-01 | lm-global-supervised-tokens: 1.182095E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.504638E-02 | lm-token-buffer-mib: 5.810068E+01 | global_load_balancing_loss: 1.000548E+00 | loss scale: 1.0 | grad norm: 0.097 | params norm: 11552.083 | params rms: 0.062551 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 12:05:19.756755 | finish at 2026-10-10 17:14:21 + [2026-10-05 05:10:26] iteration 1489/ 6855 | consumed samples: 190592 | elapsed time per iteration (ms): 84698.1 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.78% | FLOPs estimator: workload-aware | logical tokens/s: 198081.8 | valid tokens/s: 198081.8 | physical tokens/s: 198481.6 | loss tokens/s: 146967.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.021143E-01 | lm-global-supervised-tokens: 1.244787E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.339305E-02 | lm-token-buffer-mib: 5.820217E+01 | global_load_balancing_loss: 1.001642E+00 | loss scale: 1.0 | grad norm: 0.061 | params norm: 11552.004 | params rms: 0.062550 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 6:14:49.841326 | finish at 2026-10-10 11:25:16 + [2026-10-05 05:12:15] iteration 1490/ 6855 | consumed samples: 190720 | elapsed time per iteration (ms): 108726.3 | throughput per GPU (TFLOP/s/GPU): 33.6 | MFU 10.77% | FLOPs estimator: workload-aware | logical tokens/s: 154306.7 | valid tokens/s: 154306.7 | physical tokens/s: 154594.7 | loss tokens/s: 113641.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.734307E-01 | lm-global-supervised-tokens: 1.235577E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.207087E-02 | lm-token-buffer-mib: 5.825723E+01 | global_load_balancing_loss: 1.000440E+00 | loss scale: 1.0 | grad norm: 0.098 | params norm: 11551.926 | params rms: 0.062550 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 18:01:56.864283 | finish at 2026-10-11 23:14:12 + [2026-10-05 05:13:46] iteration 1491/ 6855 | consumed samples: 190848 | elapsed time per iteration (ms): 90948.5 | throughput per GPU (TFLOP/s/GPU): 41.4 | MFU 13.27% | FLOPs estimator: workload-aware | logical tokens/s: 184468.6 | valid tokens/s: 184468.6 | physical tokens/s: 184967.0 | loss tokens/s: 134376.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.204749E-01 | lm-global-supervised-tokens: 1.222136E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.146886E-02 | lm-token-buffer-mib: 5.810802E+01 | global_load_balancing_loss: 1.000175E+00 | loss scale: 1.0 | grad norm: 0.093 | params norm: 11551.850 | params rms: 0.062550 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 15:30:48.001662 | finish at 2026-10-10 20:44:34 + [2026-10-05 05:15:12] iteration 1492/ 6855 | consumed samples: 190976 | elapsed time per iteration (ms): 86388.6 | throughput per GPU (TFLOP/s/GPU): 43.2 | MFU 13.84% | FLOPs estimator: workload-aware | logical tokens/s: 194205.7 | valid tokens/s: 194205.7 | physical tokens/s: 194711.9 | loss tokens/s: 142674.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.425460E-01 | lm-global-supervised-tokens: 1.232548E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.198748E-02 | lm-token-buffer-mib: 5.813110E+01 | global_load_balancing_loss: 1.000189E+00 | loss scale: 1.0 | grad norm: 0.077 | params norm: 11551.773 | params rms: 0.062549 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 8:41:42.278485 | finish at 2026-10-10 13:56:54 + [2026-10-05 05:16:41] iteration 1493/ 6855 | consumed samples: 191104 | elapsed time per iteration (ms): 89101.2 | throughput per GPU (TFLOP/s/GPU): 40.9 | MFU 13.11% | FLOPs estimator: workload-aware | logical tokens/s: 188293.7 | valid tokens/s: 188293.7 | physical tokens/s: 188770.7 | loss tokens/s: 136161.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.280589E-01 | lm-global-supervised-tokens: 1.213212E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.155392E-02 | lm-token-buffer-mib: 5.826472E+01 | global_load_balancing_loss: 1.000356E+00 | loss scale: 1.0 | grad norm: 0.099 | params norm: 11551.696 | params rms: 0.062549 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 12:42:40.563366 | finish at 2026-10-10 17:59:22 + [2026-10-05 05:18:12] iteration 1494/ 6855 | consumed samples: 191232 | elapsed time per iteration (ms): 91258.8 | throughput per GPU (TFLOP/s/GPU): 41.1 | MFU 13.16% | FLOPs estimator: workload-aware | logical tokens/s: 183842.0 | valid tokens/s: 183842.0 | physical tokens/s: 184224.6 | loss tokens/s: 130292.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.578397E-01 | lm-global-supervised-tokens: 1.189034E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.525611E-02 | lm-token-buffer-mib: 5.808060E+01 | global_load_balancing_loss: 1.000319E+00 | loss scale: 1.0 | grad norm: 0.074 | params norm: 11551.616 | params rms: 0.062548 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 15:53:58.401396 | finish at 2026-10-10 21:12:11 + [2026-10-05 05:19:41] iteration 1495/ 6855 | consumed samples: 191360 | elapsed time per iteration (ms): 88638.7 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.58% | FLOPs estimator: workload-aware | logical tokens/s: 189276.2 | valid tokens/s: 189276.2 | physical tokens/s: 189673.8 | loss tokens/s: 136158.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.413176E-01 | lm-global-supervised-tokens: 1.206894E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.279895E-02 | lm-token-buffer-mib: 5.808514E+01 | global_load_balancing_loss: 1.000764E+00 | loss scale: 1.0 | grad norm: 0.065 | params norm: 11551.538 | params rms: 0.062548 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 11:58:23.619900 | finish at 2026-10-10 17:18:05 + [2026-10-05 05:21:10] iteration 1496/ 6855 | consumed samples: 191488 | elapsed time per iteration (ms): 88964.6 | throughput per GPU (TFLOP/s/GPU): 40.8 | MFU 13.08% | FLOPs estimator: workload-aware | logical tokens/s: 188583.0 | valid tokens/s: 188583.0 | physical tokens/s: 189168.1 | loss tokens/s: 141834.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.169975E-01 | lm-global-supervised-tokens: 1.261826E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.230974E-02 | lm-token-buffer-mib: 5.814122E+01 | global_load_balancing_loss: 1.000360E+00 | loss scale: 1.0 | grad norm: 0.120 | params norm: 11551.462 | params rms: 0.062547 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 12:26:01.281641 | finish at 2026-10-10 17:47:11 + [2026-10-05 05:22:37] iteration 1497/ 6855 | consumed samples: 191616 | elapsed time per iteration (ms): 87350.8 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.78% | FLOPs estimator: workload-aware | logical tokens/s: 192066.8 | valid tokens/s: 192066.8 | physical tokens/s: 192396.1 | loss tokens/s: 134619.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.803272E-01 | lm-global-supervised-tokens: 1.175915E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.270959E-02 | lm-token-buffer-mib: 5.803282E+01 | global_load_balancing_loss: 9.999851E-01 | loss scale: 1.0 | grad norm: 0.123 | params norm: 11551.385 | params rms: 0.062547 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 10:00:25.587878 | finish at 2026-10-10 15:23:03 + [2026-10-05 05:24:04] iteration 1498/ 6855 | consumed samples: 191744 | elapsed time per iteration (ms): 86641.1 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.42% | FLOPs estimator: workload-aware | logical tokens/s: 193639.4 | valid tokens/s: 193639.3 | physical tokens/s: 194201.1 | loss tokens/s: 142506.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.473010E-01 | lm-global-supervised-tokens: 1.234690E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.155247E-02 | lm-token-buffer-mib: 5.819421E+01 | global_load_balancing_loss: 9.988847E-01 | loss scale: 1.0 | grad norm: 0.057 | params norm: 11551.306 | params rms: 0.062547 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 8:55:36.150090 | finish at 2026-10-10 14:19:40 + [2026-10-05 05:25:29] iteration 1499/ 6855 | consumed samples: 191872 | elapsed time per iteration (ms): 85065.9 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.59% | FLOPs estimator: workload-aware | logical tokens/s: 197225.2 | valid tokens/s: 197225.1 | physical tokens/s: 197648.0 | loss tokens/s: 147556.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.392955E-01 | lm-global-supervised-tokens: 1.255206E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.804001E-02 | lm-token-buffer-mib: 5.820621E+01 | global_load_balancing_loss: 9.995205E-01 | loss scale: 1.0 | grad norm: 0.113 | params norm: 11551.230 | params rms: 0.062546 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 6:33:32.996623 | finish at 2026-10-10 11:59:02 + [2026-10-05 05:27:18] iteration 1500/ 6855 | consumed samples: 192000 | elapsed time per iteration (ms): 108837.6 | throughput per GPU (TFLOP/s/GPU): 34.3 | MFU 10.98% | FLOPs estimator: workload-aware | logical tokens/s: 154148.8 | valid tokens/s: 154148.7 | physical tokens/s: 154426.9 | loss tokens/s: 108868.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.616923E-01 | lm-global-supervised-tokens: 1.184899E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.399465E-02 | lm-token-buffer-mib: 5.803635E+01 | global_load_balancing_loss: 1.000244E+00 | loss scale: 1.0 | grad norm: 0.031 | params norm: 11551.153 | params rms: 0.062546 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 17:53:45.502446 | finish at 2026-10-11 23:21:03 + [2026-10-05 05:28:45] iteration 1501/ 6855 | consumed samples: 192128 | elapsed time per iteration (ms): 87326.7 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.70% | FLOPs estimator: workload-aware | logical tokens/s: 192119.7 | valid tokens/s: 192119.7 | physical tokens/s: 192417.3 | loss tokens/s: 144105.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.612077E-01 | lm-global-supervised-tokens: 1.258423E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.543366E-02 | lm-token-buffer-mib: 5.817260E+01 | global_load_balancing_loss: 9.995060E-01 | loss scale: 1.0 | grad norm: 0.132 | params norm: 11551.077 | params rms: 0.062545 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 9:52:27.110803 | finish at 2026-10-10 15:21:12 + [2026-10-05 05:30:10] iteration 1502/ 6855 | consumed samples: 192256 | elapsed time per iteration (ms): 85055.3 | throughput per GPU (TFLOP/s/GPU): 43.3 | MFU 13.87% | FLOPs estimator: workload-aware | logical tokens/s: 197249.9 | valid tokens/s: 197249.9 | physical tokens/s: 197768.5 | loss tokens/s: 142661.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.261233E-01 | lm-global-supervised-tokens: 1.213410E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.153462E-02 | lm-token-buffer-mib: 5.838792E+01 | global_load_balancing_loss: 1.000834E+00 | loss scale: 1.0 | grad norm: 0.089 | params norm: 11551.001 | params rms: 0.062545 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 6:28:21.159997 | finish at 2026-10-10 11:58:31 + [2026-10-05 05:31:37] iteration 1503/ 6855 | consumed samples: 192384 | elapsed time per iteration (ms): 86648.0 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.58% | FLOPs estimator: workload-aware | logical tokens/s: 193624.9 | valid tokens/s: 193624.8 | physical tokens/s: 194123.0 | loss tokens/s: 140744.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.322601E-01 | lm-global-supervised-tokens: 1.219525E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.251936E-02 | lm-token-buffer-mib: 5.809695E+01 | global_load_balancing_loss: 1.001737E+00 | loss scale: 1.0 | grad norm: 0.064 | params norm: 11550.925 | params rms: 0.062544 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 8:49:00.061558 | finish at 2026-10-10 14:20:37 + [2026-10-05 05:33:08] iteration 1504/ 6855 | consumed samples: 192512 | elapsed time per iteration (ms): 90900.7 | throughput per GPU (TFLOP/s/GPU): 41.3 | MFU 13.23% | FLOPs estimator: workload-aware | logical tokens/s: 184566.0 | valid tokens/s: 184566.0 | physical tokens/s: 184970.9 | loss tokens/s: 137310.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.244122E-01 | lm-global-supervised-tokens: 1.248163E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.879980E-02 | lm-token-buffer-mib: 5.803429E+01 | global_load_balancing_loss: 1.002636E+00 | loss scale: 1.0 | grad norm: 0.062 | params norm: 11550.849 | params rms: 0.062544 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 15:06:49.766117 | finish at 2026-10-10 20:39:58 + [2026-10-05 05:34:32] iteration 1505/ 6855 | consumed samples: 192640 | elapsed time per iteration (ms): 84503.6 | throughput per GPU (TFLOP/s/GPU): 43.4 | MFU 13.91% | FLOPs estimator: workload-aware | logical tokens/s: 198538.2 | valid tokens/s: 198538.2 | physical tokens/s: 199072.8 | loss tokens/s: 144612.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.515337E-01 | lm-global-supervised-tokens: 1.222024E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.169190E-02 | lm-token-buffer-mib: 5.823984E+01 | global_load_balancing_loss: 1.000382E+00 | loss scale: 1.0 | grad norm: 0.066 | params norm: 11550.774 | params rms: 0.062544 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 5:34:54.145846 | finish at 2026-10-10 11:09:26 + [2026-10-05 05:36:01] iteration 1506/ 6855 | consumed samples: 192768 | elapsed time per iteration (ms): 88645.7 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.62% | FLOPs estimator: workload-aware | logical tokens/s: 189261.3 | valid tokens/s: 189261.3 | physical tokens/s: 189797.7 | loss tokens/s: 142895.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.854474E-01 | lm-global-supervised-tokens: 1.266705E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.422395E-02 | lm-token-buffer-mib: 5.807259E+01 | global_load_balancing_loss: 9.989018E-01 | loss scale: 1.0 | grad norm: 0.121 | params norm: 11550.699 | params rms: 0.062543 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 11:42:45.944829 | finish at 2026-10-10 17:18:47 + [2026-10-05 05:37:26] iteration 1507/ 6855 | consumed samples: 192896 | elapsed time per iteration (ms): 84673.1 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.76% | FLOPs estimator: workload-aware | logical tokens/s: 198140.1 | valid tokens/s: 198140.0 | physical tokens/s: 198653.3 | loss tokens/s: 140038.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.327978E-01 | lm-global-supervised-tokens: 1.185745E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.541365E-02 | lm-token-buffer-mib: 5.813147E+01 | global_load_balancing_loss: 9.995227E-01 | loss scale: 1.0 | grad norm: 0.064 | params norm: 11550.624 | params rms: 0.062543 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 5:47:11.722196 | finish at 2026-10-10 11:24:37 + [2026-10-05 05:38:54] iteration 1508/ 6855 | consumed samples: 193024 | elapsed time per iteration (ms): 88379.4 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.77% | FLOPs estimator: workload-aware | logical tokens/s: 189831.0 | valid tokens/s: 189831.0 | physical tokens/s: 190368.0 | loss tokens/s: 142034.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.924741E-01 | lm-global-supervised-tokens: 1.255291E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.253509E-02 | lm-token-buffer-mib: 5.836849E+01 | global_load_balancing_loss: 9.984474E-01 | loss scale: 1.0 | grad norm: 0.062 | params norm: 11550.548 | params rms: 0.062542 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 11:16:04.686300 | finish at 2026-10-10 16:54:59 + [2026-10-05 05:40:23] iteration 1509/ 6855 | consumed samples: 193152 | elapsed time per iteration (ms): 88480.2 | throughput per GPU (TFLOP/s/GPU): 41.7 | MFU 13.37% | FLOPs estimator: workload-aware | logical tokens/s: 189615.4 | valid tokens/s: 189615.4 | physical tokens/s: 190097.9 | loss tokens/s: 141081.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.765514E-01 | lm-global-supervised-tokens: 1.248291E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.733330E-02 | lm-token-buffer-mib: 5.811935E+01 | global_load_balancing_loss: 9.986755E-01 | loss scale: 1.0 | grad norm: 0.050 | params norm: 11550.471 | params rms: 0.062542 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 11:23:35.318999 | finish at 2026-10-10 17:03:58 + [2026-10-05 05:42:11] iteration 1510/ 6855 | consumed samples: 193280 | elapsed time per iteration (ms): 108602.1 | throughput per GPU (TFLOP/s/GPU): 34.0 | MFU 10.90% | FLOPs estimator: workload-aware | logical tokens/s: 154482.7 | valid tokens/s: 154482.6 | physical tokens/s: 154815.0 | loss tokens/s: 114487.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.714323E-01 | lm-global-supervised-tokens: 1.243357E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.316429E-02 | lm-token-buffer-mib: 5.810791E+01 | global_load_balancing_loss: 9.987984E-01 | loss scale: 1.0 | grad norm: 0.064 | params norm: 11550.394 | params rms: 0.062542 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 17:14:38.006028 | finish at 2026-10-11 22:56:49 + [2026-10-05 05:43:38] iteration 1511/ 6855 | consumed samples: 193408 | elapsed time per iteration (ms): 87221.1 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.68% | FLOPs estimator: workload-aware | logical tokens/s: 192352.5 | valid tokens/s: 192352.5 | physical tokens/s: 192765.4 | loss tokens/s: 132192.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.508342E-01 | lm-global-supervised-tokens: 1.152998E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.886877E-02 | lm-token-buffer-mib: 5.810759E+01 | global_load_balancing_loss: 1.001086E+00 | loss scale: 1.0 | grad norm: 0.094 | params norm: 11550.317 | params rms: 0.062541 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 9:28:29.603485 | finish at 2026-10-10 15:12:08 + [2026-10-05 05:45:11] iteration 1512/ 6855 | consumed samples: 193536 | elapsed time per iteration (ms): 92780.3 | throughput per GPU (TFLOP/s/GPU): 41.2 | MFU 13.20% | FLOPs estimator: workload-aware | logical tokens/s: 180827.1 | valid tokens/s: 180827.1 | physical tokens/s: 181177.4 | loss tokens/s: 133553.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.146214E-01 | lm-global-supervised-tokens: 1.239112E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.144955E-02 | lm-token-buffer-mib: 5.805654E+01 | global_load_balancing_loss: 1.001129E+00 | loss scale: 1.0 | grad norm: 0.054 | params norm: 11550.242 | params rms: 0.062541 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 17:42:05.334731 | finish at 2026-10-10 23:27:16 + [2026-10-05 05:46:38] iteration 1513/ 6855 | consumed samples: 193664 | elapsed time per iteration (ms): 87301.5 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.54% | FLOPs estimator: workload-aware | logical tokens/s: 192175.4 | valid tokens/s: 192175.4 | physical tokens/s: 192556.4 | loss tokens/s: 140129.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.041167E-01 | lm-global-supervised-tokens: 1.223353E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.408626E-02 | lm-token-buffer-mib: 5.806661E+01 | global_load_balancing_loss: 1.003055E+00 | loss scale: 1.0 | grad norm: 0.201 | params norm: 11550.166 | params rms: 0.062540 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 9:32:44.670752 | finish at 2026-10-10 15:19:23 + [2026-10-05 05:48:02] iteration 1514/ 6855 | consumed samples: 193792 | elapsed time per iteration (ms): 83187.0 | throughput per GPU (TFLOP/s/GPU): 43.8 | MFU 14.05% | FLOPs estimator: workload-aware | logical tokens/s: 201679.8 | valid tokens/s: 201679.8 | physical tokens/s: 202082.3 | loss tokens/s: 155926.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.816150E-01 | lm-global-supervised-tokens: 1.297105E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.309284E-02 | lm-token-buffer-mib: 5.818763E+01 | global_load_balancing_loss: 1.002338E+00 | loss scale: 1.0 | grad norm: 0.129 | params norm: 11550.090 | params rms: 0.062540 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 3:25:01.917454 | finish at 2026-10-10 09:13:04 + [2026-10-05 05:49:27] iteration 1515/ 6855 | consumed samples: 193920 | elapsed time per iteration (ms): 85657.9 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.62% | FLOPs estimator: workload-aware | logical tokens/s: 195862.1 | valid tokens/s: 195862.1 | physical tokens/s: 196221.4 | loss tokens/s: 150399.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.197090E-01 | lm-global-supervised-tokens: 1.288287E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 7.951962E-02 | lm-token-buffer-mib: 5.804935E+01 | global_load_balancing_loss: 1.000206E+00 | loss scale: 1.0 | grad norm: 0.069 | params norm: 11550.013 | params rms: 0.062540 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 7:03:33.260350 | finish at 2026-10-10 12:53:01 + [2026-10-05 05:50:57] iteration 1516/ 6855 | consumed samples: 194048 | elapsed time per iteration (ms): 90125.3 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.45% | FLOPs estimator: workload-aware | logical tokens/s: 186154.4 | valid tokens/s: 186154.4 | physical tokens/s: 186568.4 | loss tokens/s: 142722.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.551051E-01 | lm-global-supervised-tokens: 1.286289E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.571095E-02 | lm-token-buffer-mib: 5.815668E+01 | global_load_balancing_loss: 9.996356E-01 | loss scale: 1.0 | grad norm: 0.063 | params norm: 11549.936 | params rms: 0.062539 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 13:39:38.842673 | finish at 2026-10-10 19:30:36 + [2026-10-05 05:52:26] iteration 1517/ 6855 | consumed samples: 194176 | elapsed time per iteration (ms): 88665.2 | throughput per GPU (TFLOP/s/GPU): 43.3 | MFU 13.86% | FLOPs estimator: workload-aware | logical tokens/s: 189219.4 | valid tokens/s: 189219.4 | physical tokens/s: 189473.9 | loss tokens/s: 134409.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.248426E-01 | lm-global-supervised-tokens: 1.191747E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.311025E-02 | lm-token-buffer-mib: 5.808286E+01 | global_load_balancing_loss: 9.977389E-01 | loss scale: 1.0 | grad norm: 0.062 | params norm: 11549.859 | params rms: 0.062539 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 11:28:14.912662 | finish at 2026-10-10 17:20:41 + [2026-10-05 05:53:54] iteration 1518/ 6855 | consumed samples: 194304 | elapsed time per iteration (ms): 88216.0 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.44% | FLOPs estimator: workload-aware | logical tokens/s: 190182.7 | valid tokens/s: 190182.7 | physical tokens/s: 190798.1 | loss tokens/s: 146541.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.304514E-01 | lm-global-supervised-tokens: 1.292731E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.795781E-02 | lm-token-buffer-mib: 5.827804E+01 | global_load_balancing_loss: 9.988492E-01 | loss scale: 1.0 | grad norm: 0.213 | params norm: 11549.781 | params rms: 0.062538 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 10:46:48.755527 | finish at 2026-10-10 16:40:43 + [2026-10-05 05:55:20] iteration 1519/ 6855 | consumed samples: 194432 | elapsed time per iteration (ms): 85657.1 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.43% | FLOPs estimator: workload-aware | logical tokens/s: 195864.8 | valid tokens/s: 195864.8 | physical tokens/s: 196276.7 | loss tokens/s: 145675.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.833864E-01 | lm-global-supervised-tokens: 1.247814E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.275617E-02 | lm-token-buffer-mib: 5.813636E+01 | global_load_balancing_loss: 9.990824E-01 | loss scale: 1.0 | grad norm: 0.131 | params norm: 11549.702 | params rms: 0.062538 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 6:57:46.324837 | finish at 2026-10-10 12:53:06 + [2026-10-05 05:57:06] iteration 1520/ 6855 | consumed samples: 194560 | elapsed time per iteration (ms): 105888.1 | throughput per GPU (TFLOP/s/GPU): 34.6 | MFU 11.11% | FLOPs estimator: workload-aware | logical tokens/s: 158442.7 | valid tokens/s: 158442.7 | physical tokens/s: 158749.5 | loss tokens/s: 114952.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.841419E-01 | lm-global-supervised-tokens: 1.217208E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.163884E-02 | lm-token-buffer-mib: 5.808241E+01 | global_load_balancing_loss: 1.000245E+00 | loss scale: 1.0 | grad norm: 0.165 | params norm: 11549.625 | params rms: 0.062537 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 12:55:13.061348 | finish at 2026-10-11 18:52:19 +times across ranks (ms): + save-checkpoint: + rank 0: 23839.10 + rank 1: 23839.08 + rank 2: 23839.13 + rank 3: 23839.10 + rank 4: 23839.12 + rank 5: 23839.10 + rank 6: 23839.07 + rank 7: 23839.09 + rank 8: 23839.65 + rank 9: 23839.63 + rank 10: 23839.68 + rank 11: 23839.62 + rank 12: 23839.63 + rank 13: 23839.65 + rank 14: 23839.63 + rank 15: 23839.64 + rank 16: 23839.30 + rank 17: 23839.27 + rank 18: 23839.28 + rank 19: 23839.38 + rank 20: 23839.28 + rank 21: 23839.30 + rank 22: 23839.27 + rank 23: 23839.28 + rank 24: 23839.53 + rank 25: 23839.44 + rank 26: 23839.47 + rank 27: 23839.62 + rank 28: 23839.44 + rank 29: 23839.46 + rank 30: 23839.49 + rank 31: 23839.50 + rank 32: 23839.13 + rank 33: 23839.15 + rank 34: 23839.11 + rank 35: 23839.10 + rank 36: 23839.09 + rank 37: 23839.05 + rank 38: 23839.09 + rank 39: 23839.06 + rank 40: 23839.17 + rank 41: 23839.10 + rank 42: 23839.11 + rank 43: 23839.12 + rank 44: 23839.10 + rank 45: 23839.09 + rank 46: 23839.11 + rank 47: 23839.14 + rank 48: 23839.39 + rank 49: 23839.35 + rank 50: 23839.33 + rank 51: 23839.36 + rank 52: 23839.44 + rank 53: 23839.33 + rank 54: 23839.33 + rank 55: 23839.36 + rank 56: 23839.51 + rank 57: 23839.51 + rank 58: 23839.49 + rank 59: 23839.49 + rank 60: 23839.50 + rank 61: 23839.51 + rank 62: 23839.50 + rank 63: 23839.51 + [2026-10-05 05:59:20] iteration 1521/ 6855 | consumed samples: 194688 | elapsed time per iteration (ms): 109988.5 | throughput per GPU (TFLOP/s/GPU): 33.0 | MFU 10.57% | FLOPs estimator: workload-aware | logical tokens/s: 152535.7 | valid tokens/s: 152535.7 | physical tokens/s: 152899.1 | loss tokens/s: 115959.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.680435E-01 | lm-global-supervised-tokens: 1.275420E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 9.700217E-01 | lm-token-buffer-mib: 5.813094E+01 | global_load_balancing_loss: 1.000860E+00 | loss scale: 1.0 | grad norm: 0.159 | params norm: 11549.551 | params rms: 0.062537 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 18:57:58.490491 | finish at 2026-10-12 00:57:18 + [2026-10-05 06:00:54] iteration 1522/ 6855 | consumed samples: 194816 | elapsed time per iteration (ms): 94124.6 | throughput per GPU (TFLOP/s/GPU): 40.5 | MFU 12.97% | FLOPs estimator: workload-aware | logical tokens/s: 178244.2 | valid tokens/s: 178244.2 | physical tokens/s: 178545.1 | loss tokens/s: 130691.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.815111E-01 | lm-global-supervised-tokens: 1.230131E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.768377E-02 | lm-token-buffer-mib: 5.824707E+01 | global_load_balancing_loss: 1.001447E+00 | loss scale: 1.0 | grad norm: 0.112 | params norm: 11549.475 | params rms: 0.062537 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 19:26:06.435501 | finish at 2026-10-11 01:27:01 + [2026-10-05 06:02:21] iteration 1523/ 6855 | consumed samples: 194944 | elapsed time per iteration (ms): 87286.1 | throughput per GPU (TFLOP/s/GPU): 41.5 | MFU 13.30% | FLOPs estimator: workload-aware | logical tokens/s: 192209.0 | valid tokens/s: 192209.0 | physical tokens/s: 192746.0 | loss tokens/s: 148726.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.890174E-01 | lm-global-supervised-tokens: 1.298171E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.453780E-02 | lm-token-buffer-mib: 5.820756E+01 | global_load_balancing_loss: 1.001324E+00 | loss scale: 1.0 | grad norm: 0.069 | params norm: 11549.399 | params rms: 0.062536 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 9:16:49.526675 | finish at 2026-10-10 15:19:11 + [2026-10-05 06:03:47] iteration 1524/ 6855 | consumed samples: 195072 | elapsed time per iteration (ms): 85997.3 | throughput per GPU (TFLOP/s/GPU): 43.4 | MFU 13.91% | FLOPs estimator: workload-aware | logical tokens/s: 195089.3 | valid tokens/s: 195089.3 | physical tokens/s: 195558.2 | loss tokens/s: 146908.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.272850E-01 | lm-global-supervised-tokens: 1.263371E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.445316E-02 | lm-token-buffer-mib: 5.810243E+01 | global_load_balancing_loss: 1.000700E+00 | loss scale: 1.0 | grad norm: 0.113 | params norm: 11549.324 | params rms: 0.062536 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 7:20:51.483801 | finish at 2026-10-10 13:24:39 + [2026-10-05 06:05:17] iteration 1525/ 6855 | consumed samples: 195200 | elapsed time per iteration (ms): 89395.3 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.40% | FLOPs estimator: workload-aware | logical tokens/s: 187672.4 | valid tokens/s: 187672.4 | physical tokens/s: 188302.4 | loss tokens/s: 130005.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.853467E-01 | lm-global-supervised-tokens: 1.162187E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.493746E-02 | lm-token-buffer-mib: 5.802518E+01 | global_load_balancing_loss: 1.001540E+00 | loss scale: 1.0 | grad norm: 0.098 | params norm: 11549.250 | params rms: 0.062535 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 12:21:16.724873 | finish at 2026-10-10 18:26:34 + [2026-10-05 06:06:42] iteration 1526/ 6855 | consumed samples: 195328 | elapsed time per iteration (ms): 85372.7 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.41% | FLOPs estimator: workload-aware | logical tokens/s: 196516.9 | valid tokens/s: 196516.9 | physical tokens/s: 197208.4 | loss tokens/s: 147658.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.541621E-01 | lm-global-supervised-tokens: 1.260598E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.229637E-02 | lm-token-buffer-mib: 5.817076E+01 | global_load_balancing_loss: 9.993220E-01 | loss scale: 1.0 | grad norm: 0.080 | params norm: 11549.177 | params rms: 0.062535 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 6:22:30.880582 | finish at 2026-10-10 12:29:13 + [2026-10-05 06:08:12] iteration 1527/ 6855 | consumed samples: 195456 | elapsed time per iteration (ms): 89349.0 | throughput per GPU (TFLOP/s/GPU): 41.3 | MFU 13.22% | FLOPs estimator: workload-aware | logical tokens/s: 187771.3 | valid tokens/s: 187771.3 | physical tokens/s: 188080.4 | loss tokens/s: 139426.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.565268E-01 | lm-global-supervised-tokens: 1.245765E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.252948E-02 | lm-token-buffer-mib: 5.811161E+01 | global_load_balancing_loss: 9.984828E-01 | loss scale: 1.0 | grad norm: 0.045 | params norm: 11549.104 | params rms: 0.062535 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 12:14:11.587475 | finish at 2026-10-10 18:22:23 + [2026-10-05 06:09:40] iteration 1528/ 6855 | consumed samples: 195584 | elapsed time per iteration (ms): 88618.4 | throughput per GPU (TFLOP/s/GPU): 41.3 | MFU 13.23% | FLOPs estimator: workload-aware | logical tokens/s: 189318.4 | valid tokens/s: 189318.4 | physical tokens/s: 189813.1 | loss tokens/s: 142391.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.023975E-01 | lm-global-supervised-tokens: 1.261850E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.557560E-02 | lm-token-buffer-mib: 5.826501E+01 | global_load_balancing_loss: 9.984291E-01 | loss scale: 1.0 | grad norm: 0.072 | params norm: 11549.032 | params rms: 0.062534 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 11:07:50.475137 | finish at 2026-10-10 17:17:31 + [2026-10-05 06:11:06] iteration 1529/ 6855 | consumed samples: 195712 | elapsed time per iteration (ms): 85544.6 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.67% | FLOPs estimator: workload-aware | logical tokens/s: 196122.0 | valid tokens/s: 196121.9 | physical tokens/s: 196564.7 | loss tokens/s: 150565.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.202074E-01 | lm-global-supervised-tokens: 1.288004E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.247456E-02 | lm-token-buffer-mib: 5.813067E+01 | global_load_balancing_loss: 1.000571E+00 | loss scale: 1.0 | grad norm: 0.038 | params norm: 11548.959 | params rms: 0.062534 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 6:33:30.624731 | finish at 2026-10-10 12:44:36 + [2026-10-05 06:12:52] iteration 1530/ 6855 | consumed samples: 195840 | elapsed time per iteration (ms): 106129.5 | throughput per GPU (TFLOP/s/GPU): 34.8 | MFU 11.14% | FLOPs estimator: workload-aware | logical tokens/s: 158082.2 | valid tokens/s: 158082.2 | physical tokens/s: 158376.3 | loss tokens/s: 114816.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.274384E-01 | lm-global-supervised-tokens: 1.218545E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.245539E-02 | lm-token-buffer-mib: 5.804532E+01 | global_load_balancing_loss: 1.000838E+00 | loss scale: 1.0 | grad norm: 0.078 | params norm: 11548.887 | params rms: 0.062533 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 12:58:59.632738 | finish at 2026-10-11 19:11:51 + [2026-10-05 06:14:19] iteration 1531/ 6855 | consumed samples: 195968 | elapsed time per iteration (ms): 86788.2 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.60% | FLOPs estimator: workload-aware | logical tokens/s: 193312.2 | valid tokens/s: 193312.2 | physical tokens/s: 193813.7 | loss tokens/s: 140569.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.820756E-01 | lm-global-supervised-tokens: 1.219978E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.210156E-02 | lm-token-buffer-mib: 5.815323E+01 | global_load_balancing_loss: 1.001341E+00 | loss scale: 1.0 | grad norm: 0.035 | params norm: 11548.812 | params rms: 0.062533 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 8:21:00.117331 | finish at 2026-10-10 14:35:19 + [2026-10-05 06:15:47] iteration 1532/ 6855 | consumed samples: 196096 | elapsed time per iteration (ms): 88047.1 | throughput per GPU (TFLOP/s/GPU): 41.3 | MFU 13.23% | FLOPs estimator: workload-aware | logical tokens/s: 190548.2 | valid tokens/s: 190548.2 | physical tokens/s: 191069.3 | loss tokens/s: 147372.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.634616E-01 | lm-global-supervised-tokens: 1.297569E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.421930E-02 | lm-token-buffer-mib: 5.816303E+01 | global_load_balancing_loss: 1.000635E+00 | loss scale: 1.0 | grad norm: 0.120 | params norm: 11548.737 | params rms: 0.062533 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 10:11:14.478873 | finish at 2026-10-10 16:27:01 + [2026-10-05 06:17:15] iteration 1533/ 6855 | consumed samples: 196224 | elapsed time per iteration (ms): 88742.4 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.50% | FLOPs estimator: workload-aware | logical tokens/s: 189054.1 | valid tokens/s: 189054.1 | physical tokens/s: 189563.2 | loss tokens/s: 143338.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.099258E-01 | lm-global-supervised-tokens: 1.272016E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.286502E-02 | lm-token-buffer-mib: 5.809421E+01 | global_load_balancing_loss: 9.996907E-01 | loss scale: 1.0 | grad norm: 0.323 | params norm: 11548.664 | params rms: 0.062532 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 11:11:26.991527 | finish at 2026-10-10 17:28:42 + [2026-10-05 06:18:44] iteration 1534/ 6855 | consumed samples: 196352 | elapsed time per iteration (ms): 88897.7 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.60% | FLOPs estimator: workload-aware | logical tokens/s: 188724.5 | valid tokens/s: 188724.5 | physical tokens/s: 189094.8 | loss tokens/s: 133217.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.885695E-01 | lm-global-supervised-tokens: 1.184270E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.319304E-02 | lm-token-buffer-mib: 5.813451E+01 | global_load_balancing_loss: 1.000211E+00 | loss scale: 1.0 | grad norm: 0.321 | params norm: 11548.592 | params rms: 0.062532 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 11:23:44.871403 | finish at 2026-10-10 17:42:29 + [2026-10-05 06:20:14] iteration 1535/ 6855 | consumed samples: 196480 | elapsed time per iteration (ms): 89352.7 | throughput per GPU (TFLOP/s/GPU): 41.7 | MFU 13.37% | FLOPs estimator: workload-aware | logical tokens/s: 187763.7 | valid tokens/s: 187763.7 | physical tokens/s: 188164.8 | loss tokens/s: 133208.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.448408E-01 | lm-global-supervised-tokens: 1.190258E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.188330E-02 | lm-token-buffer-mib: 5.803277E+01 | global_load_balancing_loss: 1.000041E+00 | loss scale: 1.0 | grad norm: 0.059 | params norm: 11548.518 | params rms: 0.062531 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 12:02:36.487007 | finish at 2026-10-10 18:22:50 + [2026-10-05 06:21:41] iteration 1536/ 6855 | consumed samples: 196608 | elapsed time per iteration (ms): 86958.4 | throughput per GPU (TFLOP/s/GPU): 41.5 | MFU 13.31% | FLOPs estimator: workload-aware | logical tokens/s: 192933.8 | valid tokens/s: 192933.8 | physical tokens/s: 193315.2 | loss tokens/s: 152428.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.742977E-01 | lm-global-supervised-tokens: 1.325494E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.280087E-02 | lm-token-buffer-mib: 5.811810E+01 | global_load_balancing_loss: 9.985616E-01 | loss scale: 1.0 | grad norm: 0.144 | params norm: 11548.442 | params rms: 0.062531 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 8:28:51.785457 | finish at 2026-10-10 14:50:32 + [2026-10-05 06:23:13] iteration 1537/ 6855 | consumed samples: 196736 | elapsed time per iteration (ms): 92708.2 | throughput per GPU (TFLOP/s/GPU): 41.0 | MFU 13.14% | FLOPs estimator: workload-aware | logical tokens/s: 180967.6 | valid tokens/s: 180967.6 | physical tokens/s: 181297.5 | loss tokens/s: 134028.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.925437E-01 | lm-global-supervised-tokens: 1.242558E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.405678E-02 | lm-token-buffer-mib: 5.809869E+01 | global_load_balancing_loss: 9.992195E-01 | loss scale: 1.0 | grad norm: 0.047 | params norm: 11548.367 | params rms: 0.062531 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 16:57:02.141551 | finish at 2026-10-10 23:20:15 + [2026-10-05 06:24:39] iteration 1538/ 6855 | consumed samples: 196864 | elapsed time per iteration (ms): 85710.3 | throughput per GPU (TFLOP/s/GPU): 43.7 | MFU 14.02% | FLOPs estimator: workload-aware | logical tokens/s: 195743.0 | valid tokens/s: 195743.0 | physical tokens/s: 196345.9 | loss tokens/s: 138314.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.155538E-01 | lm-global-supervised-tokens: 1.185497E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.441938E-02 | lm-token-buffer-mib: 5.826386E+01 | global_load_balancing_loss: 9.999158E-01 | loss scale: 1.0 | grad norm: 0.053 | params norm: 11548.289 | params rms: 0.062530 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 6:35:21.702964 | finish at 2026-10-10 13:00:01 + [2026-10-05 06:26:07] iteration 1539/ 6855 | consumed samples: 196992 | elapsed time per iteration (ms): 87493.0 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.51% | FLOPs estimator: workload-aware | logical tokens/s: 191754.6 | valid tokens/s: 191754.6 | physical tokens/s: 192213.2 | loss tokens/s: 144047.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.881490E-01 | lm-global-supervised-tokens: 1.260316E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 3.250643E-02 | lm-token-buffer-mib: 5.816651E+01 | global_load_balancing_loss: 1.001574E+00 | loss scale: 1.0 | grad norm: 0.046 | params norm: 11548.212 | params rms: 0.062530 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 9:11:52.766616 | finish at 2026-10-10 15:37:59 + [2026-10-05 06:27:53] iteration 1540/ 6855 | consumed samples: 197120 | elapsed time per iteration (ms): 106248.4 | throughput per GPU (TFLOP/s/GPU): 34.9 | MFU 11.17% | FLOPs estimator: workload-aware | logical tokens/s: 157905.5 | valid tokens/s: 157905.5 | physical tokens/s: 158211.2 | loss tokens/s: 113289.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.730836E-01 | lm-global-supervised-tokens: 1.203679E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.206727E-02 | lm-token-buffer-mib: 5.812080E+01 | global_load_balancing_loss: 1.002095E+00 | loss scale: 1.0 | grad norm: 0.061 | params norm: 11548.134 | params rms: 0.062529 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 12:51:50.406790 | finish at 2026-10-11 19:19:43 + [2026-10-05 06:29:19] iteration 1541/ 6855 | consumed samples: 197248 | elapsed time per iteration (ms): 86088.8 | throughput per GPU (TFLOP/s/GPU): 44.0 | MFU 14.09% | FLOPs estimator: workload-aware | logical tokens/s: 194882.1 | valid tokens/s: 194882.1 | physical tokens/s: 195279.5 | loss tokens/s: 138550.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.722703E-01 | lm-global-supervised-tokens: 1.192763E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.594022E-02 | lm-token-buffer-mib: 5.808974E+01 | global_load_balancing_loss: 1.001814E+00 | loss scale: 1.0 | grad norm: 0.053 | params norm: 11548.057 | params rms: 0.062529 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 7:04:35.918427 | finish at 2026-10-10 13:33:55 + [2026-10-05 06:30:46] iteration 1542/ 6855 | consumed samples: 197376 | elapsed time per iteration (ms): 87146.2 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.77% | FLOPs estimator: workload-aware | logical tokens/s: 192518.1 | valid tokens/s: 192518.1 | physical tokens/s: 192863.6 | loss tokens/s: 145529.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.735479E-01 | lm-global-supervised-tokens: 1.268238E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.147395E-02 | lm-token-buffer-mib: 5.820296E+01 | global_load_balancing_loss: 1.000480E+00 | loss scale: 1.0 | grad norm: 0.042 | params norm: 11547.980 | params rms: 0.062529 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 8:36:47.714688 | finish at 2026-10-10 15:07:34 + [2026-10-05 06:32:12] iteration 1543/ 6855 | consumed samples: 197504 | elapsed time per iteration (ms): 86232.8 | throughput per GPU (TFLOP/s/GPU): 43.5 | MFU 13.95% | FLOPs estimator: workload-aware | logical tokens/s: 194556.5 | valid tokens/s: 194556.5 | physical tokens/s: 194907.9 | loss tokens/s: 142086.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.943596E-01 | lm-global-supervised-tokens: 1.225251E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.596625E-02 | lm-token-buffer-mib: 5.808103E+01 | global_load_balancing_loss: 9.997332E-01 | loss scale: 1.0 | grad norm: 0.038 | params norm: 11547.905 | params rms: 0.062528 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 7:14:28.772949 | finish at 2026-10-10 13:46:41 + [2026-10-05 06:33:41] iteration 1544/ 6855 | consumed samples: 197632 | elapsed time per iteration (ms): 88535.9 | throughput per GPU (TFLOP/s/GPU): 41.4 | MFU 13.26% | FLOPs estimator: workload-aware | logical tokens/s: 189495.7 | valid tokens/s: 189495.7 | physical tokens/s: 189990.4 | loss tokens/s: 143287.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.053783E-01 | lm-global-supervised-tokens: 1.268609E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.492079E-02 | lm-token-buffer-mib: 5.817322E+01 | global_load_balancing_loss: 9.991546E-01 | loss scale: 1.0 | grad norm: 0.044 | params norm: 11547.829 | params rms: 0.062528 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 10:36:54.218698 | finish at 2026-10-10 17:10:35 + [2026-10-05 06:35:06] iteration 1545/ 6855 | consumed samples: 197760 | elapsed time per iteration (ms): 84794.4 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.66% | FLOPs estimator: workload-aware | logical tokens/s: 197856.4 | valid tokens/s: 197856.4 | physical tokens/s: 198381.3 | loss tokens/s: 150888.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.492332E-01 | lm-global-supervised-tokens: 1.279448E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.144815E-02 | lm-token-buffer-mib: 5.807437E+01 | global_load_balancing_loss: 9.980991E-01 | loss scale: 1.0 | grad norm: 0.043 | params norm: 11547.751 | params rms: 0.062527 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 5:04:18.510747 | finish at 2026-10-10 11:39:24 + [2026-10-05 06:36:33] iteration 1546/ 6855 | consumed samples: 197888 | elapsed time per iteration (ms): 87420.4 | throughput per GPU (TFLOP/s/GPU): 41.7 | MFU 13.36% | FLOPs estimator: workload-aware | logical tokens/s: 191913.3 | valid tokens/s: 191913.3 | physical tokens/s: 192418.5 | loss tokens/s: 142664.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.953102E-01 | lm-global-supervised-tokens: 1.247178E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.255043E-02 | lm-token-buffer-mib: 5.816011E+01 | global_load_balancing_loss: 1.000373E+00 | loss scale: 1.0 | grad norm: 0.034 | params norm: 11547.675 | params rms: 0.062527 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 8:55:14.790439 | finish at 2026-10-10 15:31:48 + [2026-10-05 06:38:00] iteration 1547/ 6855 | consumed samples: 198016 | elapsed time per iteration (ms): 87177.8 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.58% | FLOPs estimator: workload-aware | logical tokens/s: 192447.4 | valid tokens/s: 192447.3 | physical tokens/s: 192930.1 | loss tokens/s: 143364.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.615762E-01 | lm-global-supervised-tokens: 1.249817E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.131708E-02 | lm-token-buffer-mib: 5.806512E+01 | global_load_balancing_loss: 1.000578E+00 | loss scale: 1.0 | grad norm: 0.057 | params norm: 11547.599 | params rms: 0.062526 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 8:32:19.559598 | finish at 2026-10-10 15:10:20 + [2026-10-05 06:39:28] iteration 1548/ 6855 | consumed samples: 198144 | elapsed time per iteration (ms): 87440.0 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.56% | FLOPs estimator: workload-aware | logical tokens/s: 191870.4 | valid tokens/s: 191870.4 | physical tokens/s: 192370.7 | loss tokens/s: 139561.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.308889E-01 | lm-global-supervised-tokens: 1.220322E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.310759E-02 | lm-token-buffer-mib: 5.815671E+01 | global_load_balancing_loss: 1.000696E+00 | loss scale: 1.0 | grad norm: 0.053 | params norm: 11547.523 | params rms: 0.062526 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 8:54:04.224546 | finish at 2026-10-10 15:33:32 + [2026-10-05 06:40:55] iteration 1549/ 6855 | consumed samples: 198272 | elapsed time per iteration (ms): 87062.3 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.55% | FLOPs estimator: workload-aware | logical tokens/s: 192702.5 | valid tokens/s: 192702.5 | physical tokens/s: 192919.4 | loss tokens/s: 145139.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.840886E-01 | lm-global-supervised-tokens: 1.263615E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.156098E-02 | lm-token-buffer-mib: 5.806296E+01 | global_load_balancing_loss: 9.993741E-01 | loss scale: 1.0 | grad norm: 0.035 | params norm: 11547.448 | params rms: 0.062526 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 8:19:12.787538 | finish at 2026-10-10 15:00:07 + [2026-10-05 06:42:42] iteration 1550/ 6855 | consumed samples: 198400 | elapsed time per iteration (ms): 107731.2 | throughput per GPU (TFLOP/s/GPU): 35.1 | MFU 11.26% | FLOPs estimator: workload-aware | logical tokens/s: 155732.1 | valid tokens/s: 155732.1 | physical tokens/s: 156075.1 | loss tokens/s: 108761.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.364377E-01 | lm-global-supervised-tokens: 1.171703E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.604633E-02 | lm-token-buffer-mib: 5.805725E+01 | global_load_balancing_loss: 1.000036E+00 | loss scale: 1.0 | grad norm: 0.114 | params norm: 11547.373 | params rms: 0.062525 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 14:45:14.163876 | finish at 2026-10-11 21:27:57 + [2026-10-05 06:44:07] iteration 1551/ 6855 | consumed samples: 198528 | elapsed time per iteration (ms): 84928.2 | throughput per GPU (TFLOP/s/GPU): 43.1 | MFU 13.82% | FLOPs estimator: workload-aware | logical tokens/s: 197545.3 | valid tokens/s: 197545.2 | physical tokens/s: 198002.6 | loss tokens/s: 149651.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.636603E-01 | lm-global-supervised-tokens: 1.270962E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.219654E-02 | lm-token-buffer-mib: 5.805494E+01 | global_load_balancing_loss: 1.001187E+00 | loss scale: 1.0 | grad norm: 0.202 | params norm: 11547.297 | params rms: 0.062525 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 5:07:38.942682 | finish at 2026-10-10 11:51:46 + [2026-10-05 06:45:32] iteration 1552/ 6855 | consumed samples: 198656 | elapsed time per iteration (ms): 84855.8 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.53% | FLOPs estimator: workload-aware | logical tokens/s: 197714.1 | valid tokens/s: 197714.1 | physical tokens/s: 198234.3 | loss tokens/s: 153721.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.099863E-01 | lm-global-supervised-tokens: 1.304415E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.330405E-02 | lm-token-buffer-mib: 5.816193E+01 | global_load_balancing_loss: 1.000514E+00 | loss scale: 1.0 | grad norm: 0.152 | params norm: 11547.222 | params rms: 0.062524 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 4:59:50.215909 | finish at 2026-10-10 11:45:22 + [2026-10-05 06:46:58] iteration 1553/ 6855 | consumed samples: 198784 | elapsed time per iteration (ms): 85946.2 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.60% | FLOPs estimator: workload-aware | logical tokens/s: 195206.1 | valid tokens/s: 195206.0 | physical tokens/s: 195875.5 | loss tokens/s: 138421.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.585686E-01 | lm-global-supervised-tokens: 1.189676E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.992557E-02 | lm-token-buffer-mib: 5.815358E+01 | global_load_balancing_loss: 1.000785E+00 | loss scale: 1.0 | grad norm: 0.152 | params norm: 11547.145 | params rms: 0.062524 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 6:34:46.634277 | finish at 2026-10-10 13:21:45 + [2026-10-05 06:48:28] iteration 1554/ 6855 | consumed samples: 198912 | elapsed time per iteration (ms): 90327.4 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.69% | FLOPs estimator: workload-aware | logical tokens/s: 185737.8 | valid tokens/s: 185737.8 | physical tokens/s: 186122.9 | loss tokens/s: 129463.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.156928E-01 | lm-global-supervised-tokens: 1.169412E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.147776E-02 | lm-token-buffer-mib: 5.815626E+01 | global_load_balancing_loss: 9.993957E-01 | loss scale: 1.0 | grad norm: 0.195 | params norm: 11547.069 | params rms: 0.062524 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 13:00:25.506793 | finish at 2026-10-10 19:48:54 + [2026-10-05 06:49:56] iteration 1555/ 6855 | consumed samples: 199040 | elapsed time per iteration (ms): 87568.1 | throughput per GPU (TFLOP/s/GPU): 41.7 | MFU 13.38% | FLOPs estimator: workload-aware | logical tokens/s: 191590.2 | valid tokens/s: 191590.2 | physical tokens/s: 192055.7 | loss tokens/s: 142596.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.367153E-01 | lm-global-supervised-tokens: 1.248686E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.304800E-02 | lm-token-buffer-mib: 5.815649E+01 | global_load_balancing_loss: 1.001916E+00 | loss scale: 1.0 | grad norm: 0.155 | params norm: 11546.994 | params rms: 0.062523 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 8:55:10.819936 | finish at 2026-10-10 15:45:07 + [2026-10-05 06:51:19] iteration 1556/ 6855 | consumed samples: 199168 | elapsed time per iteration (ms): 82536.8 | throughput per GPU (TFLOP/s/GPU): 43.9 | MFU 14.09% | FLOPs estimator: workload-aware | logical tokens/s: 203268.8 | valid tokens/s: 203268.8 | physical tokens/s: 203779.5 | loss tokens/s: 154951.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.215607E-01 | lm-global-supervised-tokens: 1.278922E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.207998E-02 | lm-token-buffer-mib: 5.805753E+01 | global_load_balancing_loss: 9.997084E-01 | loss scale: 1.0 | grad norm: 0.198 | params norm: 11546.920 | params rms: 0.062523 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 1:29:22.266459 | finish at 2026-10-10 08:20:41 + [2026-10-05 06:52:46] iteration 1557/ 6855 | consumed samples: 199296 | elapsed time per iteration (ms): 87717.7 | throughput per GPU (TFLOP/s/GPU): 41.5 | MFU 13.29% | FLOPs estimator: workload-aware | logical tokens/s: 191262.7 | valid tokens/s: 191262.7 | physical tokens/s: 191814.2 | loss tokens/s: 146805.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.342885E-01 | lm-global-supervised-tokens: 1.287748E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.154577E-02 | lm-token-buffer-mib: 5.809447E+01 | global_load_balancing_loss: 9.997069E-01 | loss scale: 1.0 | grad norm: 0.176 | params norm: 11546.848 | params rms: 0.062522 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 9:05:28.579253 | finish at 2026-10-10 15:58:15 + [2026-10-05 06:54:14] iteration 1558/ 6855 | consumed samples: 199424 | elapsed time per iteration (ms): 87989.6 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.49% | FLOPs estimator: workload-aware | logical tokens/s: 190671.5 | valid tokens/s: 190671.5 | physical tokens/s: 191206.7 | loss tokens/s: 138646.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.539297E-01 | lm-global-supervised-tokens: 1.219944E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.172926E-02 | lm-token-buffer-mib: 5.833788E+01 | global_load_balancing_loss: 9.996057E-01 | loss scale: 1.0 | grad norm: 0.174 | params norm: 11546.775 | params rms: 0.062522 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 9:28:01.153376 | finish at 2026-10-10 16:22:15 + [2026-10-05 06:55:42] iteration 1559/ 6855 | consumed samples: 199552 | elapsed time per iteration (ms): 87579.0 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.77% | FLOPs estimator: workload-aware | logical tokens/s: 191566.3 | valid tokens/s: 191566.3 | physical tokens/s: 191891.8 | loss tokens/s: 135101.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.548803E-01 | lm-global-supervised-tokens: 1.183208E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.306824E-02 | lm-token-buffer-mib: 5.810841E+01 | global_load_balancing_loss: 9.998387E-01 | loss scale: 1.0 | grad norm: 0.138 | params norm: 11546.703 | params rms: 0.062522 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 8:50:18.435749 | finish at 2026-10-10 15:46:00 + [2026-10-05 06:57:30] iteration 1560/ 6855 | consumed samples: 199680 | elapsed time per iteration (ms): 108166.7 | throughput per GPU (TFLOP/s/GPU): 34.3 | MFU 11.00% | FLOPs estimator: workload-aware | logical tokens/s: 155105.2 | valid tokens/s: 155105.2 | physical tokens/s: 155546.6 | loss tokens/s: 114777.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.766265E-01 | lm-global-supervised-tokens: 1.241514E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.504319E-02 | lm-token-buffer-mib: 5.808997E+01 | global_load_balancing_loss: 1.000354E+00 | loss scale: 1.0 | grad norm: 0.069 | params norm: 11546.630 | params rms: 0.062521 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 15:05:42.576166 | finish at 2026-10-11 22:03:13 +times across ranks (ms): + save-checkpoint: + rank 0: 24896.07 + rank 1: 24896.07 + rank 2: 24896.13 + rank 3: 24896.13 + rank 4: 24896.12 + rank 5: 24896.07 + rank 6: 24896.07 + rank 7: 24896.10 + rank 8: 24894.93 + rank 9: 24894.89 + rank 10: 24894.91 + rank 11: 24894.90 + rank 12: 24894.91 + rank 13: 24894.92 + rank 14: 24894.91 + rank 15: 24894.91 + rank 16: 24896.81 + rank 17: 24896.81 + rank 18: 24896.82 + rank 19: 24896.82 + rank 20: 24896.81 + rank 21: 24896.83 + rank 22: 24896.81 + rank 23: 24896.80 + rank 24: 24895.05 + rank 25: 24895.21 + rank 26: 24895.06 + rank 27: 24895.08 + rank 28: 24895.03 + rank 29: 24895.07 + rank 30: 24895.06 + rank 31: 24895.05 + rank 32: 24894.92 + rank 33: 24894.92 + rank 34: 24894.94 + rank 35: 24894.90 + rank 36: 24894.90 + rank 37: 24894.89 + rank 38: 24894.89 + rank 39: 24894.88 + rank 40: 24895.11 + rank 41: 24895.07 + rank 42: 24895.06 + rank 43: 24895.07 + rank 44: 24895.06 + rank 45: 24895.08 + rank 46: 24895.07 + rank 47: 24895.08 + rank 48: 24894.93 + rank 49: 24894.91 + rank 50: 24894.89 + rank 51: 24894.90 + rank 52: 24894.91 + rank 53: 24894.90 + rank 54: 24894.90 + rank 55: 24894.90 + rank 56: 24895.14 + rank 57: 24895.13 + rank 58: 24895.12 + rank 59: 24895.13 + rank 60: 24895.13 + rank 61: 24895.13 + rank 62: 24895.12 + rank 63: 24895.14 + [2026-10-05 06:59:37] iteration 1561/ 6855 | consumed samples: 199808 | elapsed time per iteration (ms): 101431.4 | throughput per GPU (TFLOP/s/GPU): 35.6 | MFU 11.42% | FLOPs estimator: workload-aware | logical tokens/s: 165403.9 | valid tokens/s: 165403.9 | physical tokens/s: 165775.7 | loss tokens/s: 122039.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.577083E-01 | lm-global-supervised-tokens: 1.237862E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 5.386829E-01 | lm-token-buffer-mib: 5.805205E+01 | global_load_balancing_loss: 1.000471E+00 | loss scale: 1.0 | grad norm: 0.101 | params norm: 11546.556 | params rms: 0.062521 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 5:09:37.945518 | finish at 2026-10-11 12:09:15 + [2026-10-05 07:01:10] iteration 1562/ 6855 | consumed samples: 199936 | elapsed time per iteration (ms): 92815.0 | throughput per GPU (TFLOP/s/GPU): 38.9 | MFU 12.46% | FLOPs estimator: workload-aware | logical tokens/s: 180758.8 | valid tokens/s: 180758.8 | physical tokens/s: 181279.5 | loss tokens/s: 133480.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.646293E-01 | lm-global-supervised-tokens: 1.238902E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.349059E-02 | lm-token-buffer-mib: 5.821744E+01 | global_load_balancing_loss: 1.000053E+00 | loss scale: 1.0 | grad norm: 0.148 | params norm: 11546.483 | params rms: 0.062520 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 16:27:49.892473 | finish at 2026-10-10 23:28:59 + [2026-10-05 07:02:35] iteration 1563/ 6855 | consumed samples: 200064 | elapsed time per iteration (ms): 85973.3 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.67% | FLOPs estimator: workload-aware | logical tokens/s: 195144.2 | valid tokens/s: 195144.2 | physical tokens/s: 195587.2 | loss tokens/s: 139987.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.183152E-01 | lm-global-supervised-tokens: 1.203517E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 3.852071E-01 | lm-token-buffer-mib: 5.809162E+01 | global_load_balancing_loss: 1.001195E+00 | loss scale: 1.0 | grad norm: 0.207 | params norm: 11546.411 | params rms: 0.062520 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 6:22:50.806955 | finish at 2026-10-10 13:25:26 + [2026-10-05 07:04:04] iteration 1564/ 6855 | consumed samples: 200192 | elapsed time per iteration (ms): 88038.6 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.77% | FLOPs estimator: workload-aware | logical tokens/s: 190566.5 | valid tokens/s: 190566.5 | physical tokens/s: 190906.0 | loss tokens/s: 133186.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.999891E-01 | lm-global-supervised-tokens: 1.172556E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.242003E-02 | lm-token-buffer-mib: 5.807721E+01 | global_load_balancing_loss: 1.000019E+00 | loss scale: 1.0 | grad norm: 0.114 | params norm: 11546.340 | params rms: 0.062520 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 9:23:32.048255 | finish at 2026-10-10 16:27:36 + [2026-10-05 07:05:31] iteration 1565/ 6855 | consumed samples: 200320 | elapsed time per iteration (ms): 87984.0 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.62% | FLOPs estimator: workload-aware | logical tokens/s: 190684.9 | valid tokens/s: 190684.9 | physical tokens/s: 191057.5 | loss tokens/s: 138379.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.715031E-01 | lm-global-supervised-tokens: 1.217519E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.754727E-02 | lm-token-buffer-mib: 5.808065E+01 | global_load_balancing_loss: 9.996947E-01 | loss scale: 1.0 | grad norm: 0.151 | params norm: 11546.267 | params rms: 0.062519 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 9:17:15.200913 | finish at 2026-10-10 16:22:47 + [2026-10-05 07:07:00] iteration 1566/ 6855 | consumed samples: 200448 | elapsed time per iteration (ms): 88278.4 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.43% | FLOPs estimator: workload-aware | logical tokens/s: 190048.8 | valid tokens/s: 190048.8 | physical tokens/s: 190597.7 | loss tokens/s: 136488.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.591888E-01 | lm-global-supervised-tokens: 1.204899E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.567199E-02 | lm-token-buffer-mib: 5.816971E+01 | global_load_balancing_loss: 9.995779E-01 | loss scale: 1.0 | grad norm: 0.234 | params norm: 11546.194 | params rms: 0.062519 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 9:41:44.540531 | finish at 2026-10-10 16:48:44 + [2026-10-05 07:08:22] iteration 1567/ 6855 | consumed samples: 200576 | elapsed time per iteration (ms): 82628.0 | throughput per GPU (TFLOP/s/GPU): 44.0 | MFU 14.09% | FLOPs estimator: workload-aware | logical tokens/s: 203043.4 | valid tokens/s: 203043.4 | physical tokens/s: 203658.5 | loss tokens/s: 156424.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.165977E-01 | lm-global-supervised-tokens: 1.292504E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.325837E-02 | lm-token-buffer-mib: 5.825609E+01 | global_load_balancing_loss: 9.990773E-01 | loss scale: 1.0 | grad norm: 0.211 | params norm: 11546.122 | params rms: 0.062518 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 1:22:17.086004 | finish at 2026-10-10 08:30:39 + [2026-10-05 07:09:50] iteration 1568/ 6855 | consumed samples: 200704 | elapsed time per iteration (ms): 87633.0 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.49% | FLOPs estimator: workload-aware | logical tokens/s: 191448.5 | valid tokens/s: 191448.5 | physical tokens/s: 191780.5 | loss tokens/s: 137214.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.746386E-01 | lm-global-supervised-tokens: 1.202450E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.326673E-02 | lm-token-buffer-mib: 5.808772E+01 | global_load_balancing_loss: 9.999201E-01 | loss scale: 1.0 | grad norm: 0.160 | params norm: 11546.052 | params rms: 0.062518 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 8:41:55.717095 | finish at 2026-10-10 15:51:46 + [2026-10-05 07:11:17] iteration 1569/ 6855 | consumed samples: 200832 | elapsed time per iteration (ms): 86932.8 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.65% | FLOPs estimator: workload-aware | logical tokens/s: 192990.6 | valid tokens/s: 192990.6 | physical tokens/s: 193501.1 | loss tokens/s: 141807.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.395417E-01 | lm-global-supervised-tokens: 1.232772E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.909594E-02 | lm-token-buffer-mib: 5.821161E+01 | global_load_balancing_loss: 1.000620E+00 | loss scale: 1.0 | grad norm: 0.107 | params norm: 11545.982 | params rms: 0.062518 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 7:38:46.540375 | finish at 2026-10-10 14:50:04 + [2026-10-05 07:13:05] iteration 1570/ 6855 | consumed samples: 200960 | elapsed time per iteration (ms): 107922.9 | throughput per GPU (TFLOP/s/GPU): 34.6 | MFU 11.09% | FLOPs estimator: workload-aware | logical tokens/s: 155455.3 | valid tokens/s: 155455.3 | physical tokens/s: 155884.7 | loss tokens/s: 114006.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.485779E-01 | lm-global-supervised-tokens: 1.230395E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.247630E-02 | lm-token-buffer-mib: 5.819825E+01 | global_load_balancing_loss: 1.001419E+00 | loss scale: 1.0 | grad norm: 0.109 | params norm: 11545.910 | params rms: 0.062517 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 14:26:12.349890 | finish at 2026-10-11 21:39:17 + [2026-10-05 07:14:27] iteration 1571/ 6855 | consumed samples: 201088 | elapsed time per iteration (ms): 82458.4 | throughput per GPU (TFLOP/s/GPU): 43.6 | MFU 13.97% | FLOPs estimator: workload-aware | logical tokens/s: 203462.4 | valid tokens/s: 203462.4 | physical tokens/s: 203940.6 | loss tokens/s: 151142.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.545454E-01 | lm-global-supervised-tokens: 1.246293E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.149637E-02 | lm-token-buffer-mib: 5.823602E+01 | global_load_balancing_loss: 1.001608E+00 | loss scale: 1.0 | grad norm: 0.075 | params norm: 11545.838 | params rms: 0.062517 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 1:01:50.042041 | finish at 2026-10-10 08:16:17 + [2026-10-05 07:15:58] iteration 1572/ 6855 | consumed samples: 201216 | elapsed time per iteration (ms): 90380.7 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.43% | FLOPs estimator: workload-aware | logical tokens/s: 185627.9 | valid tokens/s: 185627.9 | physical tokens/s: 186098.7 | loss tokens/s: 130717.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.960559E-01 | lm-global-supervised-tokens: 1.181436E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.336724E-02 | lm-token-buffer-mib: 5.822460E+01 | global_load_balancing_loss: 1.001665E+00 | loss scale: 1.0 | grad norm: 0.062 | params norm: 11545.765 | params rms: 0.062517 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 12:38:01.229872 | finish at 2026-10-10 19:53:59 + [2026-10-05 07:17:26] iteration 1573/ 6855 | consumed samples: 201344 | elapsed time per iteration (ms): 87861.1 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.43% | FLOPs estimator: workload-aware | logical tokens/s: 190951.6 | valid tokens/s: 190951.6 | physical tokens/s: 191361.4 | loss tokens/s: 142978.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.926176E-01 | lm-global-supervised-tokens: 1.256220E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.270076E-02 | lm-token-buffer-mib: 5.812744E+01 | global_load_balancing_loss: 9.993950E-01 | loss scale: 1.0 | grad norm: 0.123 | params norm: 11545.692 | params rms: 0.062516 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 8:54:42.166268 | finish at 2026-10-10 16:12:08 + [2026-10-05 07:18:53] iteration 1574/ 6855 | consumed samples: 201472 | elapsed time per iteration (ms): 87369.4 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.61% | FLOPs estimator: workload-aware | logical tokens/s: 192025.8 | valid tokens/s: 192025.8 | physical tokens/s: 192465.9 | loss tokens/s: 138763.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.262416E-01 | lm-global-supervised-tokens: 1.212370E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.161015E-02 | lm-token-buffer-mib: 5.815216E+01 | global_load_balancing_loss: 9.985729E-01 | loss scale: 1.0 | grad norm: 0.110 | params norm: 11545.618 | params rms: 0.062516 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 8:09:57.909811 | finish at 2026-10-10 15:28:51 + [2026-10-05 07:20:22] iteration 1575/ 6855 | consumed samples: 201600 | elapsed time per iteration (ms): 88919.5 | throughput per GPU (TFLOP/s/GPU): 41.1 | MFU 13.18% | FLOPs estimator: workload-aware | logical tokens/s: 188677.7 | valid tokens/s: 188677.7 | physical tokens/s: 189092.0 | loss tokens/s: 144295.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.704245E-01 | lm-global-supervised-tokens: 1.283066E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.540730E-02 | lm-token-buffer-mib: 5.809389E+01 | global_load_balancing_loss: 9.982350E-01 | loss scale: 1.0 | grad norm: 0.056 | params norm: 11545.544 | params rms: 0.062515 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 10:24:54.892616 | finish at 2026-10-10 17:45:17 + [2026-10-05 07:21:52] iteration 1576/ 6855 | consumed samples: 201728 | elapsed time per iteration (ms): 89682.1 | throughput per GPU (TFLOP/s/GPU): 40.8 | MFU 13.09% | FLOPs estimator: workload-aware | logical tokens/s: 187073.6 | valid tokens/s: 187073.6 | physical tokens/s: 187534.6 | loss tokens/s: 141212.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.654959E-01 | lm-global-supervised-tokens: 1.266426E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.412510E-02 | lm-token-buffer-mib: 5.814774E+01 | global_load_balancing_loss: 1.000073E+00 | loss scale: 1.0 | grad norm: 0.090 | params norm: 11545.469 | params rms: 0.062515 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 11:30:31.663981 | finish at 2026-10-10 18:52:23 + [2026-10-05 07:23:19] iteration 1577/ 6855 | consumed samples: 201856 | elapsed time per iteration (ms): 87615.7 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.64% | FLOPs estimator: workload-aware | logical tokens/s: 191485.9 | valid tokens/s: 191485.9 | physical tokens/s: 191891.7 | loss tokens/s: 137758.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.618170E-01 | lm-global-supervised-tokens: 1.206979E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.940775E-02 | lm-token-buffer-mib: 5.808237E+01 | global_load_balancing_loss: 1.000862E+00 | loss scale: 1.0 | grad norm: 0.102 | params norm: 11545.396 | params rms: 0.062515 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 8:27:15.561448 | finish at 2026-10-10 15:50:35 + [2026-10-05 07:24:45] iteration 1578/ 6855 | consumed samples: 201984 | elapsed time per iteration (ms): 86162.9 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.79% | FLOPs estimator: workload-aware | logical tokens/s: 194713.8 | valid tokens/s: 194713.8 | physical tokens/s: 195183.4 | loss tokens/s: 140284.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.619100E-01 | lm-global-supervised-tokens: 1.208728E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.110717E-02 | lm-token-buffer-mib: 5.803224E+01 | global_load_balancing_loss: 1.001069E+00 | loss scale: 1.0 | grad norm: 0.082 | params norm: 11545.322 | params rms: 0.062514 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 6:18:01.565273 | finish at 2026-10-10 13:42:47 + [2026-10-05 07:26:10] iteration 1579/ 6855 | consumed samples: 202112 | elapsed time per iteration (ms): 84984.9 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.52% | FLOPs estimator: workload-aware | logical tokens/s: 197413.8 | valid tokens/s: 197413.8 | physical tokens/s: 197919.7 | loss tokens/s: 152795.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.274480E-01 | lm-global-supervised-tokens: 1.298528E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.158788E-02 | lm-token-buffer-mib: 5.813736E+01 | global_load_balancing_loss: 1.000990E+00 | loss scale: 1.0 | grad norm: 0.188 | params norm: 11545.247 | params rms: 0.062514 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 4:33:00.098419 | finish at 2026-10-10 11:59:10 + [2026-10-05 07:27:53] iteration 1580/ 6855 | consumed samples: 202240 | elapsed time per iteration (ms): 102864.6 | throughput per GPU (TFLOP/s/GPU): 34.8 | MFU 11.15% | FLOPs estimator: workload-aware | logical tokens/s: 163099.5 | valid tokens/s: 163099.5 | physical tokens/s: 163398.3 | loss tokens/s: 123985.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.807778E-01 | lm-global-supervised-tokens: 1.275374E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.325486E-02 | lm-token-buffer-mib: 5.807805E+01 | global_load_balancing_loss: 1.001519E+00 | loss scale: 1.0 | grad norm: 0.099 | params norm: 11545.171 | params rms: 0.062513 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 6:43:30.938257 | finish at 2026-10-11 14:11:24 + [2026-10-05 07:29:20] iteration 1581/ 6855 | consumed samples: 202368 | elapsed time per iteration (ms): 86596.2 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.74% | FLOPs estimator: workload-aware | logical tokens/s: 193740.5 | valid tokens/s: 193740.5 | physical tokens/s: 194341.9 | loss tokens/s: 137695.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.756435E-01 | lm-global-supervised-tokens: 1.192386E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.097572E-02 | lm-token-buffer-mib: 5.811008E+01 | global_load_balancing_loss: 1.000594E+00 | loss scale: 1.0 | grad norm: 0.180 | params norm: 11545.094 | params rms: 0.062513 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 6:51:48.289586 | finish at 2026-10-10 14:21:08 + [2026-10-05 07:30:49] iteration 1582/ 6855 | consumed samples: 202496 | elapsed time per iteration (ms): 89290.6 | throughput per GPU (TFLOP/s/GPU): 41.3 | MFU 13.23% | FLOPs estimator: workload-aware | logical tokens/s: 187894.2 | valid tokens/s: 187894.2 | physical tokens/s: 188293.5 | loss tokens/s: 140142.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.880798E-01 | lm-global-supervised-tokens: 1.251339E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.320546E-02 | lm-token-buffer-mib: 5.806324E+01 | global_load_balancing_loss: 9.995161E-01 | loss scale: 1.0 | grad norm: 0.263 | params norm: 11545.018 | params rms: 0.062513 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 10:47:09.138004 | finish at 2026-10-10 18:17:58 + [2026-10-05 07:32:18] iteration 1583/ 6855 | consumed samples: 202624 | elapsed time per iteration (ms): 88660.7 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.66% | FLOPs estimator: workload-aware | logical tokens/s: 189229.3 | valid tokens/s: 189229.3 | physical tokens/s: 189743.3 | loss tokens/s: 138147.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.114948E-01 | lm-global-supervised-tokens: 1.224824E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.413052E-02 | lm-token-buffer-mib: 5.815400E+01 | global_load_balancing_loss: 9.998826E-01 | loss scale: 1.0 | grad norm: 0.062 | params norm: 11544.940 | params rms: 0.062512 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 9:50:19.460968 | finish at 2026-10-10 17:22:37 + [2026-10-05 07:33:42] iteration 1584/ 6855 | consumed samples: 202752 | elapsed time per iteration (ms): 84660.8 | throughput per GPU (TFLOP/s/GPU): 43.4 | MFU 13.90% | FLOPs estimator: workload-aware | logical tokens/s: 198169.7 | valid tokens/s: 198169.7 | physical tokens/s: 198540.0 | loss tokens/s: 149674.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.494097E-01 | lm-global-supervised-tokens: 1.267157E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.632898E-02 | lm-token-buffer-mib: 5.814633E+01 | global_load_balancing_loss: 1.000516E+00 | loss scale: 1.0 | grad norm: 0.119 | params norm: 11544.862 | params rms: 0.062512 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 3:57:27.309186 | finish at 2026-10-10 11:31:10 + [2026-10-05 07:35:12] iteration 1585/ 6855 | consumed samples: 202880 | elapsed time per iteration (ms): 89522.8 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.48% | FLOPs estimator: workload-aware | logical tokens/s: 187407.2 | valid tokens/s: 187407.2 | physical tokens/s: 187724.3 | loss tokens/s: 136108.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.663907E-01 | lm-global-supervised-tokens: 1.218478E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.756245E-02 | lm-token-buffer-mib: 5.808972E+01 | global_load_balancing_loss: 9.992812E-01 | loss scale: 1.0 | grad norm: 0.029 | params norm: 11544.784 | params rms: 0.062511 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 11:03:04.959826 | finish at 2026-10-10 18:38:17 + [2026-10-05 07:36:42] iteration 1586/ 6855 | consumed samples: 203008 | elapsed time per iteration (ms): 89749.4 | throughput per GPU (TFLOP/s/GPU): 41.0 | MFU 13.13% | FLOPs estimator: workload-aware | logical tokens/s: 186932.9 | valid tokens/s: 186932.9 | physical tokens/s: 187403.3 | loss tokens/s: 137210.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.612350E-01 | lm-global-supervised-tokens: 1.231451E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.863084E-02 | lm-token-buffer-mib: 5.812191E+01 | global_load_balancing_loss: 1.000018E+00 | loss scale: 1.0 | grad norm: 0.105 | params norm: 11544.705 | params rms: 0.062511 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 11:21:29.459940 | finish at 2026-10-10 18:58:11 + [2026-10-05 07:38:08] iteration 1587/ 6855 | consumed samples: 203136 | elapsed time per iteration (ms): 86001.2 | throughput per GPU (TFLOP/s/GPU): 43.1 | MFU 13.80% | FLOPs estimator: workload-aware | logical tokens/s: 195081.2 | valid tokens/s: 195081.2 | physical tokens/s: 195551.7 | loss tokens/s: 139001.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.841230E-01 | lm-global-supervised-tokens: 1.195432E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.208271E-02 | lm-token-buffer-mib: 5.815287E+01 | global_load_balancing_loss: 9.999716E-01 | loss scale: 1.0 | grad norm: 0.057 | params norm: 11544.628 | params rms: 0.062510 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 5:50:54.235986 | finish at 2026-10-10 13:29:02 + [2026-10-05 07:39:38] iteration 1588/ 6855 | consumed samples: 203264 | elapsed time per iteration (ms): 90502.8 | throughput per GPU (TFLOP/s/GPU): 40.7 | MFU 13.05% | FLOPs estimator: workload-aware | logical tokens/s: 185377.8 | valid tokens/s: 185377.8 | physical tokens/s: 185823.7 | loss tokens/s: 141904.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.428038E-01 | lm-global-supervised-tokens: 1.284276E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.176971E-02 | lm-token-buffer-mib: 5.835751E+01 | global_load_balancing_loss: 9.987301E-01 | loss scale: 1.0 | grad norm: 0.105 | params norm: 11544.550 | params rms: 0.062510 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 12:24:38.347996 | finish at 2026-10-10 20:04:17 + [2026-10-05 07:41:05] iteration 1589/ 6855 | consumed samples: 203392 | elapsed time per iteration (ms): 86747.6 | throughput per GPU (TFLOP/s/GPU): 43.2 | MFU 13.83% | FLOPs estimator: workload-aware | logical tokens/s: 193402.7 | valid tokens/s: 193402.7 | physical tokens/s: 193754.6 | loss tokens/s: 140241.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.944455E-01 | lm-global-supervised-tokens: 1.216561E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.131447E-02 | lm-token-buffer-mib: 5.808442E+01 | global_load_balancing_loss: 1.000236E+00 | loss scale: 1.0 | grad norm: 0.052 | params norm: 11544.471 | params rms: 0.062510 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 6:53:32.795472 | finish at 2026-10-10 14:34:38 + [2026-10-05 07:42:55] iteration 1590/ 6855 | consumed samples: 203520 | elapsed time per iteration (ms): 110384.0 | throughput per GPU (TFLOP/s/GPU): 33.7 | MFU 10.81% | FLOPs estimator: workload-aware | logical tokens/s: 151989.4 | valid tokens/s: 151989.4 | physical tokens/s: 152317.4 | loss tokens/s: 114969.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.978582E-01 | lm-global-supervised-tokens: 1.269081E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.465546E-02 | lm-token-buffer-mib: 5.828004E+01 | global_load_balancing_loss: 1.000107E+00 | loss scale: 1.0 | grad norm: 0.046 | params norm: 11544.393 | params rms: 0.062509 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 17:26:11.757821 | finish at 2026-10-12 01:09:07 + [2026-10-05 07:44:23] iteration 1591/ 6855 | consumed samples: 203648 | elapsed time per iteration (ms): 87852.4 | throughput per GPU (TFLOP/s/GPU): 42.8 | MFU 13.73% | FLOPs estimator: workload-aware | logical tokens/s: 190970.5 | valid tokens/s: 190970.4 | physical tokens/s: 191291.8 | loss tokens/s: 139307.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.955475E-01 | lm-global-supervised-tokens: 1.223847E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.485009E-02 | lm-token-buffer-mib: 5.813471E+01 | global_load_balancing_loss: 1.000223E+00 | loss scale: 1.0 | grad norm: 0.039 | params norm: 11544.314 | params rms: 0.062509 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 8:27:34.928516 | finish at 2026-10-10 16:11:58 + [2026-10-05 07:45:55] iteration 1592/ 6855 | consumed samples: 203776 | elapsed time per iteration (ms): 92162.2 | throughput per GPU (TFLOP/s/GPU): 40.5 | MFU 12.98% | FLOPs estimator: workload-aware | logical tokens/s: 182040.0 | valid tokens/s: 182040.0 | physical tokens/s: 182426.0 | loss tokens/s: 136582.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.542670E-01 | lm-global-supervised-tokens: 1.258772E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.146906E-02 | lm-token-buffer-mib: 5.808430E+01 | global_load_balancing_loss: 1.001396E+00 | loss scale: 1.0 | grad norm: 0.043 | params norm: 11544.235 | params rms: 0.062508 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 14:44:09.862995 | finish at 2026-10-10 22:30:05 + [2026-10-05 07:47:23] iteration 1593/ 6855 | consumed samples: 203904 | elapsed time per iteration (ms): 87627.4 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.52% | FLOPs estimator: workload-aware | logical tokens/s: 191460.1 | valid tokens/s: 191460.1 | physical tokens/s: 192204.2 | loss tokens/s: 140876.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.755220E-01 | lm-global-supervised-tokens: 1.234464E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.154458E-02 | lm-token-buffer-mib: 5.803822E+01 | global_load_balancing_loss: 1.001621E+00 | loss scale: 1.0 | grad norm: 0.037 | params norm: 11544.156 | params rms: 0.062508 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 8:04:55.431078 | finish at 2026-10-10 15:52:18 + [2026-10-05 07:48:48] iteration 1594/ 6855 | consumed samples: 204032 | elapsed time per iteration (ms): 85303.8 | throughput per GPU (TFLOP/s/GPU): 43.3 | MFU 13.88% | FLOPs estimator: workload-aware | logical tokens/s: 196675.8 | valid tokens/s: 196675.8 | physical tokens/s: 197087.6 | loss tokens/s: 145211.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.929957E-01 | lm-global-supervised-tokens: 1.238709E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.154691E-02 | lm-token-buffer-mib: 5.813134E+01 | global_load_balancing_loss: 1.000851E+00 | loss scale: 1.0 | grad norm: 0.040 | params norm: 11544.077 | params rms: 0.062507 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 4:39:43.179469 | finish at 2026-10-10 12:28:31 + [2026-10-05 07:50:15] iteration 1595/ 6855 | consumed samples: 204160 | elapsed time per iteration (ms): 86996.8 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.45% | FLOPs estimator: workload-aware | logical tokens/s: 192848.6 | valid tokens/s: 192848.6 | physical tokens/s: 193283.9 | loss tokens/s: 146891.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.926674E-01 | lm-global-supervised-tokens: 1.277906E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.151088E-02 | lm-token-buffer-mib: 5.819955E+01 | global_load_balancing_loss: 9.997792E-01 | loss scale: 1.0 | grad norm: 0.055 | params norm: 11544.001 | params rms: 0.062507 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 7:06:43.276820 | finish at 2026-10-10 14:56:59 + [2026-10-05 07:51:40] iteration 1596/ 6855 | consumed samples: 204288 | elapsed time per iteration (ms): 84582.1 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.48% | FLOPs estimator: workload-aware | logical tokens/s: 198354.1 | valid tokens/s: 198354.1 | physical tokens/s: 199032.3 | loss tokens/s: 152037.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.587147E-01 | lm-global-supervised-tokens: 1.285968E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.157142E-02 | lm-token-buffer-mib: 5.811569E+01 | global_load_balancing_loss: 9.999570E-01 | loss scale: 1.0 | grad norm: 0.032 | params norm: 11543.926 | params rms: 0.062507 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 3:33:37.337427 | finish at 2026-10-10 11:25:17 + [2026-10-05 07:53:07] iteration 1597/ 6855 | consumed samples: 204416 | elapsed time per iteration (ms): 87631.3 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.46% | FLOPs estimator: workload-aware | logical tokens/s: 191452.0 | valid tokens/s: 191452.0 | physical tokens/s: 191952.5 | loss tokens/s: 136395.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.312932E-01 | lm-global-supervised-tokens: 1.195254E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.591292E-02 | lm-token-buffer-mib: 5.805997E+01 | global_load_balancing_loss: 1.000468E+00 | loss scale: 1.0 | grad norm: 0.036 | params norm: 11543.852 | params rms: 0.062506 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 7:59:25.559536 | finish at 2026-10-10 15:52:33 + [2026-10-05 07:54:33] iteration 1598/ 6855 | consumed samples: 204544 | elapsed time per iteration (ms): 85067.4 | throughput per GPU (TFLOP/s/GPU): 43.8 | MFU 14.03% | FLOPs estimator: workload-aware | logical tokens/s: 197222.4 | valid tokens/s: 197222.4 | physical tokens/s: 197725.9 | loss tokens/s: 146790.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.467733E-01 | lm-global-supervised-tokens: 1.248706E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.003089E-02 | lm-token-buffer-mib: 5.816047E+01 | global_load_balancing_loss: 9.998711E-01 | loss scale: 1.0 | grad norm: 0.044 | params norm: 11543.776 | params rms: 0.062506 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 4:13:19.355529 | finish at 2026-10-10 12:07:52 + [2026-10-05 07:56:00] iteration 1599/ 6855 | consumed samples: 204672 | elapsed time per iteration (ms): 87835.0 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.38% | FLOPs estimator: workload-aware | logical tokens/s: 191008.1 | valid tokens/s: 191008.1 | physical tokens/s: 191376.1 | loss tokens/s: 141780.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.778126E-01 | lm-global-supervised-tokens: 1.245332E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.544518E-02 | lm-token-buffer-mib: 5.810562E+01 | global_load_balancing_loss: 9.988891E-01 | loss scale: 1.0 | grad norm: 0.041 | params norm: 11543.701 | params rms: 0.062505 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 8:14:20.807819 | finish at 2026-10-10 16:10:21 + [2026-10-05 07:57:47] iteration 1600/ 6855 | consumed samples: 204800 | elapsed time per iteration (ms): 107049.1 | throughput per GPU (TFLOP/s/GPU): 33.9 | MFU 10.87% | FLOPs estimator: workload-aware | logical tokens/s: 156724.2 | valid tokens/s: 156724.2 | physical tokens/s: 157198.4 | loss tokens/s: 116061.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.529599E-01 | lm-global-supervised-tokens: 1.242424E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.161085E-02 | lm-token-buffer-mib: 5.816419E+01 | global_load_balancing_loss: 1.000027E+00 | loss scale: 1.0 | grad norm: 0.045 | params norm: 11543.628 | params rms: 0.062505 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 12:15:43.251876 | finish at 2026-10-11 20:13:31 +times across ranks (ms): + save-checkpoint: + rank 0: 26079.63 + rank 1: 26079.55 + rank 2: 26079.61 + rank 3: 26079.57 + rank 4: 26079.59 + rank 5: 26079.57 + rank 6: 26079.58 + rank 7: 26079.58 + rank 8: 26079.64 + rank 9: 26079.64 + rank 10: 26079.62 + rank 11: 26079.63 + rank 12: 26079.62 + rank 13: 26079.62 + rank 14: 26079.64 + rank 15: 26079.67 + rank 16: 26079.72 + rank 17: 26079.70 + rank 18: 26079.72 + rank 19: 26079.71 + rank 20: 26079.68 + rank 21: 26079.70 + rank 22: 26079.67 + rank 23: 26079.68 + rank 24: 26079.97 + rank 25: 26079.91 + rank 26: 26079.94 + rank 27: 26080.02 + rank 28: 26079.94 + rank 29: 26079.95 + rank 30: 26079.92 + rank 31: 26079.97 + rank 32: 26078.73 + rank 33: 26078.92 + rank 34: 26078.87 + rank 35: 26078.83 + rank 36: 26078.78 + rank 37: 26078.58 + rank 38: 26078.81 + rank 39: 26078.77 + rank 40: 26079.89 + rank 41: 26079.85 + rank 42: 26079.87 + rank 43: 26079.87 + rank 44: 26079.88 + rank 45: 26079.93 + rank 46: 26079.87 + rank 47: 26079.85 + rank 48: 26079.52 + rank 49: 26079.44 + rank 50: 26079.43 + rank 51: 26079.49 + rank 52: 26079.54 + rank 53: 26079.46 + rank 54: 26079.48 + rank 55: 26079.61 + rank 56: 26079.87 + rank 57: 26079.82 + rank 58: 26079.80 + rank 59: 26079.82 + rank 60: 26079.80 + rank 61: 26079.83 + rank 62: 26079.81 + rank 63: 26079.82 + [2026-10-05 07:59:42] iteration 1601/ 6855 | consumed samples: 204928 | elapsed time per iteration (ms): 87932.3 | throughput per GPU (TFLOP/s/GPU): 41.3 | MFU 13.23% | FLOPs estimator: workload-aware | logical tokens/s: 190796.8 | valid tokens/s: 190796.8 | physical tokens/s: 191364.9 | loss tokens/s: 148838.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.014230E-01 | lm-global-supervised-tokens: 1.308768E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 8.488808E-01 | lm-token-buffer-mib: 5.812547E+01 | global_load_balancing_loss: 9.990507E-01 | loss scale: 1.0 | grad norm: 0.045 | params norm: 11543.554 | params rms: 0.062505 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 8:19:56.208223 | finish at 2026-10-10 16:19:38 + [2026-10-05 08:01:10] iteration 1602/ 6855 | consumed samples: 205056 | elapsed time per iteration (ms): 88443.8 | throughput per GPU (TFLOP/s/GPU): 40.8 | MFU 13.08% | FLOPs estimator: workload-aware | logical tokens/s: 189693.3 | valid tokens/s: 189693.3 | physical tokens/s: 190032.3 | loss tokens/s: 138603.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.573634E-01 | lm-global-supervised-tokens: 1.225858E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.369430E-02 | lm-token-buffer-mib: 5.808947E+01 | global_load_balancing_loss: 1.001023E+00 | loss scale: 1.0 | grad norm: 0.050 | params norm: 11543.480 | params rms: 0.062504 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 9:03:15.337614 | finish at 2026-10-10 17:04:26 + [2026-10-05 08:02:37] iteration 1603/ 6855 | consumed samples: 205184 | elapsed time per iteration (ms): 87052.4 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.76% | FLOPs estimator: workload-aware | logical tokens/s: 192725.0 | valid tokens/s: 192725.0 | physical tokens/s: 193172.0 | loss tokens/s: 140059.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.847252E-01 | lm-global-supervised-tokens: 1.219251E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.165281E-02 | lm-token-buffer-mib: 5.805659E+01 | global_load_balancing_loss: 1.001454E+00 | loss scale: 1.0 | grad norm: 0.064 | params norm: 11543.406 | params rms: 0.062504 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 6:59:59.381946 | finish at 2026-10-10 15:02:37 + [2026-10-05 08:04:05] iteration 1604/ 6855 | consumed samples: 205312 | elapsed time per iteration (ms): 88168.6 | throughput per GPU (TFLOP/s/GPU): 41.7 | MFU 13.36% | FLOPs estimator: workload-aware | logical tokens/s: 190285.6 | valid tokens/s: 190285.6 | physical tokens/s: 190818.5 | loss tokens/s: 137624.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.762765E-01 | lm-global-supervised-tokens: 1.213416E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.615418E-02 | lm-token-buffer-mib: 5.807235E+01 | global_load_balancing_loss: 1.000036E+00 | loss scale: 1.0 | grad norm: 0.059 | params norm: 11543.334 | params rms: 0.062503 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 8:36:13.464257 | finish at 2026-10-10 16:40:19 + [2026-10-05 08:05:36] iteration 1605/ 6855 | consumed samples: 205440 | elapsed time per iteration (ms): 90506.7 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.49% | FLOPs estimator: workload-aware | logical tokens/s: 185369.6 | valid tokens/s: 185369.6 | physical tokens/s: 185756.7 | loss tokens/s: 132885.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.093385E-01 | lm-global-supervised-tokens: 1.202698E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.605693E-02 | lm-token-buffer-mib: 5.809241E+01 | global_load_balancing_loss: 9.999276E-01 | loss scale: 1.0 | grad norm: 0.077 | params norm: 11543.260 | params rms: 0.062503 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 11:59:20.056293 | finish at 2026-10-10 20:04:56 + [2026-10-05 08:07:05] iteration 1606/ 6855 | consumed samples: 205568 | elapsed time per iteration (ms): 88820.6 | throughput per GPU (TFLOP/s/GPU): 41.6 | MFU 13.32% | FLOPs estimator: workload-aware | logical tokens/s: 188888.6 | valid tokens/s: 188888.6 | physical tokens/s: 189252.3 | loss tokens/s: 146533.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.010106E-01 | lm-global-supervised-tokens: 1.301518E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.440945E-02 | lm-token-buffer-mib: 5.833672E+01 | global_load_balancing_loss: 1.000313E+00 | loss scale: 1.0 | grad norm: 0.048 | params norm: 11543.184 | params rms: 0.062503 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 9:30:19.339584 | finish at 2026-10-10 17:37:24 + [2026-10-05 08:08:32] iteration 1607/ 6855 | consumed samples: 205696 | elapsed time per iteration (ms): 87048.0 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.39% | FLOPs estimator: workload-aware | logical tokens/s: 192735.1 | valid tokens/s: 192735.1 | physical tokens/s: 193442.9 | loss tokens/s: 140265.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.910205E-01 | lm-global-supervised-tokens: 1.220981E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.254547E-02 | lm-token-buffer-mib: 5.806984E+01 | global_load_balancing_loss: 1.000929E+00 | loss scale: 1.0 | grad norm: 0.051 | params norm: 11543.111 | params rms: 0.062502 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 6:53:47.844452 | finish at 2026-10-10 15:02:20 + [2026-10-05 08:09:57] iteration 1608/ 6855 | consumed samples: 205824 | elapsed time per iteration (ms): 85286.4 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.43% | FLOPs estimator: workload-aware | logical tokens/s: 196715.8 | valid tokens/s: 196715.8 | physical tokens/s: 197279.7 | loss tokens/s: 151399.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.408157E-01 | lm-global-supervised-tokens: 1.291234E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.545049E-02 | lm-token-buffer-mib: 5.844115E+01 | global_load_balancing_loss: 9.998752E-01 | loss scale: 1.0 | grad norm: 0.076 | params norm: 11543.038 | params rms: 0.062502 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 4:18:17.833791 | finish at 2026-10-10 12:28:15 + [2026-10-05 08:11:23] iteration 1609/ 6855 | consumed samples: 205952 | elapsed time per iteration (ms): 85873.2 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.55% | FLOPs estimator: workload-aware | logical tokens/s: 195371.1 | valid tokens/s: 195371.1 | physical tokens/s: 195912.1 | loss tokens/s: 138149.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.243103E-01 | lm-global-supervised-tokens: 1.186332E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.447663E-02 | lm-token-buffer-mib: 5.814713E+01 | global_load_balancing_loss: 1.000289E+00 | loss scale: 1.0 | grad norm: 0.126 | params norm: 11542.965 | params rms: 0.062501 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 5:08:10.935710 | finish at 2026-10-10 13:19:34 + [2026-10-05 08:13:12] iteration 1610/ 6855 | consumed samples: 206080 | elapsed time per iteration (ms): 109091.4 | throughput per GPU (TFLOP/s/GPU): 34.2 | MFU 10.96% | FLOPs estimator: workload-aware | logical tokens/s: 153789.9 | valid tokens/s: 153789.9 | physical tokens/s: 154144.9 | loss tokens/s: 110807.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.166868E-01 | lm-global-supervised-tokens: 1.208817E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.150357E-02 | lm-token-buffer-mib: 5.810003E+01 | global_load_balancing_loss: 1.000305E+00 | loss scale: 1.0 | grad norm: 0.243 | params norm: 11542.891 | params rms: 0.062501 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 14:56:24.163675 | finish at 2026-10-11 23:09:36 + [2026-10-05 08:14:38] iteration 1611/ 6855 | consumed samples: 206208 | elapsed time per iteration (ms): 86203.7 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.48% | FLOPs estimator: workload-aware | logical tokens/s: 194622.1 | valid tokens/s: 194622.1 | physical tokens/s: 195214.3 | loss tokens/s: 146059.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.476330E-01 | lm-global-supervised-tokens: 1.259089E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.579818E-02 | lm-token-buffer-mib: 5.822177E+01 | global_load_balancing_loss: 1.000147E+00 | loss scale: 1.0 | grad norm: 0.401 | params norm: 11542.817 | params rms: 0.062501 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 5:34:12.064364 | finish at 2026-10-10 13:48:50 + [2026-10-05 08:16:07] iteration 1612/ 6855 | consumed samples: 206336 | elapsed time per iteration (ms): 88903.1 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.42% | FLOPs estimator: workload-aware | logical tokens/s: 188713.2 | valid tokens/s: 188713.2 | physical tokens/s: 189340.9 | loss tokens/s: 140717.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.347319E-01 | lm-global-supervised-tokens: 1.251022E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.915197E-02 | lm-token-buffer-mib: 5.819093E+01 | global_load_balancing_loss: 9.997982E-01 | loss scale: 1.0 | grad norm: 0.426 | params norm: 11542.743 | params rms: 0.062500 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 9:28:38.897121 | finish at 2026-10-10 17:44:46 + [2026-10-05 08:17:33] iteration 1613/ 6855 | consumed samples: 206464 | elapsed time per iteration (ms): 85481.2 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.48% | FLOPs estimator: workload-aware | logical tokens/s: 196267.4 | valid tokens/s: 196267.4 | physical tokens/s: 196861.5 | loss tokens/s: 146614.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.271170E-01 | lm-global-supervised-tokens: 1.253276E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.167974E-02 | lm-token-buffer-mib: 5.823104E+01 | global_load_balancing_loss: 9.997308E-01 | loss scale: 1.0 | grad norm: 0.073 | params norm: 11542.667 | params rms: 0.062500 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 4:28:12.249078 | finish at 2026-10-10 12:45:45 + [2026-10-05 08:19:01] iteration 1614/ 6855 | consumed samples: 206592 | elapsed time per iteration (ms): 88278.7 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.48% | FLOPs estimator: workload-aware | logical tokens/s: 190048.0 | valid tokens/s: 190048.0 | physical tokens/s: 190616.0 | loss tokens/s: 141623.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.746346E-01 | lm-global-supervised-tokens: 1.250232E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.288177E-02 | lm-token-buffer-mib: 5.813769E+01 | global_load_balancing_loss: 1.000416E+00 | loss scale: 1.0 | grad norm: 0.173 | params norm: 11542.592 | params rms: 0.062499 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 8:31:08.909707 | finish at 2026-10-10 16:50:10 + [2026-10-05 08:20:30] iteration 1615/ 6855 | consumed samples: 206720 | elapsed time per iteration (ms): 89147.3 | throughput per GPU (TFLOP/s/GPU): 41.4 | MFU 13.26% | FLOPs estimator: workload-aware | logical tokens/s: 188196.2 | valid tokens/s: 188196.2 | physical tokens/s: 188686.0 | loss tokens/s: 137304.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.252956E-01 | lm-global-supervised-tokens: 1.224036E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.865034E-02 | lm-token-buffer-mib: 5.818499E+01 | global_load_balancing_loss: 1.000536E+00 | loss scale: 1.0 | grad norm: 0.084 | params norm: 11542.516 | params rms: 0.062499 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 9:45:31.931982 | finish at 2026-10-10 18:06:02 + [2026-10-05 08:21:57] iteration 1616/ 6855 | consumed samples: 206848 | elapsed time per iteration (ms): 87166.2 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.44% | FLOPs estimator: workload-aware | logical tokens/s: 192473.3 | valid tokens/s: 192473.2 | physical tokens/s: 192916.3 | loss tokens/s: 143887.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.639408E-01 | lm-global-supervised-tokens: 1.254214E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.539733E-02 | lm-token-buffer-mib: 5.814577E+01 | global_load_balancing_loss: 1.000478E+00 | loss scale: 1.0 | grad norm: 0.055 | params norm: 11542.440 | params rms: 0.062499 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 6:51:03.972458 | finish at 2026-10-10 15:13:01 + [2026-10-05 08:23:25] iteration 1617/ 6855 | consumed samples: 206976 | elapsed time per iteration (ms): 87847.6 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.53% | FLOPs estimator: workload-aware | logical tokens/s: 190980.8 | valid tokens/s: 190980.8 | physical tokens/s: 191371.2 | loss tokens/s: 137602.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.283945E-01 | lm-global-supervised-tokens: 1.208801E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.137770E-02 | lm-token-buffer-mib: 5.813045E+01 | global_load_balancing_loss: 1.000415E+00 | loss scale: 1.0 | grad norm: 0.104 | params norm: 11542.364 | params rms: 0.062498 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 7:49:05.736205 | finish at 2026-10-10 16:12:31 + [2026-10-05 08:24:50] iteration 1618/ 6855 | consumed samples: 207104 | elapsed time per iteration (ms): 84575.4 | throughput per GPU (TFLOP/s/GPU): 43.2 | MFU 13.86% | FLOPs estimator: workload-aware | logical tokens/s: 198369.7 | valid tokens/s: 198369.7 | physical tokens/s: 198960.8 | loss tokens/s: 150469.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.816734E-01 | lm-global-supervised-tokens: 1.272598E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.568687E-02 | lm-token-buffer-mib: 5.819303E+01 | global_load_balancing_loss: 1.001116E+00 | loss scale: 1.0 | grad norm: 0.104 | params norm: 11542.287 | params rms: 0.062498 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 3:02:01.558936 | finish at 2026-10-10 11:26:51 + [2026-10-05 08:26:17] iteration 1619/ 6855 | consumed samples: 207232 | elapsed time per iteration (ms): 87179.5 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.39% | FLOPs estimator: workload-aware | logical tokens/s: 192443.5 | valid tokens/s: 192443.5 | physical tokens/s: 192894.5 | loss tokens/s: 138049.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.485691E-01 | lm-global-supervised-tokens: 1.203505E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.152480E-02 | lm-token-buffer-mib: 5.808363E+01 | global_load_balancing_loss: 1.000965E+00 | loss scale: 1.0 | grad norm: 0.088 | params norm: 11542.210 | params rms: 0.062497 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 6:47:51.949924 | finish at 2026-10-10 15:14:09 + [2026-10-05 08:28:05] iteration 1620/ 6855 | consumed samples: 207360 | elapsed time per iteration (ms): 108349.8 | throughput per GPU (TFLOP/s/GPU): 34.2 | MFU 10.97% | FLOPs estimator: workload-aware | logical tokens/s: 154843.0 | valid tokens/s: 154843.0 | physical tokens/s: 155175.5 | loss tokens/s: 114071.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.128460E-01 | lm-global-supervised-tokens: 1.235961E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.332122E-02 | lm-token-buffer-mib: 5.804648E+01 | global_load_balancing_loss: 9.996275E-01 | loss scale: 1.0 | grad norm: 0.068 | params norm: 11542.133 | params rms: 0.062497 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 13:33:31.217304 | finish at 2026-10-11 22:01:36 + [2026-10-05 08:29:31] iteration 1621/ 6855 | consumed samples: 207488 | elapsed time per iteration (ms): 85782.0 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.58% | FLOPs estimator: workload-aware | logical tokens/s: 195579.4 | valid tokens/s: 195579.4 | physical tokens/s: 196201.1 | loss tokens/s: 144789.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.511286E-01 | lm-global-supervised-tokens: 1.242034E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.310850E-02 | lm-token-buffer-mib: 5.809100E+01 | global_load_balancing_loss: 9.987620E-01 | loss scale: 1.0 | grad norm: 0.078 | params norm: 11542.057 | params rms: 0.062496 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 4:43:02.740011 | finish at 2026-10-10 13:12:34 + [2026-10-05 08:30:57] iteration 1622/ 6855 | consumed samples: 207616 | elapsed time per iteration (ms): 85776.2 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.69% | FLOPs estimator: workload-aware | logical tokens/s: 195592.5 | valid tokens/s: 195592.5 | physical tokens/s: 196242.0 | loss tokens/s: 146106.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.908247E-01 | lm-global-supervised-tokens: 1.253244E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.152124E-02 | lm-token-buffer-mib: 5.823692E+01 | global_load_balancing_loss: 9.985386E-01 | loss scale: 1.0 | grad norm: 0.060 | params norm: 11541.981 | params rms: 0.062496 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 4:41:06.952210 | finish at 2026-10-10 13:12:04 + [2026-10-05 08:32:24] iteration 1623/ 6855 | consumed samples: 207744 | elapsed time per iteration (ms): 87331.7 | throughput per GPU (TFLOP/s/GPU): 42.8 | MFU 13.71% | FLOPs estimator: workload-aware | logical tokens/s: 192109.1 | valid tokens/s: 192109.1 | physical tokens/s: 192430.0 | loss tokens/s: 138761.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.947786E-01 | lm-global-supervised-tokens: 1.211829E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.207620E-02 | lm-token-buffer-mib: 5.811891E+01 | global_load_balancing_loss: 9.991776E-01 | loss scale: 1.0 | grad norm: 0.062 | params norm: 11541.904 | params rms: 0.062496 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 6:55:19.227825 | finish at 2026-10-10 15:27:43 + [2026-10-05 08:33:51] iteration 1624/ 6855 | consumed samples: 207872 | elapsed time per iteration (ms): 87408.8 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.57% | FLOPs estimator: workload-aware | logical tokens/s: 191939.7 | valid tokens/s: 191939.7 | physical tokens/s: 192355.0 | loss tokens/s: 147809.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.846162E-01 | lm-global-supervised-tokens: 1.291989E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.125296E-02 | lm-token-buffer-mib: 5.822542E+01 | global_load_balancing_loss: 1.000808E+00 | loss scale: 1.0 | grad norm: 0.071 | params norm: 11541.827 | params rms: 0.062495 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 7:00:35.382321 | finish at 2026-10-10 15:34:27 + [2026-10-05 08:35:20] iteration 1625/ 6855 | consumed samples: 208000 | elapsed time per iteration (ms): 88554.6 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.41% | FLOPs estimator: workload-aware | logical tokens/s: 189455.5 | valid tokens/s: 189455.5 | physical tokens/s: 189899.3 | loss tokens/s: 140129.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.761335E-01 | lm-global-supervised-tokens: 1.240911E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.524035E-02 | lm-token-buffer-mib: 5.806989E+01 | global_load_balancing_loss: 1.001765E+00 | loss scale: 1.0 | grad norm: 0.047 | params norm: 11541.752 | params rms: 0.062495 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 8:39:00.707633 | finish at 2026-10-10 17:14:21 + [2026-10-05 08:36:46] iteration 1626/ 6855 | consumed samples: 208128 | elapsed time per iteration (ms): 85647.3 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.52% | FLOPs estimator: workload-aware | logical tokens/s: 195886.3 | valid tokens/s: 195886.3 | physical tokens/s: 196396.5 | loss tokens/s: 147295.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.134576E-01 | lm-global-supervised-tokens: 1.261548E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.101104E-02 | lm-token-buffer-mib: 5.817887E+01 | global_load_balancing_loss: 1.002872E+00 | loss scale: 1.0 | grad norm: 0.063 | params norm: 11541.676 | params rms: 0.062494 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 4:24:09.754166 | finish at 2026-10-10 13:00:55 + [2026-10-05 08:38:20] iteration 1627/ 6855 | consumed samples: 208256 | elapsed time per iteration (ms): 94078.3 | throughput per GPU (TFLOP/s/GPU): 40.8 | MFU 13.09% | FLOPs estimator: workload-aware | logical tokens/s: 178332.1 | valid tokens/s: 178332.1 | physical tokens/s: 178703.9 | loss tokens/s: 131297.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.922309E-01 | lm-global-supervised-tokens: 1.235223E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.149716E-02 | lm-token-buffer-mib: 5.817932E+01 | global_load_balancing_loss: 1.000014E+00 | loss scale: 1.0 | grad norm: 0.043 | params norm: 11541.600 | params rms: 0.062494 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 16:37:21.296306 | finish at 2026-10-11 01:15:41 + [2026-10-05 08:39:47] iteration 1628/ 6855 | consumed samples: 208384 | elapsed time per iteration (ms): 86836.5 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.69% | FLOPs estimator: workload-aware | logical tokens/s: 193204.3 | valid tokens/s: 193204.3 | physical tokens/s: 193568.5 | loss tokens/s: 136608.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.429203E-01 | lm-global-supervised-tokens: 1.186260E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.143511E-02 | lm-token-buffer-mib: 5.814722E+01 | global_load_balancing_loss: 1.000022E+00 | loss scale: 1.0 | grad norm: 0.031 | params norm: 11541.523 | params rms: 0.062494 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 6:04:54.588264 | finish at 2026-10-10 14:44:41 + [2026-10-05 08:41:14] iteration 1629/ 6855 | consumed samples: 208512 | elapsed time per iteration (ms): 87453.9 | throughput per GPU (TFLOP/s/GPU): 41.6 | MFU 13.32% | FLOPs estimator: workload-aware | logical tokens/s: 191840.7 | valid tokens/s: 191840.7 | physical tokens/s: 192434.0 | loss tokens/s: 141499.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.057214E-01 | lm-global-supervised-tokens: 1.237464E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.240838E-02 | lm-token-buffer-mib: 5.827419E+01 | global_load_balancing_loss: 9.989623E-01 | loss scale: 1.0 | grad norm: 0.056 | params norm: 11541.446 | params rms: 0.062493 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 6:57:14.014634 | finish at 2026-10-10 15:38:28 + [2026-10-05 08:43:04] iteration 1630/ 6855 | consumed samples: 208640 | elapsed time per iteration (ms): 109868.9 | throughput per GPU (TFLOP/s/GPU): 33.7 | MFU 10.81% | FLOPs estimator: workload-aware | logical tokens/s: 152702.0 | valid tokens/s: 152702.0 | physical tokens/s: 153026.2 | loss tokens/s: 113094.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.365674E-01 | lm-global-supervised-tokens: 1.242555E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.191420E-02 | lm-token-buffer-mib: 5.818789E+01 | global_load_balancing_loss: 9.990581E-01 | loss scale: 1.0 | grad norm: 0.054 | params norm: 11541.368 | params rms: 0.062493 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 15:27:45.004063 | finish at 2026-10-12 00:10:49 + [2026-10-05 08:44:30] iteration 1631/ 6855 | consumed samples: 208768 | elapsed time per iteration (ms): 85817.0 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.40% | FLOPs estimator: workload-aware | logical tokens/s: 195498.7 | valid tokens/s: 195498.6 | physical tokens/s: 196108.4 | loss tokens/s: 152483.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.758324E-01 | lm-global-supervised-tokens: 1.308566E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.463189E-02 | lm-token-buffer-mib: 5.812038E+01 | global_load_balancing_loss: 9.999757E-01 | loss scale: 1.0 | grad norm: 0.057 | params norm: 11541.291 | params rms: 0.062492 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 4:31:47.780861 | finish at 2026-10-10 13:16:18 + [2026-10-05 08:45:58] iteration 1632/ 6855 | consumed samples: 208896 | elapsed time per iteration (ms): 88600.3 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.60% | FLOPs estimator: workload-aware | logical tokens/s: 189358.5 | valid tokens/s: 189358.5 | physical tokens/s: 189811.8 | loss tokens/s: 140283.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.047243E-01 | lm-global-supervised-tokens: 1.242915E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.472849E-02 | lm-token-buffer-mib: 5.808035E+01 | global_load_balancing_loss: 1.000543E+00 | loss scale: 1.0 | grad norm: 0.054 | params norm: 11541.216 | params rms: 0.062492 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 8:32:39.169288 | finish at 2026-10-10 17:18:38 + [2026-10-05 08:47:25] iteration 1633/ 6855 | consumed samples: 209024 | elapsed time per iteration (ms): 86802.4 | throughput per GPU (TFLOP/s/GPU): 42.8 | MFU 13.71% | FLOPs estimator: workload-aware | logical tokens/s: 193280.3 | valid tokens/s: 193280.3 | physical tokens/s: 193706.3 | loss tokens/s: 141368.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.453435E-01 | lm-global-supervised-tokens: 1.227108E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.700361E-02 | lm-token-buffer-mib: 5.822135E+01 | global_load_balancing_loss: 9.997619E-01 | loss scale: 1.0 | grad norm: 0.069 | params norm: 11541.142 | params rms: 0.062492 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 5:54:42.132141 | finish at 2026-10-10 14:42:07 + [2026-10-05 08:48:50] iteration 1634/ 6855 | consumed samples: 209152 | elapsed time per iteration (ms): 84761.6 | throughput per GPU (TFLOP/s/GPU): 43.2 | MFU 13.84% | FLOPs estimator: workload-aware | logical tokens/s: 197933.8 | valid tokens/s: 197933.8 | physical tokens/s: 198363.3 | loss tokens/s: 157604.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.818977E-01 | lm-global-supervised-tokens: 1.335878E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.296897E-02 | lm-token-buffer-mib: 5.827591E+01 | global_load_balancing_loss: 1.000426E+00 | loss scale: 1.0 | grad norm: 0.032 | params norm: 11541.068 | params rms: 0.062491 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 2:55:40.145646 | finish at 2026-10-10 11:44:30 + [2026-10-05 08:50:16] iteration 1635/ 6855 | consumed samples: 209280 | elapsed time per iteration (ms): 86349.7 | throughput per GPU (TFLOP/s/GPU): 43.2 | MFU 13.84% | FLOPs estimator: workload-aware | logical tokens/s: 194293.0 | valid tokens/s: 194293.0 | physical tokens/s: 194765.9 | loss tokens/s: 141428.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.063257E-01 | lm-global-supervised-tokens: 1.221226E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.366791E-02 | lm-token-buffer-mib: 5.815208E+01 | global_load_balancing_loss: 1.001449E+00 | loss scale: 1.0 | grad norm: 0.049 | params norm: 11540.993 | params rms: 0.062491 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 5:12:25.212336 | finish at 2026-10-10 14:02:41 + [2026-10-05 08:51:40] iteration 1636/ 6855 | consumed samples: 209408 | elapsed time per iteration (ms): 83508.9 | throughput per GPU (TFLOP/s/GPU): 43.4 | MFU 13.91% | FLOPs estimator: workload-aware | logical tokens/s: 200902.0 | valid tokens/s: 200902.0 | physical tokens/s: 201262.2 | loss tokens/s: 152213.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.772600E-01 | lm-global-supervised-tokens: 1.271120E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.468841E-02 | lm-token-buffer-mib: 5.806970E+01 | global_load_balancing_loss: 1.000905E+00 | loss scale: 1.0 | grad norm: 0.052 | params norm: 11540.917 | params rms: 0.062490 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 1:03:53.005958 | finish at 2026-10-10 09:55:33 + [2026-10-05 08:53:03] iteration 1637/ 6855 | consumed samples: 209536 | elapsed time per iteration (ms): 83600.8 | throughput per GPU (TFLOP/s/GPU): 43.7 | MFU 14.00% | FLOPs estimator: workload-aware | logical tokens/s: 200681.7 | valid tokens/s: 200681.7 | physical tokens/s: 201314.9 | loss tokens/s: 152114.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.993495E-01 | lm-global-supervised-tokens: 1.271686E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.141990E-02 | lm-token-buffer-mib: 5.810140E+01 | global_load_balancing_loss: 1.000436E+00 | loss scale: 1.0 | grad norm: 0.033 | params norm: 11540.843 | params rms: 0.062490 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 1:10:28.867605 | finish at 2026-10-10 10:03:32 + [2026-10-05 08:54:28] iteration 1638/ 6855 | consumed samples: 209664 | elapsed time per iteration (ms): 84595.4 | throughput per GPU (TFLOP/s/GPU): 43.1 | MFU 13.81% | FLOPs estimator: workload-aware | logical tokens/s: 198322.4 | valid tokens/s: 198322.4 | physical tokens/s: 198743.6 | loss tokens/s: 150351.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.647506E-01 | lm-global-supervised-tokens: 1.271906E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.485884E-02 | lm-token-buffer-mib: 5.813457E+01 | global_load_balancing_loss: 9.993382E-01 | loss scale: 1.0 | grad norm: 0.048 | params norm: 11540.769 | params rms: 0.062490 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 2:35:34.321703 | finish at 2026-10-10 11:30:02 + [2026-10-05 08:55:57] iteration 1639/ 6855 | consumed samples: 209792 | elapsed time per iteration (ms): 89443.8 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.40% | FLOPs estimator: workload-aware | logical tokens/s: 187571.3 | valid tokens/s: 187571.3 | physical tokens/s: 187875.1 | loss tokens/s: 136516.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.589856E-01 | lm-global-supervised-tokens: 1.221056E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.247680E-02 | lm-token-buffer-mib: 5.806844E+01 | global_load_balancing_loss: 1.000886E+00 | loss scale: 1.0 | grad norm: 0.036 | params norm: 11540.693 | params rms: 0.062489 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 9:35:39.059631 | finish at 2026-10-10 18:31:36 + [2026-10-05 08:57:46] iteration 1640/ 6855 | consumed samples: 209920 | elapsed time per iteration (ms): 109000.5 | throughput per GPU (TFLOP/s/GPU): 33.8 | MFU 10.83% | FLOPs estimator: workload-aware | logical tokens/s: 153918.3 | valid tokens/s: 153918.3 | physical tokens/s: 154245.5 | loss tokens/s: 110059.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.864957E-01 | lm-global-supervised-tokens: 1.199655E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.405960E-02 | lm-token-buffer-mib: 5.803952E+01 | global_load_balancing_loss: 1.000159E+00 | loss scale: 1.0 | grad norm: 0.038 | params norm: 11540.617 | params rms: 0.062489 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 13:53:57.547630 | finish at 2026-10-11 22:51:44 +times across ranks (ms): + save-checkpoint: + rank 0: 24094.34 + rank 1: 24094.34 + rank 2: 24094.38 + rank 3: 24094.35 + rank 4: 24094.40 + rank 5: 24094.37 + rank 6: 24094.35 + rank 7: 24094.36 + rank 8: 24094.10 + rank 9: 24094.06 + rank 10: 24094.06 + rank 11: 24094.07 + rank 12: 24094.07 + rank 13: 24094.07 + rank 14: 24094.07 + rank 15: 24094.06 + rank 16: 24094.24 + rank 17: 24094.23 + rank 18: 24094.23 + rank 19: 24094.24 + rank 20: 24094.23 + rank 21: 24094.24 + rank 22: 24094.25 + rank 23: 24094.26 + rank 24: 24094.41 + rank 25: 24094.41 + rank 26: 24094.43 + rank 27: 24094.45 + rank 28: 24094.41 + rank 29: 24094.40 + rank 30: 24094.41 + rank 31: 24094.42 + rank 32: 24094.59 + rank 33: 24094.65 + rank 34: 24094.57 + rank 35: 24094.56 + rank 36: 24094.62 + rank 37: 24094.57 + rank 38: 24094.56 + rank 39: 24094.64 + rank 40: 24094.37 + rank 41: 24094.35 + rank 42: 24094.33 + rank 43: 24094.35 + rank 44: 24094.35 + rank 45: 24094.36 + rank 46: 24094.34 + rank 47: 24094.35 + rank 48: 24094.83 + rank 49: 24094.84 + rank 50: 24094.78 + rank 51: 24094.80 + rank 52: 24094.85 + rank 53: 24094.79 + rank 54: 24094.80 + rank 55: 24094.82 + rank 56: 24094.79 + rank 57: 24094.77 + rank 58: 24094.79 + rank 59: 24094.79 + rank 60: 24094.77 + rank 61: 24094.79 + rank 62: 24094.83 + rank 63: 24094.76 + [2026-10-05 08:59:41] iteration 1641/ 6855 | consumed samples: 210048 | elapsed time per iteration (ms): 90114.1 | throughput per GPU (TFLOP/s/GPU): 41.1 | MFU 13.17% | FLOPs estimator: workload-aware | logical tokens/s: 186177.0 | valid tokens/s: 186177.0 | physical tokens/s: 186584.2 | loss tokens/s: 129910.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 5.887226E-01 | lm-global-supervised-tokens: 1.170681E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 9.353344E-01 | lm-token-buffer-mib: 5.837608E+01 | global_load_balancing_loss: 9.991784E-01 | loss scale: 1.0 | grad norm: 0.044 | params norm: 11540.540 | params rms: 0.062488 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 10:30:55.109975 | finish at 2026-10-10 19:30:36 + [2026-10-05 09:01:15] iteration 1642/ 6855 | consumed samples: 210176 | elapsed time per iteration (ms): 94273.6 | throughput per GPU (TFLOP/s/GPU): 39.0 | MFU 12.49% | FLOPs estimator: workload-aware | logical tokens/s: 177962.6 | valid tokens/s: 177962.6 | physical tokens/s: 178334.9 | loss tokens/s: 137674.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.678815E-01 | lm-global-supervised-tokens: 1.297909E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.454234E-02 | lm-token-buffer-mib: 5.822546E+01 | global_load_balancing_loss: 9.996373E-01 | loss scale: 1.0 | grad norm: 0.094 | params norm: 11540.463 | params rms: 0.062488 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 16:30:48.182870 | finish at 2026-10-11 01:32:03 + [2026-10-05 09:02:41] iteration 1643/ 6855 | consumed samples: 210304 | elapsed time per iteration (ms): 85812.7 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.68% | FLOPs estimator: workload-aware | logical tokens/s: 195508.8 | valid tokens/s: 195508.8 | physical tokens/s: 196122.2 | loss tokens/s: 151067.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.240224E-01 | lm-global-supervised-tokens: 1.296347E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.322136E-02 | lm-token-buffer-mib: 5.821533E+01 | global_load_balancing_loss: 9.996858E-01 | loss scale: 1.0 | grad norm: 0.160 | params norm: 11540.384 | params rms: 0.062487 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 4:14:15.836065 | finish at 2026-10-10 13:16:57 + [2026-10-05 09:04:09] iteration 1644/ 6855 | consumed samples: 210432 | elapsed time per iteration (ms): 87957.1 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.46% | FLOPs estimator: workload-aware | logical tokens/s: 190743.1 | valid tokens/s: 190743.1 | physical tokens/s: 191156.6 | loss tokens/s: 140443.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.236876E-01 | lm-global-supervised-tokens: 1.235297E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.171542E-02 | lm-token-buffer-mib: 5.803881E+01 | global_load_balancing_loss: 9.998856E-01 | loss scale: 1.0 | grad norm: 0.218 | params norm: 11540.307 | params rms: 0.062487 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 7:19:04.519714 | finish at 2026-10-10 16:23:13 + [2026-10-05 09:05:39] iteration 1645/ 6855 | consumed samples: 210560 | elapsed time per iteration (ms): 90644.8 | throughput per GPU (TFLOP/s/GPU): 41.6 | MFU 13.34% | FLOPs estimator: workload-aware | logical tokens/s: 185087.2 | valid tokens/s: 185087.2 | physical tokens/s: 185535.1 | loss tokens/s: 137045.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.043784E-01 | lm-global-supervised-tokens: 1.242244E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.870073E-02 | lm-token-buffer-mib: 5.812290E+01 | global_load_balancing_loss: 1.000900E+00 | loss scale: 1.0 | grad norm: 0.094 | params norm: 11540.228 | params rms: 0.062487 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 11:10:59.396548 | finish at 2026-10-10 20:16:39 + [2026-10-05 09:07:06] iteration 1646/ 6855 | consumed samples: 210688 | elapsed time per iteration (ms): 86627.6 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.74% | FLOPs estimator: workload-aware | logical tokens/s: 193670.3 | valid tokens/s: 193670.3 | physical tokens/s: 194086.2 | loss tokens/s: 135954.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.907883E-01 | lm-global-supervised-tokens: 1.177739E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.152596E-02 | lm-token-buffer-mib: 5.810223E+01 | global_load_balancing_loss: 1.001176E+00 | loss scale: 1.0 | grad norm: 0.123 | params norm: 11540.150 | params rms: 0.062486 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 5:20:42.981875 | finish at 2026-10-10 14:27:49 + [2026-10-05 09:08:35] iteration 1647/ 6855 | consumed samples: 210816 | elapsed time per iteration (ms): 88615.6 | throughput per GPU (TFLOP/s/GPU): 41.4 | MFU 13.26% | FLOPs estimator: workload-aware | logical tokens/s: 189325.1 | valid tokens/s: 189325.1 | physical tokens/s: 189960.1 | loss tokens/s: 134065.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.293873E-01 | lm-global-supervised-tokens: 1.188032E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.957113E-02 | lm-token-buffer-mib: 5.824126E+01 | global_load_balancing_loss: 1.001543E+00 | loss scale: 1.0 | grad norm: 0.102 | params norm: 11540.071 | params rms: 0.062486 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 8:11:49.864082 | finish at 2026-10-10 17:20:25 + [2026-10-05 09:10:02] iteration 1648/ 6855 | consumed samples: 210944 | elapsed time per iteration (ms): 87642.3 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.65% | FLOPs estimator: workload-aware | logical tokens/s: 191428.1 | valid tokens/s: 191428.1 | physical tokens/s: 191900.3 | loss tokens/s: 137759.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.013265E-01 | lm-global-supervised-tokens: 1.207354E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.729434E-02 | lm-token-buffer-mib: 5.812779E+01 | global_load_balancing_loss: 1.000817E+00 | loss scale: 1.0 | grad norm: 0.054 | params norm: 11539.995 | params rms: 0.062485 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 6:45:53.335110 | finish at 2026-10-10 15:55:56 + [2026-10-05 09:11:31] iteration 1649/ 6855 | consumed samples: 211072 | elapsed time per iteration (ms): 88733.1 | throughput per GPU (TFLOP/s/GPU): 41.6 | MFU 13.34% | FLOPs estimator: workload-aware | logical tokens/s: 189075.0 | valid tokens/s: 189075.0 | physical tokens/s: 189544.4 | loss tokens/s: 143751.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.212667E-01 | lm-global-supervised-tokens: 1.275550E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.228391E-02 | lm-token-buffer-mib: 5.814170E+01 | global_load_balancing_loss: 9.992439E-01 | loss scale: 1.0 | grad norm: 0.055 | params norm: 11539.919 | params rms: 0.062485 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 8:19:04.679631 | finish at 2026-10-10 17:30:36 + [2026-10-05 09:13:17] iteration 1650/ 6855 | consumed samples: 211200 | elapsed time per iteration (ms): 105450.0 | throughput per GPU (TFLOP/s/GPU): 34.7 | MFU 11.12% | FLOPs estimator: workload-aware | logical tokens/s: 159101.2 | valid tokens/s: 159101.2 | physical tokens/s: 159494.8 | loss tokens/s: 118505.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.160957E-01 | lm-global-supervised-tokens: 1.249644E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.322731E-02 | lm-token-buffer-mib: 5.820437E+01 | global_load_balancing_loss: 9.996517E-01 | loss scale: 1.0 | grad norm: 0.057 | params norm: 11539.843 | params rms: 0.062484 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 8:27:47.118706 | finish at 2026-10-11 17:41:04 + [2026-10-05 09:14:44] iteration 1651/ 6855 | consumed samples: 211328 | elapsed time per iteration (ms): 87819.2 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.48% | FLOPs estimator: workload-aware | logical tokens/s: 191041.8 | valid tokens/s: 191041.8 | physical tokens/s: 191333.2 | loss tokens/s: 141062.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.701757E-01 | lm-global-supervised-tokens: 1.238801E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.173933E-02 | lm-token-buffer-mib: 5.806669E+01 | global_load_balancing_loss: 9.994675E-01 | loss scale: 1.0 | grad norm: 0.042 | params norm: 11539.768 | params rms: 0.062484 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 6:56:51.301871 | finish at 2026-10-10 16:11:36 + [2026-10-05 09:16:13] iteration 1652/ 6855 | consumed samples: 211456 | elapsed time per iteration (ms): 88757.2 | throughput per GPU (TFLOP/s/GPU): 41.7 | MFU 13.36% | FLOPs estimator: workload-aware | logical tokens/s: 189023.7 | valid tokens/s: 189023.7 | physical tokens/s: 189371.7 | loss tokens/s: 139197.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.452366E-01 | lm-global-supervised-tokens: 1.235481E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.524531E-02 | lm-token-buffer-mib: 5.810209E+01 | global_load_balancing_loss: 9.990875E-01 | loss scale: 1.0 | grad norm: 0.036 | params norm: 11539.694 | params rms: 0.062484 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 8:16:43.617337 | finish at 2026-10-10 17:32:57 + [2026-10-05 09:17:39] iteration 1653/ 6855 | consumed samples: 211584 | elapsed time per iteration (ms): 86356.3 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.63% | FLOPs estimator: workload-aware | logical tokens/s: 194278.8 | valid tokens/s: 194278.8 | physical tokens/s: 194619.0 | loss tokens/s: 144922.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.796907E-01 | lm-global-supervised-tokens: 1.251494E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.906390E-02 | lm-token-buffer-mib: 5.822544E+01 | global_load_balancing_loss: 9.994333E-01 | loss scale: 1.0 | grad norm: 0.052 | params norm: 11539.619 | params rms: 0.062483 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 4:47:05.387624 | finish at 2026-10-10 14:04:45 + [2026-10-05 09:19:05] iteration 1654/ 6855 | consumed samples: 211712 | elapsed time per iteration (ms): 85763.9 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.70% | FLOPs estimator: workload-aware | logical tokens/s: 195620.5 | valid tokens/s: 195620.5 | physical tokens/s: 196033.2 | loss tokens/s: 149485.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.792532E-01 | lm-global-supervised-tokens: 1.282047E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.299754E-02 | lm-token-buffer-mib: 5.811123E+01 | global_load_balancing_loss: 1.001519E+00 | loss scale: 1.0 | grad norm: 0.039 | params norm: 11539.543 | params rms: 0.062483 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 3:54:18.249959 | finish at 2026-10-10 13:13:23 + [2026-10-05 09:20:33] iteration 1655/ 6855 | consumed samples: 211840 | elapsed time per iteration (ms): 87566.3 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.65% | FLOPs estimator: workload-aware | logical tokens/s: 191593.8 | valid tokens/s: 191593.8 | physical tokens/s: 191974.6 | loss tokens/s: 144306.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.264754E-01 | lm-global-supervised-tokens: 1.263635E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.763470E-02 | lm-token-buffer-mib: 5.803445E+01 | global_load_balancing_loss: 1.000738E+00 | loss scale: 1.0 | grad norm: 0.067 | params norm: 11539.468 | params rms: 0.062482 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 6:29:04.583511 | finish at 2026-10-10 15:49:37 + [2026-10-05 09:22:00] iteration 1656/ 6855 | consumed samples: 211968 | elapsed time per iteration (ms): 87403.1 | throughput per GPU (TFLOP/s/GPU): 41.1 | MFU 13.19% | FLOPs estimator: workload-aware | logical tokens/s: 191951.8 | valid tokens/s: 191951.8 | physical tokens/s: 192478.2 | loss tokens/s: 143578.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.442201E-01 | lm-global-supervised-tokens: 1.254922E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.139523E-02 | lm-token-buffer-mib: 5.825763E+01 | global_load_balancing_loss: 1.001128E+00 | loss scale: 1.0 | grad norm: 0.055 | params norm: 11539.393 | params rms: 0.062482 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 6:13:28.579383 | finish at 2026-10-10 15:35:29 + [2026-10-05 09:23:28] iteration 1657/ 6855 | consumed samples: 212096 | elapsed time per iteration (ms): 88264.6 | throughput per GPU (TFLOP/s/GPU): 41.7 | MFU 13.36% | FLOPs estimator: workload-aware | logical tokens/s: 190078.4 | valid tokens/s: 190078.4 | physical tokens/s: 190428.8 | loss tokens/s: 138474.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.427695E-01 | lm-global-supervised-tokens: 1.222244E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.302583E-02 | lm-token-buffer-mib: 5.810876E+01 | global_load_balancing_loss: 1.001133E+00 | loss scale: 1.0 | grad norm: 0.062 | params norm: 11539.317 | params rms: 0.062482 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 7:26:39.547152 | finish at 2026-10-10 16:50:08 + [2026-10-05 09:24:55] iteration 1658/ 6855 | consumed samples: 212224 | elapsed time per iteration (ms): 86864.1 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.67% | FLOPs estimator: workload-aware | logical tokens/s: 193142.6 | valid tokens/s: 193142.6 | physical tokens/s: 193613.1 | loss tokens/s: 138341.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.350296E-01 | lm-global-supervised-tokens: 1.201688E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.391953E-02 | lm-token-buffer-mib: 5.819392E+01 | global_load_balancing_loss: 1.000574E+00 | loss scale: 1.0 | grad norm: 0.062 | params norm: 11539.240 | params rms: 0.062481 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 5:23:52.478542 | finish at 2026-10-10 14:48:48 + [2026-10-05 09:26:23] iteration 1659/ 6855 | consumed samples: 212352 | elapsed time per iteration (ms): 87336.0 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.58% | FLOPs estimator: workload-aware | logical tokens/s: 192099.5 | valid tokens/s: 192099.5 | physical tokens/s: 192535.3 | loss tokens/s: 143233.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.265289E-01 | lm-global-supervised-tokens: 1.250944E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.441719E-02 | lm-token-buffer-mib: 5.806581E+01 | global_load_balancing_loss: 1.000306E+00 | loss scale: 1.0 | grad norm: 0.041 | params norm: 11539.164 | params rms: 0.062481 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 6:03:17.917763 | finish at 2026-10-10 15:29:41 + [2026-10-05 09:28:12] iteration 1660/ 6855 | consumed samples: 212480 | elapsed time per iteration (ms): 109334.2 | throughput per GPU (TFLOP/s/GPU): 33.4 | MFU 10.69% | FLOPs estimator: workload-aware | logical tokens/s: 153448.1 | valid tokens/s: 153448.1 | physical tokens/s: 153814.5 | loss tokens/s: 113625.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.436286E-01 | lm-global-supervised-tokens: 1.242316E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.242617E-02 | lm-token-buffer-mib: 5.806023E+01 | global_load_balancing_loss: 9.993081E-01 | loss scale: 1.0 | grad norm: 0.058 | params norm: 11539.089 | params rms: 0.062480 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 13:46:31.214336 | finish at 2026-10-11 23:14:43 + [2026-10-05 09:29:35] iteration 1661/ 6855 | consumed samples: 212608 | elapsed time per iteration (ms): 83123.0 | throughput per GPU (TFLOP/s/GPU): 42.8 | MFU 13.73% | FLOPs estimator: workload-aware | logical tokens/s: 201835.5 | valid tokens/s: 201835.5 | physical tokens/s: 202365.9 | loss tokens/s: 155703.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.910764E-01 | lm-global-supervised-tokens: 1.294252E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.211758E-02 | lm-token-buffer-mib: 5.801761E+01 | global_load_balancing_loss: 9.992945E-01 | loss scale: 1.0 | grad norm: 0.051 | params norm: 11539.014 | params rms: 0.062480 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 23:55:40.974204 | finish at 2026-10-10 09:25:16 + [2026-10-05 09:31:00] iteration 1662/ 6855 | consumed samples: 212736 | elapsed time per iteration (ms): 84927.4 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.45% | FLOPs estimator: workload-aware | logical tokens/s: 197547.2 | valid tokens/s: 197547.2 | physical tokens/s: 198253.8 | loss tokens/s: 144205.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.750564E-01 | lm-global-supervised-tokens: 1.224696E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.812213E-02 | lm-token-buffer-mib: 5.818909E+01 | global_load_balancing_loss: 1.000328E+00 | loss scale: 1.0 | grad norm: 0.096 | params norm: 11538.938 | params rms: 0.062480 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 2:30:28.011068 | finish at 2026-10-10 12:01:28 + [2026-10-05 09:32:28] iteration 1663/ 6855 | consumed samples: 212864 | elapsed time per iteration (ms): 87639.1 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.69% | FLOPs estimator: workload-aware | logical tokens/s: 191434.5 | valid tokens/s: 191434.5 | physical tokens/s: 191895.8 | loss tokens/s: 137041.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.757423E-01 | lm-global-supervised-tokens: 1.201022E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.284210E-02 | lm-token-buffer-mib: 5.808324E+01 | global_load_balancing_loss: 1.000679E+00 | loss scale: 1.0 | grad norm: 0.063 | params norm: 11538.863 | params rms: 0.062479 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 6:23:42.366035 | finish at 2026-10-10 15:56:10 + [2026-10-05 09:33:57] iteration 1664/ 6855 | consumed samples: 212992 | elapsed time per iteration (ms): 88807.0 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.45% | FLOPs estimator: workload-aware | logical tokens/s: 188917.5 | valid tokens/s: 188917.5 | physical tokens/s: 189391.2 | loss tokens/s: 135763.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.938142E-01 | lm-global-supervised-tokens: 1.205674E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.291310E-02 | lm-token-buffer-mib: 5.813503E+01 | global_load_balancing_loss: 9.994590E-01 | loss scale: 1.0 | grad norm: 0.051 | params norm: 11538.787 | params rms: 0.062479 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 8:03:17.090802 | finish at 2026-10-10 17:37:14 + [2026-10-05 09:35:26] iteration 1665/ 6855 | consumed samples: 213120 | elapsed time per iteration (ms): 89120.9 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.44% | FLOPs estimator: workload-aware | logical tokens/s: 188252.1 | valid tokens/s: 188252.1 | physical tokens/s: 188732.0 | loss tokens/s: 134178.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.134350E-01 | lm-global-supervised-tokens: 1.195808E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.441627E-02 | lm-token-buffer-mib: 5.809138E+01 | global_load_balancing_loss: 1.000388E+00 | loss scale: 1.0 | grad norm: 0.046 | params norm: 11538.711 | params rms: 0.062478 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 8:28:57.459426 | finish at 2026-10-10 18:04:23 + [2026-10-05 09:36:57] iteration 1666/ 6855 | consumed samples: 213248 | elapsed time per iteration (ms): 91529.2 | throughput per GPU (TFLOP/s/GPU): 40.7 | MFU 13.04% | FLOPs estimator: workload-aware | logical tokens/s: 183299.1 | valid tokens/s: 183299.1 | physical tokens/s: 183711.5 | loss tokens/s: 129106.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.663457E-01 | lm-global-supervised-tokens: 1.181704E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.165790E-02 | lm-token-buffer-mib: 5.806269E+01 | global_load_balancing_loss: 1.000008E+00 | loss scale: 1.0 | grad norm: 0.042 | params norm: 11538.634 | params rms: 0.062478 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 11:55:44.885587 | finish at 2026-10-10 21:32:42 + [2026-10-05 09:38:27] iteration 1667/ 6855 | consumed samples: 213376 | elapsed time per iteration (ms): 89558.2 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.40% | FLOPs estimator: workload-aware | logical tokens/s: 187332.8 | valid tokens/s: 187332.8 | physical tokens/s: 187779.6 | loss tokens/s: 142092.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.103139E-01 | lm-global-supervised-tokens: 1.272558E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.163982E-02 | lm-token-buffer-mib: 5.805641E+01 | global_load_balancing_loss: 1.000325E+00 | loss scale: 1.0 | grad norm: 0.108 | params norm: 11538.557 | params rms: 0.062478 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 9:03:47.834616 | finish at 2026-10-10 18:42:15 + [2026-10-05 09:39:50] iteration 1668/ 6855 | consumed samples: 213504 | elapsed time per iteration (ms): 82918.1 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.77% | FLOPs estimator: workload-aware | logical tokens/s: 202333.9 | valid tokens/s: 202333.9 | physical tokens/s: 202799.1 | loss tokens/s: 150250.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.443326E-01 | lm-global-supervised-tokens: 1.245851E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.445594E-02 | lm-token-buffer-mib: 5.837150E+01 | global_load_balancing_loss: 9.997699E-01 | loss scale: 1.0 | grad norm: 0.204 | params norm: 11538.482 | params rms: 0.062477 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 23:28:16.029494 | finish at 2026-10-10 09:08:06 + [2026-10-05 09:41:17] iteration 1669/ 6855 | consumed samples: 213632 | elapsed time per iteration (ms): 87701.7 | throughput per GPU (TFLOP/s/GPU): 41.6 | MFU 13.34% | FLOPs estimator: workload-aware | logical tokens/s: 191298.2 | valid tokens/s: 191298.2 | physical tokens/s: 191782.3 | loss tokens/s: 141774.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.438429E-01 | lm-global-supervised-tokens: 1.243386E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.656612E-02 | lm-token-buffer-mib: 5.815379E+01 | global_load_balancing_loss: 1.000129E+00 | loss scale: 1.0 | grad norm: 0.534 | params norm: 11538.405 | params rms: 0.062477 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 6:20:20.883757 | finish at 2026-10-10 16:01:38 + [2026-10-05 09:43:07] iteration 1670/ 6855 | consumed samples: 213760 | elapsed time per iteration (ms): 109758.5 | throughput per GPU (TFLOP/s/GPU): 34.0 | MFU 10.90% | FLOPs estimator: workload-aware | logical tokens/s: 152855.5 | valid tokens/s: 152855.5 | physical tokens/s: 153227.9 | loss tokens/s: 112453.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.336913E-01 | lm-global-supervised-tokens: 1.234276E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.556619E-02 | lm-token-buffer-mib: 5.809817E+01 | global_load_balancing_loss: 1.000941E+00 | loss scale: 1.0 | grad norm: 0.607 | params norm: 11538.330 | params rms: 0.062476 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 14:04:57.982484 | finish at 2026-10-11 23:48:05 + [2026-10-05 09:44:34] iteration 1671/ 6855 | consumed samples: 213888 | elapsed time per iteration (ms): 87256.4 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.42% | FLOPs estimator: workload-aware | logical tokens/s: 192274.7 | valid tokens/s: 192274.7 | physical tokens/s: 192676.7 | loss tokens/s: 145547.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.590531E-01 | lm-global-supervised-tokens: 1.269994E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.345246E-02 | lm-token-buffer-mib: 5.811131E+01 | global_load_balancing_loss: 1.000501E+00 | loss scale: 1.0 | grad norm: 0.131 | params norm: 11538.254 | params rms: 0.062476 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 5:38:57.039734 | finish at 2026-10-10 15:23:31 + [2026-10-05 09:46:03] iteration 1672/ 6855 | consumed samples: 214016 | elapsed time per iteration (ms): 88403.1 | throughput per GPU (TFLOP/s/GPU): 41.6 | MFU 13.35% | FLOPs estimator: workload-aware | logical tokens/s: 189780.4 | valid tokens/s: 189780.4 | physical tokens/s: 190373.6 | loss tokens/s: 141057.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.924660E-01 | lm-global-supervised-tokens: 1.246994E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.329363E-02 | lm-token-buffer-mib: 5.812292E+01 | global_load_balancing_loss: 1.001459E+00 | loss scale: 1.0 | grad norm: 0.545 | params norm: 11538.176 | params rms: 0.062475 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 7:16:33.352636 | finish at 2026-10-10 17:02:36 + [2026-10-05 09:47:29] iteration 1673/ 6855 | consumed samples: 214144 | elapsed time per iteration (ms): 86573.8 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.68% | FLOPs estimator: workload-aware | logical tokens/s: 193790.8 | valid tokens/s: 193790.8 | physical tokens/s: 194202.7 | loss tokens/s: 145350.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.421533E-01 | lm-global-supervised-tokens: 1.258355E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.143014E-02 | lm-token-buffer-mib: 5.807343E+01 | global_load_balancing_loss: 1.000368E+00 | loss scale: 1.0 | grad norm: 0.044 | params norm: 11538.099 | params rms: 0.062475 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 4:37:05.287499 | finish at 2026-10-10 14:24:35 + [2026-10-05 09:48:57] iteration 1674/ 6855 | consumed samples: 214272 | elapsed time per iteration (ms): 88025.5 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.54% | FLOPs estimator: workload-aware | logical tokens/s: 190594.4 | valid tokens/s: 190594.4 | physical tokens/s: 191000.9 | loss tokens/s: 138430.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.706616E-01 | lm-global-supervised-tokens: 1.218536E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.210725E-02 | lm-token-buffer-mib: 5.803350E+01 | global_load_balancing_loss: 1.000238E+00 | loss scale: 1.0 | grad norm: 0.120 | params norm: 11538.023 | params rms: 0.062475 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 6:40:59.918167 | finish at 2026-10-10 16:29:57 + [2026-10-05 09:50:21] iteration 1675/ 6855 | consumed samples: 214400 | elapsed time per iteration (ms): 83562.5 | throughput per GPU (TFLOP/s/GPU): 43.3 | MFU 13.89% | FLOPs estimator: workload-aware | logical tokens/s: 200774.1 | valid tokens/s: 200774.1 | physical tokens/s: 201251.9 | loss tokens/s: 146966.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.253315E-01 | lm-global-supervised-tokens: 1.228090E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.284185E-02 | lm-token-buffer-mib: 5.808974E+01 | global_load_balancing_loss: 9.997076E-01 | loss scale: 1.0 | grad norm: 0.254 | params norm: 11537.946 | params rms: 0.062474 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 0:14:13.873501 | finish at 2026-10-10 10:04:35 + [2026-10-05 09:51:50] iteration 1676/ 6855 | consumed samples: 214528 | elapsed time per iteration (ms): 88592.1 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.52% | FLOPs estimator: workload-aware | logical tokens/s: 189375.7 | valid tokens/s: 189375.7 | physical tokens/s: 189790.4 | loss tokens/s: 138881.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.651552E-01 | lm-global-supervised-tokens: 1.230379E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.291703E-02 | lm-token-buffer-mib: 5.814809E+01 | global_load_balancing_loss: 1.000281E+00 | loss scale: 1.0 | grad norm: 0.088 | params norm: 11537.869 | params rms: 0.062474 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 7:26:58.234750 | finish at 2026-10-10 17:18:48 + [2026-10-05 09:53:14] iteration 1677/ 6855 | consumed samples: 214656 | elapsed time per iteration (ms): 84779.4 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.75% | FLOPs estimator: workload-aware | logical tokens/s: 197891.4 | valid tokens/s: 197891.4 | physical tokens/s: 198377.3 | loss tokens/s: 146139.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.906505E-01 | lm-global-supervised-tokens: 1.238960E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.278101E-02 | lm-token-buffer-mib: 5.821758E+01 | global_load_balancing_loss: 1.000452E+00 | loss scale: 1.0 | grad norm: 0.090 | params norm: 11537.790 | params rms: 0.062473 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 1:56:27.652292 | finish at 2026-10-10 11:49:42 + [2026-10-05 09:54:42] iteration 1678/ 6855 | consumed samples: 214784 | elapsed time per iteration (ms): 87391.6 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.48% | FLOPs estimator: workload-aware | logical tokens/s: 191976.6 | valid tokens/s: 191976.6 | physical tokens/s: 192399.6 | loss tokens/s: 140858.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.093306E-01 | lm-global-supervised-tokens: 1.230985E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.356352E-02 | lm-token-buffer-mib: 5.807728E+01 | global_load_balancing_loss: 1.000103E+00 | loss scale: 1.0 | grad norm: 0.318 | params norm: 11537.712 | params rms: 0.062473 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 5:40:26.513839 | finish at 2026-10-10 15:35:08 + [2026-10-05 09:56:11] iteration 1679/ 6855 | consumed samples: 214912 | elapsed time per iteration (ms): 89672.8 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.51% | FLOPs estimator: workload-aware | logical tokens/s: 187093.5 | valid tokens/s: 187093.5 | physical tokens/s: 187464.4 | loss tokens/s: 141427.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.660414E-01 | lm-global-supervised-tokens: 1.268218E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.169096E-02 | lm-token-buffer-mib: 5.809205E+01 | global_load_balancing_loss: 9.987603E-01 | loss scale: 1.0 | grad norm: 0.072 | params norm: 11537.635 | params rms: 0.062473 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 8:55:46.189768 | finish at 2026-10-10 18:51:58 + [2026-10-05 09:58:01] iteration 1680/ 6855 | consumed samples: 215040 | elapsed time per iteration (ms): 109856.9 | throughput per GPU (TFLOP/s/GPU): 34.0 | MFU 10.91% | FLOPs estimator: workload-aware | logical tokens/s: 152718.8 | valid tokens/s: 152718.8 | physical tokens/s: 153119.1 | loss tokens/s: 112309.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.910533E-01 | lm-global-supervised-tokens: 1.233792E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.374785E-02 | lm-token-buffer-mib: 5.806062E+01 | global_load_balancing_loss: 1.000215E+00 | loss scale: 1.0 | grad norm: 0.111 | params norm: 11537.558 | params rms: 0.062472 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 13:55:09.697974 | finish at 2026-10-11 23:53:11 +times across ranks (ms): + save-checkpoint: + rank 0: 23700.46 + rank 1: 23700.46 + rank 2: 23700.47 + rank 3: 23700.46 + rank 4: 23700.49 + rank 5: 23700.46 + rank 6: 23700.45 + rank 7: 23700.45 + rank 8: 23700.95 + rank 9: 23700.91 + rank 10: 23700.93 + rank 11: 23700.93 + rank 12: 23700.93 + rank 13: 23700.91 + rank 14: 23700.92 + rank 15: 23700.91 + rank 16: 23700.08 + rank 17: 23700.07 + rank 18: 23700.06 + rank 19: 23700.06 + rank 20: 23700.06 + rank 21: 23700.07 + rank 22: 23700.06 + rank 23: 23700.07 + rank 24: 23700.49 + rank 25: 23700.50 + rank 26: 23700.51 + rank 27: 23700.52 + rank 28: 23700.49 + rank 29: 23700.49 + rank 30: 23700.48 + rank 31: 23700.50 + rank 32: 23700.50 + rank 33: 23700.50 + rank 34: 23700.47 + rank 35: 23700.48 + rank 36: 23700.43 + rank 37: 23700.42 + rank 38: 23700.46 + rank 39: 23700.43 + rank 40: 23700.31 + rank 41: 23700.28 + rank 42: 23700.30 + rank 43: 23700.31 + rank 44: 23700.28 + rank 45: 23700.29 + rank 46: 23700.28 + rank 47: 23700.28 + rank 48: 23700.62 + rank 49: 23700.61 + rank 50: 23700.58 + rank 51: 23700.60 + rank 52: 23700.61 + rank 53: 23700.58 + rank 54: 23700.60 + rank 55: 23700.60 + rank 56: 23700.60 + rank 57: 23700.61 + rank 58: 23700.56 + rank 59: 23700.59 + rank 60: 23700.58 + rank 61: 23700.62 + rank 62: 23700.58 + rank 63: 23700.61 + [2026-10-05 09:59:55] iteration 1681/ 6855 | consumed samples: 215168 | elapsed time per iteration (ms): 90151.2 | throughput per GPU (TFLOP/s/GPU): 40.4 | MFU 12.95% | FLOPs estimator: workload-aware | logical tokens/s: 186100.5 | valid tokens/s: 186100.5 | physical tokens/s: 186599.4 | loss tokens/s: 142411.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.509163E-01 | lm-global-supervised-tokens: 1.283859E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 5.753456E-01 | lm-token-buffer-mib: 5.824578E+01 | global_load_balancing_loss: 9.996604E-01 | loss scale: 1.0 | grad norm: 0.249 | params norm: 11537.479 | params rms: 0.062472 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 9:34:02.385572 | finish at 2026-10-10 19:33:58 + [2026-10-05 10:01:33] iteration 1682/ 6855 | consumed samples: 215296 | elapsed time per iteration (ms): 97514.6 | throughput per GPU (TFLOP/s/GPU): 39.1 | MFU 12.53% | FLOPs estimator: workload-aware | logical tokens/s: 172047.3 | valid tokens/s: 172047.3 | physical tokens/s: 172464.6 | loss tokens/s: 123442.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.880754E-01 | lm-global-supervised-tokens: 1.203746E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.688663E-02 | lm-token-buffer-mib: 5.813570E+01 | global_load_balancing_loss: 1.000820E+00 | loss scale: 1.0 | grad norm: 0.178 | params norm: 11537.403 | params rms: 0.062471 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 20:07:23.146867 | finish at 2026-10-11 06:08:56 + [2026-10-05 10:02:58] iteration 1683/ 6855 | consumed samples: 215424 | elapsed time per iteration (ms): 85200.2 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.66% | FLOPs estimator: workload-aware | logical tokens/s: 196915.2 | valid tokens/s: 196915.2 | physical tokens/s: 197536.4 | loss tokens/s: 153428.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.742775E-01 | lm-global-supervised-tokens: 1.307214E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.284186E-02 | lm-token-buffer-mib: 5.820965E+01 | global_load_balancing_loss: 1.000478E+00 | loss scale: 1.0 | grad norm: 0.134 | params norm: 11537.326 | params rms: 0.062471 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 2:24:15.575392 | finish at 2026-10-10 12:27:14 + [2026-10-05 10:04:27] iteration 1684/ 6855 | consumed samples: 215552 | elapsed time per iteration (ms): 88550.3 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.39% | FLOPs estimator: workload-aware | logical tokens/s: 189464.7 | valid tokens/s: 189464.7 | physical tokens/s: 189911.9 | loss tokens/s: 141762.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.641604E-01 | lm-global-supervised-tokens: 1.255311E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.151956E-02 | lm-token-buffer-mib: 5.809332E+01 | global_load_balancing_loss: 1.000223E+00 | loss scale: 1.0 | grad norm: 0.155 | params norm: 11537.249 | params rms: 0.062470 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 7:11:33.388642 | finish at 2026-10-10 17:16:00 + [2026-10-05 10:05:54] iteration 1685/ 6855 | consumed samples: 215680 | elapsed time per iteration (ms): 87009.8 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.75% | FLOPs estimator: workload-aware | logical tokens/s: 192819.3 | valid tokens/s: 192819.3 | physical tokens/s: 193248.7 | loss tokens/s: 145058.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.450631E-01 | lm-global-supervised-tokens: 1.262154E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.702555E-02 | lm-token-buffer-mib: 5.809342E+01 | global_load_balancing_loss: 1.001130E+00 | loss scale: 1.0 | grad norm: 0.094 | params norm: 11537.173 | params rms: 0.062470 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 4:57:20.562239 | finish at 2026-10-10 15:03:14 + [2026-10-05 10:07:20] iteration 1686/ 6855 | consumed samples: 215808 | elapsed time per iteration (ms): 86543.0 | throughput per GPU (TFLOP/s/GPU): 43.4 | MFU 13.91% | FLOPs estimator: workload-aware | logical tokens/s: 193859.3 | valid tokens/s: 193859.3 | physical tokens/s: 194352.1 | loss tokens/s: 139050.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.020423E-01 | lm-global-supervised-tokens: 1.203387E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.158445E-02 | lm-token-buffer-mib: 5.808006E+01 | global_load_balancing_loss: 1.000930E+00 | loss scale: 1.0 | grad norm: 0.060 | params norm: 11537.100 | params rms: 0.062470 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 4:15:40.680644 | finish at 2026-10-10 14:23:01 + [2026-10-05 10:08:47] iteration 1687/ 6855 | consumed samples: 215936 | elapsed time per iteration (ms): 87169.9 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.52% | FLOPs estimator: workload-aware | logical tokens/s: 192465.6 | valid tokens/s: 192465.6 | physical tokens/s: 192930.9 | loss tokens/s: 137110.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.467022E-01 | lm-global-supervised-tokens: 1.195194E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.595973E-02 | lm-token-buffer-mib: 5.811899E+01 | global_load_balancing_loss: 9.999081E-01 | loss scale: 1.0 | grad norm: 0.063 | params norm: 11537.027 | params rms: 0.062469 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 5:08:14.224018 | finish at 2026-10-10 15:17:02 + [2026-10-05 10:10:14] iteration 1688/ 6855 | consumed samples: 216064 | elapsed time per iteration (ms): 86937.2 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.59% | FLOPs estimator: workload-aware | logical tokens/s: 192980.2 | valid tokens/s: 192980.2 | physical tokens/s: 193402.8 | loss tokens/s: 144388.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.708103E-01 | lm-global-supervised-tokens: 1.255272E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.530890E-02 | lm-token-buffer-mib: 5.821960E+01 | global_load_balancing_loss: 9.995070E-01 | loss scale: 1.0 | grad norm: 0.059 | params norm: 11536.954 | params rms: 0.062469 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 4:46:44.319349 | finish at 2026-10-10 14:56:59 + [2026-10-05 10:11:43] iteration 1689/ 6855 | consumed samples: 216192 | elapsed time per iteration (ms): 89018.0 | throughput per GPU (TFLOP/s/GPU): 41.5 | MFU 13.31% | FLOPs estimator: workload-aware | logical tokens/s: 188469.8 | valid tokens/s: 188469.8 | physical tokens/s: 188912.3 | loss tokens/s: 136194.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.323464E-01 | lm-global-supervised-tokens: 1.212379E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.423458E-02 | lm-token-buffer-mib: 5.817083E+01 | global_load_balancing_loss: 9.993220E-01 | loss scale: 1.0 | grad norm: 0.066 | params norm: 11536.882 | params rms: 0.062468 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 7:44:27.225131 | finish at 2026-10-10 17:56:11 + [2026-10-05 10:13:32] iteration 1690/ 6855 | consumed samples: 216320 | elapsed time per iteration (ms): 108305.3 | throughput per GPU (TFLOP/s/GPU): 34.0 | MFU 10.89% | FLOPs estimator: workload-aware | logical tokens/s: 154906.1 | valid tokens/s: 154906.1 | physical tokens/s: 155249.6 | loss tokens/s: 116404.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.748173E-01 | lm-global-supervised-tokens: 1.260724E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.160443E-02 | lm-token-buffer-mib: 5.818572E+01 | global_load_balancing_loss: 9.990268E-01 | loss scale: 1.0 | grad norm: 0.074 | params norm: 11536.810 | params rms: 0.062468 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 11:23:16.927438 | finish at 2026-10-11 21:36:49 + [2026-10-05 10:15:00] iteration 1691/ 6855 | consumed samples: 216448 | elapsed time per iteration (ms): 87973.6 | throughput per GPU (TFLOP/s/GPU): 43.3 | MFU 13.87% | FLOPs estimator: workload-aware | logical tokens/s: 190707.5 | valid tokens/s: 190707.5 | physical tokens/s: 191005.2 | loss tokens/s: 137485.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.651754E-01 | lm-global-supervised-tokens: 1.209506E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.704837E-02 | lm-token-buffer-mib: 5.801809E+01 | global_load_balancing_loss: 1.000058E+00 | loss scale: 1.0 | grad norm: 0.073 | params norm: 11536.739 | params rms: 0.062468 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 6:11:35.519733 | finish at 2026-10-10 16:26:35 + [2026-10-05 10:16:30] iteration 1692/ 6855 | consumed samples: 216576 | elapsed time per iteration (ms): 90327.6 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.49% | FLOPs estimator: workload-aware | logical tokens/s: 185736.2 | valid tokens/s: 185736.2 | physical tokens/s: 186220.7 | loss tokens/s: 131037.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.103603E-01 | lm-global-supervised-tokens: 1.183629E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.334786E-02 | lm-token-buffer-mib: 5.819537E+01 | global_load_balancing_loss: 1.000745E+00 | loss scale: 1.0 | grad norm: 0.106 | params norm: 11536.669 | params rms: 0.062467 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 9:32:41.221554 | finish at 2026-10-10 19:49:11 + [2026-10-05 10:18:05] iteration 1693/ 6855 | consumed samples: 216704 | elapsed time per iteration (ms): 94963.2 | throughput per GPU (TFLOP/s/GPU): 40.2 | MFU 12.87% | FLOPs estimator: workload-aware | logical tokens/s: 176670.4 | valid tokens/s: 176670.4 | physical tokens/s: 177052.0 | loss tokens/s: 128764.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.490810E-01 | lm-global-supervised-tokens: 1.222787E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.663405E-02 | lm-token-buffer-mib: 5.806644E+01 | global_load_balancing_loss: 1.000417E+00 | loss scale: 1.0 | grad norm: 0.052 | params norm: 11536.597 | params rms: 0.062467 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 16:10:00.123796 | finish at 2026-10-11 02:28:05 + [2026-10-05 10:19:32] iteration 1694/ 6855 | consumed samples: 216832 | elapsed time per iteration (ms): 87048.4 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.76% | FLOPs estimator: workload-aware | logical tokens/s: 192734.0 | valid tokens/s: 192734.0 | physical tokens/s: 193280.7 | loss tokens/s: 141003.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.168195E-01 | lm-global-supervised-tokens: 1.227410E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.058658E-02 | lm-token-buffer-mib: 5.816259E+01 | global_load_balancing_loss: 1.000986E+00 | loss scale: 1.0 | grad norm: 0.146 | params norm: 11536.526 | params rms: 0.062467 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 4:47:36.692345 | finish at 2026-10-10 15:07:09 + [2026-10-05 10:21:04] iteration 1695/ 6855 | consumed samples: 216960 | elapsed time per iteration (ms): 92123.4 | throughput per GPU (TFLOP/s/GPU): 41.3 | MFU 13.24% | FLOPs estimator: workload-aware | logical tokens/s: 182116.6 | valid tokens/s: 182116.6 | physical tokens/s: 182739.7 | loss tokens/s: 126858.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.215891E-01 | lm-global-supervised-tokens: 1.168659E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.514225E-02 | lm-token-buffer-mib: 5.850502E+01 | global_load_balancing_loss: 1.000952E+00 | loss scale: 1.0 | grad norm: 0.044 | params norm: 11536.457 | params rms: 0.062466 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 12:02:36.511345 | finish at 2026-10-10 22:23:41 + [2026-10-05 10:22:33] iteration 1696/ 6855 | consumed samples: 217088 | elapsed time per iteration (ms): 89074.0 | throughput per GPU (TFLOP/s/GPU): 41.5 | MFU 13.30% | FLOPs estimator: workload-aware | logical tokens/s: 188351.3 | valid tokens/s: 188351.2 | physical tokens/s: 188698.9 | loss tokens/s: 143296.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.881655E-01 | lm-global-supervised-tokens: 1.276398E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.129166E-02 | lm-token-buffer-mib: 5.815404E+01 | global_load_balancing_loss: 1.000023E+00 | loss scale: 1.0 | grad norm: 0.056 | params norm: 11536.385 | params rms: 0.062466 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 7:38:52.536610 | finish at 2026-10-10 18:01:26 + [2026-10-05 10:24:01] iteration 1697/ 6855 | consumed samples: 217216 | elapsed time per iteration (ms): 87693.2 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.61% | FLOPs estimator: workload-aware | logical tokens/s: 191317.0 | valid tokens/s: 191317.0 | physical tokens/s: 191697.8 | loss tokens/s: 138325.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.032273E-01 | lm-global-supervised-tokens: 1.213024E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.514598E-02 | lm-token-buffer-mib: 5.813292E+01 | global_load_balancing_loss: 1.000718E+00 | loss scale: 1.0 | grad norm: 0.098 | params norm: 11536.315 | params rms: 0.062465 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 5:38:41.512647 | finish at 2026-10-10 16:02:42 + [2026-10-05 10:25:29] iteration 1698/ 6855 | consumed samples: 217344 | elapsed time per iteration (ms): 87804.2 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.47% | FLOPs estimator: workload-aware | logical tokens/s: 191075.0 | valid tokens/s: 191075.0 | physical tokens/s: 191566.0 | loss tokens/s: 139568.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.738275E-01 | lm-global-supervised-tokens: 1.225474E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.167201E-02 | lm-token-buffer-mib: 5.809309E+01 | global_load_balancing_loss: 1.001444E+00 | loss scale: 1.0 | grad norm: 0.059 | params norm: 11536.244 | params rms: 0.062465 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 5:46:46.299634 | finish at 2026-10-10 16:12:15 + [2026-10-05 10:26:56] iteration 1699/ 6855 | consumed samples: 217472 | elapsed time per iteration (ms): 87517.1 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.65% | FLOPs estimator: workload-aware | logical tokens/s: 191701.8 | valid tokens/s: 191701.8 | physical tokens/s: 192153.1 | loss tokens/s: 134222.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.657603E-01 | lm-global-supervised-tokens: 1.174678E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.317380E-02 | lm-token-buffer-mib: 5.805755E+01 | global_load_balancing_loss: 1.000466E+00 | loss scale: 1.0 | grad norm: 0.035 | params norm: 11536.172 | params rms: 0.062465 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 5:20:38.342652 | finish at 2026-10-10 15:47:34 + [2026-10-05 10:28:42] iteration 1700/ 6855 | consumed samples: 217600 | elapsed time per iteration (ms): 106130.8 | throughput per GPU (TFLOP/s/GPU): 34.4 | MFU 11.01% | FLOPs estimator: workload-aware | logical tokens/s: 158080.0 | valid tokens/s: 158080.0 | physical tokens/s: 158441.6 | loss tokens/s: 118282.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.456437E-01 | lm-global-supervised-tokens: 1.255337E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.408419E-02 | lm-token-buffer-mib: 5.810866E+01 | global_load_balancing_loss: 1.000448E+00 | loss scale: 1.0 | grad norm: 0.047 | params norm: 11536.100 | params rms: 0.062464 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 7:58:24.511251 | finish at 2026-10-11 18:27:07 + [2026-10-05 10:30:07] iteration 1701/ 6855 | consumed samples: 217728 | elapsed time per iteration (ms): 84779.9 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.53% | FLOPs estimator: workload-aware | logical tokens/s: 197891.1 | valid tokens/s: 197891.1 | physical tokens/s: 198452.7 | loss tokens/s: 147746.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.480279E-01 | lm-global-supervised-tokens: 1.252594E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.385308E-02 | lm-token-buffer-mib: 5.847830E+01 | global_load_balancing_loss: 9.995378E-01 | loss scale: 1.0 | grad norm: 0.062 | params norm: 11536.030 | params rms: 0.062464 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 1:22:35.515278 | finish at 2026-10-10 11:52:43 + [2026-10-05 10:31:33] iteration 1702/ 6855 | consumed samples: 217856 | elapsed time per iteration (ms): 86141.5 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.76% | FLOPs estimator: workload-aware | logical tokens/s: 194763.4 | valid tokens/s: 194763.4 | physical tokens/s: 195176.9 | loss tokens/s: 143723.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.814300E-01 | lm-global-supervised-tokens: 1.238059E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.642388E-02 | lm-token-buffer-mib: 5.820257E+01 | global_load_balancing_loss: 9.985457E-01 | loss scale: 1.0 | grad norm: 0.041 | params norm: 11535.962 | params rms: 0.062463 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 3:18:07.178966 | finish at 2026-10-10 13:49:40 + [2026-10-05 10:33:00] iteration 1703/ 6855 | consumed samples: 217984 | elapsed time per iteration (ms): 87228.2 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.65% | FLOPs estimator: workload-aware | logical tokens/s: 192336.4 | valid tokens/s: 192336.4 | physical tokens/s: 192757.9 | loss tokens/s: 144685.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.859751E-01 | lm-global-supervised-tokens: 1.262064E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.136601E-02 | lm-token-buffer-mib: 5.817507E+01 | global_load_balancing_loss: 9.992207E-01 | loss scale: 1.0 | grad norm: 0.064 | params norm: 11535.893 | params rms: 0.062463 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 4:49:59.876579 | finish at 2026-10-10 15:23:00 + [2026-10-05 10:34:27] iteration 1704/ 6855 | consumed samples: 218112 | elapsed time per iteration (ms): 86480.8 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.43% | FLOPs estimator: workload-aware | logical tokens/s: 193998.8 | valid tokens/s: 193998.8 | physical tokens/s: 194477.3 | loss tokens/s: 139558.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.447794E-01 | lm-global-supervised-tokens: 1.206911E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.197412E-02 | lm-token-buffer-mib: 5.815845E+01 | global_load_balancing_loss: 1.000091E+00 | loss scale: 1.0 | grad norm: 0.055 | params norm: 11535.824 | params rms: 0.062463 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 3:44:22.828779 | finish at 2026-10-10 14:18:50 + [2026-10-05 10:35:54] iteration 1705/ 6855 | consumed samples: 218240 | elapsed time per iteration (ms): 86876.4 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.60% | FLOPs estimator: workload-aware | logical tokens/s: 193115.9 | valid tokens/s: 193115.9 | physical tokens/s: 193512.8 | loss tokens/s: 141035.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.511337E-01 | lm-global-supervised-tokens: 1.225264E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.967753E-02 | lm-token-buffer-mib: 5.819533E+01 | global_load_balancing_loss: 1.000347E+00 | loss scale: 1.0 | grad norm: 0.054 | params norm: 11535.756 | params rms: 0.062462 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 4:16:53.520133 | finish at 2026-10-10 14:52:47 + [2026-10-05 10:37:22] iteration 1706/ 6855 | consumed samples: 218368 | elapsed time per iteration (ms): 88707.1 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.41% | FLOPs estimator: workload-aware | logical tokens/s: 189129.4 | valid tokens/s: 189129.4 | physical tokens/s: 189594.2 | loss tokens/s: 138347.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.472946E-01 | lm-global-supervised-tokens: 1.227238E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.271089E-02 | lm-token-buffer-mib: 5.839935E+01 | global_load_balancing_loss: 1.000669E+00 | loss scale: 1.0 | grad norm: 0.045 | params norm: 11535.687 | params rms: 0.062462 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 6:52:32.857460 | finish at 2026-10-10 17:29:55 + [2026-10-05 10:38:51] iteration 1707/ 6855 | consumed samples: 218496 | elapsed time per iteration (ms): 88690.4 | throughput per GPU (TFLOP/s/GPU): 40.7 | MFU 13.06% | FLOPs estimator: workload-aware | logical tokens/s: 189165.3 | valid tokens/s: 189165.3 | physical tokens/s: 190040.2 | loss tokens/s: 142127.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.442255E-01 | lm-global-supervised-tokens: 1.260531E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.411384E-02 | lm-token-buffer-mib: 5.866119E+01 | global_load_balancing_loss: 1.000085E+00 | loss scale: 1.0 | grad norm: 0.107 | params norm: 11535.615 | params rms: 0.062462 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 6:49:38.081646 | finish at 2026-10-10 17:28:29 + [2026-10-05 10:40:17] iteration 1708/ 6855 | consumed samples: 218624 | elapsed time per iteration (ms): 86075.0 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.58% | FLOPs estimator: workload-aware | logical tokens/s: 194913.9 | valid tokens/s: 194913.9 | physical tokens/s: 195409.1 | loss tokens/s: 139949.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.210706E-01 | lm-global-supervised-tokens: 1.204614E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.367885E-02 | lm-token-buffer-mib: 5.819702E+01 | global_load_balancing_loss: 1.001520E+00 | loss scale: 1.0 | grad norm: 0.154 | params norm: 11535.541 | params rms: 0.062461 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 3:03:48.147959 | finish at 2026-10-10 13:44:05 + [2026-10-05 10:41:42] iteration 1709/ 6855 | consumed samples: 218752 | elapsed time per iteration (ms): 85242.8 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.55% | FLOPs estimator: workload-aware | logical tokens/s: 196816.0 | valid tokens/s: 196816.0 | physical tokens/s: 197325.1 | loss tokens/s: 147106.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.715723E-01 | lm-global-supervised-tokens: 1.253975E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.305859E-02 | lm-token-buffer-mib: 5.808010E+01 | global_load_balancing_loss: 1.001284E+00 | loss scale: 1.0 | grad norm: 0.091 | params norm: 11535.468 | params rms: 0.062461 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 1:50:59.419341 | finish at 2026-10-10 12:32:42 + [2026-10-05 10:43:32] iteration 1710/ 6855 | consumed samples: 218880 | elapsed time per iteration (ms): 109015.2 | throughput per GPU (TFLOP/s/GPU): 34.1 | MFU 10.95% | FLOPs estimator: workload-aware | logical tokens/s: 153897.8 | valid tokens/s: 153897.8 | physical tokens/s: 154248.6 | loss tokens/s: 109512.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.993397E-01 | lm-global-supervised-tokens: 1.193856E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.346727E-02 | lm-token-buffer-mib: 5.815130E+01 | global_load_balancing_loss: 9.998912E-01 | loss scale: 1.0 | grad norm: 0.065 | params norm: 11535.396 | params rms: 0.062460 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 11:48:02.994951 | finish at 2026-10-11 22:31:35 + [2026-10-05 10:45:01] iteration 1711/ 6855 | consumed samples: 219008 | elapsed time per iteration (ms): 89729.6 | throughput per GPU (TFLOP/s/GPU): 41.6 | MFU 13.32% | FLOPs estimator: workload-aware | logical tokens/s: 186975.0 | valid tokens/s: 186975.0 | physical tokens/s: 187408.0 | loss tokens/s: 130831.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 5.987714E-01 | lm-global-supervised-tokens: 1.173947E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.150697E-02 | lm-token-buffer-mib: 5.811284E+01 | global_load_balancing_loss: 1.000359E+00 | loss scale: 1.0 | grad norm: 0.097 | params norm: 11535.320 | params rms: 0.062460 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 8:12:49.032722 | finish at 2026-10-10 18:57:50 + [2026-10-05 10:46:30] iteration 1712/ 6855 | consumed samples: 219136 | elapsed time per iteration (ms): 88802.2 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.47% | FLOPs estimator: workload-aware | logical tokens/s: 188926.9 | valid tokens/s: 188926.9 | physical tokens/s: 189505.2 | loss tokens/s: 136829.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.771532E-01 | lm-global-supervised-tokens: 1.215073E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.599200E-02 | lm-token-buffer-mib: 5.807162E+01 | global_load_balancing_loss: 9.990236E-01 | loss scale: 1.0 | grad norm: 0.067 | params norm: 11535.243 | params rms: 0.062460 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 6:51:49.662280 | finish at 2026-10-10 17:38:20 + [2026-10-05 10:47:56] iteration 1713/ 6855 | consumed samples: 219264 | elapsed time per iteration (ms): 86458.2 | throughput per GPU (TFLOP/s/GPU): 43.2 | MFU 13.85% | FLOPs estimator: workload-aware | logical tokens/s: 194049.1 | valid tokens/s: 194049.1 | physical tokens/s: 194343.8 | loss tokens/s: 142188.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.525062E-01 | lm-global-supervised-tokens: 1.229333E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.170520E-02 | lm-token-buffer-mib: 5.808180E+01 | global_load_balancing_loss: 9.992187E-01 | loss scale: 1.0 | grad norm: 0.038 | params norm: 11535.167 | params rms: 0.062459 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 3:29:27.920850 | finish at 2026-10-10 14:17:24 + [2026-10-05 10:49:20] iteration 1714/ 6855 | consumed samples: 219392 | elapsed time per iteration (ms): 83608.6 | throughput per GPU (TFLOP/s/GPU): 43.5 | MFU 13.94% | FLOPs estimator: workload-aware | logical tokens/s: 200663.8 | valid tokens/s: 200663.8 | physical tokens/s: 201107.4 | loss tokens/s: 151028.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.234626E-01 | lm-global-supervised-tokens: 1.262729E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.492027E-02 | lm-token-buffer-mib: 5.808538E+01 | global_load_balancing_loss: 9.999778E-01 | loss scale: 1.0 | grad norm: 0.081 | params norm: 11535.091 | params rms: 0.062459 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 23:23:51.671135 | finish at 2026-10-10 10:13:12 + [2026-10-05 10:50:48] iteration 1715/ 6855 | consumed samples: 219520 | elapsed time per iteration (ms): 88053.7 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.58% | FLOPs estimator: workload-aware | logical tokens/s: 190533.2 | valid tokens/s: 190533.2 | physical tokens/s: 191044.3 | loss tokens/s: 139545.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.912552E-01 | lm-global-supervised-tokens: 1.228746E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.159139E-02 | lm-token-buffer-mib: 5.813028E+01 | global_load_balancing_loss: 1.000745E+00 | loss scale: 1.0 | grad norm: 0.032 | params norm: 11535.016 | params rms: 0.062458 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 5:43:16.169806 | finish at 2026-10-10 16:34:04 + [2026-10-05 10:52:20] iteration 1716/ 6855 | consumed samples: 219648 | elapsed time per iteration (ms): 91651.3 | throughput per GPU (TFLOP/s/GPU): 41.5 | MFU 13.32% | FLOPs estimator: workload-aware | logical tokens/s: 183054.6 | valid tokens/s: 183054.6 | physical tokens/s: 183409.8 | loss tokens/s: 133721.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.198676E-01 | lm-global-supervised-tokens: 1.225573E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.306661E-02 | lm-token-buffer-mib: 5.814866E+01 | global_load_balancing_loss: 9.995968E-01 | loss scale: 1.0 | grad norm: 0.081 | params norm: 11534.940 | params rms: 0.062458 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 10:49:55.873631 | finish at 2026-10-10 21:42:16 + [2026-10-05 10:53:44] iteration 1717/ 6855 | consumed samples: 219776 | elapsed time per iteration (ms): 84577.0 | throughput per GPU (TFLOP/s/GPU): 43.2 | MFU 13.84% | FLOPs estimator: workload-aware | logical tokens/s: 198365.8 | valid tokens/s: 198365.8 | physical tokens/s: 198762.5 | loss tokens/s: 145214.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.475962E-01 | lm-global-supervised-tokens: 1.228182E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.100262E-02 | lm-token-buffer-mib: 5.815294E+01 | global_load_balancing_loss: 1.001159E+00 | loss scale: 1.0 | grad norm: 0.027 | params norm: 11534.864 | params rms: 0.062458 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 0:42:36.572825 | finish at 2026-10-10 11:36:21 + [2026-10-05 10:55:15] iteration 1718/ 6855 | consumed samples: 219904 | elapsed time per iteration (ms): 90147.8 | throughput per GPU (TFLOP/s/GPU): 41.3 | MFU 13.24% | FLOPs estimator: workload-aware | logical tokens/s: 186107.0 | valid tokens/s: 186107.0 | physical tokens/s: 186534.1 | loss tokens/s: 135389.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.369207E-01 | lm-global-supervised-tokens: 1.220504E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.132762E-02 | lm-token-buffer-mib: 5.818214E+01 | global_load_balancing_loss: 1.001194E+00 | loss scale: 1.0 | grad norm: 0.087 | params norm: 11534.788 | params rms: 0.062457 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 8:38:08.999814 | finish at 2026-10-10 19:33:24 + [2026-10-05 10:56:42] iteration 1719/ 6855 | consumed samples: 220032 | elapsed time per iteration (ms): 87939.2 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.50% | FLOPs estimator: workload-aware | logical tokens/s: 190781.3 | valid tokens/s: 190781.3 | physical tokens/s: 191248.0 | loss tokens/s: 137551.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.017534E-01 | lm-global-supervised-tokens: 1.209612E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.351292E-02 | lm-token-buffer-mib: 5.819875E+01 | global_load_balancing_loss: 1.001119E+00 | loss scale: 1.0 | grad norm: 0.047 | params norm: 11534.712 | params rms: 0.062457 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 5:27:35.709995 | finish at 2026-10-10 16:24:18 + [2026-10-05 10:58:31] iteration 1720/ 6855 | consumed samples: 220160 | elapsed time per iteration (ms): 108547.6 | throughput per GPU (TFLOP/s/GPU): 33.3 | MFU 10.68% | FLOPs estimator: workload-aware | logical tokens/s: 154560.4 | valid tokens/s: 154560.4 | physical tokens/s: 155048.0 | loss tokens/s: 116224.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.297061E-01 | lm-global-supervised-tokens: 1.261592E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.150445E-02 | lm-token-buffer-mib: 5.824217E+01 | global_load_balancing_loss: 1.001170E+00 | loss scale: 1.0 | grad norm: 0.074 | params norm: 11534.637 | params rms: 0.062456 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 10:49:52.095109 | finish at 2026-10-11 21:48:23 +times across ranks (ms): + save-checkpoint: + rank 0: 26002.45 + rank 1: 26002.50 + rank 2: 26002.50 + rank 3: 26002.49 + rank 4: 26002.50 + rank 5: 26002.46 + rank 6: 26002.43 + rank 7: 26002.48 + rank 8: 26000.80 + rank 9: 26000.78 + rank 10: 26000.79 + rank 11: 26000.83 + rank 12: 26000.79 + rank 13: 26000.79 + rank 14: 26000.78 + rank 15: 26000.77 + rank 16: 26002.52 + rank 17: 26002.62 + rank 18: 26002.49 + rank 19: 26002.53 + rank 20: 26002.50 + rank 21: 26002.50 + rank 22: 26002.47 + rank 23: 26002.52 + rank 24: 26001.64 + rank 25: 26001.57 + rank 26: 26001.60 + rank 27: 26001.64 + rank 28: 26001.57 + rank 29: 26001.68 + rank 30: 26001.56 + rank 31: 26001.73 + rank 32: 26001.62 + rank 33: 26001.63 + rank 34: 26001.62 + rank 35: 26001.59 + rank 36: 26001.58 + rank 37: 26001.55 + rank 38: 26001.57 + rank 39: 26001.62 + rank 40: 26001.21 + rank 41: 26001.25 + rank 42: 26001.21 + rank 43: 26001.20 + rank 44: 26001.19 + rank 45: 26001.19 + rank 46: 26001.19 + rank 47: 26001.19 + rank 48: 26000.83 + rank 49: 26000.83 + rank 50: 26000.81 + rank 51: 26000.83 + rank 52: 26000.83 + rank 53: 26000.82 + rank 54: 26000.82 + rank 55: 26000.81 + rank 56: 26002.26 + rank 57: 26002.20 + rank 58: 26002.18 + rank 59: 26002.16 + rank 60: 26002.17 + rank 61: 26002.19 + rank 62: 26002.18 + rank 63: 26002.17 + [2026-10-05 11:00:57] iteration 1721/ 6855 | consumed samples: 220288 | elapsed time per iteration (ms): 120256.5 | throughput per GPU (TFLOP/s/GPU): 30.6 | MFU 9.82% | FLOPs estimator: workload-aware | logical tokens/s: 139511.6 | valid tokens/s: 139511.6 | physical tokens/s: 139822.1 | loss tokens/s: 102429.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.863275E-01 | lm-global-supervised-tokens: 1.231781E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 8.042684E-01 | lm-token-buffer-mib: 5.806712E+01 | global_load_balancing_loss: 9.995242E-01 | loss scale: 1.0 | grad norm: 0.126 | params norm: 11534.562 | params rms: 0.062456 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 7 days, 3:29:56.650702 | finish at 2026-10-12 14:30:54 + [2026-10-05 11:02:27] iteration 1722/ 6855 | consumed samples: 220416 | elapsed time per iteration (ms): 89934.0 | throughput per GPU (TFLOP/s/GPU): 40.8 | MFU 13.06% | FLOPs estimator: workload-aware | logical tokens/s: 186550.2 | valid tokens/s: 186550.2 | physical tokens/s: 186955.7 | loss tokens/s: 138943.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.412317E-01 | lm-global-supervised-tokens: 1.249576E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.964288E-02 | lm-token-buffer-mib: 5.825271E+01 | global_load_balancing_loss: 9.988814E-01 | loss scale: 1.0 | grad norm: 0.112 | params norm: 11534.485 | params rms: 0.062455 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 8:13:51.306521 | finish at 2026-10-10 19:16:19 + [2026-10-05 11:03:51] iteration 1723/ 6855 | consumed samples: 220544 | elapsed time per iteration (ms): 84041.8 | throughput per GPU (TFLOP/s/GPU): 43.5 | MFU 13.93% | FLOPs estimator: workload-aware | logical tokens/s: 199629.2 | valid tokens/s: 199629.2 | physical tokens/s: 200025.7 | loss tokens/s: 146813.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.515936E-01 | lm-global-supervised-tokens: 1.233844E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.554001E-02 | lm-token-buffer-mib: 5.813565E+01 | global_load_balancing_loss: 9.985911E-01 | loss scale: 1.0 | grad norm: 0.033 | params norm: 11534.410 | params rms: 0.062455 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 23:48:22.629058 | finish at 2026-10-10 10:52:14 + [2026-10-05 11:05:17] iteration 1724/ 6855 | consumed samples: 220672 | elapsed time per iteration (ms): 85904.6 | throughput per GPU (TFLOP/s/GPU): 43.4 | MFU 13.91% | FLOPs estimator: workload-aware | logical tokens/s: 195299.8 | valid tokens/s: 195299.8 | physical tokens/s: 195758.5 | loss tokens/s: 150162.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.189980E-01 | lm-global-supervised-tokens: 1.289962E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.121174E-02 | lm-token-buffer-mib: 5.819505E+01 | global_load_balancing_loss: 9.995716E-01 | loss scale: 1.0 | grad norm: 0.114 | params norm: 11534.334 | params rms: 0.062455 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 2:26:16.544929 | finish at 2026-10-10 13:31:34 + [2026-10-05 11:06:42] iteration 1725/ 6855 | consumed samples: 220800 | elapsed time per iteration (ms): 84933.2 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.59% | FLOPs estimator: workload-aware | logical tokens/s: 197533.2 | valid tokens/s: 197533.2 | physical tokens/s: 198133.6 | loss tokens/s: 149256.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.539213E-01 | lm-global-supervised-tokens: 1.267687E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.478733E-02 | lm-token-buffer-mib: 5.808801E+01 | global_load_balancing_loss: 1.000331E+00 | loss scale: 1.0 | grad norm: 0.140 | params norm: 11534.256 | params rms: 0.062454 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 1:01:47.458498 | finish at 2026-10-10 12:08:30 + [2026-10-05 11:08:09] iteration 1726/ 6855 | consumed samples: 220928 | elapsed time per iteration (ms): 87114.7 | throughput per GPU (TFLOP/s/GPU): 42.8 | MFU 13.72% | FLOPs estimator: workload-aware | logical tokens/s: 192586.6 | valid tokens/s: 192586.6 | physical tokens/s: 193242.7 | loss tokens/s: 143328.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.001004E-01 | lm-global-supervised-tokens: 1.248601E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.140214E-02 | lm-token-buffer-mib: 5.817924E+01 | global_load_balancing_loss: 1.001445E+00 | loss scale: 1.0 | grad norm: 0.061 | params norm: 11534.179 | params rms: 0.062454 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 4:06:51.502144 | finish at 2026-10-10 15:15:01 + [2026-10-05 11:09:37] iteration 1727/ 6855 | consumed samples: 221056 | elapsed time per iteration (ms): 88022.0 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.54% | FLOPs estimator: workload-aware | logical tokens/s: 190601.4 | valid tokens/s: 190601.3 | physical tokens/s: 191037.8 | loss tokens/s: 138838.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.326777E-01 | lm-global-supervised-tokens: 1.222088E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.479720E-02 | lm-token-buffer-mib: 5.834596E+01 | global_load_balancing_loss: 1.001107E+00 | loss scale: 1.0 | grad norm: 0.044 | params norm: 11534.101 | params rms: 0.062453 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 5:22:56.991215 | finish at 2026-10-10 16:32:34 + [2026-10-05 11:11:05] iteration 1728/ 6855 | consumed samples: 221184 | elapsed time per iteration (ms): 87582.6 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.39% | FLOPs estimator: workload-aware | logical tokens/s: 191558.1 | valid tokens/s: 191558.1 | physical tokens/s: 192115.8 | loss tokens/s: 141576.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.691762E-01 | lm-global-supervised-tokens: 1.239965E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.249757E-02 | lm-token-buffer-mib: 5.820786E+01 | global_load_balancing_loss: 1.001417E+00 | loss scale: 1.0 | grad norm: 0.050 | params norm: 11534.024 | params rms: 0.062453 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 4:43:56.031137 | finish at 2026-10-10 15:55:01 + [2026-10-05 11:12:32] iteration 1729/ 6855 | consumed samples: 221312 | elapsed time per iteration (ms): 87488.3 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.60% | FLOPs estimator: workload-aware | logical tokens/s: 191764.5 | valid tokens/s: 191764.5 | physical tokens/s: 192147.3 | loss tokens/s: 139553.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.022645E-01 | lm-global-supervised-tokens: 1.220931E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.503462E-02 | lm-token-buffer-mib: 5.813105E+01 | global_load_balancing_loss: 9.998234E-01 | loss scale: 1.0 | grad norm: 0.039 | params norm: 11533.947 | params rms: 0.062453 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 4:34:25.027458 | finish at 2026-10-10 15:46:57 + [2026-10-05 11:14:20] iteration 1730/ 6855 | consumed samples: 221440 | elapsed time per iteration (ms): 107882.0 | throughput per GPU (TFLOP/s/GPU): 34.3 | MFU 10.98% | FLOPs estimator: workload-aware | logical tokens/s: 155513.9 | valid tokens/s: 155513.9 | physical tokens/s: 155972.2 | loss tokens/s: 113580.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.474759E-01 | lm-global-supervised-tokens: 1.225332E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.350054E-02 | lm-token-buffer-mib: 5.805560E+01 | global_load_balancing_loss: 9.998779E-01 | loss scale: 1.0 | grad norm: 0.058 | params norm: 11533.872 | params rms: 0.062452 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 9:34:55.233959 | finish at 2026-10-11 20:49:16 + [2026-10-05 11:15:46] iteration 1731/ 6855 | consumed samples: 221568 | elapsed time per iteration (ms): 86165.1 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.58% | FLOPs estimator: workload-aware | logical tokens/s: 194710.0 | valid tokens/s: 194710.0 | physical tokens/s: 195118.0 | loss tokens/s: 140308.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.417815E-01 | lm-global-supervised-tokens: 1.208972E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.147373E-02 | lm-token-buffer-mib: 5.822183E+01 | global_load_balancing_loss: 9.997559E-01 | loss scale: 1.0 | grad norm: 0.062 | params norm: 11533.797 | params rms: 0.062452 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 2:38:29.801868 | finish at 2026-10-10 13:54:16 + [2026-10-05 11:17:13] iteration 1732/ 6855 | consumed samples: 221696 | elapsed time per iteration (ms): 86666.2 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.61% | FLOPs estimator: workload-aware | logical tokens/s: 193583.6 | valid tokens/s: 193583.5 | physical tokens/s: 194163.8 | loss tokens/s: 144887.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.986750E-01 | lm-global-supervised-tokens: 1.255683E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.310888E-02 | lm-token-buffer-mib: 5.813661E+01 | global_load_balancing_loss: 9.998655E-01 | loss scale: 1.0 | grad norm: 0.092 | params norm: 11533.721 | params rms: 0.062451 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 3:19:51.075338 | finish at 2026-10-10 14:37:04 + [2026-10-05 11:18:44] iteration 1733/ 6855 | consumed samples: 221824 | elapsed time per iteration (ms): 90478.4 | throughput per GPU (TFLOP/s/GPU): 40.6 | MFU 13.02% | FLOPs estimator: workload-aware | logical tokens/s: 185427.6 | valid tokens/s: 185427.6 | physical tokens/s: 185875.0 | loss tokens/s: 134736.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.597839E-01 | lm-global-supervised-tokens: 1.219070E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.769801E-02 | lm-token-buffer-mib: 5.811315E+01 | global_load_balancing_loss: 9.991456E-01 | loss scale: 1.0 | grad norm: 0.224 | params norm: 11533.647 | params rms: 0.062451 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 8:43:50.157158 | finish at 2026-10-10 20:02:34 + [2026-10-05 11:20:10] iteration 1734/ 6855 | consumed samples: 221952 | elapsed time per iteration (ms): 86030.7 | throughput per GPU (TFLOP/s/GPU): 43.4 | MFU 13.90% | FLOPs estimator: workload-aware | logical tokens/s: 195013.9 | valid tokens/s: 195013.9 | physical tokens/s: 195402.5 | loss tokens/s: 140893.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.298445E-01 | lm-global-supervised-tokens: 1.212112E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.452259E-02 | lm-token-buffer-mib: 5.810961E+01 | global_load_balancing_loss: 9.999261E-01 | loss scale: 1.0 | grad norm: 0.519 | params norm: 11533.573 | params rms: 0.062451 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 2:22:43.169363 | finish at 2026-10-10 13:42:53 + [2026-10-05 11:21:37] iteration 1735/ 6855 | consumed samples: 222080 | elapsed time per iteration (ms): 87692.7 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.47% | FLOPs estimator: workload-aware | logical tokens/s: 191318.0 | valid tokens/s: 191318.0 | physical tokens/s: 191714.7 | loss tokens/s: 141310.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.555533E-01 | lm-global-supervised-tokens: 1.239186E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.184218E-02 | lm-token-buffer-mib: 5.805880E+01 | global_load_balancing_loss: 9.996709E-01 | loss scale: 1.0 | grad norm: 0.220 | params norm: 11533.500 | params rms: 0.062450 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 4:43:06.396484 | finish at 2026-10-10 16:04:44 + [2026-10-05 11:23:10] iteration 1736/ 6855 | consumed samples: 222208 | elapsed time per iteration (ms): 92486.8 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.49% | FLOPs estimator: workload-aware | logical tokens/s: 181401.0 | valid tokens/s: 181401.0 | physical tokens/s: 181648.3 | loss tokens/s: 122717.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.691473E-01 | lm-global-supervised-tokens: 1.134978E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.160731E-02 | lm-token-buffer-mib: 5.808130E+01 | global_load_balancing_loss: 1.001329E+00 | loss scale: 1.0 | grad norm: 0.210 | params norm: 11533.425 | params rms: 0.062450 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 11:30:40.151096 | finish at 2026-10-10 22:53:50 + [2026-10-05 11:24:33] iteration 1737/ 6855 | consumed samples: 222336 | elapsed time per iteration (ms): 83534.7 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.77% | FLOPs estimator: workload-aware | logical tokens/s: 200840.8 | valid tokens/s: 200840.8 | physical tokens/s: 201393.1 | loss tokens/s: 151029.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.703732E-01 | lm-global-supervised-tokens: 1.261620E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.273407E-02 | lm-token-buffer-mib: 5.810082E+01 | global_load_balancing_loss: 1.000255E+00 | loss scale: 1.0 | grad norm: 0.053 | params norm: 11533.349 | params rms: 0.062449 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 22:45:30.539264 | finish at 2026-10-10 10:10:04 + [2026-10-05 11:26:01] iteration 1738/ 6855 | consumed samples: 222464 | elapsed time per iteration (ms): 87827.9 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.52% | FLOPs estimator: workload-aware | logical tokens/s: 191023.4 | valid tokens/s: 191023.4 | physical tokens/s: 191495.0 | loss tokens/s: 142128.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.700880E-01 | lm-global-supervised-tokens: 1.248280E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.344649E-02 | lm-token-buffer-mib: 5.805415E+01 | global_load_balancing_loss: 1.000284E+00 | loss scale: 1.0 | grad norm: 0.101 | params norm: 11533.273 | params rms: 0.062449 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 4:50:15.150459 | finish at 2026-10-10 16:16:16 + [2026-10-05 11:27:28] iteration 1739/ 6855 | consumed samples: 222592 | elapsed time per iteration (ms): 87107.5 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.44% | FLOPs estimator: workload-aware | logical tokens/s: 192603.0 | valid tokens/s: 192603.0 | physical tokens/s: 192975.7 | loss tokens/s: 148674.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.419258E-01 | lm-global-supervised-tokens: 1.295062E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.370317E-02 | lm-token-buffer-mib: 5.814826E+01 | global_load_balancing_loss: 9.998956E-01 | loss scale: 1.0 | grad norm: 0.120 | params norm: 11533.198 | params rms: 0.062449 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 3:47:22.020400 | finish at 2026-10-10 15:14:50 + [2026-10-05 11:29:16] iteration 1740/ 6855 | consumed samples: 222720 | elapsed time per iteration (ms): 107340.4 | throughput per GPU (TFLOP/s/GPU): 35.2 | MFU 11.27% | FLOPs estimator: workload-aware | logical tokens/s: 156298.9 | valid tokens/s: 156298.9 | physical tokens/s: 156553.4 | loss tokens/s: 115834.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.055444E-01 | lm-global-supervised-tokens: 1.243376E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.491086E-02 | lm-token-buffer-mib: 5.809894E+01 | global_load_balancing_loss: 9.999186E-01 | loss scale: 1.0 | grad norm: 0.050 | params norm: 11533.124 | params rms: 0.062448 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 8:30:45.943462 | finish at 2026-10-11 20:00:02 + [2026-10-05 11:30:40] iteration 1741/ 6855 | consumed samples: 222848 | elapsed time per iteration (ms): 83860.7 | throughput per GPU (TFLOP/s/GPU): 43.5 | MFU 13.93% | FLOPs estimator: workload-aware | logical tokens/s: 200060.2 | valid tokens/s: 200060.2 | physical tokens/s: 200548.5 | loss tokens/s: 147353.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.443695E-01 | lm-global-supervised-tokens: 1.235716E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.267709E-02 | lm-token-buffer-mib: 5.812022E+01 | global_load_balancing_loss: 1.001354E+00 | loss scale: 1.0 | grad norm: 0.107 | params norm: 11533.049 | params rms: 0.062448 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 23:07:43.457085 | finish at 2026-10-10 10:38:23 + [2026-10-05 11:32:07] iteration 1742/ 6855 | consumed samples: 222976 | elapsed time per iteration (ms): 87547.3 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.78% | FLOPs estimator: workload-aware | logical tokens/s: 191634.8 | valid tokens/s: 191634.8 | physical tokens/s: 192084.4 | loss tokens/s: 134061.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.385035E-01 | lm-global-supervised-tokens: 1.173668E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.552714E-02 | lm-token-buffer-mib: 5.819989E+01 | global_load_balancing_loss: 1.001500E+00 | loss scale: 1.0 | grad norm: 0.091 | params norm: 11532.973 | params rms: 0.062447 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 4:20:29.134520 | finish at 2026-10-10 15:52:36 + [2026-10-05 11:33:37] iteration 1743/ 6855 | consumed samples: 223104 | elapsed time per iteration (ms): 89571.2 | throughput per GPU (TFLOP/s/GPU): 41.5 | MFU 13.31% | FLOPs estimator: workload-aware | logical tokens/s: 187304.7 | valid tokens/s: 187304.7 | physical tokens/s: 187861.8 | loss tokens/s: 136757.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.637596E-01 | lm-global-supervised-tokens: 1.224953E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.135919E-02 | lm-token-buffer-mib: 5.853150E+01 | global_load_balancing_loss: 1.001056E+00 | loss scale: 1.0 | grad norm: 0.057 | params norm: 11532.898 | params rms: 0.062447 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 7:11:27.947044 | finish at 2026-10-10 18:45:05 + [2026-10-05 11:35:06] iteration 1744/ 6855 | consumed samples: 223232 | elapsed time per iteration (ms): 88960.3 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.57% | FLOPs estimator: workload-aware | logical tokens/s: 188591.8 | valid tokens/s: 188591.8 | physical tokens/s: 189058.2 | loss tokens/s: 142575.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.162374E-01 | lm-global-supervised-tokens: 1.268352E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.164269E-02 | lm-token-buffer-mib: 5.818112E+01 | global_load_balancing_loss: 9.991077E-01 | loss scale: 1.0 | grad norm: 0.091 | params norm: 11532.822 | params rms: 0.062446 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 6:17:56.024901 | finish at 2026-10-10 17:53:02 + [2026-10-05 11:36:32] iteration 1745/ 6855 | consumed samples: 223360 | elapsed time per iteration (ms): 86046.5 | throughput per GPU (TFLOP/s/GPU): 41.4 | MFU 13.28% | FLOPs estimator: workload-aware | logical tokens/s: 194978.2 | valid tokens/s: 194978.2 | physical tokens/s: 195604.2 | loss tokens/s: 149291.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.787346E-01 | lm-global-supervised-tokens: 1.284604E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.404121E-02 | lm-token-buffer-mib: 5.804610E+01 | global_load_balancing_loss: 9.985707E-01 | loss scale: 1.0 | grad norm: 0.043 | params norm: 11532.746 | params rms: 0.062446 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 2:08:17.521024 | finish at 2026-10-10 13:44:49 + [2026-10-05 11:37:59] iteration 1746/ 6855 | consumed samples: 223488 | elapsed time per iteration (ms): 86893.1 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.66% | FLOPs estimator: workload-aware | logical tokens/s: 193078.1 | valid tokens/s: 193078.1 | physical tokens/s: 193660.7 | loss tokens/s: 142653.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.500328E-01 | lm-global-supervised-tokens: 1.239559E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.173459E-02 | lm-token-buffer-mib: 5.807935E+01 | global_load_balancing_loss: 9.993070E-01 | loss scale: 1.0 | grad norm: 0.052 | params norm: 11532.672 | params rms: 0.062446 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 3:18:56.744482 | finish at 2026-10-10 14:56:55 + [2026-10-05 11:39:29] iteration 1747/ 6855 | consumed samples: 223616 | elapsed time per iteration (ms): 90472.9 | throughput per GPU (TFLOP/s/GPU): 41.4 | MFU 13.28% | FLOPs estimator: workload-aware | logical tokens/s: 185438.8 | valid tokens/s: 185438.8 | physical tokens/s: 185988.1 | loss tokens/s: 138333.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.011291E-01 | lm-global-supervised-tokens: 1.251539E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.385327E-02 | lm-token-buffer-mib: 5.831480E+01 | global_load_balancing_loss: 1.000394E+00 | loss scale: 1.0 | grad norm: 0.047 | params norm: 11532.597 | params rms: 0.062445 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 8:22:15.578849 | finish at 2026-10-10 20:01:45 + [2026-10-05 11:40:59] iteration 1748/ 6855 | consumed samples: 223744 | elapsed time per iteration (ms): 89978.9 | throughput per GPU (TFLOP/s/GPU): 41.4 | MFU 13.27% | FLOPs estimator: workload-aware | logical tokens/s: 186457.3 | valid tokens/s: 186457.2 | physical tokens/s: 186904.9 | loss tokens/s: 138595.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.417438E-01 | lm-global-supervised-tokens: 1.247066E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.218648E-02 | lm-token-buffer-mib: 5.821803E+01 | global_load_balancing_loss: 9.998353E-01 | loss scale: 1.0 | grad norm: 0.060 | params norm: 11532.522 | params rms: 0.062445 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 7:38:42.003424 | finish at 2026-10-10 19:19:41 + [2026-10-05 11:42:27] iteration 1749/ 6855 | consumed samples: 223872 | elapsed time per iteration (ms): 88341.4 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.62% | FLOPs estimator: workload-aware | logical tokens/s: 189913.0 | valid tokens/s: 189913.0 | physical tokens/s: 190431.1 | loss tokens/s: 141188.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.730920E-01 | lm-global-supervised-tokens: 1.247279E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.593642E-02 | lm-token-buffer-mib: 5.802399E+01 | global_load_balancing_loss: 1.000367E+00 | loss scale: 1.0 | grad norm: 0.040 | params norm: 11532.447 | params rms: 0.062444 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 5:17:50.952979 | finish at 2026-10-10 17:00:18 + [2026-10-05 11:44:17] iteration 1750/ 6855 | consumed samples: 224000 | elapsed time per iteration (ms): 109590.6 | throughput per GPU (TFLOP/s/GPU): 34.2 | MFU 10.96% | FLOPs estimator: workload-aware | logical tokens/s: 153089.1 | valid tokens/s: 153089.1 | physical tokens/s: 153487.1 | loss tokens/s: 109404.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.130447E-01 | lm-global-supervised-tokens: 1.198964E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.252585E-02 | lm-token-buffer-mib: 5.847465E+01 | global_load_balancing_loss: 1.001777E+00 | loss scale: 1.0 | grad norm: 0.039 | params norm: 11532.372 | params rms: 0.062444 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 11:24:19.848758 | finish at 2026-10-11 23:08:37 + [2026-10-05 11:45:42] iteration 1751/ 6855 | consumed samples: 224128 | elapsed time per iteration (ms): 84819.4 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.77% | FLOPs estimator: workload-aware | logical tokens/s: 197798.7 | valid tokens/s: 197798.7 | physical tokens/s: 198367.7 | loss tokens/s: 153907.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.755764E-01 | lm-global-supervised-tokens: 1.305437E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.130402E-02 | lm-token-buffer-mib: 5.804349E+01 | global_load_balancing_loss: 1.000612E+00 | loss scale: 1.0 | grad norm: 0.069 | params norm: 11532.298 | params rms: 0.062444 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 0:15:18.148605 | finish at 2026-10-10 12:01:00 + [2026-10-05 11:47:08] iteration 1752/ 6855 | consumed samples: 224256 | elapsed time per iteration (ms): 86365.7 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.41% | FLOPs estimator: workload-aware | logical tokens/s: 194257.9 | valid tokens/s: 194257.9 | physical tokens/s: 194582.3 | loss tokens/s: 145965.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.880423E-01 | lm-global-supervised-tokens: 1.260644E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.286547E-02 | lm-token-buffer-mib: 5.812743E+01 | global_load_balancing_loss: 1.000445E+00 | loss scale: 1.0 | grad norm: 0.059 | params norm: 11532.226 | params rms: 0.062443 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 2:25:23.955436 | finish at 2026-10-10 14:12:32 + [2026-10-05 11:48:37] iteration 1753/ 6855 | consumed samples: 224384 | elapsed time per iteration (ms): 88681.0 | throughput per GPU (TFLOP/s/GPU): 43.7 | MFU 14.01% | FLOPs estimator: workload-aware | logical tokens/s: 189185.6 | valid tokens/s: 189185.6 | physical tokens/s: 189599.2 | loss tokens/s: 136728.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.434513E-01 | lm-global-supervised-tokens: 1.212520E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.153242E-02 | lm-token-buffer-mib: 5.824178E+01 | global_load_balancing_loss: 9.987826E-01 | loss scale: 1.0 | grad norm: 0.059 | params norm: 11532.153 | params rms: 0.062443 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 5:40:50.708902 | finish at 2026-10-10 17:29:27 + [2026-10-05 11:50:05] iteration 1754/ 6855 | consumed samples: 224512 | elapsed time per iteration (ms): 88253.9 | throughput per GPU (TFLOP/s/GPU): 42.8 | MFU 13.73% | FLOPs estimator: workload-aware | logical tokens/s: 190101.6 | valid tokens/s: 190101.6 | physical tokens/s: 190465.3 | loss tokens/s: 139838.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.473006E-01 | lm-global-supervised-tokens: 1.234131E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.878617E-02 | lm-token-buffer-mib: 5.815504E+01 | global_load_balancing_loss: 9.991592E-01 | loss scale: 1.0 | grad norm: 0.055 | params norm: 11532.080 | params rms: 0.062442 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 5:03:03.226861 | finish at 2026-10-10 16:53:08 + [2026-10-05 11:51:35] iteration 1755/ 6855 | consumed samples: 224640 | elapsed time per iteration (ms): 90002.0 | throughput per GPU (TFLOP/s/GPU): 40.8 | MFU 13.07% | FLOPs estimator: workload-aware | logical tokens/s: 186409.0 | valid tokens/s: 186409.0 | physical tokens/s: 186858.3 | loss tokens/s: 139765.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.617514E-01 | lm-global-supervised-tokens: 1.257917E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.274641E-02 | lm-token-buffer-mib: 5.807127E+01 | global_load_balancing_loss: 1.000466E+00 | loss scale: 1.0 | grad norm: 0.061 | params norm: 11532.006 | params rms: 0.062442 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 7:30:09.979177 | finish at 2026-10-10 19:21:45 + [2026-10-05 11:53:00] iteration 1756/ 6855 | consumed samples: 224768 | elapsed time per iteration (ms): 85038.0 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.74% | FLOPs estimator: workload-aware | logical tokens/s: 197290.4 | valid tokens/s: 197290.4 | physical tokens/s: 197861.1 | loss tokens/s: 149366.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.195159E-01 | lm-global-supervised-tokens: 1.270180E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.155669E-02 | lm-token-buffer-mib: 5.811819E+01 | global_load_balancing_loss: 1.000774E+00 | loss scale: 1.0 | grad norm: 0.076 | params norm: 11531.934 | params rms: 0.062442 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 0:26:48.863447 | finish at 2026-10-10 12:19:49 + [2026-10-05 11:54:27] iteration 1757/ 6855 | consumed samples: 224896 | elapsed time per iteration (ms): 87348.9 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.75% | FLOPs estimator: workload-aware | logical tokens/s: 192071.2 | valid tokens/s: 192071.2 | physical tokens/s: 192679.5 | loss tokens/s: 131014.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.278106E-01 | lm-global-supervised-tokens: 1.144396E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.593857E-02 | lm-token-buffer-mib: 5.835574E+01 | global_load_balancing_loss: 1.001531E+00 | loss scale: 1.0 | grad norm: 0.094 | params norm: 11531.861 | params rms: 0.062441 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 3:41:44.889511 | finish at 2026-10-10 15:36:12 + [2026-10-05 11:55:56] iteration 1758/ 6855 | consumed samples: 225024 | elapsed time per iteration (ms): 88076.6 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.70% | FLOPs estimator: workload-aware | logical tokens/s: 190483.8 | valid tokens/s: 190483.8 | physical tokens/s: 190857.6 | loss tokens/s: 142707.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.799605E-01 | lm-global-supervised-tokens: 1.256918E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.124840E-02 | lm-token-buffer-mib: 5.809223E+01 | global_load_balancing_loss: 1.000242E+00 | loss scale: 1.0 | grad norm: 0.077 | params norm: 11531.787 | params rms: 0.062441 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 4:42:06.239792 | finish at 2026-10-10 16:38:02 + [2026-10-05 11:57:23] iteration 1759/ 6855 | consumed samples: 225152 | elapsed time per iteration (ms): 87349.3 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.77% | FLOPs estimator: workload-aware | logical tokens/s: 192070.0 | valid tokens/s: 192070.0 | physical tokens/s: 192460.4 | loss tokens/s: 145858.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.831412E-01 | lm-global-supervised-tokens: 1.274059E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.160690E-02 | lm-token-buffer-mib: 5.818777E+01 | global_load_balancing_loss: 1.000790E+00 | loss scale: 1.0 | grad norm: 0.089 | params norm: 11531.715 | params rms: 0.062440 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 3:38:52.018965 | finish at 2026-10-10 15:36:15 + [2026-10-05 11:59:11] iteration 1760/ 6855 | consumed samples: 225280 | elapsed time per iteration (ms): 108504.7 | throughput per GPU (TFLOP/s/GPU): 33.7 | MFU 10.80% | FLOPs estimator: workload-aware | logical tokens/s: 154621.3 | valid tokens/s: 154621.3 | physical tokens/s: 154978.7 | loss tokens/s: 119173.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.587663E-01 | lm-global-supervised-tokens: 1.293088E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.332668E-02 | lm-token-buffer-mib: 5.807411E+01 | global_load_balancing_loss: 9.995990E-01 | loss scale: 1.0 | grad norm: 0.060 | params norm: 11531.644 | params rms: 0.062440 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 9:33:51.645440 | finish at 2026-10-11 21:33:03 +times across ranks (ms): + save-checkpoint: + rank 0: 24980.11 + rank 1: 24980.11 + rank 2: 24980.18 + rank 3: 24980.14 + rank 4: 24980.13 + rank 5: 24980.14 + rank 6: 24980.10 + rank 7: 24980.12 + rank 8: 24980.15 + rank 9: 24980.14 + rank 10: 24980.12 + rank 11: 24980.10 + rank 12: 24980.12 + rank 13: 24980.16 + rank 14: 24980.15 + rank 15: 24980.13 + rank 16: 24980.09 + rank 17: 24980.10 + rank 18: 24980.07 + rank 19: 24980.09 + rank 20: 24980.07 + rank 21: 24980.09 + rank 22: 24980.07 + rank 23: 24980.13 + rank 24: 24980.52 + rank 25: 24980.46 + rank 26: 24980.52 + rank 27: 24980.25 + rank 28: 24980.54 + rank 29: 24980.52 + rank 30: 24980.44 + rank 31: 24980.49 + rank 32: 24980.79 + rank 33: 24980.72 + rank 34: 24980.74 + rank 35: 24980.69 + rank 36: 24980.71 + rank 37: 24980.69 + rank 38: 24980.69 + rank 39: 24980.66 + rank 40: 24980.27 + rank 41: 24980.28 + rank 42: 24980.27 + rank 43: 24980.27 + rank 44: 24980.29 + rank 45: 24980.24 + rank 46: 24980.24 + rank 47: 24980.23 + rank 48: 24979.60 + rank 49: 24979.76 + rank 50: 24979.74 + rank 51: 24979.59 + rank 52: 24979.69 + rank 53: 24979.72 + rank 54: 24979.66 + rank 55: 24979.79 + rank 56: 24979.69 + rank 57: 24979.68 + rank 58: 24979.68 + rank 59: 24979.67 + rank 60: 24979.67 + rank 61: 24979.69 + rank 62: 24979.68 + rank 63: 24979.75 + [2026-10-05 12:01:24] iteration 1761/ 6855 | consumed samples: 225408 | elapsed time per iteration (ms): 106899.5 | throughput per GPU (TFLOP/s/GPU): 34.5 | MFU 11.06% | FLOPs estimator: workload-aware | logical tokens/s: 156943.3 | valid tokens/s: 156943.3 | physical tokens/s: 157337.6 | loss tokens/s: 115192.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.482210E-01 | lm-global-supervised-tokens: 1.231401E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.444706E+00 | lm-token-buffer-mib: 5.811852E+01 | global_load_balancing_loss: 9.996144E-01 | loss scale: 1.0 | grad norm: 0.135 | params norm: 11531.573 | params rms: 0.062440 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 7:15:45.939507 | finish at 2026-10-11 19:17:09 + [2026-10-05 12:02:52] iteration 1762/ 6855 | consumed samples: 225536 | elapsed time per iteration (ms): 88871.8 | throughput per GPU (TFLOP/s/GPU): 40.8 | MFU 13.07% | FLOPs estimator: workload-aware | logical tokens/s: 188779.8 | valid tokens/s: 188779.8 | physical tokens/s: 189210.6 | loss tokens/s: 141816.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.722476E-01 | lm-global-supervised-tokens: 1.260345E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.998145E-02 | lm-token-buffer-mib: 5.819073E+01 | global_load_balancing_loss: 9.997286E-01 | loss scale: 1.0 | grad norm: 0.072 | params norm: 11531.501 | params rms: 0.062439 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 5:43:44.188837 | finish at 2026-10-10 17:46:37 + [2026-10-05 12:04:20] iteration 1763/ 6855 | consumed samples: 225664 | elapsed time per iteration (ms): 87697.6 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.70% | FLOPs estimator: workload-aware | logical tokens/s: 191306.1 | valid tokens/s: 191306.1 | physical tokens/s: 191798.5 | loss tokens/s: 130100.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.771527E-01 | lm-global-supervised-tokens: 1.140951E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.166617E-02 | lm-token-buffer-mib: 5.807812E+01 | global_load_balancing_loss: 1.000022E+00 | loss scale: 1.0 | grad norm: 0.101 | params norm: 11531.428 | params rms: 0.062439 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 4:02:35.929753 | finish at 2026-10-10 16:06:56 + [2026-10-05 12:05:48] iteration 1764/ 6855 | consumed samples: 225792 | elapsed time per iteration (ms): 88171.6 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.43% | FLOPs estimator: workload-aware | logical tokens/s: 190278.0 | valid tokens/s: 190278.0 | physical tokens/s: 190763.8 | loss tokens/s: 139210.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.276431E-01 | lm-global-supervised-tokens: 1.227440E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.401514E-02 | lm-token-buffer-mib: 5.817675E+01 | global_load_balancing_loss: 9.998862E-01 | loss scale: 1.0 | grad norm: 0.075 | params norm: 11531.355 | params rms: 0.062439 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 4:41:21.747216 | finish at 2026-10-10 16:47:10 + [2026-10-05 12:07:14] iteration 1765/ 6855 | consumed samples: 225920 | elapsed time per iteration (ms): 85951.5 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.54% | FLOPs estimator: workload-aware | logical tokens/s: 195194.0 | valid tokens/s: 195194.0 | physical tokens/s: 195628.1 | loss tokens/s: 150353.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.598178E-01 | lm-global-supervised-tokens: 1.292314E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.183448E-02 | lm-token-buffer-mib: 5.804456E+01 | global_load_balancing_loss: 1.000591E+00 | loss scale: 1.0 | grad norm: 0.060 | params norm: 11531.282 | params rms: 0.062438 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 1:31:33.076439 | finish at 2026-10-10 13:38:47 + [2026-10-05 12:08:42] iteration 1766/ 6855 | consumed samples: 226048 | elapsed time per iteration (ms): 87417.9 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.45% | FLOPs estimator: workload-aware | logical tokens/s: 191919.4 | valid tokens/s: 191919.4 | physical tokens/s: 192286.7 | loss tokens/s: 144392.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.813948E-01 | lm-global-supervised-tokens: 1.262248E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.168024E-02 | lm-token-buffer-mib: 5.815218E+01 | global_load_balancing_loss: 1.000758E+00 | loss scale: 1.0 | grad norm: 0.051 | params norm: 11531.208 | params rms: 0.062438 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 3:34:29.566137 | finish at 2026-10-10 15:43:11 + [2026-10-05 12:10:09] iteration 1767/ 6855 | consumed samples: 226176 | elapsed time per iteration (ms): 86998.3 | throughput per GPU (TFLOP/s/GPU): 44.3 | MFU 14.18% | FLOPs estimator: workload-aware | logical tokens/s: 192845.2 | valid tokens/s: 192845.2 | physical tokens/s: 193152.7 | loss tokens/s: 143931.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.003917E-01 | lm-global-supervised-tokens: 1.252182E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.297720E-02 | lm-token-buffer-mib: 5.812301E+01 | global_load_balancing_loss: 1.000706E+00 | loss scale: 1.0 | grad norm: 0.050 | params norm: 11531.134 | params rms: 0.062437 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 2:57:27.582481 | finish at 2026-10-10 15:07:36 + [2026-10-05 12:11:37] iteration 1768/ 6855 | consumed samples: 226304 | elapsed time per iteration (ms): 88684.1 | throughput per GPU (TFLOP/s/GPU): 41.2 | MFU 13.22% | FLOPs estimator: workload-aware | logical tokens/s: 189179.0 | valid tokens/s: 189179.0 | physical tokens/s: 189642.4 | loss tokens/s: 142140.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.470893E-01 | lm-global-supervised-tokens: 1.260556E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.844153E-02 | lm-token-buffer-mib: 5.808860E+01 | global_load_balancing_loss: 1.001454E+00 | loss scale: 1.0 | grad norm: 0.072 | params norm: 11531.062 | params rms: 0.062437 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 5:18:56.093877 | finish at 2026-10-10 17:30:33 + [2026-10-05 12:13:05] iteration 1769/ 6855 | consumed samples: 226432 | elapsed time per iteration (ms): 87799.3 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.45% | FLOPs estimator: workload-aware | logical tokens/s: 191085.5 | valid tokens/s: 191085.5 | physical tokens/s: 191522.3 | loss tokens/s: 133707.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.992832E-01 | lm-global-supervised-tokens: 1.173943E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.178836E-02 | lm-token-buffer-mib: 5.821712E+01 | global_load_balancing_loss: 9.999856E-01 | loss scale: 1.0 | grad norm: 0.056 | params norm: 11530.991 | params rms: 0.062437 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 4:02:27.215321 | finish at 2026-10-10 16:15:32 + [2026-10-05 12:14:56] iteration 1770/ 6855 | consumed samples: 226560 | elapsed time per iteration (ms): 110900.7 | throughput per GPU (TFLOP/s/GPU): 34.2 | MFU 10.98% | FLOPs estimator: workload-aware | logical tokens/s: 151281.3 | valid tokens/s: 151281.3 | physical tokens/s: 151481.8 | loss tokens/s: 105975.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.553556E-01 | lm-global-supervised-tokens: 1.175276E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.137666E-02 | lm-token-buffer-mib: 5.803926E+01 | global_load_balancing_loss: 9.998967E-01 | loss scale: 1.0 | grad norm: 0.046 | params norm: 11530.919 | params rms: 0.062436 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 12:38:50.135136 | finish at 2026-10-12 00:53:46 + [2026-10-05 12:16:26] iteration 1771/ 6855 | consumed samples: 226688 | elapsed time per iteration (ms): 89881.7 | throughput per GPU (TFLOP/s/GPU): 40.1 | MFU 12.85% | FLOPs estimator: workload-aware | logical tokens/s: 186657.7 | valid tokens/s: 186657.7 | physical tokens/s: 187138.1 | loss tokens/s: 138805.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.234885E-01 | lm-global-supervised-tokens: 1.247611E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.234883E-02 | lm-token-buffer-mib: 5.814207E+01 | global_load_balancing_loss: 9.999470E-01 | loss scale: 1.0 | grad norm: 0.061 | params norm: 11530.846 | params rms: 0.062436 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 6:55:58.556937 | finish at 2026-10-10 19:12:24 + [2026-10-05 12:17:57] iteration 1772/ 6855 | consumed samples: 226816 | elapsed time per iteration (ms): 90675.4 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.46% | FLOPs estimator: workload-aware | logical tokens/s: 185024.9 | valid tokens/s: 185024.9 | physical tokens/s: 185456.7 | loss tokens/s: 136883.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.957025E-01 | lm-global-supervised-tokens: 1.241198E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.187547E-02 | lm-token-buffer-mib: 5.811617E+01 | global_load_balancing_loss: 9.991542E-01 | loss scale: 1.0 | grad norm: 0.038 | params norm: 11530.775 | params rms: 0.062435 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 8:01:42.941954 | finish at 2026-10-10 20:19:40 + [2026-10-05 12:19:26] iteration 1773/ 6855 | consumed samples: 226944 | elapsed time per iteration (ms): 89358.2 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.39% | FLOPs estimator: workload-aware | logical tokens/s: 187751.7 | valid tokens/s: 187751.7 | physical tokens/s: 188218.0 | loss tokens/s: 137833.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.968739E-01 | lm-global-supervised-tokens: 1.231652E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.165496E-02 | lm-token-buffer-mib: 5.815363E+01 | global_load_balancing_loss: 9.983460E-01 | loss scale: 1.0 | grad norm: 0.086 | params norm: 11530.704 | params rms: 0.062435 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 6:08:38.623582 | finish at 2026-10-10 18:28:05 + [2026-10-05 12:20:50] iteration 1774/ 6855 | consumed samples: 227072 | elapsed time per iteration (ms): 84206.7 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.65% | FLOPs estimator: workload-aware | logical tokens/s: 199238.1 | valid tokens/s: 199238.0 | physical tokens/s: 199783.8 | loss tokens/s: 155037.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.088386E-01 | lm-global-supervised-tokens: 1.305519E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.198923E-02 | lm-token-buffer-mib: 5.813264E+01 | global_load_balancing_loss: 9.999107E-01 | loss scale: 1.0 | grad norm: 0.114 | params norm: 11530.630 | params rms: 0.062435 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 22:50:54.215278 | finish at 2026-10-10 11:11:44 + [2026-10-05 12:22:17] iteration 1775/ 6855 | consumed samples: 227200 | elapsed time per iteration (ms): 86368.2 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.77% | FLOPs estimator: workload-aware | logical tokens/s: 194252.2 | valid tokens/s: 194252.1 | physical tokens/s: 194644.0 | loss tokens/s: 144268.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.150626E-01 | lm-global-supervised-tokens: 1.246022E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.135590E-02 | lm-token-buffer-mib: 5.810915E+01 | global_load_balancing_loss: 1.000535E+00 | loss scale: 1.0 | grad norm: 0.155 | params norm: 11530.557 | params rms: 0.062434 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 1:52:30.610142 | finish at 2026-10-10 14:14:47 + [2026-10-05 12:23:45] iteration 1776/ 6855 | consumed samples: 227328 | elapsed time per iteration (ms): 87998.6 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.47% | FLOPs estimator: workload-aware | logical tokens/s: 190652.8 | valid tokens/s: 190652.8 | physical tokens/s: 190968.5 | loss tokens/s: 143824.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.913908E-01 | lm-global-supervised-tokens: 1.265637E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.465208E-02 | lm-token-buffer-mib: 5.805567E+01 | global_load_balancing_loss: 1.000927E+00 | loss scale: 1.0 | grad norm: 0.084 | params norm: 11530.486 | params rms: 0.062434 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 4:09:04.855525 | finish at 2026-10-10 16:32:49 + [2026-10-05 12:25:14] iteration 1777/ 6855 | consumed samples: 227456 | elapsed time per iteration (ms): 89491.3 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.50% | FLOPs estimator: workload-aware | logical tokens/s: 187473.1 | valid tokens/s: 187473.0 | physical tokens/s: 187961.3 | loss tokens/s: 130338.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.764531E-01 | lm-global-supervised-tokens: 1.166418E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.401965E-02 | lm-token-buffer-mib: 5.808980E+01 | global_load_balancing_loss: 1.002302E+00 | loss scale: 1.0 | grad norm: 0.253 | params norm: 11530.413 | params rms: 0.062433 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 6:13:57.053180 | finish at 2026-10-10 18:39:11 + [2026-10-05 12:26:38] iteration 1778/ 6855 | consumed samples: 227584 | elapsed time per iteration (ms): 84287.6 | throughput per GPU (TFLOP/s/GPU): 43.3 | MFU 13.89% | FLOPs estimator: workload-aware | logical tokens/s: 199046.5 | valid tokens/s: 199046.5 | physical tokens/s: 199576.6 | loss tokens/s: 147674.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.749319E-01 | lm-global-supervised-tokens: 1.244708E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.348537E-02 | lm-token-buffer-mib: 5.821260E+01 | global_load_balancing_loss: 1.002342E+00 | loss scale: 1.0 | grad norm: 0.409 | params norm: 11530.342 | params rms: 0.062433 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 22:52:07.914209 | finish at 2026-10-10 11:18:46 + [2026-10-05 12:28:08] iteration 1779/ 6855 | consumed samples: 227712 | elapsed time per iteration (ms): 89405.3 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.56% | FLOPs estimator: workload-aware | logical tokens/s: 187653.3 | valid tokens/s: 187653.2 | physical tokens/s: 188012.7 | loss tokens/s: 130091.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.391553E-01 | lm-global-supervised-tokens: 1.163084E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.160464E-02 | lm-token-buffer-mib: 5.817852E+01 | global_load_balancing_loss: 1.001041E+00 | loss scale: 1.0 | grad norm: 0.066 | params norm: 11530.272 | params rms: 0.062433 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 6:03:41.246633 | finish at 2026-10-10 18:31:49 + [2026-10-05 12:29:53] iteration 1780/ 6855 | consumed samples: 227840 | elapsed time per iteration (ms): 105231.4 | throughput per GPU (TFLOP/s/GPU): 34.8 | MFU 11.15% | FLOPs estimator: workload-aware | logical tokens/s: 159431.3 | valid tokens/s: 159431.2 | physical tokens/s: 159834.7 | loss tokens/s: 119869.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.001720E-01 | lm-global-supervised-tokens: 1.261400E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.165170E-02 | lm-token-buffer-mib: 5.822306E+01 | global_load_balancing_loss: 9.982715E-01 | loss scale: 1.0 | grad norm: 0.288 | params norm: 11530.202 | params rms: 0.062432 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 4:20:49.450654 | finish at 2026-10-11 16:50:42 + [2026-10-05 12:31:19] iteration 1781/ 6855 | consumed samples: 227968 | elapsed time per iteration (ms): 86022.0 | throughput per GPU (TFLOP/s/GPU): 43.4 | MFU 13.92% | FLOPs estimator: workload-aware | logical tokens/s: 195033.7 | valid tokens/s: 195033.7 | physical tokens/s: 195439.2 | loss tokens/s: 142729.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.910962E-01 | lm-global-supervised-tokens: 1.227792E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.281202E-02 | lm-token-buffer-mib: 5.810942E+01 | global_load_balancing_loss: 9.984599E-01 | loss scale: 1.0 | grad norm: 0.050 | params norm: 11530.131 | params rms: 0.062432 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 1:14:35.818306 | finish at 2026-10-10 13:45:55 + [2026-10-05 12:32:47] iteration 1782/ 6855 | consumed samples: 228096 | elapsed time per iteration (ms): 88164.6 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.50% | FLOPs estimator: workload-aware | logical tokens/s: 190293.3 | valid tokens/s: 190293.3 | physical tokens/s: 190680.6 | loss tokens/s: 144424.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.127020E-01 | lm-global-supervised-tokens: 1.273311E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.263033E-02 | lm-token-buffer-mib: 5.808810E+01 | global_load_balancing_loss: 9.987622E-01 | loss scale: 1.0 | grad norm: 0.143 | params norm: 11530.059 | params rms: 0.062432 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 4:14:19.086630 | finish at 2026-10-10 16:47:06 + [2026-10-05 12:34:13] iteration 1783/ 6855 | consumed samples: 228224 | elapsed time per iteration (ms): 86287.6 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.54% | FLOPs estimator: workload-aware | logical tokens/s: 194433.5 | valid tokens/s: 194433.5 | physical tokens/s: 194837.8 | loss tokens/s: 145609.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.576591E-01 | lm-global-supervised-tokens: 1.256428E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.113298E-02 | lm-token-buffer-mib: 5.809345E+01 | global_load_balancing_loss: 9.997917E-01 | loss scale: 1.0 | grad norm: 0.199 | params norm: 11529.988 | params rms: 0.062431 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 1:34:10.656616 | finish at 2026-10-10 14:08:24 + [2026-10-05 12:35:38] iteration 1784/ 6855 | consumed samples: 228352 | elapsed time per iteration (ms): 84285.5 | throughput per GPU (TFLOP/s/GPU): 43.4 | MFU 13.92% | FLOPs estimator: workload-aware | logical tokens/s: 199052.1 | valid tokens/s: 199052.0 | physical tokens/s: 199542.3 | loss tokens/s: 144385.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.779356E-01 | lm-global-supervised-tokens: 1.216959E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.590884E-02 | lm-token-buffer-mib: 5.817908E+01 | global_load_balancing_loss: 1.000733E+00 | loss scale: 1.0 | grad norm: 0.098 | params norm: 11529.917 | params rms: 0.062431 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 22:43:31.955732 | finish at 2026-10-10 11:19:10 + [2026-10-05 12:37:06] iteration 1785/ 6855 | consumed samples: 228480 | elapsed time per iteration (ms): 88796.9 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.60% | FLOPs estimator: workload-aware | logical tokens/s: 188938.8 | valid tokens/s: 188938.8 | physical tokens/s: 189281.7 | loss tokens/s: 131475.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.137859E-01 | lm-global-supervised-tokens: 1.167463E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.319776E-02 | lm-token-buffer-mib: 5.803495E+01 | global_load_balancing_loss: 1.001049E+00 | loss scale: 1.0 | grad norm: 0.108 | params norm: 11529.843 | params rms: 0.062430 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 5:03:20.278337 | finish at 2026-10-10 17:40:27 + [2026-10-05 12:38:35] iteration 1786/ 6855 | consumed samples: 228608 | elapsed time per iteration (ms): 88822.8 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.46% | FLOPs estimator: workload-aware | logical tokens/s: 188883.6 | valid tokens/s: 188883.6 | physical tokens/s: 189372.8 | loss tokens/s: 145524.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.083489E-01 | lm-global-supervised-tokens: 1.292586E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.969959E-02 | lm-token-buffer-mib: 5.835435E+01 | global_load_balancing_loss: 1.000670E+00 | loss scale: 1.0 | grad norm: 0.146 | params norm: 11529.771 | params rms: 0.062430 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 5:04:02.738961 | finish at 2026-10-10 17:42:38 + [2026-10-05 12:40:03] iteration 1787/ 6855 | consumed samples: 228736 | elapsed time per iteration (ms): 87463.0 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.58% | FLOPs estimator: workload-aware | logical tokens/s: 191820.2 | valid tokens/s: 191820.2 | physical tokens/s: 192255.5 | loss tokens/s: 139765.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.825054E-01 | lm-global-supervised-tokens: 1.222430E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.153023E-02 | lm-token-buffer-mib: 5.809312E+01 | global_load_balancing_loss: 1.000185E+00 | loss scale: 1.0 | grad norm: 0.100 | params norm: 11529.701 | params rms: 0.062430 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 3:07:42.303054 | finish at 2026-10-10 15:47:45 + [2026-10-05 12:41:29] iteration 1788/ 6855 | consumed samples: 228864 | elapsed time per iteration (ms): 86118.6 | throughput per GPU (TFLOP/s/GPU): 43.7 | MFU 14.01% | FLOPs estimator: workload-aware | logical tokens/s: 194814.7 | valid tokens/s: 194814.7 | physical tokens/s: 195253.6 | loss tokens/s: 138475.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.776568E-01 | lm-global-supervised-tokens: 1.192532E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.662712E-02 | lm-token-buffer-mib: 5.806002E+01 | global_load_balancing_loss: 1.000930E+00 | loss scale: 1.0 | grad norm: 0.081 | params norm: 11529.632 | params rms: 0.062429 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 1:12:43.161895 | finish at 2026-10-10 13:54:12 + [2026-10-05 12:43:01] iteration 1789/ 6855 | consumed samples: 228992 | elapsed time per iteration (ms): 91784.7 | throughput per GPU (TFLOP/s/GPU): 40.7 | MFU 13.05% | FLOPs estimator: workload-aware | logical tokens/s: 182788.5 | valid tokens/s: 182788.5 | physical tokens/s: 183245.2 | loss tokens/s: 124730.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.244774E-01 | lm-global-supervised-tokens: 1.144836E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.522064E-02 | lm-token-buffer-mib: 5.822502E+01 | global_load_balancing_loss: 1.000460E+00 | loss scale: 1.0 | grad norm: 0.105 | params norm: 11529.562 | params rms: 0.062429 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 9:09:41.197984 | finish at 2026-10-10 21:52:42 + [2026-10-05 12:44:48] iteration 1790/ 6855 | consumed samples: 229120 | elapsed time per iteration (ms): 107272.3 | throughput per GPU (TFLOP/s/GPU): 34.2 | MFU 10.95% | FLOPs estimator: workload-aware | logical tokens/s: 156398.0 | valid tokens/s: 156398.0 | physical tokens/s: 156803.0 | loss tokens/s: 116356.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.459540E-01 | lm-global-supervised-tokens: 1.248185E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.682458E-02 | lm-token-buffer-mib: 5.810442E+01 | global_load_balancing_loss: 9.995385E-01 | loss scale: 1.0 | grad norm: 0.100 | params norm: 11529.495 | params rms: 0.062428 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 6:55:34.449496 | finish at 2026-10-11 19:40:22 + [2026-10-05 12:46:16] iteration 1791/ 6855 | consumed samples: 229248 | elapsed time per iteration (ms): 87781.9 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.59% | FLOPs estimator: workload-aware | logical tokens/s: 191123.6 | valid tokens/s: 191123.6 | physical tokens/s: 191472.2 | loss tokens/s: 134504.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.451174E-01 | lm-global-supervised-tokens: 1.180706E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.755082E-02 | lm-token-buffer-mib: 5.803864E+01 | global_load_balancing_loss: 1.000644E+00 | loss scale: 1.0 | grad norm: 0.034 | params norm: 11529.430 | params rms: 0.062428 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 3:28:47.410847 | finish at 2026-10-10 16:15:03 + [2026-10-05 12:47:44] iteration 1792/ 6855 | consumed samples: 229376 | elapsed time per iteration (ms): 88488.8 | throughput per GPU (TFLOP/s/GPU): 41.4 | MFU 13.27% | FLOPs estimator: workload-aware | logical tokens/s: 189597.0 | valid tokens/s: 189597.0 | physical tokens/s: 189998.2 | loss tokens/s: 136971.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.711908E-01 | lm-global-supervised-tokens: 1.212041E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.372587E-02 | lm-token-buffer-mib: 5.821113E+01 | global_load_balancing_loss: 1.000141E+00 | loss scale: 1.0 | grad norm: 0.063 | params norm: 11529.363 | params rms: 0.062428 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 4:26:58.589438 | finish at 2026-10-10 17:14:43 + [2026-10-05 12:49:13] iteration 1793/ 6855 | consumed samples: 229504 | elapsed time per iteration (ms): 88925.6 | throughput per GPU (TFLOP/s/GPU): 41.2 | MFU 13.22% | FLOPs estimator: workload-aware | logical tokens/s: 188665.1 | valid tokens/s: 188665.1 | physical tokens/s: 189208.4 | loss tokens/s: 143027.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.404318E-01 | lm-global-supervised-tokens: 1.271882E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.209738E-02 | lm-token-buffer-mib: 5.816635E+01 | global_load_balancing_loss: 9.998187E-01 | loss scale: 1.0 | grad norm: 0.123 | params norm: 11529.295 | params rms: 0.062427 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 5:02:21.348843 | finish at 2026-10-10 17:51:34 + [2026-10-05 12:50:38] iteration 1794/ 6855 | consumed samples: 229632 | elapsed time per iteration (ms): 84828.2 | throughput per GPU (TFLOP/s/GPU): 43.2 | MFU 13.83% | FLOPs estimator: workload-aware | logical tokens/s: 197778.6 | valid tokens/s: 197778.6 | physical tokens/s: 198212.7 | loss tokens/s: 148340.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.230647E-01 | lm-global-supervised-tokens: 1.258344E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.228601E-02 | lm-token-buffer-mib: 5.808221E+01 | global_load_balancing_loss: 9.999108E-01 | loss scale: 1.0 | grad norm: 0.038 | params norm: 11529.226 | params rms: 0.062427 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 23:15:15.324034 | finish at 2026-10-10 12:05:53 + [2026-10-05 12:52:01] iteration 1795/ 6855 | consumed samples: 229760 | elapsed time per iteration (ms): 82985.4 | throughput per GPU (TFLOP/s/GPU): 44.5 | MFU 14.25% | FLOPs estimator: workload-aware | logical tokens/s: 202170.7 | valid tokens/s: 202170.7 | physical tokens/s: 202628.6 | loss tokens/s: 148166.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.832167E-01 | lm-global-supervised-tokens: 1.229567E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.170757E-02 | lm-token-buffer-mib: 5.819100E+01 | global_load_balancing_loss: 9.993072E-01 | loss scale: 1.0 | grad norm: 0.105 | params norm: 11529.158 | params rms: 0.062427 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 20:38:26.123805 | finish at 2026-10-10 09:30:27 + [2026-10-05 12:53:29] iteration 1796/ 6855 | consumed samples: 229888 | elapsed time per iteration (ms): 88045.1 | throughput per GPU (TFLOP/s/GPU): 41.5 | MFU 13.29% | FLOPs estimator: workload-aware | logical tokens/s: 190552.5 | valid tokens/s: 190552.5 | physical tokens/s: 190956.1 | loss tokens/s: 144710.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.682931E-01 | lm-global-supervised-tokens: 1.274108E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.500572E-02 | lm-token-buffer-mib: 5.816454E+01 | global_load_balancing_loss: 9.987013E-01 | loss scale: 1.0 | grad norm: 0.042 | params norm: 11529.089 | params rms: 0.062426 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 3:43:40.227106 | finish at 2026-10-10 16:37:09 + [2026-10-05 12:54:58] iteration 1797/ 6855 | consumed samples: 230016 | elapsed time per iteration (ms): 88615.1 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.59% | FLOPs estimator: workload-aware | logical tokens/s: 189326.3 | valid tokens/s: 189326.3 | physical tokens/s: 189716.2 | loss tokens/s: 137417.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.742097E-01 | lm-global-supervised-tokens: 1.217731E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.155831E-02 | lm-token-buffer-mib: 5.810003E+01 | global_load_balancing_loss: 1.001217E+00 | loss scale: 1.0 | grad norm: 0.062 | params norm: 11529.020 | params rms: 0.062426 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 4:30:15.385159 | finish at 2026-10-10 17:25:13 + [2026-10-05 12:56:24] iteration 1798/ 6855 | consumed samples: 230144 | elapsed time per iteration (ms): 86360.5 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.68% | FLOPs estimator: workload-aware | logical tokens/s: 194269.4 | valid tokens/s: 194269.3 | physical tokens/s: 194696.2 | loss tokens/s: 143583.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.777906E-01 | lm-global-supervised-tokens: 1.239993E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.586623E-02 | lm-token-buffer-mib: 5.832558E+01 | global_load_balancing_loss: 1.001254E+00 | loss scale: 1.0 | grad norm: 0.062 | params norm: 11528.949 | params rms: 0.062426 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 1:18:45.052609 | finish at 2026-10-10 14:15:09 + [2026-10-05 12:57:52] iteration 1799/ 6855 | consumed samples: 230272 | elapsed time per iteration (ms): 87695.2 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.50% | FLOPs estimator: workload-aware | logical tokens/s: 191312.5 | valid tokens/s: 191312.5 | physical tokens/s: 191615.0 | loss tokens/s: 143947.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.905452E-01 | lm-global-supervised-tokens: 1.262354E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.259494E-02 | lm-token-buffer-mib: 5.803014E+01 | global_load_balancing_loss: 1.001715E+00 | loss scale: 1.0 | grad norm: 0.039 | params norm: 11528.877 | params rms: 0.062425 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 3:09:47.164886 | finish at 2026-10-10 16:07:39 + [2026-10-05 12:59:44] iteration 1800/ 6855 | consumed samples: 230400 | elapsed time per iteration (ms): 112498.6 | throughput per GPU (TFLOP/s/GPU): 33.5 | MFU 10.74% | FLOPs estimator: workload-aware | logical tokens/s: 149132.1 | valid tokens/s: 149132.1 | physical tokens/s: 149569.1 | loss tokens/s: 111055.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.756696E-01 | lm-global-supervised-tokens: 1.249359E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.400270E-02 | lm-token-buffer-mib: 5.839764E+01 | global_load_balancing_loss: 1.001324E+00 | loss scale: 1.0 | grad norm: 0.091 | params norm: 11528.805 | params rms: 0.062425 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 13:58:00.555604 | finish at 2026-10-12 02:57:45 +times across ranks (ms): + save-checkpoint: + rank 0: 23683.02 + rank 1: 23683.03 + rank 2: 23683.08 + rank 3: 23683.05 + rank 4: 23683.08 + rank 5: 23683.01 + rank 6: 23683.01 + rank 7: 23683.03 + rank 8: 23681.96 + rank 9: 23681.95 + rank 10: 23681.96 + rank 11: 23681.94 + rank 12: 23681.94 + rank 13: 23681.94 + rank 14: 23681.97 + rank 15: 23681.94 + rank 16: 23683.68 + rank 17: 23683.70 + rank 18: 23683.67 + rank 19: 23683.67 + rank 20: 23683.68 + rank 21: 23683.70 + rank 22: 23683.67 + rank 23: 23683.67 + rank 24: 23683.50 + rank 25: 23683.50 + rank 26: 23683.51 + rank 27: 23683.61 + rank 28: 23683.46 + rank 29: 23683.51 + rank 30: 23683.50 + rank 31: 23683.55 + rank 32: 23683.33 + rank 33: 23683.33 + rank 34: 23683.31 + rank 35: 23683.30 + rank 36: 23683.28 + rank 37: 23683.31 + rank 38: 23683.31 + rank 39: 23683.27 + rank 40: 23683.75 + rank 41: 23683.73 + rank 42: 23683.73 + rank 43: 23683.73 + rank 44: 23683.73 + rank 45: 23683.76 + rank 46: 23683.72 + rank 47: 23683.72 + rank 48: 23684.22 + rank 49: 23684.28 + rank 50: 23684.22 + rank 51: 23684.20 + rank 52: 23684.20 + rank 53: 23684.19 + rank 54: 23684.35 + rank 55: 23684.21 + rank 56: 23683.18 + rank 57: 23683.15 + rank 58: 23683.16 + rank 59: 23683.16 + rank 60: 23683.17 + rank 61: 23683.15 + rank 62: 23683.16 + rank 63: 23683.17 + [2026-10-05 13:01:38] iteration 1801/ 6855 | consumed samples: 230528 | elapsed time per iteration (ms): 89799.8 | throughput per GPU (TFLOP/s/GPU): 41.6 | MFU 13.32% | FLOPs estimator: workload-aware | logical tokens/s: 186828.3 | valid tokens/s: 186828.3 | physical tokens/s: 187278.9 | loss tokens/s: 134868.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.492879E-01 | lm-global-supervised-tokens: 1.211113E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.192695E-02 | lm-token-buffer-mib: 5.808777E+01 | global_load_balancing_loss: 1.000762E+00 | loss scale: 1.0 | grad norm: 0.054 | params norm: 11528.733 | params rms: 0.062424 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 6:04:07.977922 | finish at 2026-10-10 19:05:46 + [2026-10-05 13:03:09] iteration 1802/ 6855 | consumed samples: 230656 | elapsed time per iteration (ms): 90829.0 | throughput per GPU (TFLOP/s/GPU): 40.7 | MFU 13.04% | FLOPs estimator: workload-aware | logical tokens/s: 184712.0 | valid tokens/s: 184712.0 | physical tokens/s: 185181.9 | loss tokens/s: 138339.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.946701E-01 | lm-global-supervised-tokens: 1.256525E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.758075E-02 | lm-token-buffer-mib: 5.822835E+01 | global_load_balancing_loss: 9.983359E-01 | loss scale: 1.0 | grad norm: 0.047 | params norm: 11528.661 | params rms: 0.062424 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 7:29:19.075525 | finish at 2026-10-10 20:32:28 + [2026-10-05 13:04:36] iteration 1803/ 6855 | consumed samples: 230784 | elapsed time per iteration (ms): 86909.4 | throughput per GPU (TFLOP/s/GPU): 41.3 | MFU 13.25% | FLOPs estimator: workload-aware | logical tokens/s: 193042.4 | valid tokens/s: 193042.4 | physical tokens/s: 193596.1 | loss tokens/s: 146933.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.395186E-01 | lm-global-supervised-tokens: 1.276992E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.187511E-02 | lm-token-buffer-mib: 5.808326E+01 | global_load_balancing_loss: 9.995185E-01 | loss scale: 1.0 | grad norm: 0.086 | params norm: 11528.588 | params rms: 0.062424 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 1:57:46.535651 | finish at 2026-10-10 15:02:22 + [2026-10-05 13:06:01] iteration 1804/ 6855 | consumed samples: 230912 | elapsed time per iteration (ms): 84800.6 | throughput per GPU (TFLOP/s/GPU): 43.3 | MFU 13.89% | FLOPs estimator: workload-aware | logical tokens/s: 197842.6 | valid tokens/s: 197842.6 | physical tokens/s: 198222.4 | loss tokens/s: 145798.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.235287E-01 | lm-global-supervised-tokens: 1.236380E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.139749E-02 | lm-token-buffer-mib: 5.814388E+01 | global_load_balancing_loss: 9.996132E-01 | loss scale: 1.0 | grad norm: 0.045 | params norm: 11528.515 | params rms: 0.062423 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 22:58:47.687556 | finish at 2026-10-10 12:04:48 + [2026-10-05 13:07:29] iteration 1805/ 6855 | consumed samples: 231040 | elapsed time per iteration (ms): 88473.1 | throughput per GPU (TFLOP/s/GPU): 42.8 | MFU 13.71% | FLOPs estimator: workload-aware | logical tokens/s: 189630.1 | valid tokens/s: 189630.1 | physical tokens/s: 189916.3 | loss tokens/s: 135942.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.239569E-01 | lm-global-supervised-tokens: 1.202722E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.943351E-02 | lm-token-buffer-mib: 5.808931E+01 | global_load_balancing_loss: 9.986160E-01 | loss scale: 1.0 | grad norm: 0.047 | params norm: 11528.443 | params rms: 0.062423 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 4:06:29.031923 | finish at 2026-10-10 17:13:58 + [2026-10-05 13:08:54] iteration 1806/ 6855 | consumed samples: 231168 | elapsed time per iteration (ms): 85146.7 | throughput per GPU (TFLOP/s/GPU): 43.2 | MFU 13.85% | FLOPs estimator: workload-aware | logical tokens/s: 197038.3 | valid tokens/s: 197038.3 | physical tokens/s: 197584.9 | loss tokens/s: 151668.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.877546E-01 | lm-global-supervised-tokens: 1.291406E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.348513E-02 | lm-token-buffer-mib: 5.809174E+01 | global_load_balancing_loss: 1.000243E+00 | loss scale: 1.0 | grad norm: 0.094 | params norm: 11528.370 | params rms: 0.062422 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 23:25:05.900891 | finish at 2026-10-10 12:34:00 + [2026-10-05 13:10:15] iteration 1807/ 6855 | consumed samples: 231296 | elapsed time per iteration (ms): 80418.9 | throughput per GPU (TFLOP/s/GPU): 44.0 | MFU 14.10% | FLOPs estimator: workload-aware | logical tokens/s: 208622.1 | valid tokens/s: 208622.0 | physical tokens/s: 209110.1 | loss tokens/s: 166722.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.622471E-01 | lm-global-supervised-tokens: 1.340764E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.487028E-02 | lm-token-buffer-mib: 5.806339E+01 | global_load_balancing_loss: 1.000173E+00 | loss scale: 1.0 | grad norm: 0.034 | params norm: 11528.298 | params rms: 0.062422 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 16:45:54.491726 | finish at 2026-10-10 05:56:09 + [2026-10-05 13:11:42] iteration 1808/ 6855 | consumed samples: 231424 | elapsed time per iteration (ms): 87758.1 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.58% | FLOPs estimator: workload-aware | logical tokens/s: 191175.3 | valid tokens/s: 191175.3 | physical tokens/s: 191688.7 | loss tokens/s: 139627.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.581718E-01 | lm-global-supervised-tokens: 1.225346E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.219695E-02 | lm-token-buffer-mib: 5.811852E+01 | global_load_balancing_loss: 1.000573E+00 | loss scale: 1.0 | grad norm: 0.088 | params norm: 11528.227 | params rms: 0.062422 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 3:01:55.281278 | finish at 2026-10-10 16:13:38 + [2026-10-05 13:13:07] iteration 1809/ 6855 | consumed samples: 231552 | elapsed time per iteration (ms): 85156.4 | throughput per GPU (TFLOP/s/GPU): 42.8 | MFU 13.71% | FLOPs estimator: workload-aware | logical tokens/s: 197016.3 | valid tokens/s: 197016.3 | physical tokens/s: 197701.5 | loss tokens/s: 144912.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.483253E-01 | lm-global-supervised-tokens: 1.234025E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.549757E-02 | lm-token-buffer-mib: 5.806465E+01 | global_load_balancing_loss: 1.001052E+00 | loss scale: 1.0 | grad norm: 0.230 | params norm: 11528.155 | params rms: 0.062421 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 23:21:39.268815 | finish at 2026-10-10 12:34:47 + [2026-10-05 13:14:58] iteration 1810/ 6855 | consumed samples: 231680 | elapsed time per iteration (ms): 110918.5 | throughput per GPU (TFLOP/s/GPU): 33.1 | MFU 10.60% | FLOPs estimator: workload-aware | logical tokens/s: 151256.4 | valid tokens/s: 151256.3 | physical tokens/s: 151710.3 | loss tokens/s: 112620.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.251671E-01 | lm-global-supervised-tokens: 1.249168E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.133208E-02 | lm-token-buffer-mib: 5.815552E+01 | global_load_balancing_loss: 1.000991E+00 | loss scale: 1.0 | grad norm: 0.342 | params norm: 11528.082 | params rms: 0.062421 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 11:26:23.677067 | finish at 2026-10-12 00:41:22 + [2026-10-05 13:16:26] iteration 1811/ 6855 | consumed samples: 231808 | elapsed time per iteration (ms): 87358.2 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.40% | FLOPs estimator: workload-aware | logical tokens/s: 192050.0 | valid tokens/s: 192050.0 | physical tokens/s: 192641.2 | loss tokens/s: 141786.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.303813E-01 | lm-global-supervised-tokens: 1.238617E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.159298E-02 | lm-token-buffer-mib: 5.837720E+01 | global_load_balancing_loss: 1.000936E+00 | loss scale: 1.0 | grad norm: 0.252 | params norm: 11528.012 | params rms: 0.062420 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 2:23:54.672178 | finish at 2026-10-10 15:40:20 + [2026-10-05 13:17:54] iteration 1812/ 6855 | consumed samples: 231936 | elapsed time per iteration (ms): 88120.4 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.46% | FLOPs estimator: workload-aware | logical tokens/s: 190389.0 | valid tokens/s: 190389.0 | physical tokens/s: 190782.7 | loss tokens/s: 138570.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.495695E-01 | lm-global-supervised-tokens: 1.221086E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.147800E-02 | lm-token-buffer-mib: 5.818099E+01 | global_load_balancing_loss: 1.001016E+00 | loss scale: 1.0 | grad norm: 0.151 | params norm: 11527.940 | params rms: 0.062420 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 3:26:31.386166 | finish at 2026-10-10 16:44:25 + [2026-10-05 13:19:22] iteration 1813/ 6855 | consumed samples: 232064 | elapsed time per iteration (ms): 87760.9 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.58% | FLOPs estimator: workload-aware | logical tokens/s: 191169.6 | valid tokens/s: 191169.6 | physical tokens/s: 191648.7 | loss tokens/s: 136378.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.447506E-01 | lm-global-supervised-tokens: 1.196872E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.280760E-02 | lm-token-buffer-mib: 5.818669E+01 | global_load_balancing_loss: 1.001573E+00 | loss scale: 1.0 | grad norm: 0.176 | params norm: 11527.865 | params rms: 0.062420 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 2:54:50.455500 | finish at 2026-10-10 16:14:12 + [2026-10-05 13:20:45] iteration 1814/ 6855 | consumed samples: 232192 | elapsed time per iteration (ms): 83651.2 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.66% | FLOPs estimator: workload-aware | logical tokens/s: 200561.2 | valid tokens/s: 200561.2 | physical tokens/s: 201021.4 | loss tokens/s: 154867.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.563551E-01 | lm-global-supervised-tokens: 1.295488E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.331433E-02 | lm-token-buffer-mib: 5.817585E+01 | global_load_balancing_loss: 9.995831E-01 | loss scale: 1.0 | grad norm: 0.168 | params norm: 11527.791 | params rms: 0.062419 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 21:08:05.585305 | finish at 2026-10-10 10:28:51 + [2026-10-05 13:22:15] iteration 1815/ 6855 | consumed samples: 232320 | elapsed time per iteration (ms): 89644.6 | throughput per GPU (TFLOP/s/GPU): 40.4 | MFU 12.95% | FLOPs estimator: workload-aware | logical tokens/s: 187152.6 | valid tokens/s: 187152.5 | physical tokens/s: 187695.2 | loss tokens/s: 145146.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.702383E-01 | lm-global-supervised-tokens: 1.301160E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.180040E-02 | lm-token-buffer-mib: 5.817313E+01 | global_load_balancing_loss: 9.990110E-01 | loss scale: 1.0 | grad norm: 0.102 | params norm: 11527.718 | params rms: 0.062419 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 5:30:08.689842 | finish at 2026-10-10 18:52:24 + [2026-10-05 13:23:42] iteration 1816/ 6855 | consumed samples: 232448 | elapsed time per iteration (ms): 87512.8 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.47% | FLOPs estimator: workload-aware | logical tokens/s: 191711.2 | valid tokens/s: 191711.2 | physical tokens/s: 192167.1 | loss tokens/s: 140080.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.163419E-01 | lm-global-supervised-tokens: 1.225881E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.618845E-02 | lm-token-buffer-mib: 5.813346E+01 | global_load_balancing_loss: 9.997594E-01 | loss scale: 1.0 | grad norm: 0.189 | params norm: 11527.644 | params rms: 0.062418 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 2:29:36.810472 | finish at 2026-10-10 15:53:19 + [2026-10-05 13:25:10] iteration 1817/ 6855 | consumed samples: 232576 | elapsed time per iteration (ms): 87335.1 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.68% | FLOPs estimator: workload-aware | logical tokens/s: 192101.3 | valid tokens/s: 192101.3 | physical tokens/s: 192533.7 | loss tokens/s: 137026.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.385528E-01 | lm-global-supervised-tokens: 1.196720E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.456488E-02 | lm-token-buffer-mib: 5.817943E+01 | global_load_balancing_loss: 9.994134E-01 | loss scale: 1.0 | grad norm: 0.163 | params norm: 11527.568 | params rms: 0.062418 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 2:13:13.990842 | finish at 2026-10-10 15:38:24 + [2026-10-05 13:26:37] iteration 1818/ 6855 | consumed samples: 232704 | elapsed time per iteration (ms): 86760.1 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.75% | FLOPs estimator: workload-aware | logical tokens/s: 193374.5 | valid tokens/s: 193374.5 | physical tokens/s: 193738.3 | loss tokens/s: 142388.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.077441E-01 | lm-global-supervised-tokens: 1.235367E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.194707E-02 | lm-token-buffer-mib: 5.805666E+01 | global_load_balancing_loss: 9.990222E-01 | loss scale: 1.0 | grad norm: 0.094 | params norm: 11527.492 | params rms: 0.062418 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 1:23:30.633943 | finish at 2026-10-10 14:50:07 + [2026-10-05 13:28:03] iteration 1819/ 6855 | consumed samples: 232832 | elapsed time per iteration (ms): 86896.4 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.52% | FLOPs estimator: workload-aware | logical tokens/s: 193071.0 | valid tokens/s: 193071.0 | physical tokens/s: 193773.0 | loss tokens/s: 143438.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.902069E-01 | lm-global-supervised-tokens: 1.246430E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.186375E-02 | lm-token-buffer-mib: 5.806537E+01 | global_load_balancing_loss: 9.994755E-01 | loss scale: 1.0 | grad norm: 0.085 | params norm: 11527.416 | params rms: 0.062417 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 1:33:30.302691 | finish at 2026-10-10 15:01:34 + [2026-10-05 13:29:49] iteration 1820/ 6855 | consumed samples: 232960 | elapsed time per iteration (ms): 105532.7 | throughput per GPU (TFLOP/s/GPU): 35.3 | MFU 11.32% | FLOPs estimator: workload-aware | logical tokens/s: 158976.2 | valid tokens/s: 158976.2 | physical tokens/s: 159274.6 | loss tokens/s: 115903.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.078571E-01 | lm-global-supervised-tokens: 1.223162E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.236829E-02 | lm-token-buffer-mib: 5.808945E+01 | global_load_balancing_loss: 1.000183E+00 | loss scale: 1.0 | grad norm: 0.082 | params norm: 11527.341 | params rms: 0.062417 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 3:35:57.323676 | finish at 2026-10-11 17:05:46 + [2026-10-05 13:31:15] iteration 1821/ 6855 | consumed samples: 233088 | elapsed time per iteration (ms): 85770.9 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.78% | FLOPs estimator: workload-aware | logical tokens/s: 195604.8 | valid tokens/s: 195604.8 | physical tokens/s: 196088.0 | loss tokens/s: 145298.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.981612E-01 | lm-global-supervised-tokens: 1.246237E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.308392E-02 | lm-token-buffer-mib: 5.835463E+01 | global_load_balancing_loss: 1.001750E+00 | loss scale: 1.0 | grad norm: 0.092 | params norm: 11527.266 | params rms: 0.062416 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 23:56:10.570232 | finish at 2026-10-10 13:27:25 + [2026-10-05 13:32:42] iteration 1822/ 6855 | consumed samples: 233216 | elapsed time per iteration (ms): 87416.6 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.50% | FLOPs estimator: workload-aware | logical tokens/s: 191921.8 | valid tokens/s: 191921.8 | physical tokens/s: 192371.7 | loss tokens/s: 142363.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.216127E-01 | lm-global-supervised-tokens: 1.244495E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.091940E-02 | lm-token-buffer-mib: 5.804803E+01 | global_load_balancing_loss: 1.001124E+00 | loss scale: 1.0 | grad norm: 0.065 | params norm: 11527.190 | params rms: 0.062416 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 2:12:47.702146 | finish at 2026-10-10 15:45:30 + [2026-10-05 13:34:09] iteration 1823/ 6855 | consumed samples: 233344 | elapsed time per iteration (ms): 86715.8 | throughput per GPU (TFLOP/s/GPU): 42.8 | MFU 13.73% | FLOPs estimator: workload-aware | logical tokens/s: 193473.1 | valid tokens/s: 193473.1 | physical tokens/s: 193979.1 | loss tokens/s: 143429.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.287368E-01 | lm-global-supervised-tokens: 1.243762E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.479370E-02 | lm-token-buffer-mib: 5.834403E+01 | global_load_balancing_loss: 1.000929E+00 | loss scale: 1.0 | grad norm: 0.037 | params norm: 11527.115 | params rms: 0.062416 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 1:12:33.941828 | finish at 2026-10-10 14:46:43 + [2026-10-05 13:35:37] iteration 1824/ 6855 | consumed samples: 233472 | elapsed time per iteration (ms): 88096.0 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.45% | FLOPs estimator: workload-aware | logical tokens/s: 190442.2 | valid tokens/s: 190442.2 | physical tokens/s: 190877.1 | loss tokens/s: 146084.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.203232E-01 | lm-global-supervised-tokens: 1.286942E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.554882E-02 | lm-token-buffer-mib: 5.803568E+01 | global_load_balancing_loss: 1.000057E+00 | loss scale: 1.0 | grad norm: 0.046 | params norm: 11527.042 | params rms: 0.062415 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 3:06:50.798256 | finish at 2026-10-10 16:42:28 + [2026-10-05 13:37:06] iteration 1825/ 6855 | consumed samples: 233600 | elapsed time per iteration (ms): 88629.4 | throughput per GPU (TFLOP/s/GPU): 41.2 | MFU 13.22% | FLOPs estimator: workload-aware | logical tokens/s: 189295.1 | valid tokens/s: 189295.1 | physical tokens/s: 189714.5 | loss tokens/s: 140834.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.517178E-01 | lm-global-supervised-tokens: 1.248208E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.495024E-02 | lm-token-buffer-mib: 5.813667E+01 | global_load_balancing_loss: 1.000666E+00 | loss scale: 1.0 | grad norm: 0.123 | params norm: 11526.969 | params rms: 0.062415 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 3:50:05.709383 | finish at 2026-10-10 17:27:11 + [2026-10-05 13:38:33] iteration 1826/ 6855 | consumed samples: 233728 | elapsed time per iteration (ms): 87154.3 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.56% | FLOPs estimator: workload-aware | logical tokens/s: 192499.7 | valid tokens/s: 192499.7 | physical tokens/s: 192944.9 | loss tokens/s: 141296.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.572703E-01 | lm-global-supervised-tokens: 1.231459E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.595717E-02 | lm-token-buffer-mib: 5.823139E+01 | global_load_balancing_loss: 9.995028E-01 | loss scale: 1.0 | grad norm: 0.091 | params norm: 11526.895 | params rms: 0.062414 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 1:44:59.113656 | finish at 2026-10-10 15:23:32 + [2026-10-05 13:40:04] iteration 1827/ 6855 | consumed samples: 233856 | elapsed time per iteration (ms): 90783.9 | throughput per GPU (TFLOP/s/GPU): 41.5 | MFU 13.31% | FLOPs estimator: workload-aware | logical tokens/s: 184803.9 | valid tokens/s: 184803.9 | physical tokens/s: 185052.1 | loss tokens/s: 136330.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.794792E-01 | lm-global-supervised-tokens: 1.237660E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.627707E-02 | lm-token-buffer-mib: 5.810564E+01 | global_load_balancing_loss: 1.000266E+00 | loss scale: 1.0 | grad norm: 0.034 | params norm: 11526.822 | params rms: 0.062414 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 6:47:41.333033 | finish at 2026-10-10 20:27:45 + [2026-10-05 13:41:32] iteration 1828/ 6855 | consumed samples: 233984 | elapsed time per iteration (ms): 88297.4 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.56% | FLOPs estimator: workload-aware | logical tokens/s: 190007.4 | valid tokens/s: 190007.4 | physical tokens/s: 190411.0 | loss tokens/s: 141442.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.162758E-01 | lm-global-supervised-tokens: 1.248898E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.312017E-02 | lm-token-buffer-mib: 5.824860E+01 | global_load_balancing_loss: 9.994164E-01 | loss scale: 1.0 | grad norm: 0.097 | params norm: 11526.749 | params rms: 0.062414 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 3:17:51.240730 | finish at 2026-10-10 16:59:23 + [2026-10-05 13:43:01] iteration 1829/ 6855 | consumed samples: 234112 | elapsed time per iteration (ms): 89115.9 | throughput per GPU (TFLOP/s/GPU): 41.7 | MFU 13.36% | FLOPs estimator: workload-aware | logical tokens/s: 188262.6 | valid tokens/s: 188262.6 | physical tokens/s: 188641.0 | loss tokens/s: 140326.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.835549E-01 | lm-global-supervised-tokens: 1.250535E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.157549E-02 | lm-token-buffer-mib: 5.815128E+01 | global_load_balancing_loss: 1.000119E+00 | loss scale: 1.0 | grad norm: 0.076 | params norm: 11526.674 | params rms: 0.062413 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 4:24:56.434512 | finish at 2026-10-10 18:07:57 + [2026-10-05 13:44:50] iteration 1830/ 6855 | consumed samples: 234240 | elapsed time per iteration (ms): 109129.5 | throughput per GPU (TFLOP/s/GPU): 33.7 | MFU 10.80% | FLOPs estimator: workload-aware | logical tokens/s: 153736.2 | valid tokens/s: 153736.2 | physical tokens/s: 154086.4 | loss tokens/s: 113993.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.583551E-01 | lm-global-supervised-tokens: 1.244005E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.445735E-02 | lm-token-buffer-mib: 5.827518E+01 | global_load_balancing_loss: 1.000004E+00 | loss scale: 1.0 | grad norm: 0.042 | params norm: 11526.598 | params rms: 0.062413 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 8:19:35.986254 | finish at 2026-10-11 22:04:26 + [2026-10-05 13:46:17] iteration 1831/ 6855 | consumed samples: 234368 | elapsed time per iteration (ms): 87087.4 | throughput per GPU (TFLOP/s/GPU): 41.7 | MFU 13.38% | FLOPs estimator: workload-aware | logical tokens/s: 192646.9 | valid tokens/s: 192646.9 | physical tokens/s: 193194.3 | loss tokens/s: 144473.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.241201E-01 | lm-global-supervised-tokens: 1.258178E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.488268E-02 | lm-token-buffer-mib: 5.801515E+01 | global_load_balancing_loss: 1.001137E+00 | loss scale: 1.0 | grad norm: 0.092 | params norm: 11526.524 | params rms: 0.062412 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 1:32:07.311806 | finish at 2026-10-10 15:18:24 + [2026-10-05 13:47:44] iteration 1832/ 6855 | consumed samples: 234496 | elapsed time per iteration (ms): 87323.7 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.55% | FLOPs estimator: workload-aware | logical tokens/s: 192125.6 | valid tokens/s: 192125.6 | physical tokens/s: 192618.5 | loss tokens/s: 140678.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.617125E-01 | lm-global-supervised-tokens: 1.228453E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.198092E-02 | lm-token-buffer-mib: 5.808873E+01 | global_load_balancing_loss: 9.998539E-01 | loss scale: 1.0 | grad norm: 0.073 | params norm: 11526.450 | params rms: 0.062412 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 1:50:27.043056 | finish at 2026-10-10 15:38:12 + [2026-10-05 13:49:06] iteration 1833/ 6855 | consumed samples: 234624 | elapsed time per iteration (ms): 81970.8 | throughput per GPU (TFLOP/s/GPU): 43.8 | MFU 14.03% | FLOPs estimator: workload-aware | logical tokens/s: 204672.1 | valid tokens/s: 204672.1 | physical tokens/s: 205153.7 | loss tokens/s: 151193.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.288326E-01 | lm-global-supervised-tokens: 1.239348E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.400617E-02 | lm-token-buffer-mib: 5.813499E+01 | global_load_balancing_loss: 1.000484E+00 | loss scale: 1.0 | grad norm: 0.070 | params norm: 11526.376 | params rms: 0.062412 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 18:20:57.566747 | finish at 2026-10-10 08:10:04 + [2026-10-05 13:50:31] iteration 1834/ 6855 | consumed samples: 234752 | elapsed time per iteration (ms): 84538.2 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.77% | FLOPs estimator: workload-aware | logical tokens/s: 198456.5 | valid tokens/s: 198456.5 | physical tokens/s: 198991.1 | loss tokens/s: 147223.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.648090E-01 | lm-global-supervised-tokens: 1.244601E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.829724E-02 | lm-token-buffer-mib: 5.820516E+01 | global_load_balancing_loss: 9.995549E-01 | loss scale: 1.0 | grad norm: 0.071 | params norm: 11526.301 | params rms: 0.062411 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 21:54:26.311283 | finish at 2026-10-10 11:44:57 + [2026-10-05 13:51:57] iteration 1835/ 6855 | consumed samples: 234880 | elapsed time per iteration (ms): 85958.5 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.65% | FLOPs estimator: workload-aware | logical tokens/s: 195176.7 | valid tokens/s: 195176.7 | physical tokens/s: 195515.3 | loss tokens/s: 145335.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.723431E-01 | lm-global-supervised-tokens: 1.249281E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.180888E-02 | lm-token-buffer-mib: 5.813271E+01 | global_load_balancing_loss: 9.989058E-01 | loss scale: 1.0 | grad norm: 0.034 | params norm: 11526.227 | params rms: 0.062411 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 23:51:51.485224 | finish at 2026-10-10 13:43:48 + [2026-10-05 13:53:22] iteration 1836/ 6855 | consumed samples: 235008 | elapsed time per iteration (ms): 85137.0 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.69% | FLOPs estimator: workload-aware | logical tokens/s: 197061.4 | valid tokens/s: 197061.4 | physical tokens/s: 197688.1 | loss tokens/s: 142024.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.651037E-01 | lm-global-supervised-tokens: 1.209150E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.387932E-02 | lm-token-buffer-mib: 5.814718E+01 | global_load_balancing_loss: 1.000975E+00 | loss scale: 1.0 | grad norm: 0.106 | params norm: 11526.153 | params rms: 0.062410 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 22:41:42.627354 | finish at 2026-10-10 12:35:05 + [2026-10-05 13:54:50] iteration 1837/ 6855 | consumed samples: 235136 | elapsed time per iteration (ms): 87423.4 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.64% | FLOPs estimator: workload-aware | logical tokens/s: 191907.6 | valid tokens/s: 191907.6 | physical tokens/s: 192364.1 | loss tokens/s: 138800.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.997598E-01 | lm-global-supervised-tokens: 1.213444E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.156912E-02 | lm-token-buffer-mib: 5.812679E+01 | global_load_balancing_loss: 1.000745E+00 | loss scale: 1.0 | grad norm: 0.205 | params norm: 11526.080 | params rms: 0.062410 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 1:51:30.544256 | finish at 2026-10-10 15:46:20 + [2026-10-05 13:56:17] iteration 1838/ 6855 | consumed samples: 235264 | elapsed time per iteration (ms): 87852.9 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.78% | FLOPs estimator: workload-aware | logical tokens/s: 190968.8 | valid tokens/s: 190968.8 | physical tokens/s: 191393.7 | loss tokens/s: 133686.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.535003E-01 | lm-global-supervised-tokens: 1.174470E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.148824E-02 | lm-token-buffer-mib: 5.810813E+01 | global_load_balancing_loss: 1.001036E+00 | loss scale: 1.0 | grad norm: 0.241 | params norm: 11526.005 | params rms: 0.062410 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 2:25:57.786528 | finish at 2026-10-10 16:22:15 + [2026-10-05 13:57:48] iteration 1839/ 6855 | consumed samples: 235392 | elapsed time per iteration (ms): 90247.1 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.61% | FLOPs estimator: workload-aware | logical tokens/s: 185902.7 | valid tokens/s: 185902.7 | physical tokens/s: 186431.9 | loss tokens/s: 136352.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.842715E-01 | lm-global-supervised-tokens: 1.230540E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.051603E-02 | lm-token-buffer-mib: 5.811666E+01 | global_load_balancing_loss: 1.000995E+00 | loss scale: 1.0 | grad norm: 0.197 | params norm: 11525.929 | params rms: 0.062409 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 5:44:39.592901 | finish at 2026-10-10 19:42:27 + [2026-10-05 13:59:39] iteration 1840/ 6855 | consumed samples: 235520 | elapsed time per iteration (ms): 111296.8 | throughput per GPU (TFLOP/s/GPU): 34.0 | MFU 10.90% | FLOPs estimator: workload-aware | logical tokens/s: 150742.5 | valid tokens/s: 150742.5 | physical tokens/s: 151085.4 | loss tokens/s: 106946.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.256539E-01 | lm-global-supervised-tokens: 1.190279E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.149200E-02 | lm-token-buffer-mib: 5.809652E+01 | global_load_balancing_loss: 9.999121E-01 | loss scale: 1.0 | grad norm: 0.041 | params norm: 11525.852 | params rms: 0.062409 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 11:02:33.596165 | finish at 2026-10-12 01:02:13 +times across ranks (ms): + save-checkpoint: + rank 0: 25182.58 + rank 1: 25182.47 + rank 2: 25182.50 + rank 3: 25182.50 + rank 4: 25182.51 + rank 5: 25182.50 + rank 6: 25182.46 + rank 7: 25182.47 + rank 8: 25182.92 + rank 9: 25182.88 + rank 10: 25182.88 + rank 11: 25182.88 + rank 12: 25182.90 + rank 13: 25182.88 + rank 14: 25182.92 + rank 15: 25182.87 + rank 16: 25182.83 + rank 17: 25182.83 + rank 18: 25182.81 + rank 19: 25182.83 + rank 20: 25182.80 + rank 21: 25182.85 + rank 22: 25182.83 + rank 23: 25182.65 + rank 24: 25182.56 + rank 25: 25182.52 + rank 26: 25182.54 + rank 27: 25182.58 + rank 28: 25182.46 + rank 29: 25182.51 + rank 30: 25182.68 + rank 31: 25182.56 + rank 32: 25183.07 + rank 33: 25182.99 + rank 34: 25182.99 + rank 35: 25182.96 + rank 36: 25182.88 + rank 37: 25182.87 + rank 38: 25182.86 + rank 39: 25182.95 + rank 40: 25182.58 + rank 41: 25182.55 + rank 42: 25182.56 + rank 43: 25182.57 + rank 44: 25182.54 + rank 45: 25182.56 + rank 46: 25182.56 + rank 47: 25182.54 + rank 48: 25182.52 + rank 49: 25182.48 + rank 50: 25182.62 + rank 51: 25182.62 + rank 52: 25182.67 + rank 53: 25182.54 + rank 54: 25182.58 + rank 55: 25182.75 + rank 56: 25182.96 + rank 57: 25182.93 + rank 58: 25182.90 + rank 59: 25182.90 + rank 60: 25182.89 + rank 61: 25182.92 + rank 62: 25182.89 + rank 63: 25182.94 + [2026-10-05 14:01:31] iteration 1841/ 6855 | consumed samples: 235648 | elapsed time per iteration (ms): 86182.2 | throughput per GPU (TFLOP/s/GPU): 43.1 | MFU 13.82% | FLOPs estimator: workload-aware | logical tokens/s: 194671.5 | valid tokens/s: 194671.5 | physical tokens/s: 195014.4 | loss tokens/s: 145283.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.079065E-01 | lm-global-supervised-tokens: 1.252088E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.008795E+00 | lm-token-buffer-mib: 5.823377E+01 | global_load_balancing_loss: 9.993235E-01 | loss scale: 1.0 | grad norm: 0.092 | params norm: 11525.777 | params rms: 0.062408 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 0:01:57.396364 | finish at 2026-10-10 14:03:28 + [2026-10-05 14:03:01] iteration 1842/ 6855 | consumed samples: 235776 | elapsed time per iteration (ms): 90294.3 | throughput per GPU (TFLOP/s/GPU): 41.1 | MFU 13.17% | FLOPs estimator: workload-aware | logical tokens/s: 185805.8 | valid tokens/s: 185805.8 | physical tokens/s: 186200.9 | loss tokens/s: 136128.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.971645E-01 | lm-global-supervised-tokens: 1.229158E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.374089E-02 | lm-token-buffer-mib: 5.817236E+01 | global_load_balancing_loss: 1.000529E+00 | loss scale: 1.0 | grad norm: 0.051 | params norm: 11525.702 | params rms: 0.062408 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 5:44:05.256977 | finish at 2026-10-10 19:47:06 + [2026-10-05 14:04:24] iteration 1843/ 6855 | consumed samples: 235904 | elapsed time per iteration (ms): 83092.8 | throughput per GPU (TFLOP/s/GPU): 43.6 | MFU 13.96% | FLOPs estimator: workload-aware | logical tokens/s: 201908.9 | valid tokens/s: 201908.9 | physical tokens/s: 202395.3 | loss tokens/s: 151370.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.396750E-01 | lm-global-supervised-tokens: 1.257782E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.163056E-02 | lm-token-buffer-mib: 5.808945E+01 | global_load_balancing_loss: 9.993145E-01 | loss scale: 1.0 | grad norm: 0.053 | params norm: 11525.629 | params rms: 0.062408 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 19:41:01.248138 | finish at 2026-10-10 09:45:25 + [2026-10-05 14:05:49] iteration 1844/ 6855 | consumed samples: 236032 | elapsed time per iteration (ms): 85403.2 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.68% | FLOPs estimator: workload-aware | logical tokens/s: 196447.0 | valid tokens/s: 196446.9 | physical tokens/s: 197019.2 | loss tokens/s: 142656.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.475497E-01 | lm-global-supervised-tokens: 1.218334E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.158804E-02 | lm-token-buffer-mib: 5.815919E+01 | global_load_balancing_loss: 1.000117E+00 | loss scale: 1.0 | grad norm: 0.052 | params norm: 11525.556 | params rms: 0.062407 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 22:52:35.365461 | finish at 2026-10-10 12:58:25 + [2026-10-05 14:07:16] iteration 1845/ 6855 | consumed samples: 236160 | elapsed time per iteration (ms): 86640.0 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.78% | FLOPs estimator: workload-aware | logical tokens/s: 193642.9 | valid tokens/s: 193642.9 | physical tokens/s: 194288.5 | loss tokens/s: 146145.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.271983E-01 | lm-global-supervised-tokens: 1.266206E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 7.819153E-02 | lm-token-buffer-mib: 5.819843E+01 | global_load_balancing_loss: 9.997037E-01 | loss scale: 1.0 | grad norm: 0.036 | params norm: 11525.484 | params rms: 0.062407 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 0:34:26.287050 | finish at 2026-10-10 14:41:42 + [2026-10-05 14:08:41] iteration 1846/ 6855 | consumed samples: 236288 | elapsed time per iteration (ms): 85343.3 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.79% | FLOPs estimator: workload-aware | logical tokens/s: 196584.2 | valid tokens/s: 196584.2 | physical tokens/s: 197013.6 | loss tokens/s: 143880.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.997420E-01 | lm-global-supervised-tokens: 1.227924E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.219561E-02 | lm-token-buffer-mib: 5.812907E+01 | global_load_balancing_loss: 1.000783E+00 | loss scale: 1.0 | grad norm: 0.036 | params norm: 11525.412 | params rms: 0.062406 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 22:44:44.586639 | finish at 2026-10-10 12:53:26 + [2026-10-05 14:10:08] iteration 1847/ 6855 | consumed samples: 236416 | elapsed time per iteration (ms): 86581.2 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.74% | FLOPs estimator: workload-aware | logical tokens/s: 193774.2 | valid tokens/s: 193774.2 | physical tokens/s: 194158.1 | loss tokens/s: 148624.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.401115E-01 | lm-global-supervised-tokens: 1.286809E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.435930E-02 | lm-token-buffer-mib: 5.808585E+01 | global_load_balancing_loss: 9.999686E-01 | loss scale: 1.0 | grad norm: 0.036 | params norm: 11525.339 | params rms: 0.062406 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 0:26:38.877075 | finish at 2026-10-10 14:36:47 + [2026-10-05 14:11:37] iteration 1848/ 6855 | consumed samples: 236544 | elapsed time per iteration (ms): 88607.6 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.44% | FLOPs estimator: workload-aware | logical tokens/s: 189342.0 | valid tokens/s: 189342.0 | physical tokens/s: 190004.9 | loss tokens/s: 140219.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.829941E-01 | lm-global-supervised-tokens: 1.242446E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.862950E-02 | lm-token-buffer-mib: 5.852694E+01 | global_load_balancing_loss: 1.000514E+00 | loss scale: 1.0 | grad norm: 0.030 | params norm: 11525.264 | params rms: 0.062406 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 3:14:18.089523 | finish at 2026-10-10 17:25:55 + [2026-10-05 14:13:01] iteration 1849/ 6855 | consumed samples: 236672 | elapsed time per iteration (ms): 84453.2 | throughput per GPU (TFLOP/s/GPU): 43.7 | MFU 14.00% | FLOPs estimator: workload-aware | logical tokens/s: 198656.2 | valid tokens/s: 198656.2 | physical tokens/s: 199067.2 | loss tokens/s: 146840.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.991127E-01 | lm-global-supervised-tokens: 1.240111E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.523503E-02 | lm-token-buffer-mib: 5.814468E+01 | global_load_balancing_loss: 1.000501E+00 | loss scale: 1.0 | grad norm: 0.041 | params norm: 11525.189 | params rms: 0.062405 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 21:26:12.584842 | finish at 2026-10-10 11:39:14 + [2026-10-05 14:14:49] iteration 1850/ 6855 | consumed samples: 236800 | elapsed time per iteration (ms): 108355.3 | throughput per GPU (TFLOP/s/GPU): 33.6 | MFU 10.78% | FLOPs estimator: workload-aware | logical tokens/s: 154835.1 | valid tokens/s: 154835.1 | physical tokens/s: 155319.9 | loss tokens/s: 115393.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.765934E-01 | lm-global-supervised-tokens: 1.250353E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.463805E-02 | lm-token-buffer-mib: 5.818568E+01 | global_load_balancing_loss: 1.001626E+00 | loss scale: 1.0 | grad norm: 0.038 | params norm: 11525.116 | params rms: 0.062405 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 6:38:38.156919 | finish at 2026-10-11 20:53:28 + [2026-10-05 14:16:17] iteration 1851/ 6855 | consumed samples: 236928 | elapsed time per iteration (ms): 87136.6 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.55% | FLOPs estimator: workload-aware | logical tokens/s: 192538.8 | valid tokens/s: 192538.8 | physical tokens/s: 193271.1 | loss tokens/s: 139154.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.532296E-01 | lm-global-supervised-tokens: 1.212547E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.219982E-02 | lm-token-buffer-mib: 5.810748E+01 | global_load_balancing_loss: 1.001201E+00 | loss scale: 1.0 | grad norm: 0.041 | params norm: 11525.043 | params rms: 0.062404 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 1:07:11.306685 | finish at 2026-10-10 15:23:28 + [2026-10-05 14:17:45] iteration 1852/ 6855 | consumed samples: 237056 | elapsed time per iteration (ms): 88102.0 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.52% | FLOPs estimator: workload-aware | logical tokens/s: 190429.1 | valid tokens/s: 190429.1 | physical tokens/s: 190833.0 | loss tokens/s: 140018.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.865774E-01 | lm-global-supervised-tokens: 1.233587E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.172488E-02 | lm-token-buffer-mib: 5.808244E+01 | global_load_balancing_loss: 1.000252E+00 | loss scale: 1.0 | grad norm: 0.043 | params norm: 11524.967 | params rms: 0.062404 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 2:26:14.391873 | finish at 2026-10-10 16:43:59 + [2026-10-05 14:19:10] iteration 1853/ 6855 | consumed samples: 237184 | elapsed time per iteration (ms): 85736.4 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.52% | FLOPs estimator: workload-aware | logical tokens/s: 195683.6 | valid tokens/s: 195683.6 | physical tokens/s: 196147.6 | loss tokens/s: 151393.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.382145E-01 | lm-global-supervised-tokens: 1.297995E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.184356E-02 | lm-token-buffer-mib: 5.813092E+01 | global_load_balancing_loss: 9.993828E-01 | loss scale: 1.0 | grad norm: 0.089 | params norm: 11524.893 | params rms: 0.062404 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 23:07:33.570035 | finish at 2026-10-10 13:26:44 + [2026-10-05 14:20:39] iteration 1854/ 6855 | consumed samples: 237312 | elapsed time per iteration (ms): 88238.5 | throughput per GPU (TFLOP/s/GPU): 41.5 | MFU 13.32% | FLOPs estimator: workload-aware | logical tokens/s: 190134.8 | valid tokens/s: 190134.8 | physical tokens/s: 190464.9 | loss tokens/s: 138127.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.247519E-01 | lm-global-supervised-tokens: 1.218812E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.814122E-02 | lm-token-buffer-mib: 5.807673E+01 | global_load_balancing_loss: 9.992011E-01 | loss scale: 1.0 | grad norm: 0.062 | params norm: 11524.819 | params rms: 0.062403 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 2:34:40.685436 | finish at 2026-10-10 16:55:19 + [2026-10-05 14:22:03] iteration 1855/ 6855 | consumed samples: 237440 | elapsed time per iteration (ms): 84849.5 | throughput per GPU (TFLOP/s/GPU): 43.1 | MFU 13.80% | FLOPs estimator: workload-aware | logical tokens/s: 197729.1 | valid tokens/s: 197729.1 | physical tokens/s: 198326.1 | loss tokens/s: 149903.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.448583E-01 | lm-global-supervised-tokens: 1.271924E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.128031E-02 | lm-token-buffer-mib: 5.810837E+01 | global_load_balancing_loss: 9.986349E-01 | loss scale: 1.0 | grad norm: 0.161 | params norm: 11524.747 | params rms: 0.062403 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 21:50:47.437716 | finish at 2026-10-10 12:12:51 + [2026-10-05 14:23:32] iteration 1856/ 6855 | consumed samples: 237568 | elapsed time per iteration (ms): 88086.6 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.47% | FLOPs estimator: workload-aware | logical tokens/s: 190462.0 | valid tokens/s: 190462.0 | physical tokens/s: 190898.2 | loss tokens/s: 136307.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.846578E-01 | lm-global-supervised-tokens: 1.200684E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.508800E-02 | lm-token-buffer-mib: 5.824623E+01 | global_load_balancing_loss: 9.994586E-01 | loss scale: 1.0 | grad norm: 0.249 | params norm: 11524.672 | params rms: 0.062402 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 2:19:04.758785 | finish at 2026-10-10 16:42:36 + [2026-10-05 14:25:01] iteration 1857/ 6855 | consumed samples: 237696 | elapsed time per iteration (ms): 89044.3 | throughput per GPU (TFLOP/s/GPU): 41.5 | MFU 13.30% | FLOPs estimator: workload-aware | logical tokens/s: 188413.8 | valid tokens/s: 188413.8 | physical tokens/s: 188789.0 | loss tokens/s: 140715.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.164446E-01 | lm-global-supervised-tokens: 1.252990E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.269571E-02 | lm-token-buffer-mib: 5.814035E+01 | global_load_balancing_loss: 1.000792E+00 | loss scale: 1.0 | grad norm: 0.084 | params norm: 11524.596 | params rms: 0.062402 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 3:37:23.548832 | finish at 2026-10-10 18:02:24 + [2026-10-05 14:26:25] iteration 1858/ 6855 | consumed samples: 237824 | elapsed time per iteration (ms): 84358.7 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.77% | FLOPs estimator: workload-aware | logical tokens/s: 198878.8 | valid tokens/s: 198878.8 | physical tokens/s: 199376.4 | loss tokens/s: 144270.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.417938E-01 | lm-global-supervised-tokens: 1.217044E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.320536E-02 | lm-token-buffer-mib: 5.806380E+01 | global_load_balancing_loss: 1.001399E+00 | loss scale: 1.0 | grad norm: 0.190 | params norm: 11524.518 | params rms: 0.062402 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 21:05:40.531309 | finish at 2026-10-10 11:32:05 + [2026-10-05 14:27:52] iteration 1859/ 6855 | consumed samples: 237952 | elapsed time per iteration (ms): 86729.9 | throughput per GPU (TFLOP/s/GPU): 43.7 | MFU 14.00% | FLOPs estimator: workload-aware | logical tokens/s: 193441.0 | valid tokens/s: 193441.0 | physical tokens/s: 194090.1 | loss tokens/s: 135658.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.502521E-01 | lm-global-supervised-tokens: 1.176567E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.547718E-02 | lm-token-buffer-mib: 5.810278E+01 | global_load_balancing_loss: 1.000718E+00 | loss scale: 1.0 | grad norm: 0.236 | params norm: 11524.442 | params rms: 0.062401 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 0:21:42.764443 | finish at 2026-10-10 14:49:34 + [2026-10-05 14:29:44] iteration 1860/ 6855 | consumed samples: 238080 | elapsed time per iteration (ms): 112359.2 | throughput per GPU (TFLOP/s/GPU): 32.7 | MFU 10.48% | FLOPs estimator: workload-aware | logical tokens/s: 149317.3 | valid tokens/s: 149317.3 | physical tokens/s: 149679.7 | loss tokens/s: 112089.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.713530E-01 | lm-global-supervised-tokens: 1.259432E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.159890E-02 | lm-token-buffer-mib: 5.833655E+01 | global_load_balancing_loss: 1.001306E+00 | loss scale: 1.0 | grad norm: 0.040 | params norm: 11524.367 | params rms: 0.062401 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 11:53:54.153986 | finish at 2026-10-12 02:23:38 + [2026-10-05 14:31:11] iteration 1861/ 6855 | consumed samples: 238208 | elapsed time per iteration (ms): 87347.9 | throughput per GPU (TFLOP/s/GPU): 42.8 | MFU 13.72% | FLOPs estimator: workload-aware | logical tokens/s: 192072.9 | valid tokens/s: 192072.9 | physical tokens/s: 192459.8 | loss tokens/s: 139907.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.886113E-01 | lm-global-supervised-tokens: 1.222063E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 4.749888E-02 | lm-token-buffer-mib: 5.804487E+01 | global_load_balancing_loss: 1.000399E+00 | loss scale: 1.0 | grad norm: 0.229 | params norm: 11524.291 | params rms: 0.062400 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 1:10:15.412169 | finish at 2026-10-10 15:41:27 + [2026-10-05 14:32:42] iteration 1862/ 6855 | consumed samples: 238336 | elapsed time per iteration (ms): 90933.9 | throughput per GPU (TFLOP/s/GPU): 41.4 | MFU 13.26% | FLOPs estimator: workload-aware | logical tokens/s: 184498.9 | valid tokens/s: 184498.9 | physical tokens/s: 184911.8 | loss tokens/s: 135043.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.255064E-01 | lm-global-supervised-tokens: 1.228003E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.529790E-02 | lm-token-buffer-mib: 5.808736E+01 | global_load_balancing_loss: 9.997330E-01 | loss scale: 1.0 | grad norm: 0.078 | params norm: 11524.213 | params rms: 0.062400 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 6:07:13.083521 | finish at 2026-10-10 20:39:55 + [2026-10-05 14:34:12] iteration 1863/ 6855 | consumed samples: 238464 | elapsed time per iteration (ms): 89706.1 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.39% | FLOPs estimator: workload-aware | logical tokens/s: 187023.9 | valid tokens/s: 187023.9 | physical tokens/s: 187502.5 | loss tokens/s: 138301.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.440606E-01 | lm-global-supervised-tokens: 1.240645E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.418116E-02 | lm-token-buffer-mib: 5.814782E+01 | global_load_balancing_loss: 9.996857E-01 | loss scale: 1.0 | grad norm: 0.283 | params norm: 11524.135 | params rms: 0.062399 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 4:23:32.665466 | finish at 2026-10-10 18:57:45 + [2026-10-05 14:35:40] iteration 1864/ 6855 | consumed samples: 238592 | elapsed time per iteration (ms): 88262.6 | throughput per GPU (TFLOP/s/GPU): 41.7 | MFU 13.36% | FLOPs estimator: workload-aware | logical tokens/s: 190081.8 | valid tokens/s: 190081.8 | physical tokens/s: 190543.9 | loss tokens/s: 136158.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.732305E-01 | lm-global-supervised-tokens: 1.201768E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.226811E-02 | lm-token-buffer-mib: 5.825838E+01 | global_load_balancing_loss: 9.997806E-01 | loss scale: 1.0 | grad norm: 0.067 | params norm: 11524.059 | params rms: 0.062399 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 2:21:58.727952 | finish at 2026-10-10 16:57:39 + [2026-10-05 14:37:06] iteration 1865/ 6855 | consumed samples: 238720 | elapsed time per iteration (ms): 85350.4 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.55% | FLOPs estimator: workload-aware | logical tokens/s: 196568.3 | valid tokens/s: 196568.3 | physical tokens/s: 196964.8 | loss tokens/s: 146613.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.648897E-01 | lm-global-supervised-tokens: 1.251352E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.155000E-02 | lm-token-buffer-mib: 5.805268E+01 | global_load_balancing_loss: 9.998122E-01 | loss scale: 1.0 | grad norm: 0.314 | params norm: 11523.982 | params rms: 0.062399 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 22:18:18.523219 | finish at 2026-10-10 12:55:24 + [2026-10-05 14:38:35] iteration 1866/ 6855 | consumed samples: 238848 | elapsed time per iteration (ms): 89175.1 | throughput per GPU (TFLOP/s/GPU): 40.7 | MFU 13.03% | FLOPs estimator: workload-aware | logical tokens/s: 188137.4 | valid tokens/s: 188137.4 | physical tokens/s: 188666.2 | loss tokens/s: 142405.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.379160E-01 | lm-global-supervised-tokens: 1.269899E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.149362E-02 | lm-token-buffer-mib: 5.818359E+01 | global_load_balancing_loss: 9.993060E-01 | loss scale: 1.0 | grad norm: 0.053 | params norm: 11523.904 | params rms: 0.062398 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 3:34:54.789634 | finish at 2026-10-10 18:13:30 + [2026-10-05 14:40:02] iteration 1867/ 6855 | consumed samples: 238976 | elapsed time per iteration (ms): 86987.9 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.64% | FLOPs estimator: workload-aware | logical tokens/s: 192868.5 | valid tokens/s: 192868.5 | physical tokens/s: 193409.4 | loss tokens/s: 138805.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.321313E-01 | lm-global-supervised-tokens: 1.207437E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.142188E-02 | lm-token-buffer-mib: 5.815443E+01 | global_load_balancing_loss: 1.001835E+00 | loss scale: 1.0 | grad norm: 0.130 | params norm: 11523.827 | params rms: 0.062398 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 0:31:35.418150 | finish at 2026-10-10 15:11:37 + [2026-10-05 14:41:30] iteration 1868/ 6855 | consumed samples: 239104 | elapsed time per iteration (ms): 88368.6 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.39% | FLOPs estimator: workload-aware | logical tokens/s: 189854.7 | valid tokens/s: 189854.7 | physical tokens/s: 190368.0 | loss tokens/s: 133831.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.380271E-01 | lm-global-supervised-tokens: 1.182648E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.257649E-02 | lm-token-buffer-mib: 5.847781E+01 | global_load_balancing_loss: 1.001948E+00 | loss scale: 1.0 | grad norm: 0.153 | params norm: 11523.749 | params rms: 0.062397 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 2:24:54.035256 | finish at 2026-10-10 17:06:24 + [2026-10-05 14:42:57] iteration 1869/ 6855 | consumed samples: 239232 | elapsed time per iteration (ms): 86746.6 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.62% | FLOPs estimator: workload-aware | logical tokens/s: 193404.6 | valid tokens/s: 193404.6 | physical tokens/s: 193765.4 | loss tokens/s: 138831.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.369189E-01 | lm-global-supervised-tokens: 1.204318E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.154236E-02 | lm-token-buffer-mib: 5.816878E+01 | global_load_balancing_loss: 1.000861E+00 | loss scale: 1.0 | grad norm: 0.041 | params norm: 11523.673 | params rms: 0.062397 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 0:08:38.704080 | finish at 2026-10-10 14:51:36 + [2026-10-05 14:44:45] iteration 1870/ 6855 | consumed samples: 239360 | elapsed time per iteration (ms): 107833.4 | throughput per GPU (TFLOP/s/GPU): 34.1 | MFU 10.93% | FLOPs estimator: workload-aware | logical tokens/s: 155583.9 | valid tokens/s: 155583.9 | physical tokens/s: 156010.0 | loss tokens/s: 115179.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.127515E-01 | lm-global-supervised-tokens: 1.242020E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.544104E-02 | lm-token-buffer-mib: 5.848134E+01 | global_load_balancing_loss: 9.995968E-01 | loss scale: 1.0 | grad norm: 0.127 | params norm: 11523.598 | params rms: 0.062397 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 5:19:09.639300 | finish at 2026-10-11 20:03:54 + [2026-10-05 14:46:12] iteration 1871/ 6855 | consumed samples: 239488 | elapsed time per iteration (ms): 87120.3 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.77% | FLOPs estimator: workload-aware | logical tokens/s: 192575.3 | valid tokens/s: 192575.3 | physical tokens/s: 192874.7 | loss tokens/s: 143784.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.556796E-01 | lm-global-supervised-tokens: 1.252658E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.324157E-02 | lm-token-buffer-mib: 5.802943E+01 | global_load_balancing_loss: 9.983365E-01 | loss scale: 1.0 | grad norm: 0.051 | params norm: 11523.520 | params rms: 0.062396 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 0:36:47.407925 | finish at 2026-10-10 15:22:59 + [2026-10-05 14:47:38] iteration 1872/ 6855 | consumed samples: 239616 | elapsed time per iteration (ms): 86253.4 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.67% | FLOPs estimator: workload-aware | logical tokens/s: 194510.6 | valid tokens/s: 194510.6 | physical tokens/s: 195018.6 | loss tokens/s: 143347.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.863775E-01 | lm-global-supervised-tokens: 1.236425E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.455893E-02 | lm-token-buffer-mib: 5.826619E+01 | global_load_balancing_loss: 9.982255E-01 | loss scale: 1.0 | grad norm: 0.102 | params norm: 11523.442 | params rms: 0.062396 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 23:23:20.934995 | finish at 2026-10-10 14:10:59 + [2026-10-05 14:49:02] iteration 1873/ 6855 | consumed samples: 239744 | elapsed time per iteration (ms): 84365.5 | throughput per GPU (TFLOP/s/GPU): 42.8 | MFU 13.73% | FLOPs estimator: workload-aware | logical tokens/s: 198862.8 | valid tokens/s: 198862.8 | physical tokens/s: 199302.2 | loss tokens/s: 150449.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.474357E-01 | lm-global-supervised-tokens: 1.269276E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.126675E-02 | lm-token-buffer-mib: 5.809567E+01 | global_load_balancing_loss: 1.000169E+00 | loss scale: 1.0 | grad norm: 0.084 | params norm: 11523.365 | params rms: 0.062395 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 20:45:08.732007 | finish at 2026-10-10 11:34:11 + [2026-10-05 14:50:27] iteration 1874/ 6855 | consumed samples: 239872 | elapsed time per iteration (ms): 84698.0 | throughput per GPU (TFLOP/s/GPU): 43.5 | MFU 13.96% | FLOPs estimator: workload-aware | logical tokens/s: 198082.1 | valid tokens/s: 198082.1 | physical tokens/s: 198477.0 | loss tokens/s: 144926.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.566034E-01 | lm-global-supervised-tokens: 1.227495E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.467774E-02 | lm-token-buffer-mib: 5.808205E+01 | global_load_balancing_loss: 1.000804E+00 | loss scale: 1.0 | grad norm: 0.088 | params norm: 11523.290 | params rms: 0.062395 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 21:11:20.501447 | finish at 2026-10-10 12:01:48 + [2026-10-05 14:51:54] iteration 1875/ 6855 | consumed samples: 240000 | elapsed time per iteration (ms): 87054.3 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.77% | FLOPs estimator: workload-aware | logical tokens/s: 192720.9 | valid tokens/s: 192720.9 | physical tokens/s: 193151.7 | loss tokens/s: 145994.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.271940E-01 | lm-global-supervised-tokens: 1.270941E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.534181E-02 | lm-token-buffer-mib: 5.827487E+01 | global_load_balancing_loss: 1.000876E+00 | loss scale: 1.0 | grad norm: 0.050 | params norm: 11523.216 | params rms: 0.062394 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 0:25:30.452342 | finish at 2026-10-10 15:17:25 + [2026-10-05 14:53:20] iteration 1876/ 6855 | consumed samples: 240128 | elapsed time per iteration (ms): 85868.5 | throughput per GPU (TFLOP/s/GPU): 43.3 | MFU 13.87% | FLOPs estimator: workload-aware | logical tokens/s: 195382.2 | valid tokens/s: 195382.2 | physical tokens/s: 195670.4 | loss tokens/s: 148772.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.033098E-01 | lm-global-supervised-tokens: 1.277485E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.502346E-02 | lm-token-buffer-mib: 5.809104E+01 | global_load_balancing_loss: 1.000160E+00 | loss scale: 1.0 | grad norm: 0.047 | params norm: 11523.140 | params rms: 0.062394 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 22:45:39.268594 | finish at 2026-10-10 13:38:59 + [2026-10-05 14:54:46] iteration 1877/ 6855 | consumed samples: 240256 | elapsed time per iteration (ms): 85870.6 | throughput per GPU (TFLOP/s/GPU): 43.2 | MFU 13.84% | FLOPs estimator: workload-aware | logical tokens/s: 195377.7 | valid tokens/s: 195377.7 | physical tokens/s: 195772.8 | loss tokens/s: 139598.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.260022E-01 | lm-global-supervised-tokens: 1.198740E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.158429E-02 | lm-token-buffer-mib: 5.806017E+01 | global_load_balancing_loss: 1.001154E+00 | loss scale: 1.0 | grad norm: 0.121 | params norm: 11523.063 | params rms: 0.062394 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 22:44:23.959476 | finish at 2026-10-10 13:39:10 + [2026-10-05 14:56:10] iteration 1878/ 6855 | consumed samples: 240384 | elapsed time per iteration (ms): 84259.2 | throughput per GPU (TFLOP/s/GPU): 43.7 | MFU 13.99% | FLOPs estimator: workload-aware | logical tokens/s: 199113.6 | valid tokens/s: 199113.6 | physical tokens/s: 199684.4 | loss tokens/s: 150810.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.766115E-01 | lm-global-supervised-tokens: 1.270716E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.156138E-02 | lm-token-buffer-mib: 5.821078E+01 | global_load_balancing_loss: 1.000608E+00 | loss scale: 1.0 | grad norm: 0.044 | params norm: 11522.989 | params rms: 0.062393 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 20:29:18.041252 | finish at 2026-10-10 11:25:28 + [2026-10-05 14:57:35] iteration 1879/ 6855 | consumed samples: 240512 | elapsed time per iteration (ms): 84611.2 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.76% | FLOPs estimator: workload-aware | logical tokens/s: 198285.7 | valid tokens/s: 198285.7 | physical tokens/s: 198767.7 | loss tokens/s: 150976.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.286912E-01 | lm-global-supervised-tokens: 1.277432E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.566871E-02 | lm-token-buffer-mib: 5.821430E+01 | global_load_balancing_loss: 1.000876E+00 | loss scale: 1.0 | grad norm: 0.111 | params norm: 11522.914 | params rms: 0.062393 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 20:57:05.485897 | finish at 2026-10-10 11:54:40 + [2026-10-05 14:59:27] iteration 1880/ 6855 | consumed samples: 240640 | elapsed time per iteration (ms): 111910.3 | throughput per GPU (TFLOP/s/GPU): 34.4 | MFU 11.03% | FLOPs estimator: workload-aware | logical tokens/s: 149915.8 | valid tokens/s: 149915.8 | physical tokens/s: 150245.4 | loss tokens/s: 104005.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.927032E-01 | lm-global-supervised-tokens: 1.163927E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.286827E-02 | lm-token-buffer-mib: 5.822308E+01 | global_load_balancing_loss: 9.995883E-01 | loss scale: 1.0 | grad norm: 0.062 | params norm: 11522.841 | params rms: 0.062392 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 10:39:13.558731 | finish at 2026-10-12 01:38:40 +times across ranks (ms): + save-checkpoint: + rank 0: 25766.43 + rank 1: 25766.32 + rank 2: 25766.37 + rank 3: 25766.19 + rank 4: 25766.34 + rank 5: 25766.40 + rank 6: 25766.31 + rank 7: 25766.34 + rank 8: 25767.51 + rank 9: 25767.46 + rank 10: 25767.46 + rank 11: 25767.50 + rank 12: 25767.47 + rank 13: 25767.47 + rank 14: 25767.47 + rank 15: 25767.47 + rank 16: 25769.02 + rank 17: 25768.97 + rank 18: 25768.99 + rank 19: 25768.98 + rank 20: 25768.97 + rank 21: 25768.97 + rank 22: 25768.95 + rank 23: 25768.94 + rank 24: 25770.72 + rank 25: 25770.71 + rank 26: 25770.77 + rank 27: 25770.80 + rank 28: 25770.67 + rank 29: 25770.76 + rank 30: 25770.62 + rank 31: 25770.78 + rank 32: 25767.49 + rank 33: 25767.48 + rank 34: 25767.52 + rank 35: 25767.45 + rank 36: 25767.44 + rank 37: 25767.41 + rank 38: 25767.44 + rank 39: 25767.38 + rank 40: 25767.48 + rank 41: 25767.47 + rank 42: 25767.46 + rank 43: 25767.47 + rank 44: 25767.47 + rank 45: 25767.44 + rank 46: 25767.47 + rank 47: 25767.48 + rank 48: 25767.54 + rank 49: 25767.54 + rank 50: 25767.83 + rank 51: 25767.59 + rank 52: 25767.76 + rank 53: 25767.63 + rank 54: 25767.68 + rank 55: 25767.70 + rank 56: 25767.02 + rank 57: 25766.99 + rank 58: 25766.97 + rank 59: 25766.99 + rank 60: 25766.98 + rank 61: 25767.00 + rank 62: 25766.98 + rank 63: 25766.98 + [2026-10-05 15:01:42] iteration 1881/ 6855 | consumed samples: 240768 | elapsed time per iteration (ms): 109138.8 | throughput per GPU (TFLOP/s/GPU): 33.9 | MFU 10.87% | FLOPs estimator: workload-aware | logical tokens/s: 153723.3 | valid tokens/s: 153723.3 | physical tokens/s: 154123.1 | loss tokens/s: 115312.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.956412E-01 | lm-global-supervised-tokens: 1.258505E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 7.692623E-01 | lm-token-buffer-mib: 5.820509E+01 | global_load_balancing_loss: 9.997733E-01 | loss scale: 1.0 | grad norm: 0.057 | params norm: 11522.766 | params rms: 0.062392 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 6:47:36.593519 | finish at 2026-10-11 21:49:18 + [2026-10-05 15:03:09] iteration 1882/ 6855 | consumed samples: 240896 | elapsed time per iteration (ms): 86984.3 | throughput per GPU (TFLOP/s/GPU): 40.9 | MFU 13.10% | FLOPs estimator: workload-aware | logical tokens/s: 192875.2 | valid tokens/s: 192875.2 | physical tokens/s: 193413.0 | loss tokens/s: 144469.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.252472E-01 | lm-global-supervised-tokens: 1.256659E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.065757E-02 | lm-token-buffer-mib: 5.832922E+01 | global_load_balancing_loss: 9.993162E-01 | loss scale: 1.0 | grad norm: 0.099 | params norm: 11522.691 | params rms: 0.062392 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 0:09:32.854625 | finish at 2026-10-10 15:12:42 + [2026-10-05 15:04:38] iteration 1883/ 6855 | consumed samples: 241024 | elapsed time per iteration (ms): 89117.5 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.43% | FLOPs estimator: workload-aware | logical tokens/s: 188258.2 | valid tokens/s: 188258.2 | physical tokens/s: 188631.3 | loss tokens/s: 131852.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.560552E-01 | lm-global-supervised-tokens: 1.175036E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.477581E-02 | lm-token-buffer-mib: 5.813536E+01 | global_load_balancing_loss: 1.001438E+00 | loss scale: 1.0 | grad norm: 0.041 | params norm: 11522.619 | params rms: 0.062391 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 3:04:52.168842 | finish at 2026-10-10 18:09:30 + [2026-10-05 15:06:03] iteration 1884/ 6855 | consumed samples: 241152 | elapsed time per iteration (ms): 84600.7 | throughput per GPU (TFLOP/s/GPU): 43.5 | MFU 13.94% | FLOPs estimator: workload-aware | logical tokens/s: 198309.9 | valid tokens/s: 198309.8 | physical tokens/s: 198638.7 | loss tokens/s: 144957.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.521811E-01 | lm-global-supervised-tokens: 1.226354E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.352617E-02 | lm-token-buffer-mib: 5.808497E+01 | global_load_balancing_loss: 1.000558E+00 | loss scale: 1.0 | grad norm: 0.054 | params norm: 11522.547 | params rms: 0.062391 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 20:49:10.095803 | finish at 2026-10-10 11:55:13 + [2026-10-05 15:07:30] iteration 1885/ 6855 | consumed samples: 241280 | elapsed time per iteration (ms): 87588.3 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.70% | FLOPs estimator: workload-aware | logical tokens/s: 191545.8 | valid tokens/s: 191545.8 | physical tokens/s: 191904.9 | loss tokens/s: 136562.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.782190E-01 | lm-global-supervised-tokens: 1.196125E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.161908E-02 | lm-token-buffer-mib: 5.818474E+01 | global_load_balancing_loss: 9.990821E-01 | loss scale: 1.0 | grad norm: 0.055 | params norm: 11522.474 | params rms: 0.062390 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 0:55:13.922045 | finish at 2026-10-10 16:02:44 + [2026-10-05 15:08:59] iteration 1886/ 6855 | consumed samples: 241408 | elapsed time per iteration (ms): 88745.8 | throughput per GPU (TFLOP/s/GPU): 41.7 | MFU 13.37% | FLOPs estimator: workload-aware | logical tokens/s: 189048.1 | valid tokens/s: 189048.1 | physical tokens/s: 189500.4 | loss tokens/s: 140848.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.856403E-01 | lm-global-supervised-tokens: 1.249971E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.134516E-02 | lm-token-buffer-mib: 5.816833E+01 | global_load_balancing_loss: 9.990845E-01 | loss scale: 1.0 | grad norm: 0.057 | params norm: 11522.402 | params rms: 0.062390 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 2:29:37.725095 | finish at 2026-10-10 17:38:37 + [2026-10-05 15:10:28] iteration 1887/ 6855 | consumed samples: 241536 | elapsed time per iteration (ms): 89158.2 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.57% | FLOPs estimator: workload-aware | logical tokens/s: 188173.5 | valid tokens/s: 188173.5 | physical tokens/s: 188869.3 | loss tokens/s: 134113.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.751609E-01 | lm-global-supervised-tokens: 1.195734E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.156191E-02 | lm-token-buffer-mib: 5.808421E+01 | global_load_balancing_loss: 1.000162E+00 | loss scale: 1.0 | grad norm: 0.064 | params norm: 11522.331 | params rms: 0.062390 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 3:02:18.039591 | finish at 2026-10-10 18:12:46 + [2026-10-05 15:11:54] iteration 1888/ 6855 | consumed samples: 241664 | elapsed time per iteration (ms): 86348.2 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.67% | FLOPs estimator: workload-aware | logical tokens/s: 194297.2 | valid tokens/s: 194297.2 | physical tokens/s: 194733.3 | loss tokens/s: 139133.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.592083E-01 | lm-global-supervised-tokens: 1.201392E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.404826E-02 | lm-token-buffer-mib: 5.809415E+01 | global_load_balancing_loss: 1.000990E+00 | loss scale: 1.0 | grad norm: 0.035 | params norm: 11522.261 | params rms: 0.062389 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 23:08:11.275335 | finish at 2026-10-10 14:20:06 + [2026-10-05 15:13:23] iteration 1889/ 6855 | consumed samples: 241792 | elapsed time per iteration (ms): 88155.5 | throughput per GPU (TFLOP/s/GPU): 40.7 | MFU 13.05% | FLOPs estimator: workload-aware | logical tokens/s: 190313.5 | valid tokens/s: 190313.5 | physical tokens/s: 190904.6 | loss tokens/s: 144846.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.722381E-01 | lm-global-supervised-tokens: 1.276901E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.586252E-02 | lm-token-buffer-mib: 5.850090E+01 | global_load_balancing_loss: 9.999275E-01 | loss scale: 1.0 | grad norm: 0.040 | params norm: 11522.190 | params rms: 0.062389 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 1:36:20.380804 | finish at 2026-10-10 16:49:43 + [2026-10-05 15:15:10] iteration 1890/ 6855 | consumed samples: 241920 | elapsed time per iteration (ms): 107179.9 | throughput per GPU (TFLOP/s/GPU): 34.5 | MFU 11.05% | FLOPs estimator: workload-aware | logical tokens/s: 156532.9 | valid tokens/s: 156532.9 | physical tokens/s: 156823.4 | loss tokens/s: 117608.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.558840E-01 | lm-global-supervised-tokens: 1.260527E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.306976E-02 | lm-token-buffer-mib: 5.810036E+01 | global_load_balancing_loss: 1.000242E+00 | loss scale: 1.0 | grad norm: 0.036 | params norm: 11522.119 | params rms: 0.062389 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 3:49:08.381903 | finish at 2026-10-11 19:04:18 + [2026-10-05 15:16:38] iteration 1891/ 6855 | consumed samples: 242048 | elapsed time per iteration (ms): 88539.4 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.49% | FLOPs estimator: workload-aware | logical tokens/s: 189488.0 | valid tokens/s: 189488.0 | physical tokens/s: 190027.4 | loss tokens/s: 137530.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.490892E-01 | lm-global-supervised-tokens: 1.217688E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.543144E-02 | lm-token-buffer-mib: 5.814910E+01 | global_load_balancing_loss: 1.000039E+00 | loss scale: 1.0 | grad norm: 0.044 | params norm: 11522.050 | params rms: 0.062388 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 2:05:09.685065 | finish at 2026-10-10 17:21:48 + [2026-10-05 15:18:11] iteration 1892/ 6855 | consumed samples: 242176 | elapsed time per iteration (ms): 92318.9 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.46% | FLOPs estimator: workload-aware | logical tokens/s: 181730.6 | valid tokens/s: 181730.6 | physical tokens/s: 182233.3 | loss tokens/s: 124009.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.545430E-01 | lm-global-supervised-tokens: 1.144840E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.578408E-02 | lm-token-buffer-mib: 5.816298E+01 | global_load_balancing_loss: 1.000976E+00 | loss scale: 1.0 | grad norm: 0.056 | params norm: 11521.981 | params rms: 0.062388 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 7:16:18.794716 | finish at 2026-10-10 22:34:29 + [2026-10-05 15:19:35] iteration 1893/ 6855 | consumed samples: 242304 | elapsed time per iteration (ms): 84377.1 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.57% | FLOPs estimator: workload-aware | logical tokens/s: 198836.2 | valid tokens/s: 198836.2 | physical tokens/s: 199264.4 | loss tokens/s: 150553.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.531067E-01 | lm-global-supervised-tokens: 1.270329E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.249379E-02 | lm-token-buffer-mib: 5.808616E+01 | global_load_balancing_loss: 1.000857E+00 | loss scale: 1.0 | grad norm: 0.045 | params norm: 11521.912 | params rms: 0.062387 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 20:17:58.940646 | finish at 2026-10-10 11:37:34 + [2026-10-05 15:21:02] iteration 1894/ 6855 | consumed samples: 242432 | elapsed time per iteration (ms): 87182.9 | throughput per GPU (TFLOP/s/GPU): 43.3 | MFU 13.87% | FLOPs estimator: workload-aware | logical tokens/s: 192436.4 | valid tokens/s: 192436.4 | physical tokens/s: 192928.6 | loss tokens/s: 140328.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.907948E-01 | lm-global-supervised-tokens: 1.223423E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.140096E-02 | lm-token-buffer-mib: 5.820496E+01 | global_load_balancing_loss: 1.000790E+00 | loss scale: 1.0 | grad norm: 0.135 | params norm: 11521.843 | params rms: 0.062387 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 0:08:34.418705 | finish at 2026-10-10 15:29:37 + [2026-10-05 15:22:30] iteration 1895/ 6855 | consumed samples: 242560 | elapsed time per iteration (ms): 88333.5 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.57% | FLOPs estimator: workload-aware | logical tokens/s: 189930.1 | valid tokens/s: 189930.1 | physical tokens/s: 190407.6 | loss tokens/s: 138016.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.633380E-01 | lm-global-supervised-tokens: 1.219152E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.310152E-02 | lm-token-buffer-mib: 5.818026E+01 | global_load_balancing_loss: 9.999790E-01 | loss scale: 1.0 | grad norm: 0.135 | params norm: 11521.772 | params rms: 0.062387 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 1:42:14.020004 | finish at 2026-10-10 17:04:44 + [2026-10-05 15:23:57] iteration 1896/ 6855 | consumed samples: 242688 | elapsed time per iteration (ms): 86650.9 | throughput per GPU (TFLOP/s/GPU): 43.8 | MFU 14.05% | FLOPs estimator: workload-aware | logical tokens/s: 193618.3 | valid tokens/s: 193618.3 | physical tokens/s: 194021.9 | loss tokens/s: 140487.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.130386E-01 | lm-global-supervised-tokens: 1.217337E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.235540E-02 | lm-token-buffer-mib: 5.814738E+01 | global_load_balancing_loss: 9.989451E-01 | loss scale: 1.0 | grad norm: 0.041 | params norm: 11521.700 | params rms: 0.062386 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 23:21:41.923678 | finish at 2026-10-10 14:45:39 + [2026-10-05 15:25:25] iteration 1897/ 6855 | consumed samples: 242816 | elapsed time per iteration (ms): 87593.4 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.78% | FLOPs estimator: workload-aware | logical tokens/s: 191535.1 | valid tokens/s: 191535.1 | physical tokens/s: 191846.5 | loss tokens/s: 139875.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.001411E-01 | lm-global-supervised-tokens: 1.225217E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.256835E-02 | lm-token-buffer-mib: 5.810123E+01 | global_load_balancing_loss: 1.000012E+00 | loss scale: 1.0 | grad norm: 0.103 | params norm: 11521.629 | params rms: 0.062386 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 0:38:08.298256 | finish at 2026-10-10 16:03:33 + [2026-10-05 15:26:50] iteration 1898/ 6855 | consumed samples: 242944 | elapsed time per iteration (ms): 85408.3 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.60% | FLOPs estimator: workload-aware | logical tokens/s: 196435.5 | valid tokens/s: 196435.5 | physical tokens/s: 196810.8 | loss tokens/s: 144629.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.285270E-01 | lm-global-supervised-tokens: 1.235255E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.110161E-02 | lm-token-buffer-mib: 5.803848E+01 | global_load_balancing_loss: 1.000341E+00 | loss scale: 1.0 | grad norm: 0.048 | params norm: 11521.557 | params rms: 0.062385 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 21:36:08.720532 | finish at 2026-10-10 13:02:59 + [2026-10-05 15:28:17] iteration 1899/ 6855 | consumed samples: 243072 | elapsed time per iteration (ms): 86974.9 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.48% | FLOPs estimator: workload-aware | logical tokens/s: 192896.1 | valid tokens/s: 192896.1 | physical tokens/s: 193428.0 | loss tokens/s: 142551.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.702545E-01 | lm-global-supervised-tokens: 1.239837E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.046165E-02 | lm-token-buffer-mib: 5.807432E+01 | global_load_balancing_loss: 9.998322E-01 | loss scale: 1.0 | grad norm: 0.099 | params norm: 11521.486 | params rms: 0.062385 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 23:44:07.792311 | finish at 2026-10-10 15:12:25 + [2026-10-05 15:30:09] iteration 1900/ 6855 | consumed samples: 243200 | elapsed time per iteration (ms): 111952.1 | throughput per GPU (TFLOP/s/GPU): 33.3 | MFU 10.67% | FLOPs estimator: workload-aware | logical tokens/s: 149860.4 | valid tokens/s: 149860.3 | physical tokens/s: 150200.5 | loss tokens/s: 108848.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.561466E-01 | lm-global-supervised-tokens: 1.218581E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.144274E-02 | lm-token-buffer-mib: 5.825977E+01 | global_load_balancing_loss: 1.000526E+00 | loss scale: 1.0 | grad norm: 0.201 | params norm: 11521.416 | params rms: 0.062385 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 10:05:22.795092 | finish at 2026-10-12 01:35:32 + [2026-10-05 15:31:35] iteration 1901/ 6855 | consumed samples: 243328 | elapsed time per iteration (ms): 86160.8 | throughput per GPU (TFLOP/s/GPU): 43.4 | MFU 13.92% | FLOPs estimator: workload-aware | logical tokens/s: 194719.3 | valid tokens/s: 194719.3 | physical tokens/s: 195199.8 | loss tokens/s: 140534.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.724812E-01 | lm-global-supervised-tokens: 1.210858E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.258955E-02 | lm-token-buffer-mib: 5.812425E+01 | global_load_balancing_loss: 1.000583E+00 | loss scale: 1.0 | grad norm: 0.147 | params norm: 11521.344 | params rms: 0.062384 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 22:34:00.422390 | finish at 2026-10-10 14:05:36 + [2026-10-05 15:33:02] iteration 1902/ 6855 | consumed samples: 243456 | elapsed time per iteration (ms): 87288.6 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.57% | FLOPs estimator: workload-aware | logical tokens/s: 192203.8 | valid tokens/s: 192203.8 | physical tokens/s: 192740.0 | loss tokens/s: 143273.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.942552E-01 | lm-global-supervised-tokens: 1.250618E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.335474E-02 | lm-token-buffer-mib: 5.820240E+01 | global_load_balancing_loss: 1.000969E+00 | loss scale: 1.0 | grad norm: 0.116 | params norm: 11521.273 | params rms: 0.062384 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 0:05:40.678904 | finish at 2026-10-10 15:38:43 + [2026-10-05 15:34:31] iteration 1903/ 6855 | consumed samples: 243584 | elapsed time per iteration (ms): 88849.4 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.46% | FLOPs estimator: workload-aware | logical tokens/s: 188827.0 | valid tokens/s: 188827.0 | physical tokens/s: 189259.6 | loss tokens/s: 135059.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.791275E-01 | lm-global-supervised-tokens: 1.199997E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.299224E-02 | lm-token-buffer-mib: 5.810263E+01 | global_load_balancing_loss: 1.000015E+00 | loss scale: 1.0 | grad norm: 0.222 | params norm: 11521.199 | params rms: 0.062384 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 2:13:02.142828 | finish at 2026-10-10 17:47:33 + [2026-10-05 15:36:05] iteration 1904/ 6855 | consumed samples: 243712 | elapsed time per iteration (ms): 93462.0 | throughput per GPU (TFLOP/s/GPU): 40.7 | MFU 13.06% | FLOPs estimator: workload-aware | logical tokens/s: 179508.5 | valid tokens/s: 179508.5 | physical tokens/s: 179912.7 | loss tokens/s: 127757.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.601794E-01 | lm-global-supervised-tokens: 1.194044E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.103757E-02 | lm-token-buffer-mib: 5.833234E+01 | global_load_balancing_loss: 1.001358E+00 | loss scale: 1.0 | grad norm: 0.049 | params norm: 11521.127 | params rms: 0.062383 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 8:32:10.146636 | finish at 2026-10-11 00:08:15 + [2026-10-05 15:37:32] iteration 1905/ 6855 | consumed samples: 243840 | elapsed time per iteration (ms): 87480.6 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.63% | FLOPs estimator: workload-aware | logical tokens/s: 191781.6 | valid tokens/s: 191781.6 | physical tokens/s: 192172.7 | loss tokens/s: 143156.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.660735E-01 | lm-global-supervised-tokens: 1.252338E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.172398E-02 | lm-token-buffer-mib: 5.807446E+01 | global_load_balancing_loss: 1.000666E+00 | loss scale: 1.0 | grad norm: 0.134 | params norm: 11521.055 | params rms: 0.062383 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 0:17:09.140532 | finish at 2026-10-10 15:54:41 + [2026-10-05 15:39:00] iteration 1906/ 6855 | consumed samples: 243968 | elapsed time per iteration (ms): 87539.4 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.57% | FLOPs estimator: workload-aware | logical tokens/s: 191653.3 | valid tokens/s: 191653.3 | physical tokens/s: 192064.9 | loss tokens/s: 145185.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.640657E-01 | lm-global-supervised-tokens: 1.270942E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.607518E-02 | lm-token-buffer-mib: 5.807009E+01 | global_load_balancing_loss: 9.985651E-01 | loss scale: 1.0 | grad norm: 0.050 | params norm: 11520.984 | params rms: 0.062382 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 0:20:32.433282 | finish at 2026-10-10 15:59:32 + [2026-10-05 15:40:26] iteration 1907/ 6855 | consumed samples: 244096 | elapsed time per iteration (ms): 86229.6 | throughput per GPU (TFLOP/s/GPU): 43.1 | MFU 13.80% | FLOPs estimator: workload-aware | logical tokens/s: 194564.2 | valid tokens/s: 194564.2 | physical tokens/s: 195000.5 | loss tokens/s: 143394.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.609824E-01 | lm-global-supervised-tokens: 1.236482E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.822995E-02 | lm-token-buffer-mib: 5.808445E+01 | global_load_balancing_loss: 9.997658E-01 | loss scale: 1.0 | grad norm: 0.117 | params norm: 11520.912 | params rms: 0.062382 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 22:31:04.074722 | finish at 2026-10-10 14:11:30 + [2026-10-05 15:41:52] iteration 1908/ 6855 | consumed samples: 244224 | elapsed time per iteration (ms): 86297.6 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.54% | FLOPs estimator: workload-aware | logical tokens/s: 194410.5 | valid tokens/s: 194410.5 | physical tokens/s: 194860.6 | loss tokens/s: 145549.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 5.947547E-01 | lm-global-supervised-tokens: 1.256054E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.122896E-02 | lm-token-buffer-mib: 5.806400E+01 | global_load_balancing_loss: 1.000290E+00 | loss scale: 1.0 | grad norm: 0.032 | params norm: 11520.839 | params rms: 0.062382 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 22:35:14.208482 | finish at 2026-10-10 14:17:07 + [2026-10-05 15:43:16] iteration 1909/ 6855 | consumed samples: 244352 | elapsed time per iteration (ms): 84086.7 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.69% | FLOPs estimator: workload-aware | logical tokens/s: 199522.3 | valid tokens/s: 199522.3 | physical tokens/s: 200003.4 | loss tokens/s: 149117.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.436314E-01 | lm-global-supervised-tokens: 1.253880E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.360635E-02 | lm-token-buffer-mib: 5.802955E+01 | global_load_balancing_loss: 1.000168E+00 | loss scale: 1.0 | grad norm: 0.114 | params norm: 11520.766 | params rms: 0.062381 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 19:31:32.954804 | finish at 2026-10-10 11:14:49 + [2026-10-05 15:45:08] iteration 1910/ 6855 | consumed samples: 244480 | elapsed time per iteration (ms): 111222.6 | throughput per GPU (TFLOP/s/GPU): 33.1 | MFU 10.61% | FLOPs estimator: workload-aware | logical tokens/s: 150843.3 | valid tokens/s: 150843.3 | physical tokens/s: 151164.6 | loss tokens/s: 108176.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.755530E-01 | lm-global-supervised-tokens: 1.203163E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.289249E-02 | lm-token-buffer-mib: 5.813213E+01 | global_load_balancing_loss: 9.992995E-01 | loss scale: 1.0 | grad norm: 0.040 | params norm: 11520.692 | params rms: 0.062381 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 8:46:35.664957 | finish at 2026-10-12 00:31:43 + [2026-10-05 15:46:38] iteration 1911/ 6855 | consumed samples: 244608 | elapsed time per iteration (ms): 90476.6 | throughput per GPU (TFLOP/s/GPU): 41.5 | MFU 13.30% | FLOPs estimator: workload-aware | logical tokens/s: 185431.2 | valid tokens/s: 185431.2 | physical tokens/s: 185851.6 | loss tokens/s: 134166.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.741927E-01 | lm-global-supervised-tokens: 1.213895E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.283591E-02 | lm-token-buffer-mib: 5.816099E+01 | global_load_balancing_loss: 1.000733E+00 | loss scale: 1.0 | grad norm: 0.109 | params norm: 11520.616 | params rms: 0.062380 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 4:15:16.232265 | finish at 2026-10-10 20:01:54 + [2026-10-05 15:48:06] iteration 1912/ 6855 | consumed samples: 244736 | elapsed time per iteration (ms): 87417.5 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.52% | FLOPs estimator: workload-aware | logical tokens/s: 191918.9 | valid tokens/s: 191918.9 | physical tokens/s: 192412.3 | loss tokens/s: 142784.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.691139E-01 | lm-global-supervised-tokens: 1.248189E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.221051E-02 | lm-token-buffer-mib: 5.810468E+01 | global_load_balancing_loss: 9.996384E-01 | loss scale: 1.0 | grad norm: 0.057 | params norm: 11520.540 | params rms: 0.062380 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 0:01:44.841658 | finish at 2026-10-10 15:49:50 + [2026-10-05 15:49:32] iteration 1913/ 6855 | consumed samples: 244864 | elapsed time per iteration (ms): 86752.2 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.63% | FLOPs estimator: workload-aware | logical tokens/s: 193391.8 | valid tokens/s: 193391.8 | physical tokens/s: 193849.8 | loss tokens/s: 143360.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.322584E-01 | lm-global-supervised-tokens: 1.243683E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.357637E-02 | lm-token-buffer-mib: 5.822155E+01 | global_load_balancing_loss: 1.000811E+00 | loss scale: 1.0 | grad norm: 0.094 | params norm: 11520.463 | params rms: 0.062380 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 23:05:29.309400 | finish at 2026-10-10 14:55:02 + [2026-10-05 15:50:59] iteration 1914/ 6855 | consumed samples: 244992 | elapsed time per iteration (ms): 86432.9 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.48% | FLOPs estimator: workload-aware | logical tokens/s: 194106.4 | valid tokens/s: 194106.4 | physical tokens/s: 194680.8 | loss tokens/s: 145488.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.051008E-01 | lm-global-supervised-tokens: 1.257500E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.425174E-02 | lm-token-buffer-mib: 5.820699E+01 | global_load_balancing_loss: 1.000556E+00 | loss scale: 1.0 | grad norm: 0.066 | params norm: 11520.388 | params rms: 0.062379 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 22:37:45.041124 | finish at 2026-10-10 14:28:44 + [2026-10-05 15:52:25] iteration 1915/ 6855 | consumed samples: 245120 | elapsed time per iteration (ms): 85855.5 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.74% | FLOPs estimator: workload-aware | logical tokens/s: 195412.0 | valid tokens/s: 195412.0 | physical tokens/s: 195847.0 | loss tokens/s: 146414.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.640591E-01 | lm-global-supervised-tokens: 1.257053E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.149433E-02 | lm-token-buffer-mib: 5.819005E+01 | global_load_balancing_loss: 1.000526E+00 | loss scale: 1.0 | grad norm: 0.065 | params norm: 11520.311 | params rms: 0.062379 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 21:48:46.200538 | finish at 2026-10-10 13:41:11 + [2026-10-05 15:53:54] iteration 1916/ 6855 | consumed samples: 245248 | elapsed time per iteration (ms): 89320.0 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.44% | FLOPs estimator: workload-aware | logical tokens/s: 187832.1 | valid tokens/s: 187832.1 | physical tokens/s: 188124.6 | loss tokens/s: 132013.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.685592E-01 | lm-global-supervised-tokens: 1.179147E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.173322E-02 | lm-token-buffer-mib: 5.809687E+01 | global_load_balancing_loss: 1.001651E+00 | loss scale: 1.0 | grad norm: 0.036 | params norm: 11520.237 | params rms: 0.062378 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 2:32:31.675143 | finish at 2026-10-10 18:26:26 + [2026-10-05 15:55:17] iteration 1917/ 6855 | consumed samples: 245376 | elapsed time per iteration (ms): 82625.2 | throughput per GPU (TFLOP/s/GPU): 43.6 | MFU 13.98% | FLOPs estimator: workload-aware | logical tokens/s: 203051.2 | valid tokens/s: 203051.2 | physical tokens/s: 203555.0 | loss tokens/s: 152543.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.949915E-01 | lm-global-supervised-tokens: 1.260390E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.246516E-02 | lm-token-buffer-mib: 5.816431E+01 | global_load_balancing_loss: 9.996951E-01 | loss scale: 1.0 | grad norm: 0.074 | params norm: 11520.164 | params rms: 0.062378 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 17:20:03.203622 | finish at 2026-10-10 09:15:20 + [2026-10-05 15:56:43] iteration 1918/ 6855 | consumed samples: 245504 | elapsed time per iteration (ms): 86342.9 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.78% | FLOPs estimator: workload-aware | logical tokens/s: 194309.0 | valid tokens/s: 194309.0 | physical tokens/s: 194802.8 | loss tokens/s: 145356.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.318337E-01 | lm-global-supervised-tokens: 1.255049E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.595602E-02 | lm-token-buffer-mib: 5.820302E+01 | global_load_balancing_loss: 9.996841E-01 | loss scale: 1.0 | grad norm: 0.046 | params norm: 11520.091 | params rms: 0.062378 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 22:24:34.868060 | finish at 2026-10-10 14:21:18 + [2026-10-05 15:58:11] iteration 1919/ 6855 | consumed samples: 245632 | elapsed time per iteration (ms): 88500.0 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.49% | FLOPs estimator: workload-aware | logical tokens/s: 189572.5 | valid tokens/s: 189572.5 | physical tokens/s: 189895.6 | loss tokens/s: 140007.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.473601E-01 | lm-global-supervised-tokens: 1.239065E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.140216E-02 | lm-token-buffer-mib: 5.807800E+01 | global_load_balancing_loss: 9.991832E-01 | loss scale: 1.0 | grad norm: 0.093 | params norm: 11520.016 | params rms: 0.062377 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 1:20:36.016476 | finish at 2026-10-10 17:18:47 + [2026-10-05 16:00:04] iteration 1920/ 6855 | consumed samples: 245760 | elapsed time per iteration (ms): 112904.1 | throughput per GPU (TFLOP/s/GPU): 33.2 | MFU 10.65% | FLOPs estimator: workload-aware | logical tokens/s: 148597.0 | valid tokens/s: 148597.0 | physical tokens/s: 148955.7 | loss tokens/s: 103917.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.292600E-01 | lm-global-supervised-tokens: 1.173268E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.240389E-02 | lm-token-buffer-mib: 5.824551E+01 | global_load_balancing_loss: 9.990509E-01 | loss scale: 1.0 | grad norm: 0.065 | params norm: 11519.943 | params rms: 0.062377 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 10:46:21.748506 | finish at 2026-10-12 02:46:26 +times across ranks (ms): + save-checkpoint: + rank 0: 24202.13 + rank 1: 24202.12 + rank 2: 24202.19 + rank 3: 24202.19 + rank 4: 24202.17 + rank 5: 24202.15 + rank 6: 24202.12 + rank 7: 24202.13 + rank 8: 24202.38 + rank 9: 24202.37 + rank 10: 24202.35 + rank 11: 24202.34 + rank 12: 24202.36 + rank 13: 24202.37 + rank 14: 24202.34 + rank 15: 24202.36 + rank 16: 24201.45 + rank 17: 24201.45 + rank 18: 24201.44 + rank 19: 24201.47 + rank 20: 24201.43 + rank 21: 24201.47 + rank 22: 24201.43 + rank 23: 24201.45 + rank 24: 24201.21 + rank 25: 24201.26 + rank 26: 24201.22 + rank 27: 24201.23 + rank 28: 24201.18 + rank 29: 24201.19 + rank 30: 24201.15 + rank 31: 24201.21 + rank 32: 24201.74 + rank 33: 24201.74 + rank 34: 24201.75 + rank 35: 24201.74 + rank 36: 24201.71 + rank 37: 24201.72 + rank 38: 24201.73 + rank 39: 24201.73 + rank 40: 24201.91 + rank 41: 24201.90 + rank 42: 24201.96 + rank 43: 24201.91 + rank 44: 24201.90 + rank 45: 24201.89 + rank 46: 24201.90 + rank 47: 24201.91 + rank 48: 24201.35 + rank 49: 24201.33 + rank 50: 24201.31 + rank 51: 24201.34 + rank 52: 24201.32 + rank 53: 24201.32 + rank 54: 24201.33 + rank 55: 24201.33 + rank 56: 24201.07 + rank 57: 24201.06 + rank 58: 24201.06 + rank 59: 24201.03 + rank 60: 24201.06 + rank 61: 24201.07 + rank 62: 24201.06 + rank 63: 24201.06 + [2026-10-05 16:02:03] iteration 1921/ 6855 | consumed samples: 245888 | elapsed time per iteration (ms): 94426.8 | throughput per GPU (TFLOP/s/GPU): 39.6 | MFU 12.69% | FLOPs estimator: workload-aware | logical tokens/s: 177673.7 | valid tokens/s: 177673.7 | physical tokens/s: 178138.8 | loss tokens/s: 130989.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.635234E-01 | lm-global-supervised-tokens: 1.236891E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 5.432352E-01 | lm-token-buffer-mib: 5.825907E+01 | global_load_balancing_loss: 1.000927E+00 | loss scale: 1.0 | grad norm: 0.150 | params norm: 11519.870 | params rms: 0.062376 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 9:25:02.011245 | finish at 2026-10-11 01:27:05 + [2026-10-05 16:03:37] iteration 1922/ 6855 | consumed samples: 246016 | elapsed time per iteration (ms): 93382.0 | throughput per GPU (TFLOP/s/GPU): 39.8 | MFU 12.77% | FLOPs estimator: workload-aware | logical tokens/s: 179661.6 | valid tokens/s: 179661.6 | physical tokens/s: 180032.0 | loss tokens/s: 127078.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.773021E-01 | lm-global-supervised-tokens: 1.186687E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.762569E-02 | lm-token-buffer-mib: 5.808002E+01 | global_load_balancing_loss: 1.001854E+00 | loss scale: 1.0 | grad norm: 0.166 | params norm: 11519.797 | params rms: 0.062376 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 7:57:33.374094 | finish at 2026-10-11 00:01:10 + [2026-10-05 16:04:59] iteration 1923/ 6855 | consumed samples: 246144 | elapsed time per iteration (ms): 82045.0 | throughput per GPU (TFLOP/s/GPU): 43.5 | MFU 13.94% | FLOPs estimator: workload-aware | logical tokens/s: 204487.3 | valid tokens/s: 204487.3 | physical tokens/s: 205187.6 | loss tokens/s: 154659.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.321886E-01 | lm-global-supervised-tokens: 1.268908E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.204673E-02 | lm-token-buffer-mib: 5.816073E+01 | global_load_balancing_loss: 9.997343E-01 | loss scale: 1.0 | grad norm: 0.105 | params norm: 11519.725 | params rms: 0.062376 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 16:24:06.048557 | finish at 2026-10-10 08:29:05 + [2026-10-05 16:06:26] iteration 1924/ 6855 | consumed samples: 246272 | elapsed time per iteration (ms): 87206.4 | throughput per GPU (TFLOP/s/GPU): 43.1 | MFU 13.80% | FLOPs estimator: workload-aware | logical tokens/s: 192383.7 | valid tokens/s: 192383.7 | physical tokens/s: 192812.9 | loss tokens/s: 143810.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.028943E-01 | lm-global-supervised-tokens: 1.254116E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.304484E-02 | lm-token-buffer-mib: 5.821696E+01 | global_load_balancing_loss: 1.000643E+00 | loss scale: 1.0 | grad norm: 0.223 | params norm: 11519.654 | params rms: 0.062375 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 23:26:54.620444 | finish at 2026-10-10 15:33:20 + [2026-10-05 16:07:55] iteration 1925/ 6855 | consumed samples: 246400 | elapsed time per iteration (ms): 88762.4 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.40% | FLOPs estimator: workload-aware | logical tokens/s: 189012.5 | valid tokens/s: 189012.4 | physical tokens/s: 189350.8 | loss tokens/s: 137461.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.486850E-01 | lm-global-supervised-tokens: 1.220144E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.182917E-02 | lm-token-buffer-mib: 5.812359E+01 | global_load_balancing_loss: 1.000514E+00 | loss scale: 1.0 | grad norm: 0.031 | params norm: 11519.582 | params rms: 0.062375 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 1:33:18.579848 | finish at 2026-10-10 17:41:13 + [2026-10-05 16:09:21] iteration 1926/ 6855 | consumed samples: 246528 | elapsed time per iteration (ms): 85984.7 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.51% | FLOPs estimator: workload-aware | logical tokens/s: 195117.9 | valid tokens/s: 195117.9 | physical tokens/s: 195612.2 | loss tokens/s: 148945.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.600682E-01 | lm-global-supervised-tokens: 1.280703E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.234514E-02 | lm-token-buffer-mib: 5.817916E+01 | global_load_balancing_loss: 9.996967E-01 | loss scale: 1.0 | grad norm: 0.285 | params norm: 11519.509 | params rms: 0.062374 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 21:43:38.751823 | finish at 2026-10-10 13:52:59 + [2026-10-05 16:10:47] iteration 1927/ 6855 | consumed samples: 246656 | elapsed time per iteration (ms): 86508.3 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.47% | FLOPs estimator: workload-aware | logical tokens/s: 193937.3 | valid tokens/s: 193937.3 | physical tokens/s: 194438.3 | loss tokens/s: 146301.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.886246E-01 | lm-global-supervised-tokens: 1.265630E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.164644E-02 | lm-token-buffer-mib: 5.808080E+01 | global_load_balancing_loss: 9.990861E-01 | loss scale: 1.0 | grad norm: 0.123 | params norm: 11519.437 | params rms: 0.062374 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 22:25:12.858078 | finish at 2026-10-10 14:36:00 + [2026-10-05 16:12:13] iteration 1928/ 6855 | consumed samples: 246784 | elapsed time per iteration (ms): 86151.0 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.64% | FLOPs estimator: workload-aware | logical tokens/s: 194742.0 | valid tokens/s: 194741.9 | physical tokens/s: 195217.7 | loss tokens/s: 140638.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.278883E-01 | lm-global-supervised-tokens: 1.211616E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.910312E-02 | lm-token-buffer-mib: 5.828175E+01 | global_load_balancing_loss: 1.000312E+00 | loss scale: 1.0 | grad norm: 0.224 | params norm: 11519.365 | params rms: 0.062374 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 21:54:25.865517 | finish at 2026-10-10 14:06:39 + [2026-10-05 16:13:42] iteration 1929/ 6855 | consumed samples: 246912 | elapsed time per iteration (ms): 88789.6 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.49% | FLOPs estimator: workload-aware | logical tokens/s: 188954.5 | valid tokens/s: 188954.5 | physical tokens/s: 189401.8 | loss tokens/s: 140357.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.010515E-01 | lm-global-supervised-tokens: 1.246227E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.156817E-02 | lm-token-buffer-mib: 5.836639E+01 | global_load_balancing_loss: 1.000554E+00 | loss scale: 1.0 | grad norm: 0.052 | params norm: 11519.292 | params rms: 0.062373 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 1:29:37.700624 | finish at 2026-10-10 17:43:20 + [2026-10-05 16:15:33] iteration 1930/ 6855 | consumed samples: 247040 | elapsed time per iteration (ms): 111082.4 | throughput per GPU (TFLOP/s/GPU): 34.8 | MFU 11.17% | FLOPs estimator: workload-aware | logical tokens/s: 151033.8 | valid tokens/s: 151033.8 | physical tokens/s: 151278.9 | loss tokens/s: 103119.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.493406E-01 | lm-global-supervised-tokens: 1.145475E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.216071E-02 | lm-token-buffer-mib: 5.804483E+01 | global_load_balancing_loss: 1.000934E+00 | loss scale: 1.0 | grad norm: 0.133 | params norm: 11519.219 | params rms: 0.062373 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 7:58:01.008285 | finish at 2026-10-12 00:13:34 + [2026-10-05 16:17:04] iteration 1931/ 6855 | consumed samples: 247168 | elapsed time per iteration (ms): 90968.8 | throughput per GPU (TFLOP/s/GPU): 41.2 | MFU 13.20% | FLOPs estimator: workload-aware | logical tokens/s: 184428.1 | valid tokens/s: 184428.1 | physical tokens/s: 184964.2 | loss tokens/s: 139496.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.295014E-01 | lm-global-supervised-tokens: 1.268983E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.289154E-02 | lm-token-buffer-mib: 5.819619E+01 | global_load_balancing_loss: 1.000216E+00 | loss scale: 1.0 | grad norm: 0.226 | params norm: 11519.144 | params rms: 0.062372 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 4:25:30.284660 | finish at 2026-10-10 20:42:34 + [2026-10-05 16:18:31] iteration 1932/ 6855 | consumed samples: 247296 | elapsed time per iteration (ms): 87300.8 | throughput per GPU (TFLOP/s/GPU): 41.7 | MFU 13.38% | FLOPs estimator: workload-aware | logical tokens/s: 192176.4 | valid tokens/s: 192176.4 | physical tokens/s: 192753.6 | loss tokens/s: 144344.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.651004E-01 | lm-global-supervised-tokens: 1.260139E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.428097E-02 | lm-token-buffer-mib: 5.846482E+01 | global_load_balancing_loss: 9.995407E-01 | loss scale: 1.0 | grad norm: 0.136 | params norm: 11519.068 | params rms: 0.062372 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 23:23:01.699144 | finish at 2026-10-10 15:41:33 + [2026-10-05 16:20:01] iteration 1933/ 6855 | consumed samples: 247424 | elapsed time per iteration (ms): 89355.0 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.40% | FLOPs estimator: workload-aware | logical tokens/s: 187759.1 | valid tokens/s: 187759.1 | physical tokens/s: 188077.6 | loss tokens/s: 135493.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.981360E-01 | lm-global-supervised-tokens: 1.210705E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.675623E-02 | lm-token-buffer-mib: 5.806314E+01 | global_load_balancing_loss: 1.000224E+00 | loss scale: 1.0 | grad norm: 0.162 | params norm: 11518.994 | params rms: 0.062372 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 2:10:05.409841 | finish at 2026-10-10 18:30:06 + [2026-10-05 16:21:31] iteration 1934/ 6855 | consumed samples: 247552 | elapsed time per iteration (ms): 90351.5 | throughput per GPU (TFLOP/s/GPU): 41.2 | MFU 13.22% | FLOPs estimator: workload-aware | logical tokens/s: 185687.8 | valid tokens/s: 185687.8 | physical tokens/s: 186090.9 | loss tokens/s: 135555.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.050695E-01 | lm-global-supervised-tokens: 1.224766E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.547090E-02 | lm-token-buffer-mib: 5.809112E+01 | global_load_balancing_loss: 1.000187E+00 | loss scale: 1.0 | grad norm: 0.150 | params norm: 11518.919 | params rms: 0.062371 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 3:30:19.870113 | finish at 2026-10-10 19:51:51 + [2026-10-05 16:23:01] iteration 1935/ 6855 | consumed samples: 247680 | elapsed time per iteration (ms): 89550.2 | throughput per GPU (TFLOP/s/GPU): 41.7 | MFU 13.36% | FLOPs estimator: workload-aware | logical tokens/s: 187349.4 | valid tokens/s: 187349.4 | physical tokens/s: 187624.5 | loss tokens/s: 132306.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.414470E-01 | lm-global-supervised-tokens: 1.184806E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.165506E-02 | lm-token-buffer-mib: 5.803719E+01 | global_load_balancing_loss: 9.999556E-01 | loss scale: 1.0 | grad norm: 0.061 | params norm: 11518.845 | params rms: 0.062371 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 2:23:07.212667 | finish at 2026-10-10 18:46:08 + [2026-10-05 16:24:31] iteration 1936/ 6855 | consumed samples: 247808 | elapsed time per iteration (ms): 90315.6 | throughput per GPU (TFLOP/s/GPU): 40.9 | MFU 13.10% | FLOPs estimator: workload-aware | logical tokens/s: 185761.6 | valid tokens/s: 185761.6 | physical tokens/s: 186210.3 | loss tokens/s: 133743.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.555183E-01 | lm-global-supervised-tokens: 1.207911E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.889877E-02 | lm-token-buffer-mib: 5.812256E+01 | global_load_balancing_loss: 1.000802E+00 | loss scale: 1.0 | grad norm: 0.222 | params norm: 11518.770 | params rms: 0.062370 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 3:24:22.557968 | finish at 2026-10-10 19:48:54 + [2026-10-05 16:25:56] iteration 1937/ 6855 | consumed samples: 247936 | elapsed time per iteration (ms): 85298.1 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.66% | FLOPs estimator: workload-aware | logical tokens/s: 196689.2 | valid tokens/s: 196689.1 | physical tokens/s: 197213.6 | loss tokens/s: 146439.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.810745E-01 | lm-global-supervised-tokens: 1.249101E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.675209E-02 | lm-token-buffer-mib: 5.818875E+01 | global_load_balancing_loss: 1.001691E+00 | loss scale: 1.0 | grad norm: 0.115 | params norm: 11518.696 | params rms: 0.062370 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 20:31:35.823610 | finish at 2026-10-10 12:57:32 + [2026-10-05 16:27:23] iteration 1938/ 6855 | consumed samples: 248064 | elapsed time per iteration (ms): 86837.6 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.57% | FLOPs estimator: workload-aware | logical tokens/s: 193202.0 | valid tokens/s: 193202.0 | physical tokens/s: 193632.5 | loss tokens/s: 136526.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.369980E-01 | lm-global-supervised-tokens: 1.185562E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.164239E-02 | lm-token-buffer-mib: 5.832504E+01 | global_load_balancing_loss: 1.001886E+00 | loss scale: 1.0 | grad norm: 0.048 | params norm: 11518.622 | params rms: 0.062370 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 22:36:20.635081 | finish at 2026-10-10 15:03:44 + [2026-10-05 16:28:52] iteration 1939/ 6855 | consumed samples: 248192 | elapsed time per iteration (ms): 89125.8 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.50% | FLOPs estimator: workload-aware | logical tokens/s: 188241.5 | valid tokens/s: 188241.5 | physical tokens/s: 188670.0 | loss tokens/s: 139720.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.705319E-01 | lm-global-supervised-tokens: 1.245267E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.263600E-02 | lm-token-buffer-mib: 5.815166E+01 | global_load_balancing_loss: 9.999253E-01 | loss scale: 1.0 | grad norm: 0.113 | params norm: 11518.549 | params rms: 0.062369 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 1:42:22.611607 | finish at 2026-10-10 18:11:15 + [2026-10-05 16:30:41] iteration 1940/ 6855 | consumed samples: 248320 | elapsed time per iteration (ms): 108656.6 | throughput per GPU (TFLOP/s/GPU): 34.4 | MFU 11.03% | FLOPs estimator: workload-aware | logical tokens/s: 154405.2 | valid tokens/s: 154405.2 | physical tokens/s: 154855.4 | loss tokens/s: 114158.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.580551E-01 | lm-global-supervised-tokens: 1.240410E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.702048E-02 | lm-token-buffer-mib: 5.812269E+01 | global_load_balancing_loss: 9.980596E-01 | loss scale: 1.0 | grad norm: 0.113 | params norm: 11518.476 | params rms: 0.062369 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 4:20:47.310863 | finish at 2026-10-11 20:51:28 + [2026-10-05 16:32:10] iteration 1941/ 6855 | consumed samples: 248448 | elapsed time per iteration (ms): 89211.7 | throughput per GPU (TFLOP/s/GPU): 43.1 | MFU 13.83% | FLOPs estimator: workload-aware | logical tokens/s: 188060.1 | valid tokens/s: 188060.1 | physical tokens/s: 188423.7 | loss tokens/s: 133754.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.834218E-01 | lm-global-supervised-tokens: 1.193244E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.165377E-02 | lm-token-buffer-mib: 5.811788E+01 | global_load_balancing_loss: 9.981867E-01 | loss scale: 1.0 | grad norm: 0.054 | params norm: 11518.404 | params rms: 0.062368 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 1:46:26.388515 | finish at 2026-10-10 18:18:37 + [2026-10-05 16:33:34] iteration 1942/ 6855 | consumed samples: 248576 | elapsed time per iteration (ms): 83745.2 | throughput per GPU (TFLOP/s/GPU): 42.8 | MFU 13.71% | FLOPs estimator: workload-aware | logical tokens/s: 200336.0 | valid tokens/s: 200336.0 | physical tokens/s: 200886.5 | loss tokens/s: 149432.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.410810E-01 | lm-global-supervised-tokens: 1.251423E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.459976E-02 | lm-token-buffer-mib: 5.807101E+01 | global_load_balancing_loss: 9.988773E-01 | loss scale: 1.0 | grad norm: 0.082 | params norm: 11518.334 | params rms: 0.062368 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 18:17:20.199999 | finish at 2026-10-10 10:50:54 + [2026-10-05 16:34:58] iteration 1943/ 6855 | consumed samples: 248704 | elapsed time per iteration (ms): 83663.1 | throughput per GPU (TFLOP/s/GPU): 44.0 | MFU 14.10% | FLOPs estimator: workload-aware | logical tokens/s: 200532.7 | valid tokens/s: 200532.7 | physical tokens/s: 200985.5 | loss tokens/s: 147616.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.699703E-01 | lm-global-supervised-tokens: 1.235008E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.174484E-02 | lm-token-buffer-mib: 5.817158E+01 | global_load_balancing_loss: 9.999198E-01 | loss scale: 1.0 | grad norm: 0.097 | params norm: 11518.264 | params rms: 0.062368 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 18:09:13.277214 | finish at 2026-10-10 10:44:11 + [2026-10-05 16:36:25] iteration 1944/ 6855 | consumed samples: 248832 | elapsed time per iteration (ms): 87221.8 | throughput per GPU (TFLOP/s/GPU): 42.8 | MFU 13.71% | FLOPs estimator: workload-aware | logical tokens/s: 192351.0 | valid tokens/s: 192351.0 | physical tokens/s: 192783.7 | loss tokens/s: 139425.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.579950E-01 | lm-global-supervised-tokens: 1.216091E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.028840E-02 | lm-token-buffer-mib: 5.815033E+01 | global_load_balancing_loss: 1.002223E+00 | loss scale: 1.0 | grad norm: 0.056 | params norm: 11518.192 | params rms: 0.062367 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 22:59:06.398400 | finish at 2026-10-10 15:35:31 + [2026-10-05 16:37:52] iteration 1945/ 6855 | consumed samples: 248960 | elapsed time per iteration (ms): 86958.0 | throughput per GPU (TFLOP/s/GPU): 43.5 | MFU 13.93% | FLOPs estimator: workload-aware | logical tokens/s: 192934.1 | valid tokens/s: 192934.1 | physical tokens/s: 193375.5 | loss tokens/s: 136216.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.926263E-01 | lm-global-supervised-tokens: 1.184515E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.148660E-02 | lm-token-buffer-mib: 5.816743E+01 | global_load_balancing_loss: 1.002379E+00 | loss scale: 1.0 | grad norm: 0.080 | params norm: 11518.121 | params rms: 0.062367 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 22:36:03.945959 | finish at 2026-10-10 15:13:56 + [2026-10-05 16:39:20] iteration 1946/ 6855 | consumed samples: 249088 | elapsed time per iteration (ms): 87837.4 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.65% | FLOPs estimator: workload-aware | logical tokens/s: 191002.7 | valid tokens/s: 191002.7 | physical tokens/s: 191379.3 | loss tokens/s: 142801.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.760240E-01 | lm-global-supervised-tokens: 1.254329E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.281892E-02 | lm-token-buffer-mib: 5.809821E+01 | global_load_balancing_loss: 1.001110E+00 | loss scale: 1.0 | grad norm: 0.059 | params norm: 11518.051 | params rms: 0.062367 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 23:46:34.007073 | finish at 2026-10-10 16:25:54 + [2026-10-05 16:40:50] iteration 1947/ 6855 | consumed samples: 249216 | elapsed time per iteration (ms): 90485.1 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.46% | FLOPs estimator: workload-aware | logical tokens/s: 185414.1 | valid tokens/s: 185414.1 | physical tokens/s: 185688.0 | loss tokens/s: 132552.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.115824E-01 | lm-global-supervised-tokens: 1.199404E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.181747E-02 | lm-token-buffer-mib: 5.809111E+01 | global_load_balancing_loss: 9.995897E-01 | loss scale: 1.0 | grad norm: 0.069 | params norm: 11517.980 | params rms: 0.062366 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 3:21:40.861590 | finish at 2026-10-10 20:02:31 + [2026-10-05 16:42:18] iteration 1948/ 6855 | consumed samples: 249344 | elapsed time per iteration (ms): 88023.5 | throughput per GPU (TFLOP/s/GPU): 43.2 | MFU 13.86% | FLOPs estimator: workload-aware | logical tokens/s: 190598.7 | valid tokens/s: 190598.7 | physical tokens/s: 190915.0 | loss tokens/s: 132846.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.713689E-01 | lm-global-supervised-tokens: 1.169364E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.154170E-02 | lm-token-buffer-mib: 5.802094E+01 | global_load_balancing_loss: 1.000268E+00 | loss scale: 1.0 | grad norm: 0.081 | params norm: 11517.905 | params rms: 0.062366 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 23:58:51.385697 | finish at 2026-10-10 16:41:09 + [2026-10-05 16:43:46] iteration 1949/ 6855 | consumed samples: 249472 | elapsed time per iteration (ms): 87926.4 | throughput per GPU (TFLOP/s/GPU): 43.5 | MFU 13.94% | FLOPs estimator: workload-aware | logical tokens/s: 190809.1 | valid tokens/s: 190809.1 | physical tokens/s: 191163.8 | loss tokens/s: 142513.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.448857E-01 | lm-global-supervised-tokens: 1.253070E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.287907E-02 | lm-token-buffer-mib: 5.805020E+01 | global_load_balancing_loss: 9.982173E-01 | loss scale: 1.0 | grad norm: 0.074 | params norm: 11517.835 | params rms: 0.062365 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 23:49:26.695897 | finish at 2026-10-10 16:33:13 + [2026-10-05 16:45:34] iteration 1950/ 6855 | consumed samples: 249600 | elapsed time per iteration (ms): 108287.0 | throughput per GPU (TFLOP/s/GPU): 34.1 | MFU 10.94% | FLOPs estimator: workload-aware | logical tokens/s: 154932.9 | valid tokens/s: 154932.9 | physical tokens/s: 155328.8 | loss tokens/s: 111531.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.563022E-01 | lm-global-supervised-tokens: 1.207737E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.245470E-02 | lm-token-buffer-mib: 5.810656E+01 | global_load_balancing_loss: 1.000062E+00 | loss scale: 1.0 | grad norm: 0.226 | params norm: 11517.767 | params rms: 0.062365 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 3:32:27.781488 | finish at 2026-10-11 20:18:02 + [2026-10-05 16:47:05] iteration 1951/ 6855 | consumed samples: 249728 | elapsed time per iteration (ms): 91023.1 | throughput per GPU (TFLOP/s/GPU): 41.0 | MFU 13.14% | FLOPs estimator: workload-aware | logical tokens/s: 184317.6 | valid tokens/s: 184317.6 | physical tokens/s: 184813.1 | loss tokens/s: 127600.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.579421E-01 | lm-global-supervised-tokens: 1.161456E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.736736E-02 | lm-token-buffer-mib: 5.812936E+01 | global_load_balancing_loss: 1.000382E+00 | loss scale: 1.0 | grad norm: 0.132 | params norm: 11517.699 | params rms: 0.062365 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 3:59:37.150955 | finish at 2026-10-10 20:46:42 + [2026-10-05 16:48:33] iteration 1952/ 6855 | consumed samples: 249856 | elapsed time per iteration (ms): 87680.8 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.46% | FLOPs estimator: workload-aware | logical tokens/s: 191343.9 | valid tokens/s: 191343.9 | physical tokens/s: 191829.4 | loss tokens/s: 145132.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.725080E-01 | lm-global-supervised-tokens: 1.272536E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.513724E-02 | lm-token-buffer-mib: 5.806391E+01 | global_load_balancing_loss: 1.000165E+00 | loss scale: 1.0 | grad norm: 0.378 | params norm: 11517.630 | params rms: 0.062364 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 23:24:58.784526 | finish at 2026-10-10 16:13:32 + [2026-10-05 16:49:58] iteration 1953/ 6855 | consumed samples: 249984 | elapsed time per iteration (ms): 84935.5 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.79% | FLOPs estimator: workload-aware | logical tokens/s: 197528.9 | valid tokens/s: 197528.9 | physical tokens/s: 198013.3 | loss tokens/s: 147602.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.604953E-01 | lm-global-supervised-tokens: 1.253668E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.960992E-02 | lm-token-buffer-mib: 5.818042E+01 | global_load_balancing_loss: 1.000208E+00 | loss scale: 1.0 | grad norm: 0.174 | params norm: 11517.561 | params rms: 0.062364 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 19:39:13.609002 | finish at 2026-10-10 12:29:12 + [2026-10-05 16:51:25] iteration 1954/ 6855 | consumed samples: 250112 | elapsed time per iteration (ms): 87393.3 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.66% | FLOPs estimator: workload-aware | logical tokens/s: 191973.4 | valid tokens/s: 191973.3 | physical tokens/s: 192397.9 | loss tokens/s: 139605.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.458302E-01 | lm-global-supervised-tokens: 1.220055E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.332544E-02 | lm-token-buffer-mib: 5.811180E+01 | global_load_balancing_loss: 1.001903E+00 | loss scale: 1.0 | grad norm: 0.304 | params norm: 11517.492 | params rms: 0.062363 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 22:58:34.667572 | finish at 2026-10-10 15:50:00 + [2026-10-05 16:52:52] iteration 1955/ 6855 | consumed samples: 250240 | elapsed time per iteration (ms): 86309.5 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.45% | FLOPs estimator: workload-aware | logical tokens/s: 194383.8 | valid tokens/s: 194383.8 | physical tokens/s: 194699.8 | loss tokens/s: 143589.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.771770E-01 | lm-global-supervised-tokens: 1.239317E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.171589E-02 | lm-token-buffer-mib: 5.814707E+01 | global_load_balancing_loss: 1.000037E+00 | loss scale: 1.0 | grad norm: 0.054 | params norm: 11517.425 | params rms: 0.062363 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 21:28:36.755509 | finish at 2026-10-10 14:21:28 + [2026-10-05 16:54:19] iteration 1956/ 6855 | consumed samples: 250368 | elapsed time per iteration (ms): 87229.2 | throughput per GPU (TFLOP/s/GPU): 41.7 | MFU 13.37% | FLOPs estimator: workload-aware | logical tokens/s: 192334.6 | valid tokens/s: 192334.6 | physical tokens/s: 192944.2 | loss tokens/s: 141099.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.294340E-01 | lm-global-supervised-tokens: 1.230798E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.452191E-02 | lm-token-buffer-mib: 5.828070E+01 | global_load_balancing_loss: 1.000103E+00 | loss scale: 1.0 | grad norm: 0.222 | params norm: 11517.359 | params rms: 0.062363 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 22:42:15.747458 | finish at 2026-10-10 15:36:35 + [2026-10-05 16:55:47] iteration 1957/ 6855 | consumed samples: 250496 | elapsed time per iteration (ms): 88406.8 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.39% | FLOPs estimator: workload-aware | logical tokens/s: 189773.0 | valid tokens/s: 189772.9 | physical tokens/s: 190278.8 | loss tokens/s: 143420.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.963388E-01 | lm-global-supervised-tokens: 1.267935E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.257520E-02 | lm-token-buffer-mib: 5.818736E+01 | global_load_balancing_loss: 9.994830E-01 | loss scale: 1.0 | grad norm: 0.110 | params norm: 11517.292 | params rms: 0.062362 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 0:16:56.400288 | finish at 2026-10-10 17:12:44 + [2026-10-05 16:57:13] iteration 1958/ 6855 | consumed samples: 250624 | elapsed time per iteration (ms): 86074.5 | throughput per GPU (TFLOP/s/GPU): 43.1 | MFU 13.81% | FLOPs estimator: workload-aware | logical tokens/s: 194914.5 | valid tokens/s: 194914.5 | physical tokens/s: 195359.6 | loss tokens/s: 144124.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.388704E-01 | lm-global-supervised-tokens: 1.240543E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.145694E-02 | lm-token-buffer-mib: 5.820216E+01 | global_load_balancing_loss: 9.998097E-01 | loss scale: 1.0 | grad norm: 0.092 | params norm: 11517.222 | params rms: 0.062362 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 21:05:06.679801 | finish at 2026-10-10 14:02:20 + [2026-10-05 16:58:41] iteration 1959/ 6855 | consumed samples: 250752 | elapsed time per iteration (ms): 87717.2 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.62% | FLOPs estimator: workload-aware | logical tokens/s: 191264.3 | valid tokens/s: 191264.3 | physical tokens/s: 191793.9 | loss tokens/s: 136509.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.654311E-01 | lm-global-supervised-tokens: 1.197426E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.731187E-02 | lm-token-buffer-mib: 5.813011E+01 | global_load_balancing_loss: 9.999840E-01 | loss scale: 1.0 | grad norm: 0.189 | params norm: 11517.151 | params rms: 0.062362 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 23:17:43.620346 | finish at 2026-10-10 16:16:25 + [2026-10-05 17:00:28] iteration 1960/ 6855 | consumed samples: 250880 | elapsed time per iteration (ms): 106853.0 | throughput per GPU (TFLOP/s/GPU): 34.7 | MFU 11.13% | FLOPs estimator: workload-aware | logical tokens/s: 157011.8 | valid tokens/s: 157011.8 | physical tokens/s: 157317.5 | loss tokens/s: 115334.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.885003E-01 | lm-global-supervised-tokens: 1.232384E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.233008E-02 | lm-token-buffer-mib: 5.812021E+01 | global_load_balancing_loss: 1.000186E+00 | loss scale: 1.0 | grad norm: 0.066 | params norm: 11517.079 | params rms: 0.062361 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 1:17:25.291088 | finish at 2026-10-11 18:17:53 +times across ranks (ms): + save-checkpoint: + rank 0: 26393.81 + rank 1: 26393.83 + rank 2: 26393.86 + rank 3: 26393.82 + rank 4: 26393.80 + rank 5: 26393.79 + rank 6: 26393.79 + rank 7: 26393.84 + rank 8: 26394.84 + rank 9: 26394.79 + rank 10: 26394.80 + rank 11: 26394.79 + rank 12: 26394.79 + rank 13: 26394.83 + rank 14: 26394.79 + rank 15: 26394.78 + rank 16: 26394.09 + rank 17: 26394.09 + rank 18: 26394.06 + rank 19: 26394.10 + rank 20: 26394.06 + rank 21: 26394.11 + rank 22: 26394.06 + rank 23: 26394.07 + rank 24: 26394.58 + rank 25: 26394.48 + rank 26: 26394.57 + rank 27: 26394.62 + rank 28: 26394.60 + rank 29: 26394.58 + rank 30: 26394.51 + rank 31: 26394.55 + rank 32: 26394.08 + rank 33: 26394.26 + rank 34: 26394.28 + rank 35: 26394.23 + rank 36: 26395.41 + rank 37: 26394.18 + rank 38: 26394.42 + rank 39: 26394.13 + rank 40: 26393.58 + rank 41: 26393.53 + rank 42: 26393.55 + rank 43: 26393.57 + rank 44: 26393.52 + rank 45: 26393.53 + rank 46: 26393.55 + rank 47: 26393.38 + rank 48: 26393.36 + rank 49: 26393.33 + rank 50: 26393.31 + rank 51: 26393.25 + rank 52: 26393.31 + rank 53: 26393.18 + rank 54: 26393.26 + rank 55: 26393.29 + rank 56: 26393.90 + rank 57: 26393.85 + rank 58: 26393.84 + rank 59: 26393.84 + rank 60: 26393.83 + rank 61: 26393.84 + rank 62: 26393.84 + rank 63: 26393.85 + [2026-10-05 17:02:19] iteration 1961/ 6855 | consumed samples: 251008 | elapsed time per iteration (ms): 84184.4 | throughput per GPU (TFLOP/s/GPU): 43.2 | MFU 13.84% | FLOPs estimator: workload-aware | logical tokens/s: 199290.9 | valid tokens/s: 199290.9 | physical tokens/s: 199754.9 | loss tokens/s: 149255.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.579610E-01 | lm-global-supervised-tokens: 1.256499E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 6.686787E-01 | lm-token-buffer-mib: 5.805917E+01 | global_load_balancing_loss: 9.997911E-01 | loss scale: 1.0 | grad norm: 0.163 | params norm: 11517.007 | params rms: 0.062361 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 18:26:38.648859 | finish at 2026-10-10 11:28:57 + [2026-10-05 17:03:49] iteration 1962/ 6855 | consumed samples: 251136 | elapsed time per iteration (ms): 90658.9 | throughput per GPU (TFLOP/s/GPU): 40.7 | MFU 13.04% | FLOPs estimator: workload-aware | logical tokens/s: 185058.5 | valid tokens/s: 185058.5 | physical tokens/s: 185469.8 | loss tokens/s: 135827.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.329712E-01 | lm-global-supervised-tokens: 1.231394E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.434182E-02 | lm-token-buffer-mib: 5.808671E+01 | global_load_balancing_loss: 1.000584E+00 | loss scale: 1.0 | grad norm: 0.062 | params norm: 11516.936 | params rms: 0.062360 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 3:13:13.857820 | finish at 2026-10-10 20:17:03 + [2026-10-05 17:05:16] iteration 1963/ 6855 | consumed samples: 251264 | elapsed time per iteration (ms): 86979.7 | throughput per GPU (TFLOP/s/GPU): 42.8 | MFU 13.73% | FLOPs estimator: workload-aware | logical tokens/s: 192885.4 | valid tokens/s: 192885.4 | physical tokens/s: 193333.5 | loss tokens/s: 142074.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.247937E-01 | lm-global-supervised-tokens: 1.235758E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.465781E-02 | lm-token-buffer-mib: 5.829552E+01 | global_load_balancing_loss: 1.001095E+00 | loss scale: 1.0 | grad norm: 0.113 | params norm: 11516.866 | params rms: 0.062360 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 22:11:44.514382 | finish at 2026-10-10 15:17:01 + [2026-10-05 17:06:43] iteration 1964/ 6855 | consumed samples: 251392 | elapsed time per iteration (ms): 86566.6 | throughput per GPU (TFLOP/s/GPU): 43.9 | MFU 14.06% | FLOPs estimator: workload-aware | logical tokens/s: 193806.2 | valid tokens/s: 193806.2 | physical tokens/s: 194212.6 | loss tokens/s: 140386.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.792217E-01 | lm-global-supervised-tokens: 1.215275E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.159921E-02 | lm-token-buffer-mib: 5.820313E+01 | global_load_balancing_loss: 9.998043E-01 | loss scale: 1.0 | grad norm: 0.094 | params norm: 11516.796 | params rms: 0.062360 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 21:36:37.478898 | finish at 2026-10-10 14:43:20 + [2026-10-05 17:08:10] iteration 1965/ 6855 | consumed samples: 251520 | elapsed time per iteration (ms): 87542.6 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.59% | FLOPs estimator: workload-aware | logical tokens/s: 191645.8 | valid tokens/s: 191645.8 | physical tokens/s: 192079.3 | loss tokens/s: 141449.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.963919E-01 | lm-global-supervised-tokens: 1.238282E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.562986E-02 | lm-token-buffer-mib: 5.806380E+01 | global_load_balancing_loss: 1.000888E+00 | loss scale: 1.0 | grad norm: 0.052 | params norm: 11516.725 | params rms: 0.062359 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 22:54:43.373051 | finish at 2026-10-10 16:02:54 + [2026-10-05 17:09:35] iteration 1966/ 6855 | consumed samples: 251648 | elapsed time per iteration (ms): 84605.1 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.54% | FLOPs estimator: workload-aware | logical tokens/s: 198299.7 | valid tokens/s: 198299.7 | physical tokens/s: 198659.8 | loss tokens/s: 148160.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.483396E-01 | lm-global-supervised-tokens: 1.253510E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.557009E-02 | lm-token-buffer-mib: 5.808896E+01 | global_load_balancing_loss: 9.998255E-01 | loss scale: 1.0 | grad norm: 0.109 | params norm: 11516.654 | params rms: 0.062359 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 18:53:54.413920 | finish at 2026-10-10 12:03:29 + [2026-10-05 17:11:03] iteration 1967/ 6855 | consumed samples: 251776 | elapsed time per iteration (ms): 88064.6 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.66% | FLOPs estimator: workload-aware | logical tokens/s: 190509.7 | valid tokens/s: 190509.7 | physical tokens/s: 190901.8 | loss tokens/s: 135087.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.087827E-01 | lm-global-supervised-tokens: 1.189645E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.383697E-02 | lm-token-buffer-mib: 5.820096E+01 | global_load_balancing_loss: 1.000519E+00 | loss scale: 1.0 | grad norm: 0.063 | params norm: 11516.583 | params rms: 0.062359 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 23:34:20.004826 | finish at 2026-10-10 16:45:23 + [2026-10-05 17:12:29] iteration 1968/ 6855 | consumed samples: 251904 | elapsed time per iteration (ms): 85556.1 | throughput per GPU (TFLOP/s/GPU): 43.5 | MFU 13.95% | FLOPs estimator: workload-aware | logical tokens/s: 196096.1 | valid tokens/s: 196096.1 | physical tokens/s: 196442.8 | loss tokens/s: 144457.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.576942E-01 | lm-global-supervised-tokens: 1.235920E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.990229E-02 | lm-token-buffer-mib: 5.808089E+01 | global_load_balancing_loss: 9.996176E-01 | loss scale: 1.0 | grad norm: 0.031 | params norm: 11516.511 | params rms: 0.062358 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 20:08:32.645024 | finish at 2026-10-10 13:21:01 + [2026-10-05 17:13:54] iteration 1969/ 6855 | consumed samples: 252032 | elapsed time per iteration (ms): 85711.6 | throughput per GPU (TFLOP/s/GPU): 43.5 | MFU 13.95% | FLOPs estimator: workload-aware | logical tokens/s: 195739.6 | valid tokens/s: 195739.6 | physical tokens/s: 196341.5 | loss tokens/s: 145055.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.380888E-01 | lm-global-supervised-tokens: 1.243298E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.320588E-02 | lm-token-buffer-mib: 5.810984E+01 | global_load_balancing_loss: 1.000137E+00 | loss scale: 1.0 | grad norm: 0.137 | params norm: 11516.440 | params rms: 0.062358 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 20:19:47.044501 | finish at 2026-10-10 13:33:41 + [2026-10-05 17:15:47] iteration 1970/ 6855 | consumed samples: 252160 | elapsed time per iteration (ms): 112858.4 | throughput per GPU (TFLOP/s/GPU): 32.8 | MFU 10.52% | FLOPs estimator: workload-aware | logical tokens/s: 148657.2 | valid tokens/s: 148657.2 | physical tokens/s: 149109.0 | loss tokens/s: 116276.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.396978E-01 | lm-global-supervised-tokens: 1.312283E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.155198E-02 | lm-token-buffer-mib: 5.816174E+01 | global_load_balancing_loss: 9.996386E-01 | loss scale: 1.0 | grad norm: 0.037 | params norm: 11516.368 | params rms: 0.062357 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 9:08:33.311307 | finish at 2026-10-12 02:24:21 + [2026-10-05 17:17:19] iteration 1971/ 6855 | consumed samples: 252288 | elapsed time per iteration (ms): 91381.2 | throughput per GPU (TFLOP/s/GPU): 41.3 | MFU 13.24% | FLOPs estimator: workload-aware | logical tokens/s: 183595.7 | valid tokens/s: 183595.7 | physical tokens/s: 183985.9 | loss tokens/s: 126952.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.058693E-01 | lm-global-supervised-tokens: 1.160105E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.263019E-02 | lm-token-buffer-mib: 5.826501E+01 | global_load_balancing_loss: 1.001379E+00 | loss scale: 1.0 | grad norm: 0.143 | params norm: 11516.294 | params rms: 0.062357 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 3:58:25.672874 | finish at 2026-10-10 21:15:44 + [2026-10-05 17:18:47] iteration 1972/ 6855 | consumed samples: 252416 | elapsed time per iteration (ms): 87995.8 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.49% | FLOPs estimator: workload-aware | logical tokens/s: 190658.6 | valid tokens/s: 190658.6 | physical tokens/s: 191174.8 | loss tokens/s: 141526.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.697421E-01 | lm-global-supervised-tokens: 1.245371E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.208100E-02 | lm-token-buffer-mib: 5.815997E+01 | global_load_balancing_loss: 9.999294E-01 | loss scale: 1.0 | grad norm: 0.084 | params norm: 11516.221 | params rms: 0.062357 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 23:21:23.345965 | finish at 2026-10-10 16:40:10 + [2026-10-05 17:20:12] iteration 1973/ 6855 | consumed samples: 252544 | elapsed time per iteration (ms): 85213.7 | throughput per GPU (TFLOP/s/GPU): 43.5 | MFU 13.94% | FLOPs estimator: workload-aware | logical tokens/s: 196883.3 | valid tokens/s: 196883.3 | physical tokens/s: 197258.4 | loss tokens/s: 148629.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.028242E-01 | lm-global-supervised-tokens: 1.266528E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.401433E-02 | lm-token-buffer-mib: 5.807058E+01 | global_load_balancing_loss: 1.000572E+00 | loss scale: 1.0 | grad norm: 0.056 | params norm: 11516.150 | params rms: 0.062356 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 19:33:33.286002 | finish at 2026-10-10 12:53:45 + [2026-10-05 17:21:38] iteration 1974/ 6855 | consumed samples: 252672 | elapsed time per iteration (ms): 86504.1 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.56% | FLOPs estimator: workload-aware | logical tokens/s: 193946.7 | valid tokens/s: 193946.7 | physical tokens/s: 194379.1 | loss tokens/s: 148440.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.633750E-01 | lm-global-supervised-tokens: 1.284070E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.208620E-02 | lm-token-buffer-mib: 5.809801E+01 | global_load_balancing_loss: 9.998553E-01 | loss scale: 1.0 | grad norm: 0.049 | params norm: 11516.079 | params rms: 0.062356 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 21:17:06.532295 | finish at 2026-10-10 14:38:45 + [2026-10-05 17:23:05] iteration 1975/ 6855 | consumed samples: 252800 | elapsed time per iteration (ms): 86966.4 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.46% | FLOPs estimator: workload-aware | logical tokens/s: 192916.0 | valid tokens/s: 192916.0 | physical tokens/s: 193478.3 | loss tokens/s: 145742.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.035920E-01 | lm-global-supervised-tokens: 1.267467E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.969892E-02 | lm-token-buffer-mib: 5.826332E+01 | global_load_balancing_loss: 1.000050E+00 | loss scale: 1.0 | grad norm: 0.069 | params norm: 11516.007 | params rms: 0.062355 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 21:53:16.051331 | finish at 2026-10-10 15:16:21 + [2026-10-05 17:24:34] iteration 1976/ 6855 | consumed samples: 252928 | elapsed time per iteration (ms): 88929.8 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.55% | FLOPs estimator: workload-aware | logical tokens/s: 188656.8 | valid tokens/s: 188656.8 | physical tokens/s: 189039.1 | loss tokens/s: 137415.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.949567E-01 | lm-global-supervised-tokens: 1.222032E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.633762E-02 | lm-token-buffer-mib: 5.807041E+01 | global_load_balancing_loss: 9.997563E-01 | loss scale: 1.0 | grad norm: 0.085 | params norm: 11515.934 | params rms: 0.062355 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 0:31:28.355938 | finish at 2026-10-10 17:56:03 + [2026-10-05 17:26:01] iteration 1977/ 6855 | consumed samples: 253056 | elapsed time per iteration (ms): 87128.2 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.45% | FLOPs estimator: workload-aware | logical tokens/s: 192556.7 | valid tokens/s: 192556.7 | physical tokens/s: 193325.3 | loss tokens/s: 142056.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 5.905055E-01 | lm-global-supervised-tokens: 1.237712E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.264933E-02 | lm-token-buffer-mib: 5.817072E+01 | global_load_balancing_loss: 9.992842E-01 | loss scale: 1.0 | grad norm: 0.063 | params norm: 11515.862 | params rms: 0.062355 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 22:03:31.534315 | finish at 2026-10-10 15:29:33 + [2026-10-05 17:27:28] iteration 1978/ 6855 | consumed samples: 253184 | elapsed time per iteration (ms): 86774.1 | throughput per GPU (TFLOP/s/GPU): 42.8 | MFU 13.71% | FLOPs estimator: workload-aware | logical tokens/s: 193343.4 | valid tokens/s: 193343.4 | physical tokens/s: 194064.7 | loss tokens/s: 141225.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.792427E-01 | lm-global-supervised-tokens: 1.225474E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.301178E-02 | lm-token-buffer-mib: 5.825890E+01 | global_load_balancing_loss: 1.000454E+00 | loss scale: 1.0 | grad norm: 0.150 | params norm: 11515.790 | params rms: 0.062354 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 21:33:17.256949 | finish at 2026-10-10 15:00:45 + [2026-10-05 17:28:53] iteration 1979/ 6855 | consumed samples: 253312 | elapsed time per iteration (ms): 85270.9 | throughput per GPU (TFLOP/s/GPU): 43.7 | MFU 14.01% | FLOPs estimator: workload-aware | logical tokens/s: 196751.9 | valid tokens/s: 196751.9 | physical tokens/s: 197215.6 | loss tokens/s: 146812.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.629499E-01 | lm-global-supervised-tokens: 1.251885E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.156492E-02 | lm-token-buffer-mib: 5.812698E+01 | global_load_balancing_loss: 1.000624E+00 | loss scale: 1.0 | grad norm: 0.035 | params norm: 11515.716 | params rms: 0.062354 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 19:29:41.002619 | finish at 2026-10-10 12:58:34 + [2026-10-05 17:30:42] iteration 1980/ 6855 | consumed samples: 253440 | elapsed time per iteration (ms): 108286.0 | throughput per GPU (TFLOP/s/GPU): 34.0 | MFU 10.91% | FLOPs estimator: workload-aware | logical tokens/s: 154934.0 | valid tokens/s: 154934.0 | physical tokens/s: 155395.4 | loss tokens/s: 110998.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.366783E-01 | lm-global-supervised-tokens: 1.201955E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.290557E-02 | lm-token-buffer-mib: 5.808141E+01 | global_load_balancing_loss: 1.000466E+00 | loss scale: 1.0 | grad norm: 0.114 | params norm: 11515.643 | params rms: 0.062353 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 2:38:14.096643 | finish at 2026-10-11 20:08:56 + [2026-10-05 17:32:11] iteration 1981/ 6855 | consumed samples: 253568 | elapsed time per iteration (ms): 89540.6 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.62% | FLOPs estimator: workload-aware | logical tokens/s: 187369.7 | valid tokens/s: 187369.6 | physical tokens/s: 187756.7 | loss tokens/s: 136737.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.026232E-01 | lm-global-supervised-tokens: 1.224356E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.157650E-02 | lm-token-buffer-mib: 5.816054E+01 | global_load_balancing_loss: 9.997739E-01 | loss scale: 1.0 | grad norm: 0.039 | params norm: 11515.569 | params rms: 0.062353 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 1:13:40.992770 | finish at 2026-10-10 18:45:52 + [2026-10-05 17:33:37] iteration 1982/ 6855 | consumed samples: 253696 | elapsed time per iteration (ms): 85728.2 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.76% | FLOPs estimator: workload-aware | logical tokens/s: 195702.0 | valid tokens/s: 195702.0 | physical tokens/s: 196127.3 | loss tokens/s: 134706.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 5.914287E-01 | lm-global-supervised-tokens: 1.154816E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.233952E-02 | lm-token-buffer-mib: 5.803320E+01 | global_load_balancing_loss: 1.001378E+00 | loss scale: 1.0 | grad norm: 0.114 | params norm: 11515.497 | params rms: 0.062353 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 20:02:33.715908 | finish at 2026-10-10 13:36:11 + [2026-10-05 17:35:06] iteration 1983/ 6855 | consumed samples: 253824 | elapsed time per iteration (ms): 89543.3 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.43% | FLOPs estimator: workload-aware | logical tokens/s: 187364.2 | valid tokens/s: 187364.2 | physical tokens/s: 187678.1 | loss tokens/s: 131571.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.488024E-01 | lm-global-supervised-tokens: 1.178130E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.163098E-02 | lm-token-buffer-mib: 5.807408E+01 | global_load_balancing_loss: 1.000642E+00 | loss scale: 1.0 | grad norm: 0.044 | params norm: 11515.426 | params rms: 0.062352 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 1:10:54.738802 | finish at 2026-10-10 18:46:01 + [2026-10-05 17:36:32] iteration 1984/ 6855 | consumed samples: 253952 | elapsed time per iteration (ms): 85548.1 | throughput per GPU (TFLOP/s/GPU): 42.8 | MFU 13.72% | FLOPs estimator: workload-aware | logical tokens/s: 196114.5 | valid tokens/s: 196114.5 | physical tokens/s: 196522.1 | loss tokens/s: 144640.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.667520E-01 | lm-global-supervised-tokens: 1.237369E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.600420E-02 | lm-token-buffer-mib: 5.817482E+01 | global_load_balancing_loss: 9.992605E-01 | loss scale: 1.0 | grad norm: 0.060 | params norm: 11515.355 | params rms: 0.062352 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 19:45:04.555839 | finish at 2026-10-10 13:21:37 + [2026-10-05 17:38:02] iteration 1985/ 6855 | consumed samples: 254080 | elapsed time per iteration (ms): 89678.6 | throughput per GPU (TFLOP/s/GPU): 41.3 | MFU 13.25% | FLOPs estimator: workload-aware | logical tokens/s: 187081.2 | valid tokens/s: 187081.1 | physical tokens/s: 187722.8 | loss tokens/s: 137022.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.540741E-01 | lm-global-supervised-tokens: 1.228794E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.261554E-02 | lm-token-buffer-mib: 5.802349E+01 | global_load_balancing_loss: 9.990768E-01 | loss scale: 1.0 | grad norm: 0.078 | params norm: 11515.282 | params rms: 0.062352 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 1:18:54.723139 | finish at 2026-10-10 18:56:56 + [2026-10-05 17:39:31] iteration 1986/ 6855 | consumed samples: 254208 | elapsed time per iteration (ms): 89680.0 | throughput per GPU (TFLOP/s/GPU): 41.3 | MFU 13.25% | FLOPs estimator: workload-aware | logical tokens/s: 187078.3 | valid tokens/s: 187078.3 | physical tokens/s: 187607.1 | loss tokens/s: 140177.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.075179E-01 | lm-global-supervised-tokens: 1.257109E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.142349E-02 | lm-token-buffer-mib: 5.838814E+01 | global_load_balancing_loss: 1.000955E+00 | loss scale: 1.0 | grad norm: 0.050 | params norm: 11515.209 | params rms: 0.062351 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 1:17:31.891304 | finish at 2026-10-10 18:57:03 + [2026-10-05 17:41:00] iteration 1987/ 6855 | consumed samples: 254336 | elapsed time per iteration (ms): 88334.7 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.47% | FLOPs estimator: workload-aware | logical tokens/s: 189927.4 | valid tokens/s: 189927.4 | physical tokens/s: 190373.7 | loss tokens/s: 142042.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.529490E-01 | lm-global-supervised-tokens: 1.254729E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.290685E-02 | lm-token-buffer-mib: 5.818684E+01 | global_load_balancing_loss: 1.000847E+00 | loss scale: 1.0 | grad norm: 0.113 | params norm: 11515.136 | params rms: 0.062351 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 23:26:53.506984 | finish at 2026-10-10 17:07:53 + [2026-10-05 17:42:24] iteration 1988/ 6855 | consumed samples: 254464 | elapsed time per iteration (ms): 84036.2 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.63% | FLOPs estimator: workload-aware | logical tokens/s: 199642.6 | valid tokens/s: 199642.6 | physical tokens/s: 200094.0 | loss tokens/s: 156477.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.821541E-01 | lm-global-supervised-tokens: 1.314974E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.205356E-02 | lm-token-buffer-mib: 5.812611E+01 | global_load_balancing_loss: 9.998774E-01 | loss scale: 1.0 | grad norm: 0.103 | params norm: 11515.063 | params rms: 0.062350 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 17:36:44.020852 | finish at 2026-10-10 11:19:08 + [2026-10-05 17:43:50] iteration 1989/ 6855 | consumed samples: 254592 | elapsed time per iteration (ms): 86410.5 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.44% | FLOPs estimator: workload-aware | logical tokens/s: 194156.9 | valid tokens/s: 194156.9 | physical tokens/s: 194682.0 | loss tokens/s: 147646.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.618413E-01 | lm-global-supervised-tokens: 1.275823E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.532728E-02 | lm-token-buffer-mib: 5.822123E+01 | global_load_balancing_loss: 1.000394E+00 | loss scale: 1.0 | grad norm: 0.035 | params norm: 11514.988 | params rms: 0.062350 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 20:47:53.351704 | finish at 2026-10-10 14:31:44 + [2026-10-05 17:45:35] iteration 1990/ 6855 | consumed samples: 254720 | elapsed time per iteration (ms): 104924.3 | throughput per GPU (TFLOP/s/GPU): 34.4 | MFU 11.02% | FLOPs estimator: workload-aware | logical tokens/s: 159897.9 | valid tokens/s: 159897.9 | physical tokens/s: 160287.7 | loss tokens/s: 120401.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.344844E-01 | lm-global-supervised-tokens: 1.263308E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.431305E-02 | lm-token-buffer-mib: 5.818512E+01 | global_load_balancing_loss: 9.996905E-01 | loss scale: 1.0 | grad norm: 0.056 | params norm: 11514.913 | params rms: 0.062350 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 21:47:36.917627 | finish at 2026-10-11 15:33:12 + [2026-10-05 17:47:05] iteration 1991/ 6855 | consumed samples: 254848 | elapsed time per iteration (ms): 89522.2 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.39% | FLOPs estimator: workload-aware | logical tokens/s: 187407.9 | valid tokens/s: 187407.9 | physical tokens/s: 187850.9 | loss tokens/s: 140923.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.072645E-01 | lm-global-supervised-tokens: 1.261580E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.410226E-02 | lm-token-buffer-mib: 5.835822E+01 | global_load_balancing_loss: 9.993855E-01 | loss scale: 1.0 | grad norm: 0.060 | params norm: 11514.838 | params rms: 0.062349 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 0:57:16.049744 | finish at 2026-10-10 18:44:21 + [2026-10-05 17:48:32] iteration 1992/ 6855 | consumed samples: 254976 | elapsed time per iteration (ms): 86938.9 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.53% | FLOPs estimator: workload-aware | logical tokens/s: 192976.7 | valid tokens/s: 192976.7 | physical tokens/s: 193440.2 | loss tokens/s: 147265.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.693847E-01 | lm-global-supervised-tokens: 1.280307E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.896073E-02 | lm-token-buffer-mib: 5.814913E+01 | global_load_balancing_loss: 1.000625E+00 | loss scale: 1.0 | grad norm: 0.047 | params norm: 11514.763 | params rms: 0.062349 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 21:26:24.020237 | finish at 2026-10-10 15:14:56 + [2026-10-05 17:50:00] iteration 1993/ 6855 | consumed samples: 255104 | elapsed time per iteration (ms): 88240.1 | throughput per GPU (TFLOP/s/GPU): 41.1 | MFU 13.16% | FLOPs estimator: workload-aware | logical tokens/s: 190130.3 | valid tokens/s: 190130.3 | physical tokens/s: 190663.4 | loss tokens/s: 147546.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.481602E-01 | lm-global-supervised-tokens: 1.301948E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.337880E-02 | lm-token-buffer-mib: 5.807172E+01 | global_load_balancing_loss: 1.000222E+00 | loss scale: 1.0 | grad norm: 0.131 | params norm: 11514.687 | params rms: 0.062348 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 23:10:23.427185 | finish at 2026-10-10 17:00:23 + [2026-10-05 17:51:29] iteration 1994/ 6855 | consumed samples: 255232 | elapsed time per iteration (ms): 89381.1 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.51% | FLOPs estimator: workload-aware | logical tokens/s: 187702.9 | valid tokens/s: 187702.9 | physical tokens/s: 188083.7 | loss tokens/s: 134048.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.751570E-01 | lm-global-supervised-tokens: 1.198143E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.152684E-02 | lm-token-buffer-mib: 5.809193E+01 | global_load_balancing_loss: 1.000698E+00 | loss scale: 1.0 | grad norm: 0.272 | params norm: 11514.609 | params rms: 0.062348 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 0:41:21.393526 | finish at 2026-10-10 18:32:51 + [2026-10-05 17:52:57] iteration 1995/ 6855 | consumed samples: 255360 | elapsed time per iteration (ms): 87361.5 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.68% | FLOPs estimator: workload-aware | logical tokens/s: 192042.5 | valid tokens/s: 192042.5 | physical tokens/s: 192560.9 | loss tokens/s: 133644.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.396579E-01 | lm-global-supervised-tokens: 1.167540E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.098752E-02 | lm-token-buffer-mib: 5.849431E+01 | global_load_balancing_loss: 1.001475E+00 | loss scale: 1.0 | grad norm: 0.293 | params norm: 11514.529 | params rms: 0.062347 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 21:56:16.713996 | finish at 2026-10-10 15:49:13 + [2026-10-05 17:54:26] iteration 1996/ 6855 | consumed samples: 255488 | elapsed time per iteration (ms): 89163.3 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.50% | FLOPs estimator: workload-aware | logical tokens/s: 188162.6 | valid tokens/s: 188162.6 | physical tokens/s: 188562.8 | loss tokens/s: 135633.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.871358E-01 | lm-global-supervised-tokens: 1.209348E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.273988E-02 | lm-token-buffer-mib: 5.816557E+01 | global_load_balancing_loss: 9.998879E-01 | loss scale: 1.0 | grad norm: 0.130 | params norm: 11514.451 | params rms: 0.062347 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 0:20:44.406532 | finish at 2026-10-10 18:15:10 + [2026-10-05 17:55:54] iteration 1997/ 6855 | consumed samples: 255616 | elapsed time per iteration (ms): 87847.5 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.47% | FLOPs estimator: workload-aware | logical tokens/s: 190980.4 | valid tokens/s: 190980.4 | physical tokens/s: 191418.4 | loss tokens/s: 144563.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.622685E-01 | lm-global-supervised-tokens: 1.269952E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.553366E-02 | lm-token-buffer-mib: 5.824773E+01 | global_load_balancing_loss: 9.999554E-01 | loss scale: 1.0 | grad norm: 0.168 | params norm: 11514.373 | params rms: 0.062347 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 22:32:43.326836 | finish at 2026-10-10 16:28:37 + [2026-10-05 17:57:20] iteration 1998/ 6855 | consumed samples: 255744 | elapsed time per iteration (ms): 86335.6 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.77% | FLOPs estimator: workload-aware | logical tokens/s: 194324.9 | valid tokens/s: 194324.9 | physical tokens/s: 194847.2 | loss tokens/s: 147699.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.064584E-01 | lm-global-supervised-tokens: 1.275174E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.167175E-02 | lm-token-buffer-mib: 5.818448E+01 | global_load_balancing_loss: 9.998799E-01 | loss scale: 1.0 | grad norm: 0.080 | params norm: 11514.295 | params rms: 0.062346 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 20:28:51.806061 | finish at 2026-10-10 14:26:12 + [2026-10-05 17:58:52] iteration 1999/ 6855 | consumed samples: 255872 | elapsed time per iteration (ms): 91633.7 | throughput per GPU (TFLOP/s/GPU): 41.5 | MFU 13.30% | FLOPs estimator: workload-aware | logical tokens/s: 183089.9 | valid tokens/s: 183089.9 | physical tokens/s: 183755.8 | loss tokens/s: 136281.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.008998E-01 | lm-global-supervised-tokens: 1.248798E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.412445E-02 | lm-token-buffer-mib: 5.812776E+01 | global_load_balancing_loss: 9.990257E-01 | loss scale: 1.0 | grad norm: 0.185 | params norm: 11514.215 | params rms: 0.062346 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 3:36:13.361303 | finish at 2026-10-10 21:35:05 + [2026-10-05 18:00:43] iteration 2000/ 6855 | consumed samples: 256000 | elapsed time per iteration (ms): 111300.5 | throughput per GPU (TFLOP/s/GPU): 33.5 | MFU 10.73% | FLOPs estimator: workload-aware | logical tokens/s: 150737.9 | valid tokens/s: 150737.9 | physical tokens/s: 151054.6 | loss tokens/s: 108486.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.109785E-01 | lm-global-supervised-tokens: 1.207462E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.349480E-02 | lm-token-buffer-mib: 5.813686E+01 | global_load_balancing_loss: 1.000345E+00 | loss scale: 1.0 | grad norm: 0.073 | params norm: 11514.139 | params rms: 0.062345 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 6:06:04.141234 | finish at 2026-10-12 00:06:47 +times across ranks (ms): + save-checkpoint: + rank 0: 24055.95 + rank 1: 24055.96 + rank 2: 24055.97 + rank 3: 24055.96 + rank 4: 24055.96 + rank 5: 24055.96 + rank 6: 24055.95 + rank 7: 24055.94 + rank 8: 24055.59 + rank 9: 24055.56 + rank 10: 24055.57 + rank 11: 24055.56 + rank 12: 24055.56 + rank 13: 24055.57 + rank 14: 24055.56 + rank 15: 24055.57 + rank 16: 24055.39 + rank 17: 24055.35 + rank 18: 24055.34 + rank 19: 24055.34 + rank 20: 24055.34 + rank 21: 24055.35 + rank 22: 24055.33 + rank 23: 24055.33 + rank 24: 24055.56 + rank 25: 24055.52 + rank 26: 24055.52 + rank 27: 24055.52 + rank 28: 24055.48 + rank 29: 24055.52 + rank 30: 24055.50 + rank 31: 24055.54 + rank 32: 24054.89 + rank 33: 24054.89 + rank 34: 24054.88 + rank 35: 24054.88 + rank 36: 24054.90 + rank 37: 24054.85 + rank 38: 24054.87 + rank 39: 24054.87 + rank 40: 24055.27 + rank 41: 24055.25 + rank 42: 24055.24 + rank 43: 24055.25 + rank 44: 24055.25 + rank 45: 24055.25 + rank 46: 24055.24 + rank 47: 24055.28 + rank 48: 24055.72 + rank 49: 24055.71 + rank 50: 24055.71 + rank 51: 24055.71 + rank 52: 24055.69 + rank 53: 24055.66 + rank 54: 24055.69 + rank 55: 24055.69 + rank 56: 24055.40 + rank 57: 24055.38 + rank 58: 24055.36 + rank 59: 24055.36 + rank 60: 24055.36 + rank 61: 24055.38 + rank 62: 24055.36 + rank 63: 24055.38 + [2026-10-05 18:02:38] iteration 2001/ 6855 | consumed samples: 256128 | elapsed time per iteration (ms): 90930.5 | throughput per GPU (TFLOP/s/GPU): 39.2 | MFU 12.55% | FLOPs estimator: workload-aware | logical tokens/s: 184505.9 | valid tokens/s: 184505.9 | physical tokens/s: 184944.0 | loss tokens/s: 135659.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.259952E-01 | lm-global-supervised-tokens: 1.233559E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 6.832076E-01 | lm-token-buffer-mib: 5.820980E+01 | global_load_balancing_loss: 1.000384E+00 | loss scale: 1.0 | grad norm: 0.113 | params norm: 11514.063 | params rms: 0.062345 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 2:36:16.691125 | finish at 2026-10-10 20:38:55 +times across ranks (ms): + forward-backward: + rank 0: 90123.98 + rank 1: 90128.00 + rank 2: 90125.86 + rank 3: 90127.20 + rank 4: 90129.02 + rank 5: 90125.81 + rank 6: 90126.86 + rank 7: 90125.98 + rank 8: 90118.30 + rank 9: 90122.70 + rank 10: 90121.07 + rank 11: 90122.30 + rank 12: 90124.33 + rank 13: 90121.54 + rank 14: 90121.48 + rank 15: 90120.84 + rank 16: 90113.94 + rank 17: 90118.54 + rank 18: 90115.67 + rank 19: 90117.16 + rank 20: 90120.44 + rank 21: 90117.47 + rank 22: 90117.29 + rank 23: 90116.59 + rank 24: 90113.63 + rank 25: 90118.78 + rank 26: 90116.55 + rank 27: 90117.72 + rank 28: 90118.99 + rank 29: 90116.64 + rank 30: 90115.52 + rank 31: 90115.43 + rank 32: 90107.25 + rank 33: 90109.91 + rank 34: 90108.18 + rank 35: 90108.89 + rank 36: 90112.98 + rank 37: 90109.04 + rank 38: 90111.54 + rank 39: 90111.52 + rank 40: 90112.41 + rank 41: 90116.03 + rank 42: 90112.20 + rank 43: 90113.38 + rank 44: 90114.34 + rank 45: 90111.49 + rank 46: 90117.15 + rank 47: 90115.77 + rank 48: 90125.00 + rank 49: 90128.93 + rank 50: 90125.86 + rank 51: 90127.57 + rank 52: 90129.16 + rank 53: 90126.20 + rank 54: 90129.36 + rank 55: 90128.02 + rank 56: 90122.56 + rank 57: 90126.58 + rank 58: 90122.13 + rank 59: 90123.57 + rank 60: 90124.81 + rank 61: 90120.93 + rank 62: 90126.93 + rank 63: 90128.15 + forward-compute: + rank 0: 38569.26 + rank 1: 38577.31 + rank 2: 38569.95 + rank 3: 38566.55 + rank 4: 38572.24 + rank 5: 38566.83 + rank 6: 38571.30 + rank 7: 38581.29 + rank 8: 37906.56 + rank 9: 37913.36 + rank 10: 37911.38 + rank 11: 37912.46 + rank 12: 37914.28 + rank 13: 37914.39 + rank 14: 37911.81 + rank 15: 37913.35 + rank 16: 37195.46 + rank 17: 37197.42 + rank 18: 37191.17 + rank 19: 37191.81 + rank 20: 37195.83 + rank 21: 37191.32 + rank 22: 37191.69 + rank 23: 37191.39 + rank 24: 36662.20 + rank 25: 36823.34 + rank 26: 36670.50 + rank 27: 36669.61 + rank 28: 36680.20 + rank 29: 36675.51 + rank 30: 36675.75 + rank 31: 36673.31 + rank 32: 37582.28 + rank 33: 37592.11 + rank 34: 37586.23 + rank 35: 37590.64 + rank 36: 37595.66 + rank 37: 37595.15 + rank 38: 37596.93 + rank 39: 37596.13 + rank 40: 41394.25 + rank 41: 41399.67 + rank 42: 41396.60 + rank 43: 41397.64 + rank 44: 41399.32 + rank 45: 41396.01 + rank 46: 41399.19 + rank 47: 41398.02 + rank 48: 37679.84 + rank 49: 37683.77 + rank 50: 37679.42 + rank 51: 37680.05 + rank 52: 37688.36 + rank 53: 37682.62 + rank 54: 37682.27 + rank 55: 37684.21 + rank 56: 39836.11 + rank 57: 39841.52 + rank 58: 39832.07 + rank 59: 39831.36 + rank 60: 39838.49 + rank 61: 39836.03 + rank 62: 39840.77 + rank 63: 39851.90 + backward-compute: + rank 0: 51514.55 + rank 1: 51515.25 + rank 2: 51516.95 + rank 3: 51522.73 + rank 4: 51521.44 + rank 5: 51521.06 + rank 6: 51519.51 + rank 7: 51509.32 + rank 8: 52175.82 + rank 9: 52173.51 + rank 10: 52173.95 + rank 11: 52173.75 + rank 12: 52171.17 + rank 13: 52170.79 + rank 14: 52171.81 + rank 15: 52172.72 + rank 16: 52883.84 + rank 17: 52885.81 + rank 18: 52889.31 + rank 19: 52890.14 + rank 20: 52888.78 + rank 21: 52891.96 + rank 22: 52890.14 + rank 23: 52891.95 + rank 24: 53410.81 + rank 25: 53255.84 + rank 26: 53403.73 + rank 27: 53408.23 + rank 28: 53394.51 + rank 29: 53401.12 + rank 30: 53398.63 + rank 31: 53401.72 + rank 32: 52482.29 + rank 33: 52479.71 + rank 34: 52481.88 + rank 35: 52479.95 + rank 36: 52476.70 + rank 37: 52474.44 + rank 38: 52475.83 + rank 39: 52476.74 + rank 40: 48679.69 + rank 41: 48677.84 + rank 42: 48677.35 + rank 43: 48678.21 + rank 44: 48675.62 + rank 45: 48678.18 + rank 46: 48681.49 + rank 47: 48679.92 + rank 48: 52403.15 + rank 49: 52401.33 + rank 50: 52404.62 + rank 51: 52402.83 + rank 52: 52392.95 + rank 53: 52402.78 + rank 54: 52404.57 + rank 55: 52400.59 + rank 56: 50249.28 + rank 57: 50246.49 + rank 58: 50253.12 + rank 59: 50253.75 + rank 60: 50248.57 + rank 61: 50247.26 + rank 62: 50246.30 + rank 63: 50232.59 + batch-generator: + rank 0: 26.31 + rank 1: 25.36 + rank 2: 28.11 + rank 3: 27.94 + rank 4: 24.76 + rank 5: 24.91 + rank 6: 26.40 + rank 7: 25.75 + rank 8: 25.24 + rank 9: 24.88 + rank 10: 24.92 + rank 11: 25.82 + rank 12: 24.98 + rank 13: 25.03 + rank 14: 25.02 + rank 15: 24.59 + rank 16: 23.66 + rank 17: 24.08 + rank 18: 24.43 + rank 19: 24.26 + rank 20: 24.06 + rank 21: 24.34 + rank 22: 25.27 + rank 23: 24.87 + rank 24: 34.28 + rank 25: 31.92 + rank 26: 33.07 + rank 27: 32.32 + rank 28: 33.85 + rank 29: 32.40 + rank 30: 28.97 + rank 31: 31.64 + rank 32: 35.07 + rank 33: 29.48 + rank 34: 35.64 + rank 35: 28.31 + rank 36: 34.45 + rank 37: 33.08 + rank 38: 31.93 + rank 39: 34.30 + rank 40: 26.69 + rank 41: 23.17 + rank 42: 24.80 + rank 43: 25.27 + rank 44: 25.24 + rank 45: 24.94 + rank 46: 25.03 + rank 47: 24.18 + rank 48: 34.43 + rank 49: 37.85 + rank 50: 30.29 + rank 51: 33.13 + rank 52: 41.30 + rank 53: 34.35 + rank 54: 32.09 + rank 55: 39.23 + rank 56: 25.16 + rank 57: 25.11 + rank 58: 24.60 + rank 59: 25.38 + rank 60: 26.09 + rank 61: 25.11 + rank 62: 24.93 + rank 63: 25.81 + embedding-grads-all-reduce: + rank 0: 0.03 + rank 1: 0.03 + rank 2: 0.03 + rank 3: 0.03 + rank 4: 0.03 + rank 5: 0.03 + rank 6: 0.03 + rank 7: 0.03 + rank 8: 0.03 + rank 9: 0.03 + rank 10: 0.02 + rank 11: 0.03 + rank 12: 0.03 + rank 13: 0.03 + rank 14: 0.02 + rank 15: 0.03 + rank 16: 0.03 + rank 17: 0.02 + rank 18: 0.03 + rank 19: 0.03 + rank 20: 0.03 + rank 21: 0.03 + rank 22: 0.03 + rank 23: 0.02 + rank 24: 0.04 + rank 25: 0.04 + rank 26: 0.04 + rank 27: 0.05 + rank 28: 0.04 + rank 29: 0.06 + rank 30: 0.04 + rank 31: 0.04 + rank 32: 0.04 + rank 33: 0.03 + rank 34: 0.04 + rank 35: 0.03 + rank 36: 0.03 + rank 37: 0.03 + rank 38: 0.03 + rank 39: 0.03 + rank 40: 0.03 + rank 41: 0.03 + rank 42: 0.02 + rank 43: 0.03 + rank 44: 0.03 + rank 45: 0.03 + rank 46: 0.03 + rank 47: 0.03 + rank 48: 0.03 + rank 49: 0.04 + rank 50: 0.03 + rank 51: 0.03 + rank 52: 0.03 + rank 53: 0.03 + rank 54: 0.03 + rank 55: 0.03 + rank 56: 0.03 + rank 57: 0.02 + rank 58: 0.03 + rank 59: 0.03 + rank 60: 0.03 + rank 61: 0.03 + rank 62: 0.03 + rank 63: 0.03 + all-grads-sync: + rank 0: 1.20 + rank 1: 0.77 + rank 2: 0.79 + rank 3: 0.80 + rank 4: 0.86 + rank 5: 0.80 + rank 6: 0.76 + rank 7: 0.84 + rank 8: 0.84 + rank 9: 0.75 + rank 10: 0.95 + rank 11: 0.79 + rank 12: 0.78 + rank 13: 0.78 + rank 14: 0.82 + rank 15: 0.85 + rank 16: 0.80 + rank 17: 0.73 + rank 18: 0.86 + rank 19: 0.74 + rank 20: 0.78 + rank 21: 0.77 + rank 22: 0.79 + rank 23: 0.76 + rank 24: 1.79 + rank 25: 1.30 + rank 26: 1.40 + rank 27: 1.26 + rank 28: 1.12 + rank 29: 1.19 + rank 30: 1.03 + rank 31: 1.18 + rank 32: 1.28 + rank 33: 0.98 + rank 34: 1.29 + rank 35: 0.88 + rank 36: 1.17 + rank 37: 1.18 + rank 38: 1.12 + rank 39: 1.40 + rank 40: 0.80 + rank 41: 0.93 + rank 42: 0.77 + rank 43: 0.79 + rank 44: 0.76 + rank 45: 0.83 + rank 46: 0.76 + rank 47: 0.78 + rank 48: 1.01 + rank 49: 1.11 + rank 50: 0.89 + rank 51: 0.90 + rank 52: 1.17 + rank 53: 1.00 + rank 54: 1.08 + rank 55: 1.09 + rank 56: 0.93 + rank 57: 0.82 + rank 58: 0.77 + rank 59: 0.82 + rank 60: 1.04 + rank 61: 0.78 + rank 62: 0.93 + rank 63: 1.98 + params-all-gather: + rank 0: 0.02 + rank 1: 0.02 + rank 2: 0.02 + rank 3: 0.02 + rank 4: 0.02 + rank 5: 0.04 + rank 6: 0.02 + rank 7: 0.02 + rank 8: 0.02 + rank 9: 0.02 + rank 10: 0.02 + rank 11: 0.02 + rank 12: 0.02 + rank 13: 0.02 + rank 14: 0.02 + rank 15: 0.02 + rank 16: 0.02 + rank 17: 0.02 + rank 18: 0.02 + rank 19: 0.02 + rank 20: 0.02 + rank 21: 0.02 + rank 22: 0.02 + rank 23: 0.02 + rank 24: 0.04 + rank 25: 0.03 + rank 26: 0.04 + rank 27: 0.05 + rank 28: 0.03 + rank 29: 0.04 + rank 30: 0.04 + rank 31: 0.07 + rank 32: 0.02 + rank 33: 0.03 + rank 34: 0.02 + rank 35: 0.02 + rank 36: 0.02 + rank 37: 0.02 + rank 38: 0.03 + rank 39: 0.02 + rank 40: 0.02 + rank 41: 0.03 + rank 42: 0.02 + rank 43: 0.02 + rank 44: 0.02 + rank 45: 0.02 + rank 46: 0.02 + rank 47: 0.02 + rank 48: 0.03 + rank 49: 0.02 + rank 50: 0.03 + rank 51: 0.03 + rank 52: 0.03 + rank 53: 0.03 + rank 54: 0.03 + rank 55: 0.03 + rank 56: 0.02 + rank 57: 0.02 + rank 58: 0.02 + rank 59: 0.02 + rank 60: 0.02 + rank 61: 0.02 + rank 62: 0.02 + rank 63: 0.02 + optimizer-copy-to-main-grad: + rank 0: 2.29 + rank 1: 1.93 + rank 2: 1.72 + rank 3: 2.20 + rank 4: 1.71 + rank 5: 1.78 + rank 6: 1.89 + rank 7: 1.84 + rank 8: 1.80 + rank 9: 2.11 + rank 10: 1.81 + rank 11: 1.73 + rank 12: 1.76 + rank 13: 1.78 + rank 14: 1.80 + rank 15: 2.09 + rank 16: 2.47 + rank 17: 2.09 + rank 18: 2.17 + rank 19: 2.15 + rank 20: 2.27 + rank 21: 2.31 + rank 22: 2.54 + rank 23: 2.30 + rank 24: 2.08 + rank 25: 2.25 + rank 26: 2.16 + rank 27: 2.06 + rank 28: 2.26 + rank 29: 2.30 + rank 30: 2.33 + rank 31: 2.04 + rank 32: 2.20 + rank 33: 1.77 + rank 34: 2.01 + rank 35: 2.20 + rank 36: 2.11 + rank 37: 1.97 + rank 38: 2.30 + rank 39: 1.93 + rank 40: 1.75 + rank 41: 1.86 + rank 42: 1.81 + rank 43: 1.80 + rank 44: 1.85 + rank 45: 2.09 + rank 46: 1.73 + rank 47: 1.68 + rank 48: 1.95 + rank 49: 1.89 + rank 50: 2.26 + rank 51: 1.86 + rank 52: 1.93 + rank 53: 1.72 + rank 54: 1.82 + rank 55: 1.81 + rank 56: 1.95 + rank 57: 2.61 + rank 58: 1.95 + rank 59: 1.91 + rank 60: 1.90 + rank 61: 1.82 + rank 62: 1.98 + rank 63: 2.08 + optimizer-inner-step: + rank 0: 48.33 + rank 1: 48.31 + rank 2: 48.11 + rank 3: 48.22 + rank 4: 48.01 + rank 5: 48.15 + rank 6: 48.51 + rank 7: 48.52 + rank 8: 48.20 + rank 9: 48.21 + rank 10: 48.06 + rank 11: 48.19 + rank 12: 48.20 + rank 13: 48.39 + rank 14: 48.19 + rank 15: 48.10 + rank 16: 48.47 + rank 17: 48.29 + rank 18: 48.37 + rank 19: 48.28 + rank 20: 48.30 + rank 21: 48.15 + rank 22: 48.62 + rank 23: 48.38 + rank 24: 47.95 + rank 25: 48.18 + rank 26: 47.94 + rank 27: 48.30 + rank 28: 48.62 + rank 29: 48.04 + rank 30: 48.06 + rank 31: 48.35 + rank 32: 48.27 + rank 33: 48.07 + rank 34: 48.23 + rank 35: 48.56 + rank 36: 48.32 + rank 37: 48.29 + rank 38: 48.13 + rank 39: 48.08 + rank 40: 48.20 + rank 41: 48.32 + rank 42: 48.41 + rank 43: 48.41 + rank 44: 47.97 + rank 45: 48.62 + rank 46: 48.24 + rank 47: 47.89 + rank 48: 47.18 + rank 49: 46.38 + rank 50: 58.04 + rank 51: 46.17 + rank 52: 46.57 + rank 53: 46.38 + rank 54: 46.62 + rank 55: 46.52 + rank 56: 47.76 + rank 57: 49.07 + rank 58: 48.11 + rank 59: 47.96 + rank 60: 48.27 + rank 61: 47.89 + rank 62: 48.02 + rank 63: 47.94 + optimizer-copy-main-to-model-params: + rank 0: 4.44 + rank 1: 3.88 + rank 2: 3.78 + rank 3: 4.46 + rank 4: 3.74 + rank 5: 3.94 + rank 6: 3.95 + rank 7: 4.02 + rank 8: 3.87 + rank 9: 4.45 + rank 10: 3.79 + rank 11: 3.76 + rank 12: 3.93 + rank 13: 3.82 + rank 14: 3.98 + rank 15: 4.24 + rank 16: 4.41 + rank 17: 3.99 + rank 18: 4.01 + rank 19: 4.01 + rank 20: 4.05 + rank 21: 4.23 + rank 22: 4.56 + rank 23: 4.20 + rank 24: 4.08 + rank 25: 4.79 + rank 26: 4.00 + rank 27: 4.17 + rank 28: 4.39 + rank 29: 4.55 + rank 30: 4.38 + rank 31: 4.11 + rank 32: 4.16 + rank 33: 3.91 + rank 34: 3.99 + rank 35: 4.50 + rank 36: 4.26 + rank 37: 4.05 + rank 38: 4.59 + rank 39: 3.92 + rank 40: 3.76 + rank 41: 3.92 + rank 42: 3.95 + rank 43: 4.01 + rank 44: 3.98 + rank 45: 4.24 + rank 46: 3.76 + rank 47: 3.76 + rank 48: 4.14 + rank 49: 3.97 + rank 50: 4.97 + rank 51: 3.94 + rank 52: 4.20 + rank 53: 3.86 + rank 54: 3.85 + rank 55: 3.87 + rank 56: 3.99 + rank 57: 5.00 + rank 58: 4.11 + rank 59: 3.90 + rank 60: 3.96 + rank 61: 3.94 + rank 62: 4.11 + rank 63: 3.90 + optimizer: + rank 0: 60.30 + rank 1: 59.51 + rank 2: 59.11 + rank 3: 60.16 + rank 4: 58.96 + rank 5: 59.43 + rank 6: 59.80 + rank 7: 59.88 + rank 8: 59.35 + rank 9: 60.19 + rank 10: 59.15 + rank 11: 59.20 + rank 12: 59.42 + rank 13: 59.49 + rank 14: 59.49 + rank 15: 59.77 + rank 16: 60.65 + rank 17: 59.84 + rank 18: 59.95 + rank 19: 59.88 + rank 20: 59.92 + rank 21: 60.09 + rank 22: 60.97 + rank 23: 60.19 + rank 24: 59.40 + rank 25: 60.45 + rank 26: 59.49 + rank 27: 59.87 + rank 28: 60.44 + rank 29: 60.09 + rank 30: 59.89 + rank 31: 59.84 + rank 32: 59.73 + rank 33: 59.19 + rank 34: 59.46 + rank 35: 60.50 + rank 36: 59.93 + rank 37: 59.63 + rank 38: 60.18 + rank 39: 59.23 + rank 40: 59.21 + rank 41: 59.59 + rank 42: 59.69 + rank 43: 59.76 + rank 44: 59.28 + rank 45: 60.30 + rank 46: 59.23 + rank 47: 58.89 + rank 48: 60.39 + rank 49: 59.33 + rank 50: 72.61 + rank 51: 59.09 + rank 52: 59.83 + rank 53: 59.16 + rank 54: 59.40 + rank 55: 59.32 + rank 56: 59.11 + rank 57: 61.81 + rank 58: 59.59 + rank 59: 59.16 + rank 60: 59.60 + rank 61: 59.14 + rank 62: 59.52 + rank 63: 59.13 + interval-time: + rank 0: 0.85 + rank 1: 0.82 + rank 2: 0.74 + rank 3: 0.73 + rank 4: 0.82 + rank 5: 0.72 + rank 6: 0.81 + rank 7: 0.73 + rank 8: 0.83 + rank 9: 0.75 + rank 10: 0.84 + rank 11: 0.81 + rank 12: 0.83 + rank 13: 0.81 + rank 14: 0.74 + rank 15: 0.74 + rank 16: 0.73 + rank 17: 0.82 + rank 18: 0.81 + rank 19: 0.82 + rank 20: 0.81 + rank 21: 0.74 + rank 22: 0.74 + rank 23: 0.72 + rank 24: 0.89 + rank 25: 0.94 + rank 26: 0.94 + rank 27: 1.13 + rank 28: 0.83 + rank 29: 1.10 + rank 30: 0.89 + rank 31: 0.85 + rank 32: 0.94 + rank 33: 0.93 + rank 34: 0.84 + rank 35: 0.92 + rank 36: 0.79 + rank 37: 0.89 + rank 38: 0.94 + rank 39: 0.90 + rank 40: 0.75 + rank 41: 0.83 + rank 42: 0.82 + rank 43: 0.84 + rank 44: 0.83 + rank 45: 0.79 + rank 46: 0.83 + rank 47: 0.74 + rank 48: 7.31 + rank 49: 7.30 + rank 50: 7.28 + rank 51: 7.28 + rank 52: 7.29 + rank 53: 7.26 + rank 54: 7.26 + rank 55: 7.29 + rank 56: 0.76 + rank 57: 0.81 + rank 58: 0.81 + rank 59: 0.83 + rank 60: 0.81 + rank 61: 0.80 + rank 62: 0.80 + rank 63: 3.57 + [2026-10-05 18:04:10] iteration 2002/ 6855 | consumed samples: 256256 | elapsed time per iteration (ms): 92007.1 | throughput per GPU (TFLOP/s/GPU): 40.3 | MFU 12.93% | FLOPs estimator: workload-aware | logical tokens/s: 182346.6 | valid tokens/s: 182346.6 | physical tokens/s: 182691.3 | loss tokens/s: 130787.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.406768E-01 | lm-global-supervised-tokens: 1.203338E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.243523E-02 | lm-token-buffer-mib: 5.812614E+01 | global_load_balancing_loss: 1.001482E+00 | loss scale: 1.0 | grad norm: 0.062 | params norm: 11513.987 | params rms: 0.062344 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 4:01:50.474167 | finish at 2026-10-10 22:06:00 +times across ranks (ms): + forward-backward: + rank 0: 86116.13 + rank 1: 86227.45 + rank 2: 86227.78 + rank 3: 86228.09 + rank 4: 86227.83 + rank 5: 86219.13 + rank 6: 86227.01 + rank 7: 86225.32 + rank 8: 86109.42 + rank 9: 86221.55 + rank 10: 86223.52 + rank 11: 86224.23 + rank 12: 86223.50 + rank 13: 86215.71 + rank 14: 86219.95 + rank 15: 86219.16 + rank 16: 86105.81 + rank 17: 86217.45 + rank 18: 86217.80 + rank 19: 86218.71 + rank 20: 86217.38 + rank 21: 86210.29 + rank 22: 86216.05 + rank 23: 86214.68 + rank 24: 86104.28 + rank 25: 86218.36 + rank 26: 86218.09 + rank 27: 86217.87 + rank 28: 86217.16 + rank 29: 86209.85 + rank 30: 86214.78 + rank 31: 86213.80 + rank 32: 86098.41 + rank 33: 86210.07 + rank 34: 86212.54 + rank 35: 86212.03 + rank 36: 86211.29 + rank 37: 86202.60 + rank 38: 86209.83 + rank 39: 86210.59 + rank 40: 86103.23 + rank 41: 86215.08 + rank 42: 86214.70 + rank 43: 86214.78 + rank 44: 86212.64 + rank 45: 86204.37 + rank 46: 86216.19 + rank 47: 86214.37 + rank 48: 86108.89 + rank 49: 86221.29 + rank 50: 86219.78 + rank 51: 86220.64 + rank 52: 86219.03 + rank 53: 86209.65 + rank 54: 86221.70 + rank 55: 86220.43 + rank 56: 86113.34 + rank 57: 86224.65 + rank 58: 86224.09 + rank 59: 86223.49 + rank 60: 86222.02 + rank 61: 86214.09 + rank 62: 86224.71 + rank 63: 86225.14 + forward-compute: + rank 0: 32089.33 + rank 1: 32204.47 + rank 2: 32201.78 + rank 3: 32202.78 + rank 4: 32197.61 + rank 5: 32188.93 + rank 6: 32192.91 + rank 7: 32200.49 + rank 8: 33546.67 + rank 9: 33649.21 + rank 10: 33651.19 + rank 11: 33654.61 + rank 12: 33653.94 + rank 13: 33646.46 + rank 14: 33654.67 + rank 15: 33651.77 + rank 16: 28927.34 + rank 17: 29036.11 + rank 18: 29041.44 + rank 19: 29044.66 + rank 20: 29039.77 + rank 21: 29028.65 + rank 22: 29039.31 + rank 23: 29047.25 + rank 24: 28791.49 + rank 25: 28909.49 + rank 26: 28910.27 + rank 27: 28904.79 + rank 28: 28917.93 + rank 29: 28903.94 + rank 30: 28909.25 + rank 31: 28908.64 + rank 32: 32477.99 + rank 33: 32597.50 + rank 34: 32595.23 + rank 35: 32595.39 + rank 36: 32595.59 + rank 37: 32589.69 + rank 38: 32598.56 + rank 39: 32592.89 + rank 40: 33348.07 + rank 41: 33464.54 + rank 42: 33465.84 + rank 43: 33465.83 + rank 44: 33463.69 + rank 45: 33457.75 + rank 46: 33465.93 + rank 47: 33464.77 + rank 48: 32922.41 + rank 49: 33035.88 + rank 50: 33037.97 + rank 51: 33042.39 + rank 52: 33045.96 + rank 53: 33037.53 + rank 54: 33043.23 + rank 55: 33039.35 + rank 56: 30677.37 + rank 57: 30797.39 + rank 58: 30799.65 + rank 59: 30797.44 + rank 60: 30798.64 + rank 61: 30800.20 + rank 62: 30802.62 + rank 63: 30809.04 + backward-compute: + rank 0: 53989.97 + rank 1: 53988.27 + rank 2: 53984.97 + rank 3: 53989.07 + rank 4: 53994.70 + rank 5: 53994.88 + rank 6: 53996.38 + rank 7: 53989.55 + rank 8: 52524.93 + rank 9: 52538.91 + rank 10: 52536.82 + rank 11: 52533.63 + rank 12: 52533.61 + rank 13: 52532.14 + rank 14: 52530.02 + rank 15: 52533.65 + rank 16: 57144.70 + rank 17: 57148.97 + rank 18: 57144.76 + rank 19: 57135.95 + rank 20: 57146.11 + rank 21: 57148.87 + rank 22: 57143.13 + rank 23: 57135.25 + rank 24: 57274.35 + rank 25: 57266.15 + rank 26: 57267.01 + rank 27: 57275.08 + rank 28: 57261.80 + rank 29: 57265.54 + rank 30: 57267.22 + rank 31: 57266.50 + rank 32: 53579.04 + rank 33: 53574.68 + rank 34: 53573.67 + rank 35: 53576.36 + rank 36: 53578.38 + rank 37: 53572.88 + rank 38: 53573.16 + rank 39: 53579.21 + rank 40: 52719.80 + rank 41: 52717.86 + rank 42: 52713.95 + rank 43: 52711.98 + rank 44: 52715.08 + rank 45: 52711.27 + rank 46: 52714.46 + rank 47: 52714.09 + rank 48: 53152.43 + rank 49: 53146.61 + rank 50: 53146.47 + rank 51: 53139.31 + rank 52: 53133.01 + rank 53: 53135.36 + rank 54: 53142.88 + rank 55: 53146.00 + rank 56: 55402.15 + rank 57: 55393.77 + rank 58: 55391.49 + rank 59: 55388.96 + rank 60: 55390.10 + rank 61: 55379.12 + rank 62: 55388.74 + rank 63: 55380.74 + batch-generator: + rank 0: 24.80 + rank 1: 25.03 + rank 2: 24.97 + rank 3: 23.14 + rank 4: 24.10 + rank 5: 22.94 + rank 6: 23.71 + rank 7: 24.59 + rank 8: 36.72 + rank 9: 36.89 + rank 10: 37.72 + rank 11: 37.40 + rank 12: 36.63 + rank 13: 37.96 + rank 14: 36.31 + rank 15: 38.09 + rank 16: 23.32 + rank 17: 22.12 + rank 18: 23.02 + rank 19: 24.72 + rank 20: 24.12 + rank 21: 23.16 + rank 22: 23.64 + rank 23: 22.50 + rank 24: 32.67 + rank 25: 37.16 + rank 26: 35.85 + rank 27: 32.06 + rank 28: 31.69 + rank 29: 35.32 + rank 30: 24.86 + rank 31: 31.67 + rank 32: 35.44 + rank 33: 25.44 + rank 34: 36.51 + rank 35: 32.92 + rank 36: 33.59 + rank 37: 31.87 + rank 38: 30.53 + rank 39: 31.10 + rank 40: 24.08 + rank 41: 22.79 + rank 42: 22.60 + rank 43: 22.58 + rank 44: 23.95 + rank 45: 23.61 + rank 46: 21.06 + rank 47: 21.88 + rank 48: 23.87 + rank 49: 32.14 + rank 50: 28.13 + rank 51: 28.09 + rank 52: 32.33 + rank 53: 26.19 + rank 54: 26.84 + rank 55: 24.90 + rank 56: 25.03 + rank 57: 23.42 + rank 58: 22.67 + rank 59: 23.55 + rank 60: 21.95 + rank 61: 24.22 + rank 62: 21.70 + rank 63: 27.51 + embedding-grads-all-reduce: + rank 0: 0.03 + rank 1: 0.03 + rank 2: 0.03 + rank 3: 0.02 + rank 4: 0.02 + rank 5: 0.03 + rank 6: 0.03 + rank 7: 0.03 + rank 8: 0.02 + rank 9: 0.02 + rank 10: 0.03 + rank 11: 0.03 + rank 12: 0.03 + rank 13: 0.03 + rank 14: 0.02 + rank 15: 0.02 + rank 16: 0.03 + rank 17: 0.03 + rank 18: 0.03 + rank 19: 0.03 + rank 20: 0.03 + rank 21: 0.03 + rank 22: 0.03 + rank 23: 0.02 + rank 24: 0.04 + rank 25: 0.05 + rank 26: 0.04 + rank 27: 0.03 + rank 28: 0.04 + rank 29: 0.04 + rank 30: 0.04 + rank 31: 0.04 + rank 32: 0.03 + rank 33: 0.03 + rank 34: 0.03 + rank 35: 0.03 + rank 36: 0.03 + rank 37: 0.03 + rank 38: 0.03 + rank 39: 0.03 + rank 40: 0.03 + rank 41: 0.03 + rank 42: 0.03 + rank 43: 0.03 + rank 44: 0.03 + rank 45: 0.03 + rank 46: 0.03 + rank 47: 0.03 + rank 48: 0.03 + rank 49: 0.03 + rank 50: 0.03 + rank 51: 0.03 + rank 52: 0.03 + rank 53: 0.03 + rank 54: 0.03 + rank 55: 0.03 + rank 56: 0.03 + rank 57: 0.02 + rank 58: 0.02 + rank 59: 0.03 + rank 60: 0.03 + rank 61: 0.03 + rank 62: 0.03 + rank 63: 0.03 + all-grads-sync: + rank 0: 2.27 + rank 1: 0.78 + rank 2: 0.92 + rank 3: 1.06 + rank 4: 1.17 + rank 5: 0.83 + rank 6: 1.94 + rank 7: 1.44 + rank 8: 0.80 + rank 9: 0.81 + rank 10: 0.92 + rank 11: 0.81 + rank 12: 0.86 + rank 13: 1.05 + rank 14: 0.78 + rank 15: 0.99 + rank 16: 0.76 + rank 17: 0.76 + rank 18: 0.77 + rank 19: 0.81 + rank 20: 0.81 + rank 21: 0.76 + rank 22: 0.80 + rank 23: 0.86 + rank 24: 1.38 + rank 25: 1.44 + rank 26: 1.36 + rank 27: 1.23 + rank 28: 1.04 + rank 29: 1.33 + rank 30: 1.26 + rank 31: 1.42 + rank 32: 1.22 + rank 33: 1.18 + rank 34: 1.28 + rank 35: 2.11 + rank 36: 1.26 + rank 37: 1.31 + rank 38: 1.16 + rank 39: 1.32 + rank 40: 0.91 + rank 41: 0.95 + rank 42: 1.07 + rank 43: 0.78 + rank 44: 1.18 + rank 45: 0.80 + rank 46: 1.25 + rank 47: 0.81 + rank 48: 0.90 + rank 49: 1.17 + rank 50: 0.83 + rank 51: 0.83 + rank 52: 1.16 + rank 53: 0.89 + rank 54: 0.86 + rank 55: 0.99 + rank 56: 1.18 + rank 57: 0.79 + rank 58: 0.78 + rank 59: 0.79 + rank 60: 0.79 + rank 61: 0.77 + rank 62: 0.86 + rank 63: 1.10 + params-all-gather: + rank 0: 0.02 + rank 1: 0.02 + rank 2: 0.02 + rank 3: 0.02 + rank 4: 0.02 + rank 5: 0.03 + rank 6: 0.02 + rank 7: 0.02 + rank 8: 0.02 + rank 9: 0.02 + rank 10: 0.02 + rank 11: 0.02 + rank 12: 0.02 + rank 13: 0.02 + rank 14: 0.03 + rank 15: 0.02 + rank 16: 0.02 + rank 17: 0.02 + rank 18: 0.02 + rank 19: 0.02 + rank 20: 0.03 + rank 21: 0.02 + rank 22: 0.02 + rank 23: 0.03 + rank 24: 0.03 + rank 25: 0.03 + rank 26: 0.03 + rank 27: 0.04 + rank 28: 0.04 + rank 29: 0.03 + rank 30: 0.03 + rank 31: 0.04 + rank 32: 0.02 + rank 33: 0.03 + rank 34: 0.02 + rank 35: 0.02 + rank 36: 0.02 + rank 37: 0.02 + rank 38: 0.02 + rank 39: 0.02 + rank 40: 0.02 + rank 41: 0.02 + rank 42: 0.02 + rank 43: 0.02 + rank 44: 0.02 + rank 45: 0.02 + rank 46: 0.02 + rank 47: 0.02 + rank 48: 0.03 + rank 49: 0.03 + rank 50: 0.03 + rank 51: 0.03 + rank 52: 0.03 + rank 53: 0.03 + rank 54: 0.03 + rank 55: 0.02 + rank 56: 0.02 + rank 57: 0.02 + rank 58: 0.02 + rank 59: 0.02 + rank 60: 0.02 + rank 61: 0.02 + rank 62: 0.02 + rank 63: 0.02 + optimizer-copy-to-main-grad: + rank 0: 2.21 + rank 1: 1.84 + rank 2: 1.73 + rank 3: 2.13 + rank 4: 1.73 + rank 5: 1.81 + rank 6: 1.89 + rank 7: 1.88 + rank 8: 1.74 + rank 9: 2.14 + rank 10: 1.78 + rank 11: 1.70 + rank 12: 1.78 + rank 13: 1.75 + rank 14: 1.87 + rank 15: 1.99 + rank 16: 2.39 + rank 17: 2.05 + rank 18: 2.12 + rank 19: 2.16 + rank 20: 2.16 + rank 21: 2.27 + rank 22: 2.64 + rank 23: 2.30 + rank 24: 1.84 + rank 25: 2.32 + rank 26: 2.15 + rank 27: 2.10 + rank 28: 2.30 + rank 29: 2.34 + rank 30: 2.28 + rank 31: 1.92 + rank 32: 2.29 + rank 33: 2.09 + rank 34: 2.27 + rank 35: 2.49 + rank 36: 2.20 + rank 37: 2.23 + rank 38: 2.43 + rank 39: 2.00 + rank 40: 1.70 + rank 41: 1.88 + rank 42: 1.85 + rank 43: 1.88 + rank 44: 1.92 + rank 45: 2.01 + rank 46: 1.80 + rank 47: 1.68 + rank 48: 1.93 + rank 49: 1.80 + rank 50: 2.26 + rank 51: 1.82 + rank 52: 1.95 + rank 53: 1.72 + rank 54: 1.77 + rank 55: 1.78 + rank 56: 1.74 + rank 57: 2.32 + rank 58: 1.80 + rank 59: 1.65 + rank 60: 1.73 + rank 61: 1.61 + rank 62: 1.83 + rank 63: 1.82 + optimizer-inner-step: + rank 0: 42.04 + rank 1: 41.89 + rank 2: 41.71 + rank 3: 41.79 + rank 4: 41.64 + rank 5: 41.80 + rank 6: 42.11 + rank 7: 42.09 + rank 8: 41.83 + rank 9: 41.78 + rank 10: 41.65 + rank 11: 41.90 + rank 12: 41.81 + rank 13: 41.98 + rank 14: 41.82 + rank 15: 41.67 + rank 16: 42.11 + rank 17: 41.92 + rank 18: 42.00 + rank 19: 41.93 + rank 20: 41.90 + rank 21: 41.79 + rank 22: 42.27 + rank 23: 41.99 + rank 24: 41.28 + rank 25: 41.70 + rank 26: 41.68 + rank 27: 41.80 + rank 28: 45.54 + rank 29: 41.49 + rank 30: 41.51 + rank 31: 41.91 + rank 32: 41.98 + rank 33: 41.84 + rank 34: 41.70 + rank 35: 47.52 + rank 36: 42.06 + rank 37: 41.93 + rank 38: 41.86 + rank 39: 41.82 + rank 40: 41.79 + rank 41: 41.89 + rank 42: 42.00 + rank 43: 42.04 + rank 44: 41.66 + rank 45: 42.19 + rank 46: 41.90 + rank 47: 41.50 + rank 48: 42.30 + rank 49: 41.74 + rank 50: 53.52 + rank 51: 41.53 + rank 52: 41.93 + rank 53: 54.47 + rank 54: 42.03 + rank 55: 41.84 + rank 56: 41.34 + rank 57: 47.03 + rank 58: 41.77 + rank 59: 41.65 + rank 60: 41.88 + rank 61: 41.63 + rank 62: 41.77 + rank 63: 41.61 + optimizer-copy-main-to-model-params: + rank 0: 4.41 + rank 1: 3.85 + rank 2: 3.69 + rank 3: 4.34 + rank 4: 3.78 + rank 5: 3.98 + rank 6: 3.96 + rank 7: 3.97 + rank 8: 3.90 + rank 9: 4.47 + rank 10: 3.80 + rank 11: 3.81 + rank 12: 4.00 + rank 13: 3.77 + rank 14: 4.02 + rank 15: 4.21 + rank 16: 4.38 + rank 17: 4.01 + rank 18: 4.01 + rank 19: 4.04 + rank 20: 4.03 + rank 21: 4.20 + rank 22: 4.53 + rank 23: 4.17 + rank 24: 3.92 + rank 25: 4.31 + rank 26: 3.98 + rank 27: 4.10 + rank 28: 4.36 + rank 29: 4.32 + rank 30: 4.35 + rank 31: 3.97 + rank 32: 4.08 + rank 33: 3.90 + rank 34: 4.41 + rank 35: 4.53 + rank 36: 4.53 + rank 37: 4.30 + rank 38: 4.45 + rank 39: 3.90 + rank 40: 3.77 + rank 41: 3.99 + rank 42: 3.95 + rank 43: 4.04 + rank 44: 3.97 + rank 45: 4.23 + rank 46: 3.73 + rank 47: 3.74 + rank 48: 4.05 + rank 49: 3.90 + rank 50: 4.85 + rank 51: 3.97 + rank 52: 4.19 + rank 53: 3.85 + rank 54: 3.87 + rank 55: 3.85 + rank 56: 3.80 + rank 57: 4.72 + rank 58: 3.90 + rank 59: 3.86 + rank 60: 3.77 + rank 61: 3.67 + rank 62: 3.92 + rank 63: 3.72 + optimizer: + rank 0: 53.54 + rank 1: 52.67 + rank 2: 52.25 + rank 3: 53.25 + rank 4: 52.27 + rank 5: 52.72 + rank 6: 53.04 + rank 7: 53.03 + rank 8: 52.63 + rank 9: 53.39 + rank 10: 52.39 + rank 11: 52.56 + rank 12: 52.74 + rank 13: 52.67 + rank 14: 52.77 + rank 15: 52.95 + rank 16: 53.86 + rank 17: 53.08 + rank 18: 53.19 + rank 19: 53.13 + rank 20: 53.08 + rank 21: 53.30 + rank 22: 54.20 + rank 23: 53.40 + rank 24: 52.34 + rank 25: 53.10 + rank 26: 52.60 + rank 27: 53.01 + rank 28: 57.11 + rank 29: 53.04 + rank 30: 53.04 + rank 31: 52.87 + rank 32: 52.96 + rank 33: 52.58 + rank 34: 53.22 + rank 35: 59.13 + rank 36: 53.59 + rank 37: 53.16 + rank 38: 53.36 + rank 39: 52.56 + rank 40: 52.44 + rank 41: 52.84 + rank 42: 52.88 + rank 43: 53.03 + rank 44: 52.58 + rank 45: 53.51 + rank 46: 52.46 + rank 47: 52.09 + rank 48: 53.30 + rank 49: 52.50 + rank 50: 65.61 + rank 51: 52.38 + rank 52: 53.09 + rank 53: 65.16 + rank 54: 52.76 + rank 55: 52.53 + rank 56: 52.03 + rank 57: 59.03 + rank 58: 52.58 + rank 59: 52.35 + rank 60: 52.54 + rank 61: 52.12 + rank 62: 52.61 + rank 63: 52.19 + interval-time: + rank 0: 0.76 + rank 1: 0.82 + rank 2: 0.74 + rank 3: 0.72 + rank 4: 0.84 + rank 5: 0.72 + rank 6: 0.81 + rank 7: 0.74 + rank 8: 0.83 + rank 9: 0.74 + rank 10: 0.83 + rank 11: 0.82 + rank 12: 0.83 + rank 13: 0.80 + rank 14: 0.83 + rank 15: 0.73 + rank 16: 0.74 + rank 17: 0.82 + rank 18: 0.82 + rank 19: 0.83 + rank 20: 0.82 + rank 21: 0.77 + rank 22: 0.74 + rank 23: 0.73 + rank 24: 0.87 + rank 25: 0.86 + rank 26: 1.02 + rank 27: 0.96 + rank 28: 0.83 + rank 29: 1.19 + rank 30: 0.96 + rank 31: 0.86 + rank 32: 0.92 + rank 33: 0.93 + rank 34: 0.83 + rank 35: 0.91 + rank 36: 0.83 + rank 37: 0.92 + rank 38: 0.80 + rank 39: 0.89 + rank 40: 0.76 + rank 41: 0.83 + rank 42: 0.81 + rank 43: 0.84 + rank 44: 0.82 + rank 45: 0.81 + rank 46: 0.83 + rank 47: 0.73 + rank 48: 0.91 + rank 49: 0.93 + rank 50: 0.90 + rank 51: 0.89 + rank 52: 0.83 + rank 53: 0.88 + rank 54: 0.89 + rank 55: 0.91 + rank 56: 0.75 + rank 57: 0.81 + rank 58: 0.81 + rank 59: 0.83 + rank 60: 0.80 + rank 61: 0.80 + rank 62: 0.81 + rank 63: 3.00 + [2026-10-05 18:05:35] iteration 2003/ 6855 | consumed samples: 256384 | elapsed time per iteration (ms): 85416.2 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.52% | FLOPs estimator: workload-aware | logical tokens/s: 196417.0 | valid tokens/s: 196417.0 | physical tokens/s: 196876.5 | loss tokens/s: 142591.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 5.934445E-01 | lm-global-supervised-tokens: 1.217958E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.169266E-02 | lm-token-buffer-mib: 5.809134E+01 | global_load_balancing_loss: 1.001509E+00 | loss scale: 1.0 | grad norm: 0.079 | params norm: 11513.913 | params rms: 0.062344 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 19:07:19.191485 | finish at 2026-10-10 13:12:55 +times across ranks (ms): + forward-backward: + rank 0: 85201.12 + rank 1: 85205.20 + rank 2: 85202.78 + rank 3: 85203.20 + rank 4: 85205.54 + rank 5: 85203.95 + rank 6: 85205.53 + rank 7: 85205.22 + rank 8: 85188.78 + rank 9: 85193.46 + rank 10: 85191.09 + rank 11: 85192.52 + rank 12: 85194.31 + rank 13: 85194.48 + rank 14: 85191.91 + rank 15: 85192.27 + rank 16: 85184.24 + rank 17: 85189.27 + rank 18: 85186.39 + rank 19: 85187.77 + rank 20: 85190.55 + rank 21: 85190.25 + rank 22: 85188.05 + rank 23: 85188.28 + rank 24: 85183.99 + rank 25: 85189.60 + rank 26: 85186.82 + rank 27: 85187.14 + rank 28: 85189.85 + rank 29: 85189.93 + rank 30: 85187.64 + rank 31: 85187.98 + rank 32: 85178.49 + rank 33: 85182.78 + rank 34: 85179.81 + rank 35: 85181.03 + rank 36: 85184.20 + rank 37: 85182.85 + rank 38: 85183.54 + rank 39: 85184.35 + rank 40: 85183.13 + rank 41: 85186.74 + rank 42: 85183.55 + rank 43: 85183.97 + rank 44: 85185.70 + rank 45: 85183.41 + rank 46: 85188.16 + rank 47: 85187.56 + rank 48: 85187.17 + rank 49: 85192.17 + rank 50: 85186.98 + rank 51: 85189.04 + rank 52: 85190.53 + rank 53: 85187.76 + rank 54: 85192.61 + rank 55: 85193.10 + rank 56: 85192.31 + rank 57: 85196.23 + rank 58: 85192.90 + rank 59: 85193.14 + rank 60: 85193.94 + rank 61: 85193.11 + rank 62: 85196.61 + rank 63: 85198.20 + forward-compute: + rank 0: 36301.45 + rank 1: 36307.50 + rank 2: 36305.78 + rank 3: 36306.09 + rank 4: 36307.93 + rank 5: 36306.21 + rank 6: 36308.84 + rank 7: 36309.11 + rank 8: 33257.61 + rank 9: 33262.49 + rank 10: 33261.59 + rank 11: 33262.38 + rank 12: 33264.84 + rank 13: 33264.04 + rank 14: 33261.13 + rank 15: 33263.31 + rank 16: 30690.47 + rank 17: 30692.72 + rank 18: 30733.58 + rank 19: 30690.23 + rank 20: 30680.91 + rank 21: 30684.73 + rank 22: 30677.55 + rank 23: 30683.54 + rank 24: 33179.21 + rank 25: 33185.80 + rank 26: 33183.34 + rank 27: 33181.29 + rank 28: 33188.47 + rank 29: 33184.30 + rank 30: 33186.77 + rank 31: 33185.82 + rank 32: 34449.31 + rank 33: 34455.88 + rank 34: 34452.47 + rank 35: 34455.16 + rank 36: 34458.10 + rank 37: 34456.20 + rank 38: 34456.39 + rank 39: 34453.82 + rank 40: 30431.41 + rank 41: 30428.26 + rank 42: 30434.50 + rank 43: 30437.25 + rank 44: 30435.02 + rank 45: 30439.95 + rank 46: 30434.09 + rank 47: 30440.52 + rank 48: 30495.36 + rank 49: 30509.34 + rank 50: 30509.18 + rank 51: 30510.41 + rank 52: 30525.73 + rank 53: 30523.65 + rank 54: 30513.29 + rank 55: 30511.57 + rank 56: 33242.90 + rank 57: 33249.62 + rank 58: 33247.32 + rank 59: 33248.78 + rank 60: 33251.11 + rank 61: 33249.07 + rank 62: 33250.28 + rank 63: 33250.71 + backward-compute: + rank 0: 48836.45 + rank 1: 48836.08 + rank 2: 48833.64 + rank 3: 48836.29 + rank 4: 48833.86 + rank 5: 48837.11 + rank 6: 48835.06 + rank 7: 48835.61 + rank 8: 51894.82 + rank 9: 51894.73 + rank 10: 51893.40 + rank 11: 51895.66 + rank 12: 51894.38 + rank 13: 51893.69 + rank 14: 51895.90 + rank 15: 51894.08 + rank 16: 54462.39 + rank 17: 54464.20 + rank 18: 54420.73 + rank 19: 54462.28 + rank 20: 54475.80 + rank 21: 54471.62 + rank 22: 54478.47 + rank 23: 54471.58 + rank 24: 51951.61 + rank 25: 51948.43 + rank 26: 51945.74 + rank 27: 51950.62 + rank 28: 51947.80 + rank 29: 51950.15 + rank 30: 51946.81 + rank 31: 51949.22 + rank 32: 50689.74 + rank 33: 50687.39 + rank 34: 50687.15 + rank 35: 50683.78 + rank 36: 50686.01 + rank 37: 50687.88 + rank 38: 50687.23 + rank 39: 50691.00 + rank 40: 54717.57 + rank 41: 54722.10 + rank 42: 54716.50 + rank 43: 54712.59 + rank 44: 54717.70 + rank 45: 54710.89 + rank 46: 54720.20 + rank 47: 54715.71 + rank 48: 54659.37 + rank 49: 54637.40 + rank 50: 54645.93 + rank 51: 54645.19 + rank 52: 54627.12 + rank 53: 54633.14 + rank 54: 54645.00 + rank 55: 54647.50 + rank 56: 51913.42 + rank 57: 51911.77 + rank 58: 51912.41 + rank 59: 51907.79 + rank 60: 51909.27 + rank 61: 51909.63 + rank 62: 51913.16 + rank 63: 51909.64 + batch-generator: + rank 0: 28.78 + rank 1: 26.76 + rank 2: 29.68 + rank 3: 28.78 + rank 4: 27.58 + rank 5: 26.94 + rank 6: 28.23 + rank 7: 28.81 + rank 8: 22.11 + rank 9: 21.16 + rank 10: 22.56 + rank 11: 21.56 + rank 12: 21.87 + rank 13: 21.65 + rank 14: 21.70 + rank 15: 21.63 + rank 16: 22.27 + rank 17: 21.65 + rank 18: 24.38 + rank 19: 21.91 + rank 20: 22.98 + rank 21: 23.06 + rank 22: 22.39 + rank 23: 21.65 + rank 24: 31.06 + rank 25: 34.90 + rank 26: 35.74 + rank 27: 35.43 + rank 28: 24.33 + rank 29: 33.86 + rank 30: 34.01 + rank 31: 31.16 + rank 32: 34.21 + rank 33: 32.01 + rank 34: 33.09 + rank 35: 30.52 + rank 36: 30.95 + rank 37: 32.41 + rank 38: 32.89 + rank 39: 29.50 + rank 40: 24.20 + rank 41: 22.41 + rank 42: 22.50 + rank 43: 22.98 + rank 44: 22.55 + rank 45: 22.09 + rank 46: 21.08 + rank 47: 22.43 + rank 48: 22.16 + rank 49: 42.04 + rank 50: 24.07 + rank 51: 22.21 + rank 52: 32.28 + rank 53: 24.89 + rank 54: 21.48 + rank 55: 22.67 + rank 56: 23.79 + rank 57: 23.11 + rank 58: 22.39 + rank 59: 22.30 + rank 60: 22.45 + rank 61: 23.37 + rank 62: 21.87 + rank 63: 25.37 + embedding-grads-all-reduce: + rank 0: 0.03 + rank 1: 0.02 + rank 2: 0.03 + rank 3: 0.02 + rank 4: 0.03 + rank 5: 0.02 + rank 6: 0.03 + rank 7: 0.03 + rank 8: 0.02 + rank 9: 0.03 + rank 10: 0.02 + rank 11: 0.03 + rank 12: 0.02 + rank 13: 0.03 + rank 14: 0.03 + rank 15: 0.02 + rank 16: 0.03 + rank 17: 0.03 + rank 18: 0.03 + rank 19: 0.02 + rank 20: 0.03 + rank 21: 0.03 + rank 22: 0.03 + rank 23: 0.02 + rank 24: 0.04 + rank 25: 0.04 + rank 26: 0.04 + rank 27: 0.03 + rank 28: 0.04 + rank 29: 0.04 + rank 30: 0.04 + rank 31: 0.04 + rank 32: 0.03 + rank 33: 0.03 + rank 34: 0.03 + rank 35: 0.03 + rank 36: 0.03 + rank 37: 0.03 + rank 38: 0.03 + rank 39: 0.03 + rank 40: 0.03 + rank 41: 0.02 + rank 42: 0.03 + rank 43: 0.03 + rank 44: 0.03 + rank 45: 0.03 + rank 46: 0.03 + rank 47: 0.03 + rank 48: 0.04 + rank 49: 0.05 + rank 50: 0.03 + rank 51: 0.03 + rank 52: 0.03 + rank 53: 0.03 + rank 54: 0.02 + rank 55: 0.03 + rank 56: 0.03 + rank 57: 0.02 + rank 58: 0.02 + rank 59: 0.02 + rank 60: 0.02 + rank 61: 0.03 + rank 62: 0.03 + rank 63: 0.03 + all-grads-sync: + rank 0: 0.78 + rank 1: 0.83 + rank 2: 0.78 + rank 3: 0.93 + rank 4: 0.88 + rank 5: 0.82 + rank 6: 0.80 + rank 7: 0.77 + rank 8: 0.95 + rank 9: 0.74 + rank 10: 0.77 + rank 11: 0.79 + rank 12: 0.76 + rank 13: 1.24 + rank 14: 0.77 + rank 15: 0.91 + rank 16: 0.83 + rank 17: 0.77 + rank 18: 0.74 + rank 19: 0.75 + rank 20: 0.77 + rank 21: 0.80 + rank 22: 0.77 + rank 23: 0.75 + rank 24: 1.26 + rank 25: 1.23 + rank 26: 1.20 + rank 27: 1.26 + rank 28: 1.12 + rank 29: 1.25 + rank 30: 1.26 + rank 31: 1.32 + rank 32: 1.14 + rank 33: 1.27 + rank 34: 0.95 + rank 35: 1.39 + rank 36: 1.28 + rank 37: 1.37 + rank 38: 1.19 + rank 39: 1.50 + rank 40: 0.97 + rank 41: 0.87 + rank 42: 1.15 + rank 43: 0.77 + rank 44: 0.75 + rank 45: 0.86 + rank 46: 1.08 + rank 47: 0.77 + rank 48: 0.83 + rank 49: 1.16 + rank 50: 0.88 + rank 51: 0.96 + rank 52: 1.13 + rank 53: 0.84 + rank 54: 1.01 + rank 55: 1.09 + rank 56: 0.79 + rank 57: 0.82 + rank 58: 0.76 + rank 59: 0.76 + rank 60: 0.81 + rank 61: 0.80 + rank 62: 0.78 + rank 63: 1.26 + params-all-gather: + rank 0: 0.02 + rank 1: 0.02 + rank 2: 0.02 + rank 3: 0.02 + rank 4: 0.02 + rank 5: 0.02 + rank 6: 0.02 + rank 7: 0.02 + rank 8: 0.02 + rank 9: 0.02 + rank 10: 0.02 + rank 11: 0.02 + rank 12: 0.02 + rank 13: 0.02 + rank 14: 0.02 + rank 15: 0.02 + rank 16: 0.02 + rank 17: 0.02 + rank 18: 0.02 + rank 19: 0.02 + rank 20: 0.02 + rank 21: 0.02 + rank 22: 0.02 + rank 23: 0.02 + rank 24: 0.03 + rank 25: 0.03 + rank 26: 0.04 + rank 27: 0.03 + rank 28: 0.04 + rank 29: 0.03 + rank 30: 0.04 + rank 31: 0.04 + rank 32: 0.02 + rank 33: 0.02 + rank 34: 0.02 + rank 35: 0.02 + rank 36: 0.02 + rank 37: 0.02 + rank 38: 0.02 + rank 39: 0.02 + rank 40: 0.02 + rank 41: 0.02 + rank 42: 0.02 + rank 43: 0.02 + rank 44: 0.02 + rank 45: 0.02 + rank 46: 0.02 + rank 47: 0.02 + rank 48: 0.03 + rank 49: 0.03 + rank 50: 0.02 + rank 51: 0.03 + rank 52: 0.03 + rank 53: 0.02 + rank 54: 0.02 + rank 55: 0.03 + rank 56: 0.02 + rank 57: 0.02 + rank 58: 0.02 + rank 59: 0.02 + rank 60: 0.02 + rank 61: 0.02 + rank 62: 0.02 + rank 63: 0.02 + optimizer-copy-to-main-grad: + rank 0: 2.15 + rank 1: 1.79 + rank 2: 1.69 + rank 3: 2.13 + rank 4: 1.65 + rank 5: 1.76 + rank 6: 1.89 + rank 7: 1.92 + rank 8: 1.74 + rank 9: 2.11 + rank 10: 1.69 + rank 11: 1.67 + rank 12: 1.52 + rank 13: 1.72 + rank 14: 1.85 + rank 15: 1.94 + rank 16: 2.41 + rank 17: 2.04 + rank 18: 2.11 + rank 19: 2.21 + rank 20: 2.21 + rank 21: 2.34 + rank 22: 2.49 + rank 23: 2.26 + rank 24: 1.88 + rank 25: 2.31 + rank 26: 1.86 + rank 27: 2.04 + rank 28: 2.31 + rank 29: 2.33 + rank 30: 2.36 + rank 31: 2.49 + rank 32: 2.08 + rank 33: 1.97 + rank 34: 1.91 + rank 35: 2.20 + rank 36: 2.00 + rank 37: 2.00 + rank 38: 2.41 + rank 39: 1.99 + rank 40: 1.71 + rank 41: 1.84 + rank 42: 1.80 + rank 43: 1.82 + rank 44: 1.87 + rank 45: 1.99 + rank 46: 1.72 + rank 47: 1.66 + rank 48: 1.94 + rank 49: 2.51 + rank 50: 2.26 + rank 51: 1.86 + rank 52: 1.97 + rank 53: 1.73 + rank 54: 1.80 + rank 55: 1.80 + rank 56: 1.72 + rank 57: 2.34 + rank 58: 1.85 + rank 59: 1.67 + rank 60: 1.72 + rank 61: 1.67 + rank 62: 1.89 + rank 63: 1.83 + optimizer-inner-step: + rank 0: 41.83 + rank 1: 41.69 + rank 2: 41.47 + rank 3: 41.63 + rank 4: 41.42 + rank 5: 41.61 + rank 6: 41.88 + rank 7: 41.92 + rank 8: 41.62 + rank 9: 41.68 + rank 10: 41.49 + rank 11: 41.65 + rank 12: 41.63 + rank 13: 41.86 + rank 14: 41.61 + rank 15: 41.49 + rank 16: 41.96 + rank 17: 41.77 + rank 18: 41.80 + rank 19: 41.74 + rank 20: 41.77 + rank 21: 41.63 + rank 22: 42.11 + rank 23: 41.80 + rank 24: 41.38 + rank 25: 44.29 + rank 26: 41.63 + rank 27: 42.55 + rank 28: 45.91 + rank 29: 43.58 + rank 30: 42.25 + rank 31: 43.33 + rank 32: 41.74 + rank 33: 41.53 + rank 34: 41.67 + rank 35: 47.25 + rank 36: 41.78 + rank 37: 41.76 + rank 38: 41.57 + rank 39: 41.58 + rank 40: 41.69 + rank 41: 41.75 + rank 42: 41.81 + rank 43: 41.86 + rank 44: 41.43 + rank 45: 42.04 + rank 46: 41.70 + rank 47: 41.41 + rank 48: 42.40 + rank 49: 41.62 + rank 50: 53.36 + rank 51: 41.40 + rank 52: 44.09 + rank 53: 41.56 + rank 54: 41.79 + rank 55: 41.70 + rank 56: 41.33 + rank 57: 47.33 + rank 58: 41.65 + rank 59: 41.49 + rank 60: 41.81 + rank 61: 41.39 + rank 62: 41.59 + rank 63: 41.44 + optimizer-copy-main-to-model-params: + rank 0: 4.44 + rank 1: 3.85 + rank 2: 3.77 + rank 3: 4.33 + rank 4: 3.73 + rank 5: 3.92 + rank 6: 3.92 + rank 7: 4.01 + rank 8: 3.84 + rank 9: 4.48 + rank 10: 3.77 + rank 11: 3.75 + rank 12: 3.92 + rank 13: 3.78 + rank 14: 4.01 + rank 15: 4.21 + rank 16: 4.40 + rank 17: 3.99 + rank 18: 3.99 + rank 19: 4.00 + rank 20: 4.12 + rank 21: 4.33 + rank 22: 4.55 + rank 23: 4.19 + rank 24: 3.99 + rank 25: 4.33 + rank 26: 3.99 + rank 27: 4.17 + rank 28: 4.32 + rank 29: 4.44 + rank 30: 4.41 + rank 31: 4.46 + rank 32: 4.34 + rank 33: 3.87 + rank 34: 4.00 + rank 35: 4.49 + rank 36: 4.21 + rank 37: 4.02 + rank 38: 4.51 + rank 39: 3.91 + rank 40: 3.76 + rank 41: 3.96 + rank 42: 3.86 + rank 43: 4.00 + rank 44: 3.93 + rank 45: 4.24 + rank 46: 3.73 + rank 47: 3.76 + rank 48: 4.12 + rank 49: 3.99 + rank 50: 5.02 + rank 51: 3.96 + rank 52: 4.45 + rank 53: 3.85 + rank 54: 3.87 + rank 55: 4.00 + rank 56: 3.80 + rank 57: 4.72 + rank 58: 3.90 + rank 59: 3.73 + rank 60: 3.75 + rank 61: 3.68 + rank 62: 3.88 + rank 63: 3.68 + optimizer: + rank 0: 53.31 + rank 1: 52.43 + rank 2: 52.07 + rank 3: 53.01 + rank 4: 51.98 + rank 5: 52.39 + rank 6: 52.70 + rank 7: 52.81 + rank 8: 52.30 + rank 9: 53.21 + rank 10: 52.09 + rank 11: 52.19 + rank 12: 52.39 + rank 13: 52.48 + rank 14: 52.51 + rank 15: 52.69 + rank 16: 53.67 + rank 17: 52.85 + rank 18: 52.89 + rank 19: 52.84 + rank 20: 52.97 + rank 21: 53.19 + rank 22: 53.99 + rank 23: 53.16 + rank 24: 52.19 + rank 25: 55.53 + rank 26: 52.44 + rank 27: 53.59 + rank 28: 57.21 + rank 29: 54.98 + rank 30: 53.61 + rank 31: 54.65 + rank 32: 52.90 + rank 33: 52.15 + rank 34: 52.47 + rank 35: 58.71 + rank 36: 52.87 + rank 37: 52.62 + rank 38: 53.06 + rank 39: 52.24 + rank 40: 52.26 + rank 41: 52.60 + rank 42: 52.53 + rank 43: 52.73 + rank 44: 52.22 + rank 45: 53.26 + rank 46: 52.19 + rank 47: 51.93 + rank 48: 53.41 + rank 49: 52.46 + rank 50: 65.55 + rank 51: 52.17 + rank 52: 55.65 + rank 53: 52.17 + rank 54: 52.43 + rank 55: 52.49 + rank 56: 51.91 + rank 57: 59.24 + rank 58: 52.37 + rank 59: 51.97 + rank 60: 52.36 + rank 61: 51.84 + rank 62: 52.29 + rank 63: 51.88 + interval-time: + rank 0: 0.82 + rank 1: 0.83 + rank 2: 0.75 + rank 3: 0.73 + rank 4: 0.80 + rank 5: 0.73 + rank 6: 0.81 + rank 7: 0.73 + rank 8: 0.78 + rank 9: 0.73 + rank 10: 0.82 + rank 11: 0.81 + rank 12: 0.80 + rank 13: 0.79 + rank 14: 0.80 + rank 15: 0.72 + rank 16: 0.74 + rank 17: 0.80 + rank 18: 0.81 + rank 19: 0.81 + rank 20: 0.81 + rank 21: 0.74 + rank 22: 0.72 + rank 23: 0.72 + rank 24: 2.91 + rank 25: 2.89 + rank 26: 3.01 + rank 27: 3.08 + rank 28: 2.89 + rank 29: 3.00 + rank 30: 2.99 + rank 31: 2.95 + rank 32: 0.92 + rank 33: 0.94 + rank 34: 0.85 + rank 35: 0.92 + rank 36: 0.80 + rank 37: 0.89 + rank 38: 0.81 + rank 39: 0.90 + rank 40: 0.74 + rank 41: 0.82 + rank 42: 0.81 + rank 43: 0.82 + rank 44: 0.80 + rank 45: 0.76 + rank 46: 0.83 + rank 47: 0.74 + rank 48: 0.93 + rank 49: 0.99 + rank 50: 0.81 + rank 51: 0.90 + rank 52: 0.84 + rank 53: 0.90 + rank 54: 0.90 + rank 55: 0.94 + rank 56: 0.75 + rank 57: 0.84 + rank 58: 0.81 + rank 59: 0.82 + rank 60: 0.81 + rank 61: 0.83 + rank 62: 0.81 + rank 63: 3.82 + [2026-10-05 18:07:02] iteration 2004/ 6855 | consumed samples: 256512 | elapsed time per iteration (ms): 87025.4 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.60% | FLOPs estimator: workload-aware | logical tokens/s: 192784.5 | valid tokens/s: 192784.5 | physical tokens/s: 193246.0 | loss tokens/s: 144132.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.068738E-01 | lm-global-supervised-tokens: 1.254318E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.975543E-02 | lm-token-buffer-mib: 5.803262E+01 | global_load_balancing_loss: 1.000576E+00 | loss scale: 1.0 | grad norm: 0.058 | params norm: 11513.839 | params rms: 0.062344 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 21:16:00.247412 | finish at 2026-10-10 15:23:03 +times across ranks (ms): + forward-backward: + rank 0: 86806.82 + rank 1: 86807.70 + rank 2: 86808.28 + rank 3: 86808.58 + rank 4: 86809.15 + rank 5: 86809.40 + rank 6: 86807.27 + rank 7: 86806.32 + rank 8: 86802.62 + rank 9: 86803.55 + rank 10: 86802.41 + rank 11: 86804.89 + rank 12: 86804.23 + rank 13: 86805.73 + rank 14: 86802.02 + rank 15: 86801.33 + rank 16: 86798.59 + rank 17: 86799.86 + rank 18: 86799.16 + rank 19: 86806.66 + rank 20: 86800.87 + rank 21: 86801.54 + rank 22: 86798.14 + rank 23: 86798.31 + rank 24: 86798.32 + rank 25: 86798.62 + rank 26: 86798.27 + rank 27: 86799.43 + rank 28: 86800.73 + rank 29: 86800.59 + rank 30: 86798.86 + rank 31: 86796.89 + rank 32: 86792.78 + rank 33: 86793.55 + rank 34: 86795.15 + rank 35: 86792.96 + rank 36: 86795.43 + rank 37: 86794.20 + rank 38: 86793.44 + rank 39: 86792.97 + rank 40: 86797.43 + rank 41: 86796.68 + rank 42: 86795.90 + rank 43: 86796.87 + rank 44: 86796.02 + rank 45: 86794.02 + rank 46: 86798.19 + rank 47: 86796.35 + rank 48: 86802.20 + rank 49: 86801.93 + rank 50: 86799.12 + rank 51: 86801.18 + rank 52: 86800.94 + rank 53: 86799.91 + rank 54: 86803.38 + rank 55: 86802.23 + rank 56: 86806.89 + rank 57: 86806.09 + rank 58: 86804.79 + rank 59: 86804.48 + rank 60: 86804.88 + rank 61: 86804.42 + rank 62: 86807.36 + rank 63: 86806.33 + forward-compute: + rank 0: 34859.07 + rank 1: 34858.71 + rank 2: 34859.16 + rank 3: 34855.64 + rank 4: 34854.66 + rank 5: 34852.77 + rank 6: 34852.79 + rank 7: 34859.27 + rank 8: 29600.68 + rank 9: 29608.30 + rank 10: 29607.18 + rank 11: 29606.58 + rank 12: 29606.18 + rank 13: 29612.12 + rank 14: 29609.09 + rank 15: 29605.16 + rank 16: 33591.03 + rank 17: 33586.96 + rank 18: 33590.20 + rank 19: 33587.36 + rank 20: 33590.02 + rank 21: 33591.02 + rank 22: 33585.98 + rank 23: 33588.96 + rank 24: 36849.02 + rank 25: 36851.35 + rank 26: 36851.28 + rank 27: 36852.65 + rank 28: 36853.00 + rank 29: 36853.82 + rank 30: 36853.92 + rank 31: 36851.53 + rank 32: 36540.00 + rank 33: 36543.99 + rank 34: 36544.53 + rank 35: 36546.58 + rank 36: 36549.70 + rank 37: 36549.96 + rank 38: 36551.45 + rank 39: 36547.02 + rank 40: 35129.22 + rank 41: 35131.78 + rank 42: 35132.45 + rank 43: 35132.97 + rank 44: 35132.60 + rank 45: 35134.85 + rank 46: 35134.72 + rank 47: 35131.84 + rank 48: 30629.59 + rank 49: 30633.79 + rank 50: 30635.19 + rank 51: 30633.11 + rank 52: 30648.06 + rank 53: 30646.12 + rank 54: 30639.47 + rank 55: 30637.98 + rank 56: 32817.66 + rank 57: 32818.47 + rank 58: 32809.72 + rank 59: 32813.37 + rank 60: 32820.24 + rank 61: 32821.94 + rank 62: 32820.06 + rank 63: 32826.46 + backward-compute: + rank 0: 51914.09 + rank 1: 51914.89 + rank 2: 51914.53 + rank 3: 51918.10 + rank 4: 51918.98 + rank 5: 51922.43 + rank 6: 51919.30 + rank 7: 51912.04 + rank 8: 57165.59 + rank 9: 57164.15 + rank 10: 57162.96 + rank 11: 57164.59 + rank 12: 57165.13 + rank 13: 57160.55 + rank 14: 57158.80 + rank 15: 57165.00 + rank 16: 53170.54 + rank 17: 53179.81 + rank 18: 53171.16 + rank 19: 53179.56 + rank 20: 53176.61 + rank 21: 53173.86 + rank 22: 53179.57 + rank 23: 53175.18 + rank 24: 49908.24 + rank 25: 49904.86 + rank 26: 49903.38 + rank 27: 49900.68 + rank 28: 49907.48 + rank 29: 49899.70 + rank 30: 49900.97 + rank 31: 49902.81 + rank 32: 50214.83 + rank 33: 50209.63 + rank 34: 50206.46 + rank 35: 50207.82 + rank 36: 50206.69 + rank 37: 50206.95 + rank 38: 50201.83 + rank 39: 50204.44 + rank 40: 51631.92 + rank 41: 51628.33 + rank 42: 51629.65 + rank 43: 51627.35 + rank 44: 51628.20 + rank 45: 51626.64 + rank 46: 51628.02 + rank 47: 51629.30 + rank 48: 56137.90 + rank 49: 56131.75 + rank 50: 56130.04 + rank 51: 56134.05 + rank 52: 56116.02 + rank 53: 56120.44 + rank 54: 56130.38 + rank 55: 56127.06 + rank 56: 53953.39 + rank 57: 53951.97 + rank 58: 53960.95 + rank 59: 53957.39 + rank 60: 53950.44 + rank 61: 53945.45 + rank 62: 53953.97 + rank 63: 53944.03 + batch-generator: + rank 0: 23.56 + rank 1: 22.63 + rank 2: 24.94 + rank 3: 22.83 + rank 4: 24.58 + rank 5: 22.68 + rank 6: 22.82 + rank 7: 21.47 + rank 8: 21.66 + rank 9: 22.06 + rank 10: 22.58 + rank 11: 23.06 + rank 12: 21.95 + rank 13: 21.31 + rank 14: 21.47 + rank 15: 22.54 + rank 16: 20.81 + rank 17: 21.01 + rank 18: 23.41 + rank 19: 21.25 + rank 20: 22.10 + rank 21: 22.17 + rank 22: 23.20 + rank 23: 24.15 + rank 24: 34.05 + rank 25: 31.08 + rank 26: 34.29 + rank 27: 34.49 + rank 28: 29.54 + rank 29: 35.50 + rank 30: 33.86 + rank 31: 33.92 + rank 32: 34.25 + rank 33: 32.93 + rank 34: 33.06 + rank 35: 33.31 + rank 36: 25.97 + rank 37: 31.91 + rank 38: 32.64 + rank 39: 33.19 + rank 40: 23.79 + rank 41: 22.96 + rank 42: 23.35 + rank 43: 22.93 + rank 44: 22.22 + rank 45: 23.89 + rank 46: 21.50 + rank 47: 23.17 + rank 48: 26.31 + rank 49: 32.29 + rank 50: 24.85 + rank 51: 28.65 + rank 52: 33.18 + rank 53: 21.77 + rank 54: 27.82 + rank 55: 28.58 + rank 56: 24.54 + rank 57: 23.22 + rank 58: 23.19 + rank 59: 21.17 + rank 60: 22.93 + rank 61: 23.06 + rank 62: 20.63 + rank 63: 25.56 + embedding-grads-all-reduce: + rank 0: 0.03 + rank 1: 0.03 + rank 2: 0.03 + rank 3: 0.03 + rank 4: 0.03 + rank 5: 0.02 + rank 6: 0.03 + rank 7: 0.03 + rank 8: 0.02 + rank 9: 0.03 + rank 10: 0.02 + rank 11: 0.03 + rank 12: 0.03 + rank 13: 0.03 + rank 14: 0.03 + rank 15: 0.02 + rank 16: 0.03 + rank 17: 0.03 + rank 18: 0.03 + rank 19: 0.03 + rank 20: 0.03 + rank 21: 0.03 + rank 22: 0.03 + rank 23: 0.03 + rank 24: 0.04 + rank 25: 0.03 + rank 26: 0.04 + rank 27: 0.04 + rank 28: 0.04 + rank 29: 0.04 + rank 30: 0.04 + rank 31: 0.04 + rank 32: 0.03 + rank 33: 0.03 + rank 34: 0.05 + rank 35: 0.03 + rank 36: 0.03 + rank 37: 0.03 + rank 38: 0.03 + rank 39: 0.03 + rank 40: 0.03 + rank 41: 0.03 + rank 42: 0.03 + rank 43: 0.03 + rank 44: 0.03 + rank 45: 0.03 + rank 46: 0.02 + rank 47: 0.03 + rank 48: 0.03 + rank 49: 0.03 + rank 50: 0.03 + rank 51: 0.03 + rank 52: 0.03 + rank 53: 0.03 + rank 54: 0.03 + rank 55: 0.03 + rank 56: 0.03 + rank 57: 0.03 + rank 58: 0.02 + rank 59: 0.02 + rank 60: 0.02 + rank 61: 0.03 + rank 62: 0.03 + rank 63: 0.03 + all-grads-sync: + rank 0: 0.88 + rank 1: 0.74 + rank 2: 1.06 + rank 3: 0.90 + rank 4: 0.84 + rank 5: 0.98 + rank 6: 1.04 + rank 7: 1.04 + rank 8: 0.80 + rank 9: 0.78 + rank 10: 0.84 + rank 11: 1.10 + rank 12: 0.85 + rank 13: 1.30 + rank 14: 0.80 + rank 15: 1.01 + rank 16: 0.88 + rank 17: 0.81 + rank 18: 0.78 + rank 19: 1.40 + rank 20: 0.82 + rank 21: 0.85 + rank 22: 0.84 + rank 23: 0.77 + rank 24: 1.28 + rank 25: 1.19 + rank 26: 1.18 + rank 27: 1.71 + rank 28: 1.33 + rank 29: 1.33 + rank 30: 1.26 + rank 31: 1.27 + rank 32: 1.32 + rank 33: 1.24 + rank 34: 1.73 + rank 35: 1.26 + rank 36: 1.68 + rank 37: 1.45 + rank 38: 1.21 + rank 39: 1.43 + rank 40: 0.82 + rank 41: 0.81 + rank 42: 0.79 + rank 43: 1.37 + rank 44: 1.38 + rank 45: 0.84 + rank 46: 0.79 + rank 47: 0.80 + rank 48: 1.09 + rank 49: 1.12 + rank 50: 0.86 + rank 51: 1.11 + rank 52: 1.38 + rank 53: 0.91 + rank 54: 1.02 + rank 55: 0.93 + rank 56: 0.86 + rank 57: 0.90 + rank 58: 0.86 + rank 59: 0.80 + rank 60: 0.74 + rank 61: 1.03 + rank 62: 0.91 + rank 63: 0.98 + params-all-gather: + rank 0: 0.02 + rank 1: 0.02 + rank 2: 0.02 + rank 3: 0.02 + rank 4: 0.02 + rank 5: 0.02 + rank 6: 0.02 + rank 7: 0.02 + rank 8: 0.02 + rank 9: 0.02 + rank 10: 0.02 + rank 11: 0.02 + rank 12: 0.02 + rank 13: 0.02 + rank 14: 0.02 + rank 15: 0.02 + rank 16: 0.02 + rank 17: 0.02 + rank 18: 0.02 + rank 19: 0.03 + rank 20: 0.02 + rank 21: 0.02 + rank 22: 0.02 + rank 23: 0.02 + rank 24: 0.03 + rank 25: 0.03 + rank 26: 0.04 + rank 27: 0.04 + rank 28: 0.03 + rank 29: 0.04 + rank 30: 0.03 + rank 31: 0.04 + rank 32: 0.02 + rank 33: 0.02 + rank 34: 0.03 + rank 35: 0.03 + rank 36: 0.02 + rank 37: 0.02 + rank 38: 0.02 + rank 39: 0.02 + rank 40: 0.02 + rank 41: 0.02 + rank 42: 0.02 + rank 43: 0.02 + rank 44: 0.02 + rank 45: 0.02 + rank 46: 0.02 + rank 47: 0.02 + rank 48: 0.03 + rank 49: 0.03 + rank 50: 0.03 + rank 51: 0.03 + rank 52: 0.03 + rank 53: 0.03 + rank 54: 0.02 + rank 55: 0.03 + rank 56: 0.02 + rank 57: 0.02 + rank 58: 0.02 + rank 59: 0.02 + rank 60: 0.02 + rank 61: 0.02 + rank 62: 0.02 + rank 63: 0.02 + optimizer-copy-to-main-grad: + rank 0: 2.17 + rank 1: 1.78 + rank 2: 1.76 + rank 3: 2.16 + rank 4: 1.66 + rank 5: 1.84 + rank 6: 1.92 + rank 7: 1.88 + rank 8: 1.73 + rank 9: 2.15 + rank 10: 1.67 + rank 11: 1.70 + rank 12: 1.86 + rank 13: 1.70 + rank 14: 1.89 + rank 15: 1.96 + rank 16: 2.47 + rank 17: 2.11 + rank 18: 2.18 + rank 19: 2.86 + rank 20: 2.26 + rank 21: 2.36 + rank 22: 2.49 + rank 23: 2.33 + rank 24: 1.94 + rank 25: 2.22 + rank 26: 2.15 + rank 27: 2.19 + rank 28: 2.28 + rank 29: 2.35 + rank 30: 2.31 + rank 31: 1.94 + rank 32: 2.07 + rank 33: 1.86 + rank 34: 2.19 + rank 35: 2.26 + rank 36: 2.02 + rank 37: 1.90 + rank 38: 2.25 + rank 39: 1.88 + rank 40: 1.70 + rank 41: 1.80 + rank 42: 1.81 + rank 43: 1.95 + rank 44: 1.84 + rank 45: 1.92 + rank 46: 1.72 + rank 47: 1.64 + rank 48: 1.91 + rank 49: 1.61 + rank 50: 2.24 + rank 51: 1.77 + rank 52: 1.93 + rank 53: 1.72 + rank 54: 1.73 + rank 55: 1.77 + rank 56: 1.76 + rank 57: 2.29 + rank 58: 1.82 + rank 59: 1.66 + rank 60: 1.73 + rank 61: 1.78 + rank 62: 1.82 + rank 63: 1.81 + optimizer-inner-step: + rank 0: 42.16 + rank 1: 42.02 + rank 2: 41.79 + rank 3: 41.94 + rank 4: 41.73 + rank 5: 41.86 + rank 6: 42.23 + rank 7: 42.22 + rank 8: 41.95 + rank 9: 41.91 + rank 10: 41.80 + rank 11: 41.93 + rank 12: 41.95 + rank 13: 42.16 + rank 14: 41.87 + rank 15: 41.78 + rank 16: 42.25 + rank 17: 42.06 + rank 18: 42.06 + rank 19: 41.24 + rank 20: 42.07 + rank 21: 41.91 + rank 22: 42.37 + rank 23: 42.08 + rank 24: 41.63 + rank 25: 42.00 + rank 26: 41.90 + rank 27: 43.45 + rank 28: 45.82 + rank 29: 41.84 + rank 30: 41.82 + rank 31: 42.11 + rank 32: 42.02 + rank 33: 41.79 + rank 34: 42.03 + rank 35: 47.20 + rank 36: 42.03 + rank 37: 42.02 + rank 38: 41.84 + rank 39: 41.83 + rank 40: 41.94 + rank 41: 42.05 + rank 42: 42.12 + rank 43: 42.16 + rank 44: 41.77 + rank 45: 42.29 + rank 46: 41.95 + rank 47: 41.64 + rank 48: 45.61 + rank 49: 41.84 + rank 50: 53.55 + rank 51: 41.65 + rank 52: 41.96 + rank 53: 41.86 + rank 54: 42.02 + rank 55: 41.89 + rank 56: 41.60 + rank 57: 47.10 + rank 58: 41.94 + rank 59: 41.75 + rank 60: 42.10 + rank 61: 41.68 + rank 62: 41.87 + rank 63: 41.76 + optimizer-copy-main-to-model-params: + rank 0: 4.48 + rank 1: 3.89 + rank 2: 3.77 + rank 3: 4.36 + rank 4: 3.73 + rank 5: 3.92 + rank 6: 3.95 + rank 7: 3.99 + rank 8: 3.87 + rank 9: 4.51 + rank 10: 3.76 + rank 11: 3.76 + rank 12: 3.92 + rank 13: 3.74 + rank 14: 4.03 + rank 15: 4.23 + rank 16: 4.44 + rank 17: 4.07 + rank 18: 4.03 + rank 19: 5.30 + rank 20: 4.02 + rank 21: 4.29 + rank 22: 4.52 + rank 23: 4.16 + rank 24: 3.95 + rank 25: 4.23 + rank 26: 3.90 + rank 27: 4.16 + rank 28: 4.35 + rank 29: 4.39 + rank 30: 4.36 + rank 31: 4.04 + rank 32: 4.09 + rank 33: 3.85 + rank 34: 4.45 + rank 35: 4.45 + rank 36: 4.21 + rank 37: 4.03 + rank 38: 4.50 + rank 39: 3.92 + rank 40: 3.77 + rank 41: 3.97 + rank 42: 3.91 + rank 43: 3.98 + rank 44: 4.01 + rank 45: 4.21 + rank 46: 3.72 + rank 47: 3.78 + rank 48: 4.16 + rank 49: 3.88 + rank 50: 4.87 + rank 51: 3.91 + rank 52: 4.20 + rank 53: 3.86 + rank 54: 3.86 + rank 55: 3.86 + rank 56: 3.84 + rank 57: 4.72 + rank 58: 3.89 + rank 59: 3.73 + rank 60: 3.75 + rank 61: 3.70 + rank 62: 3.84 + rank 63: 3.71 + optimizer: + rank 0: 54.91 + rank 1: 53.97 + rank 2: 53.56 + rank 3: 54.54 + rank 4: 53.44 + rank 5: 53.88 + rank 6: 54.29 + rank 7: 54.29 + rank 8: 53.85 + rank 9: 54.71 + rank 10: 53.59 + rank 11: 53.72 + rank 12: 53.94 + rank 13: 53.93 + rank 14: 53.99 + rank 15: 54.19 + rank 16: 55.20 + rank 17: 54.44 + rank 18: 54.40 + rank 19: 55.79 + rank 20: 54.39 + rank 21: 54.66 + rank 22: 55.43 + rank 23: 54.63 + rank 24: 53.63 + rank 25: 54.34 + rank 26: 53.81 + rank 27: 55.71 + rank 28: 58.36 + rank 29: 54.41 + rank 30: 54.34 + rank 31: 54.23 + rank 32: 54.17 + rank 33: 53.62 + rank 34: 54.54 + rank 35: 59.86 + rank 36: 54.35 + rank 37: 54.10 + rank 38: 54.55 + rank 39: 53.73 + rank 40: 53.74 + rank 41: 54.14 + rank 42: 54.10 + rank 43: 54.23 + rank 44: 53.84 + rank 45: 54.73 + rank 46: 53.66 + rank 47: 53.42 + rank 48: 57.90 + rank 49: 53.72 + rank 50: 66.89 + rank 51: 53.58 + rank 52: 54.28 + rank 53: 53.70 + rank 54: 53.87 + rank 55: 53.75 + rank 56: 53.48 + rank 57: 60.23 + rank 58: 53.88 + rank 59: 53.45 + rank 60: 53.84 + rank 61: 53.40 + rank 62: 53.74 + rank 63: 53.45 + interval-time: + rank 0: 0.81 + rank 1: 0.82 + rank 2: 0.74 + rank 3: 0.72 + rank 4: 0.82 + rank 5: 0.72 + rank 6: 0.81 + rank 7: 0.73 + rank 8: 0.82 + rank 9: 0.73 + rank 10: 0.82 + rank 11: 0.80 + rank 12: 0.82 + rank 13: 0.80 + rank 14: 0.81 + rank 15: 0.73 + rank 16: 0.76 + rank 17: 0.83 + rank 18: 0.82 + rank 19: 0.84 + rank 20: 0.81 + rank 21: 0.77 + rank 22: 0.72 + rank 23: 0.72 + rank 24: 0.88 + rank 25: 0.84 + rank 26: 0.99 + rank 27: 0.98 + rank 28: 0.81 + rank 29: 0.94 + rank 30: 0.90 + rank 31: 0.87 + rank 32: 0.93 + rank 33: 0.97 + rank 34: 0.88 + rank 35: 0.91 + rank 36: 0.79 + rank 37: 0.88 + rank 38: 0.91 + rank 39: 0.89 + rank 40: 0.74 + rank 41: 0.83 + rank 42: 0.79 + rank 43: 0.82 + rank 44: 0.82 + rank 45: 0.80 + rank 46: 0.81 + rank 47: 0.75 + rank 48: 0.93 + rank 49: 0.94 + rank 50: 0.81 + rank 51: 0.89 + rank 52: 0.84 + rank 53: 0.89 + rank 54: 0.91 + rank 55: 0.93 + rank 56: 0.76 + rank 57: 0.91 + rank 58: 0.91 + rank 59: 0.83 + rank 60: 0.80 + rank 61: 0.85 + rank 62: 0.80 + rank 63: 3.37 + [2026-10-05 18:08:30] iteration 2005/ 6855 | consumed samples: 256640 | elapsed time per iteration (ms): 87210.7 | throughput per GPU (TFLOP/s/GPU): 43.6 | MFU 13.96% | FLOPs estimator: workload-aware | logical tokens/s: 192375.2 | valid tokens/s: 192375.2 | physical tokens/s: 192862.9 | loss tokens/s: 137396.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.002088E-01 | lm-global-supervised-tokens: 1.198247E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.148853E-02 | lm-token-buffer-mib: 5.809461E+01 | global_load_balancing_loss: 9.987372E-01 | loss scale: 1.0 | grad norm: 0.070 | params norm: 11513.767 | params rms: 0.062343 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 21:29:31.966863 | finish at 2026-10-10 15:38:02 +times across ranks (ms): + forward-backward: + rank 0: 86990.01 + rank 1: 86988.17 + rank 2: 86991.78 + rank 3: 86992.12 + rank 4: 86992.67 + rank 5: 86992.26 + rank 6: 86989.23 + rank 7: 86990.85 + rank 8: 86985.56 + rank 9: 86983.83 + rank 10: 86987.06 + rank 11: 86988.23 + rank 12: 86987.89 + rank 13: 86989.09 + rank 14: 86984.08 + rank 15: 86986.16 + rank 16: 86981.59 + rank 17: 86980.01 + rank 18: 86982.29 + rank 19: 86983.41 + rank 20: 86984.44 + rank 21: 86985.49 + rank 22: 86980.84 + rank 23: 86982.18 + rank 24: 86980.55 + rank 25: 86978.80 + rank 26: 86982.36 + rank 27: 86982.29 + rank 28: 86984.24 + rank 29: 86984.84 + rank 30: 86980.65 + rank 31: 86980.75 + rank 32: 86975.27 + rank 33: 86973.92 + rank 34: 86976.14 + rank 35: 86976.45 + rank 36: 86978.38 + rank 37: 86976.25 + rank 38: 86975.39 + rank 39: 86977.43 + rank 40: 86980.12 + rank 41: 86977.03 + rank 42: 86979.84 + rank 43: 86979.69 + rank 44: 86979.19 + rank 45: 86977.39 + rank 46: 86980.95 + rank 47: 86981.73 + rank 48: 86986.52 + rank 49: 86982.55 + rank 50: 86983.77 + rank 51: 86984.55 + rank 52: 86984.39 + rank 53: 86984.04 + rank 54: 86985.02 + rank 55: 86987.52 + rank 56: 86990.09 + rank 57: 86986.59 + rank 58: 86989.27 + rank 59: 86988.05 + rank 60: 86987.99 + rank 61: 86988.98 + rank 62: 86989.94 + rank 63: 86990.94 + forward-compute: + rank 0: 34330.20 + rank 1: 34323.75 + rank 2: 34332.42 + rank 3: 34328.77 + rank 4: 34330.80 + rank 5: 34332.60 + rank 6: 34329.75 + rank 7: 34345.47 + rank 8: 29738.25 + rank 9: 29731.72 + rank 10: 29733.59 + rank 11: 29739.19 + rank 12: 29739.62 + rank 13: 29738.45 + rank 14: 29729.77 + rank 15: 29730.84 + rank 16: 33816.91 + rank 17: 33809.09 + rank 18: 33813.23 + rank 19: 33814.24 + rank 20: 33812.82 + rank 21: 33814.58 + rank 22: 33808.59 + rank 23: 33814.14 + rank 24: 34416.43 + rank 25: 34415.27 + rank 26: 34418.70 + rank 27: 34418.38 + rank 28: 34423.01 + rank 29: 34007.54 + rank 30: 34420.08 + rank 31: 34419.87 + rank 32: 34222.27 + rank 33: 34221.23 + rank 34: 34226.82 + rank 35: 34229.52 + rank 36: 34240.67 + rank 37: 34234.86 + rank 38: 34233.79 + rank 39: 34231.49 + rank 40: 29150.07 + rank 41: 29123.54 + rank 42: 29136.67 + rank 43: 29128.67 + rank 44: 29127.62 + rank 45: 29129.00 + rank 46: 29131.84 + rank 47: 29131.29 + rank 48: 32238.51 + rank 49: 32237.62 + rank 50: 32245.45 + rank 51: 32236.39 + rank 52: 32246.10 + rank 53: 32246.68 + rank 54: 32243.96 + rank 55: 32238.63 + rank 56: 33294.43 + rank 57: 33292.37 + rank 58: 33296.64 + rank 59: 33295.84 + rank 60: 33295.14 + rank 61: 33298.93 + rank 62: 33298.19 + rank 63: 33298.66 + backward-compute: + rank 0: 52620.55 + rank 1: 52630.02 + rank 2: 52621.54 + rank 3: 52625.66 + rank 4: 52625.29 + rank 5: 52625.50 + rank 6: 52624.02 + rank 7: 52608.51 + rank 8: 57213.84 + rank 9: 57220.20 + rank 10: 57220.84 + rank 11: 57215.52 + rank 12: 57214.92 + rank 13: 57217.77 + rank 14: 57221.27 + rank 15: 57222.03 + rank 16: 53131.29 + rank 17: 53138.43 + rank 18: 53131.89 + rank 19: 53135.15 + rank 20: 53138.20 + rank 21: 53135.27 + rank 22: 53137.43 + rank 23: 53132.75 + rank 24: 52523.18 + rank 25: 52522.79 + rank 26: 52522.52 + rank 27: 52516.76 + rank 28: 52520.65 + rank 29: 52930.39 + rank 30: 52522.11 + rank 31: 52521.82 + rank 32: 52709.59 + rank 33: 52709.39 + rank 34: 52707.23 + rank 35: 52707.48 + rank 36: 52700.14 + rank 37: 52704.86 + rank 38: 52701.46 + rank 39: 52706.36 + rank 40: 57793.30 + rank 41: 57821.50 + rank 42: 57810.67 + rank 43: 57816.31 + rank 44: 57819.87 + rank 45: 57815.52 + rank 46: 57818.21 + rank 47: 57818.20 + rank 48: 54712.92 + rank 49: 54707.82 + rank 50: 54702.12 + rank 51: 54715.79 + rank 52: 54699.71 + rank 53: 54704.71 + rank 54: 54704.69 + rank 55: 54710.12 + rank 56: 53659.11 + rank 57: 53659.25 + rank 58: 53657.33 + rank 59: 53659.56 + rank 60: 53658.98 + rank 61: 53653.86 + rank 62: 53658.12 + rank 63: 53651.85 + batch-generator: + rank 0: 25.67 + rank 1: 24.90 + rank 2: 25.79 + rank 3: 24.86 + rank 4: 27.13 + rank 5: 25.88 + rank 6: 26.27 + rank 7: 24.26 + rank 8: 23.86 + rank 9: 22.33 + rank 10: 22.66 + rank 11: 22.76 + rank 12: 23.49 + rank 13: 23.26 + rank 14: 23.44 + rank 15: 22.50 + rank 16: 22.70 + rank 17: 22.70 + rank 18: 21.92 + rank 19: 22.86 + rank 20: 23.13 + rank 21: 23.55 + rank 22: 21.91 + rank 23: 22.53 + rank 24: 32.72 + rank 25: 30.48 + rank 26: 34.49 + rank 27: 36.78 + rank 28: 31.10 + rank 29: 37.47 + rank 30: 32.74 + rank 31: 32.00 + rank 32: 35.06 + rank 33: 33.69 + rank 34: 37.21 + rank 35: 32.87 + rank 36: 33.02 + rank 37: 31.92 + rank 38: 31.35 + rank 39: 31.13 + rank 40: 22.84 + rank 41: 22.57 + rank 42: 23.47 + rank 43: 21.95 + rank 44: 23.21 + rank 45: 23.45 + rank 46: 22.55 + rank 47: 23.53 + rank 48: 30.23 + rank 49: 31.38 + rank 50: 27.24 + rank 51: 22.88 + rank 52: 33.13 + rank 53: 22.78 + rank 54: 29.32 + rank 55: 31.38 + rank 56: 23.19 + rank 57: 23.23 + rank 58: 21.99 + rank 59: 22.26 + rank 60: 21.94 + rank 61: 23.12 + rank 62: 22.03 + rank 63: 23.55 + embedding-grads-all-reduce: + rank 0: 0.03 + rank 1: 0.03 + rank 2: 0.03 + rank 3: 0.03 + rank 4: 0.03 + rank 5: 0.03 + rank 6: 0.03 + rank 7: 0.03 + rank 8: 0.02 + rank 9: 0.03 + rank 10: 0.03 + rank 11: 0.03 + rank 12: 0.03 + rank 13: 0.03 + rank 14: 0.03 + rank 15: 0.02 + rank 16: 0.03 + rank 17: 0.03 + rank 18: 0.03 + rank 19: 0.02 + rank 20: 0.03 + rank 21: 0.03 + rank 22: 0.03 + rank 23: 0.02 + rank 24: 0.04 + rank 25: 0.04 + rank 26: 0.04 + rank 27: 0.04 + rank 28: 0.04 + rank 29: 0.03 + rank 30: 0.04 + rank 31: 0.03 + rank 32: 0.03 + rank 33: 0.03 + rank 34: 0.03 + rank 35: 0.03 + rank 36: 0.03 + rank 37: 0.03 + rank 38: 0.03 + rank 39: 0.03 + rank 40: 0.03 + rank 41: 0.02 + rank 42: 0.03 + rank 43: 0.03 + rank 44: 0.02 + rank 45: 0.03 + rank 46: 0.03 + rank 47: 0.03 + rank 48: 0.03 + rank 49: 0.03 + rank 50: 0.03 + rank 51: 0.03 + rank 52: 0.03 + rank 53: 0.03 + rank 54: 0.03 + rank 55: 0.03 + rank 56: 0.03 + rank 57: 0.02 + rank 58: 0.03 + rank 59: 0.02 + rank 60: 0.02 + rank 61: 0.03 + rank 62: 0.03 + rank 63: 0.03 + all-grads-sync: + rank 0: 0.79 + rank 1: 0.72 + rank 2: 0.95 + rank 3: 0.86 + rank 4: 0.93 + rank 5: 0.77 + rank 6: 0.93 + rank 7: 0.77 + rank 8: 0.87 + rank 9: 0.76 + rank 10: 0.90 + rank 11: 0.91 + rank 12: 0.87 + rank 13: 0.78 + rank 14: 0.77 + rank 15: 0.94 + rank 16: 0.85 + rank 17: 0.76 + rank 18: 0.82 + rank 19: 0.77 + rank 20: 0.81 + rank 21: 0.92 + rank 22: 0.79 + rank 23: 0.77 + rank 24: 1.34 + rank 25: 1.18 + rank 26: 1.25 + rank 27: 1.30 + rank 28: 1.32 + rank 29: 1.25 + rank 30: 1.22 + rank 31: 1.07 + rank 32: 1.18 + rank 33: 1.23 + rank 34: 1.28 + rank 35: 1.24 + rank 36: 1.11 + rank 37: 1.30 + rank 38: 1.28 + rank 39: 1.28 + rank 40: 0.80 + rank 41: 0.77 + rank 42: 0.81 + rank 43: 0.79 + rank 44: 0.76 + rank 45: 0.84 + rank 46: 0.78 + rank 47: 0.78 + rank 48: 1.46 + rank 49: 1.12 + rank 50: 1.16 + rank 51: 0.91 + rank 52: 1.17 + rank 53: 1.06 + rank 54: 0.85 + rank 55: 1.08 + rank 56: 0.84 + rank 57: 0.84 + rank 58: 1.07 + rank 59: 0.77 + rank 60: 0.75 + rank 61: 1.24 + rank 62: 1.03 + rank 63: 0.90 + params-all-gather: + rank 0: 0.02 + rank 1: 0.02 + rank 2: 0.02 + rank 3: 0.02 + rank 4: 0.02 + rank 5: 0.02 + rank 6: 0.02 + rank 7: 0.02 + rank 8: 0.02 + rank 9: 0.02 + rank 10: 0.02 + rank 11: 0.02 + rank 12: 0.02 + rank 13: 0.02 + rank 14: 0.02 + rank 15: 0.02 + rank 16: 0.02 + rank 17: 0.02 + rank 18: 0.02 + rank 19: 0.02 + rank 20: 0.02 + rank 21: 0.02 + rank 22: 0.02 + rank 23: 0.02 + rank 24: 0.04 + rank 25: 0.03 + rank 26: 0.04 + rank 27: 0.03 + rank 28: 0.03 + rank 29: 0.03 + rank 30: 0.03 + rank 31: 0.04 + rank 32: 0.02 + rank 33: 0.02 + rank 34: 0.03 + rank 35: 0.03 + rank 36: 0.02 + rank 37: 0.03 + rank 38: 0.03 + rank 39: 0.02 + rank 40: 0.02 + rank 41: 0.02 + rank 42: 0.02 + rank 43: 0.02 + rank 44: 0.02 + rank 45: 0.02 + rank 46: 0.02 + rank 47: 0.02 + rank 48: 0.03 + rank 49: 0.03 + rank 50: 0.03 + rank 51: 0.03 + rank 52: 0.03 + rank 53: 0.03 + rank 54: 0.03 + rank 55: 0.03 + rank 56: 0.02 + rank 57: 0.02 + rank 58: 0.02 + rank 59: 0.02 + rank 60: 0.02 + rank 61: 0.02 + rank 62: 0.02 + rank 63: 0.02 + optimizer-copy-to-main-grad: + rank 0: 2.16 + rank 1: 1.78 + rank 2: 1.68 + rank 3: 2.12 + rank 4: 1.70 + rank 5: 1.81 + rank 6: 1.83 + rank 7: 1.89 + rank 8: 1.79 + rank 9: 2.09 + rank 10: 1.72 + rank 11: 1.69 + rank 12: 1.79 + rank 13: 1.70 + rank 14: 1.86 + rank 15: 1.96 + rank 16: 2.46 + rank 17: 2.07 + rank 18: 2.13 + rank 19: 2.10 + rank 20: 2.21 + rank 21: 2.36 + rank 22: 2.56 + rank 23: 2.29 + rank 24: 2.02 + rank 25: 2.29 + rank 26: 2.12 + rank 27: 2.21 + rank 28: 2.24 + rank 29: 2.41 + rank 30: 2.32 + rank 31: 1.96 + rank 32: 2.03 + rank 33: 1.93 + rank 34: 2.09 + rank 35: 2.28 + rank 36: 2.06 + rank 37: 2.09 + rank 38: 2.28 + rank 39: 1.91 + rank 40: 1.68 + rank 41: 1.86 + rank 42: 1.84 + rank 43: 1.85 + rank 44: 1.88 + rank 45: 2.05 + rank 46: 1.79 + rank 47: 1.68 + rank 48: 1.96 + rank 49: 1.95 + rank 50: 2.24 + rank 51: 1.79 + rank 52: 2.06 + rank 53: 1.76 + rank 54: 1.75 + rank 55: 1.79 + rank 56: 1.81 + rank 57: 2.39 + rank 58: 1.81 + rank 59: 1.71 + rank 60: 1.72 + rank 61: 1.72 + rank 62: 1.86 + rank 63: 1.91 + optimizer-inner-step: + rank 0: 42.00 + rank 1: 41.79 + rank 2: 41.51 + rank 3: 41.74 + rank 4: 41.51 + rank 5: 41.67 + rank 6: 42.06 + rank 7: 42.01 + rank 8: 41.75 + rank 9: 41.82 + rank 10: 41.64 + rank 11: 41.76 + rank 12: 41.75 + rank 13: 41.99 + rank 14: 41.78 + rank 15: 41.63 + rank 16: 42.12 + rank 17: 41.84 + rank 18: 41.89 + rank 19: 41.85 + rank 20: 41.89 + rank 21: 41.76 + rank 22: 42.17 + rank 23: 41.90 + rank 24: 41.44 + rank 25: 41.79 + rank 26: 41.69 + rank 27: 41.93 + rank 28: 45.88 + rank 29: 41.55 + rank 30: 41.69 + rank 31: 41.84 + rank 32: 41.80 + rank 33: 41.62 + rank 34: 41.69 + rank 35: 47.47 + rank 36: 41.89 + rank 37: 41.88 + rank 38: 41.68 + rank 39: 41.68 + rank 40: 41.73 + rank 41: 41.82 + rank 42: 41.97 + rank 43: 41.96 + rank 44: 41.54 + rank 45: 42.10 + rank 46: 41.78 + rank 47: 41.39 + rank 48: 42.37 + rank 49: 41.72 + rank 50: 53.44 + rank 51: 41.47 + rank 52: 41.82 + rank 53: 41.65 + rank 54: 41.85 + rank 55: 41.78 + rank 56: 41.39 + rank 57: 47.32 + rank 58: 41.72 + rank 59: 41.61 + rank 60: 41.91 + rank 61: 41.52 + rank 62: 41.65 + rank 63: 41.56 + optimizer-copy-main-to-model-params: + rank 0: 4.40 + rank 1: 3.84 + rank 2: 3.75 + rank 3: 4.36 + rank 4: 3.69 + rank 5: 3.93 + rank 6: 3.90 + rank 7: 4.01 + rank 8: 3.85 + rank 9: 4.43 + rank 10: 3.80 + rank 11: 3.78 + rank 12: 3.88 + rank 13: 3.80 + rank 14: 4.02 + rank 15: 4.22 + rank 16: 4.42 + rank 17: 4.01 + rank 18: 3.98 + rank 19: 4.00 + rank 20: 3.98 + rank 21: 4.24 + rank 22: 4.59 + rank 23: 4.18 + rank 24: 3.90 + rank 25: 4.22 + rank 26: 3.92 + rank 27: 4.37 + rank 28: 4.34 + rank 29: 4.36 + rank 30: 4.68 + rank 31: 4.02 + rank 32: 4.13 + rank 33: 3.94 + rank 34: 4.07 + rank 35: 4.52 + rank 36: 4.20 + rank 37: 4.07 + rank 38: 4.46 + rank 39: 3.92 + rank 40: 3.76 + rank 41: 4.01 + rank 42: 3.95 + rank 43: 4.05 + rank 44: 3.99 + rank 45: 4.26 + rank 46: 3.75 + rank 47: 3.74 + rank 48: 4.08 + rank 49: 3.90 + rank 50: 4.87 + rank 51: 3.90 + rank 52: 4.20 + rank 53: 3.85 + rank 54: 3.85 + rank 55: 3.87 + rank 56: 3.83 + rank 57: 4.80 + rank 58: 3.85 + rank 59: 3.73 + rank 60: 3.73 + rank 61: 3.70 + rank 62: 3.92 + rank 63: 3.75 + optimizer: + rank 0: 53.97 + rank 1: 52.99 + rank 2: 52.60 + rank 3: 53.62 + rank 4: 52.47 + rank 5: 52.98 + rank 6: 53.36 + rank 7: 53.40 + rank 8: 52.92 + rank 9: 53.79 + rank 10: 52.74 + rank 11: 52.82 + rank 12: 52.96 + rank 13: 53.08 + rank 14: 53.15 + rank 15: 53.31 + rank 16: 54.26 + rank 17: 53.42 + rank 18: 53.48 + rank 19: 53.45 + rank 20: 53.40 + rank 21: 53.65 + rank 22: 54.56 + rank 23: 53.75 + rank 24: 52.67 + rank 25: 53.41 + rank 26: 52.93 + rank 27: 53.66 + rank 28: 57.76 + rank 29: 53.46 + rank 30: 53.81 + rank 31: 53.31 + rank 32: 53.23 + rank 33: 52.80 + rank 34: 53.10 + rank 35: 59.44 + rank 36: 53.45 + rank 37: 53.24 + rank 38: 53.58 + rank 39: 52.83 + rank 40: 52.80 + rank 41: 53.21 + rank 42: 53.27 + rank 43: 53.38 + rank 44: 52.89 + rank 45: 53.85 + rank 46: 52.79 + rank 47: 52.43 + rank 48: 53.83 + rank 49: 52.88 + rank 50: 65.96 + rank 51: 52.64 + rank 52: 53.38 + rank 53: 52.74 + rank 54: 52.94 + rank 55: 52.90 + rank 56: 52.52 + rank 57: 59.84 + rank 58: 52.87 + rank 59: 52.56 + rank 60: 52.90 + rank 61: 52.47 + rank 62: 52.88 + rank 63: 52.53 + interval-time: + rank 0: 0.98 + rank 1: 0.98 + rank 2: 0.93 + rank 3: 0.93 + rank 4: 1.01 + rank 5: 0.93 + rank 6: 0.96 + rank 7: 0.88 + rank 8: 0.86 + rank 9: 1.78 + rank 10: 0.84 + rank 11: 0.85 + rank 12: 0.86 + rank 13: 1.77 + rank 14: 1.86 + rank 15: 0.79 + rank 16: 0.80 + rank 17: 0.79 + rank 18: 0.81 + rank 19: 0.75 + rank 20: 0.81 + rank 21: 0.82 + rank 22: 0.73 + rank 23: 0.80 + rank 24: 0.92 + rank 25: 0.85 + rank 26: 0.98 + rank 27: 1.00 + rank 28: 0.82 + rank 29: 0.92 + rank 30: 0.94 + rank 31: 0.87 + rank 32: 0.93 + rank 33: 0.93 + rank 34: 0.86 + rank 35: 0.90 + rank 36: 0.79 + rank 37: 0.88 + rank 38: 0.80 + rank 39: 0.91 + rank 40: 0.74 + rank 41: 0.83 + rank 42: 0.80 + rank 43: 0.84 + rank 44: 0.82 + rank 45: 0.73 + rank 46: 0.81 + rank 47: 0.75 + rank 48: 0.93 + rank 49: 0.93 + rank 50: 0.83 + rank 51: 0.90 + rank 52: 0.84 + rank 53: 0.93 + rank 54: 1.10 + rank 55: 0.93 + rank 56: 0.81 + rank 57: 0.84 + rank 58: 0.82 + rank 59: 0.82 + rank 60: 0.80 + rank 61: 0.81 + rank 62: 0.81 + rank 63: 3.36 + [2026-10-05 18:09:56] iteration 2006/ 6855 | consumed samples: 256768 | elapsed time per iteration (ms): 85940.7 | throughput per GPU (TFLOP/s/GPU): 43.1 | MFU 13.82% | FLOPs estimator: workload-aware | logical tokens/s: 195218.4 | valid tokens/s: 195218.4 | physical tokens/s: 195592.9 | loss tokens/s: 144903.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.819754E-01 | lm-global-supervised-tokens: 1.245309E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.482330E-02 | lm-token-buffer-mib: 5.816965E+01 | global_load_balancing_loss: 9.990172E-01 | loss scale: 1.0 | grad norm: 0.043 | params norm: 11513.693 | params rms: 0.062343 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 19:45:26.572484 | finish at 2026-10-10 13:55:22 + [2026-10-05 18:11:20] iteration 2007/ 6855 | consumed samples: 256896 | elapsed time per iteration (ms): 84408.6 | throughput per GPU (TFLOP/s/GPU): 43.1 | MFU 13.83% | FLOPs estimator: workload-aware | logical tokens/s: 198761.5 | valid tokens/s: 198761.5 | physical tokens/s: 199335.0 | loss tokens/s: 144882.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.021006E-01 | lm-global-supervised-tokens: 1.222932E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.433965E-02 | lm-token-buffer-mib: 5.818355E+01 | global_load_balancing_loss: 9.997878E-01 | loss scale: 1.0 | grad norm: 0.105 | params norm: 11513.620 | params rms: 0.062343 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 17:40:12.789219 | finish at 2026-10-10 11:51:33 + [2026-10-05 18:12:50] iteration 2008/ 6855 | consumed samples: 257024 | elapsed time per iteration (ms): 89852.9 | throughput per GPU (TFLOP/s/GPU): 41.3 | MFU 13.25% | FLOPs estimator: workload-aware | logical tokens/s: 186718.5 | valid tokens/s: 186718.5 | physical tokens/s: 187159.0 | loss tokens/s: 133746.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.835463E-01 | lm-global-supervised-tokens: 1.201753E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.511720E-02 | lm-token-buffer-mib: 5.805550E+01 | global_load_balancing_loss: 9.996364E-01 | loss scale: 1.0 | grad norm: 0.073 | params norm: 11513.548 | params rms: 0.062342 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 0:58:37.121998 | finish at 2026-10-10 19:11:27 + [2026-10-05 18:14:19] iteration 2009/ 6855 | consumed samples: 257152 | elapsed time per iteration (ms): 89073.1 | throughput per GPU (TFLOP/s/GPU): 41.6 | MFU 13.32% | FLOPs estimator: workload-aware | logical tokens/s: 188353.0 | valid tokens/s: 188353.0 | physical tokens/s: 188882.1 | loss tokens/s: 138866.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.603800E-01 | lm-global-supervised-tokens: 1.236923E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.174479E-02 | lm-token-buffer-mib: 5.809521E+01 | global_load_balancing_loss: 1.000419E+00 | loss scale: 1.0 | grad norm: 0.107 | params norm: 11513.475 | params rms: 0.062342 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 23:54:08.312359 | finish at 2026-10-10 18:08:27 + [2026-10-05 18:16:08] iteration 2010/ 6855 | consumed samples: 257280 | elapsed time per iteration (ms): 109262.9 | throughput per GPU (TFLOP/s/GPU): 33.7 | MFU 10.81% | FLOPs estimator: workload-aware | logical tokens/s: 153548.8 | valid tokens/s: 153548.8 | physical tokens/s: 153923.6 | loss tokens/s: 115081.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.508095E-01 | lm-global-supervised-tokens: 1.257408E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.283739E-02 | lm-token-buffer-mib: 5.805484E+01 | global_load_balancing_loss: 1.000999E+00 | loss scale: 1.0 | grad norm: 0.062 | params norm: 11513.404 | params rms: 0.062341 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 3:02:58.747587 | finish at 2026-10-11 21:19:07 + [2026-10-05 18:17:37] iteration 2011/ 6855 | consumed samples: 257408 | elapsed time per iteration (ms): 88863.2 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.69% | FLOPs estimator: workload-aware | logical tokens/s: 188796.9 | valid tokens/s: 188796.9 | physical tokens/s: 189189.3 | loss tokens/s: 136236.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.806479E-01 | lm-global-supervised-tokens: 1.210638E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.344129E-02 | lm-token-buffer-mib: 5.810287E+01 | global_load_balancing_loss: 1.001040E+00 | loss scale: 1.0 | grad norm: 0.170 | params norm: 11513.335 | params rms: 0.062341 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 23:34:13.364807 | finish at 2026-10-10 17:51:50 + [2026-10-05 18:19:04] iteration 2012/ 6855 | consumed samples: 257536 | elapsed time per iteration (ms): 86658.7 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.41% | FLOPs estimator: workload-aware | logical tokens/s: 193600.0 | valid tokens/s: 193600.0 | physical tokens/s: 194112.0 | loss tokens/s: 145849.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.692615E-01 | lm-global-supervised-tokens: 1.263917E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.589620E-02 | lm-token-buffer-mib: 5.806596E+01 | global_load_balancing_loss: 1.000601E+00 | loss scale: 1.0 | grad norm: 0.076 | params norm: 11513.265 | params rms: 0.062341 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 20:34:48.280341 | finish at 2026-10-10 14:53:52 + [2026-10-05 18:20:29] iteration 2013/ 6855 | consumed samples: 257664 | elapsed time per iteration (ms): 85571.4 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.69% | FLOPs estimator: workload-aware | logical tokens/s: 196060.4 | valid tokens/s: 196060.4 | physical tokens/s: 196509.3 | loss tokens/s: 141416.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 5.877118E-01 | lm-global-supervised-tokens: 1.210119E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.878191E-02 | lm-token-buffer-mib: 5.813995E+01 | global_load_balancing_loss: 1.000461E+00 | loss scale: 1.0 | grad norm: 0.138 | params norm: 11513.195 | params rms: 0.062340 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 19:05:36.876603 | finish at 2026-10-10 13:26:06 + [2026-10-05 18:22:00] iteration 2014/ 6855 | consumed samples: 257792 | elapsed time per iteration (ms): 90469.0 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.44% | FLOPs estimator: workload-aware | logical tokens/s: 185447.0 | valid tokens/s: 185447.0 | physical tokens/s: 185820.6 | loss tokens/s: 132381.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.993369E-01 | lm-global-supervised-tokens: 1.197638E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.215267E-02 | lm-token-buffer-mib: 5.818106E+01 | global_load_balancing_loss: 1.000623E+00 | loss scale: 1.0 | grad norm: 0.048 | params norm: 11513.122 | params rms: 0.062340 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 1:39:20.577225 | finish at 2026-10-10 20:01:20 + [2026-10-05 18:23:25] iteration 2015/ 6855 | consumed samples: 257920 | elapsed time per iteration (ms): 85459.8 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.53% | FLOPs estimator: workload-aware | logical tokens/s: 196316.8 | valid tokens/s: 196316.8 | physical tokens/s: 196733.7 | loss tokens/s: 146541.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.782144E-01 | lm-global-supervised-tokens: 1.252344E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.790198E-02 | lm-token-buffer-mib: 5.814046E+01 | global_load_balancing_loss: 1.000331E+00 | loss scale: 1.0 | grad norm: 0.131 | params norm: 11513.049 | params rms: 0.062339 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 18:53:45.600500 | finish at 2026-10-10 13:17:11 + [2026-10-05 18:24:51] iteration 2016/ 6855 | consumed samples: 258048 | elapsed time per iteration (ms): 85787.2 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.52% | FLOPs estimator: workload-aware | logical tokens/s: 195567.3 | valid tokens/s: 195567.3 | physical tokens/s: 196255.2 | loss tokens/s: 149699.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.345799E-01 | lm-global-supervised-tokens: 1.284226E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.380377E-02 | lm-token-buffer-mib: 5.810976E+01 | global_load_balancing_loss: 1.001043E+00 | loss scale: 1.0 | grad norm: 0.045 | params norm: 11512.978 | params rms: 0.062339 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 19:18:44.127998 | finish at 2026-10-10 13:43:35 + [2026-10-05 18:26:16] iteration 2017/ 6855 | consumed samples: 258176 | elapsed time per iteration (ms): 84642.6 | throughput per GPU (TFLOP/s/GPU): 43.7 | MFU 13.99% | FLOPs estimator: workload-aware | logical tokens/s: 198211.3 | valid tokens/s: 198211.3 | physical tokens/s: 198493.9 | loss tokens/s: 148597.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.868478E-01 | lm-global-supervised-tokens: 1.257770E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.135507E-02 | lm-token-buffer-mib: 5.805186E+01 | global_load_balancing_loss: 9.990092E-01 | loss scale: 1.0 | grad norm: 0.078 | params norm: 11512.907 | params rms: 0.062339 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 17:45:00.817192 | finish at 2026-10-10 12:11:16 + [2026-10-05 18:27:44] iteration 2018/ 6855 | consumed samples: 258304 | elapsed time per iteration (ms): 88297.6 | throughput per GPU (TFLOP/s/GPU): 41.7 | MFU 13.37% | FLOPs estimator: workload-aware | logical tokens/s: 190007.2 | valid tokens/s: 190007.2 | physical tokens/s: 190471.4 | loss tokens/s: 138431.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.948696E-01 | lm-global-supervised-tokens: 1.222313E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.717401E-02 | lm-token-buffer-mib: 5.829023E+01 | global_load_balancing_loss: 9.983714E-01 | loss scale: 1.0 | grad norm: 0.058 | params norm: 11512.836 | params rms: 0.062338 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 22:38:15.606673 | finish at 2026-10-10 17:06:00 + [2026-10-05 18:29:14] iteration 2019/ 6855 | consumed samples: 258432 | elapsed time per iteration (ms): 89663.2 | throughput per GPU (TFLOP/s/GPU): 41.5 | MFU 13.30% | FLOPs estimator: workload-aware | logical tokens/s: 187113.0 | valid tokens/s: 187113.0 | physical tokens/s: 187466.8 | loss tokens/s: 132631.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.472184E-01 | lm-global-supervised-tokens: 1.189217E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.509962E-02 | lm-token-buffer-mib: 5.806418E+01 | global_load_balancing_loss: 9.996161E-01 | loss scale: 1.0 | grad norm: 0.039 | params norm: 11512.765 | params rms: 0.062338 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 0:26:51.024879 | finish at 2026-10-10 18:56:05 + [2026-10-05 18:31:03] iteration 2020/ 6855 | consumed samples: 258560 | elapsed time per iteration (ms): 108974.0 | throughput per GPU (TFLOP/s/GPU): 33.5 | MFU 10.73% | FLOPs estimator: workload-aware | logical tokens/s: 153955.5 | valid tokens/s: 153955.5 | physical tokens/s: 154409.4 | loss tokens/s: 115917.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.463459E-01 | lm-global-supervised-tokens: 1.263194E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.141700E-02 | lm-token-buffer-mib: 5.814142E+01 | global_load_balancing_loss: 9.992890E-01 | loss scale: 1.0 | grad norm: 0.055 | params norm: 11512.693 | params rms: 0.062337 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 2:21:29.146948 | finish at 2026-10-11 20:52:32 + [2026-10-05 18:32:32] iteration 2021/ 6855 | consumed samples: 258688 | elapsed time per iteration (ms): 89207.3 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.44% | FLOPs estimator: workload-aware | logical tokens/s: 188069.8 | valid tokens/s: 188069.8 | physical tokens/s: 188495.6 | loss tokens/s: 137089.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.895700E-01 | lm-global-supervised-tokens: 1.222939E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.326022E-02 | lm-token-buffer-mib: 5.803271E+01 | global_load_balancing_loss: 9.998600E-01 | loss scale: 1.0 | grad norm: 0.053 | params norm: 11512.620 | params rms: 0.062337 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 23:47:08.154793 | finish at 2026-10-10 18:19:40 + [2026-10-05 18:33:58] iteration 2022/ 6855 | consumed samples: 258816 | elapsed time per iteration (ms): 86389.1 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.61% | FLOPs estimator: workload-aware | logical tokens/s: 194204.9 | valid tokens/s: 194204.9 | physical tokens/s: 194821.8 | loss tokens/s: 148073.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.818208E-01 | lm-global-supervised-tokens: 1.279191E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.577936E-02 | lm-token-buffer-mib: 5.806612E+01 | global_load_balancing_loss: 1.001966E+00 | loss scale: 1.0 | grad norm: 0.052 | params norm: 11512.548 | params rms: 0.062337 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 19:58:38.573666 | finish at 2026-10-10 14:32:37 + [2026-10-05 18:35:25] iteration 2023/ 6855 | consumed samples: 258944 | elapsed time per iteration (ms): 86965.6 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.56% | FLOPs estimator: workload-aware | logical tokens/s: 192917.5 | valid tokens/s: 192917.5 | physical tokens/s: 193289.1 | loss tokens/s: 137021.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.658192E-01 | lm-global-supervised-tokens: 1.191614E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.176508E-02 | lm-token-buffer-mib: 5.805972E+01 | global_load_balancing_loss: 1.002398E+00 | loss scale: 1.0 | grad norm: 0.163 | params norm: 11512.476 | params rms: 0.062336 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 20:43:37.707840 | finish at 2026-10-10 15:19:03 + [2026-10-05 18:36:51] iteration 2024/ 6855 | consumed samples: 259072 | elapsed time per iteration (ms): 85831.1 | throughput per GPU (TFLOP/s/GPU): 43.1 | MFU 13.82% | FLOPs estimator: workload-aware | logical tokens/s: 195467.1 | valid tokens/s: 195467.0 | physical tokens/s: 195854.7 | loss tokens/s: 151303.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.218310E-01 | lm-global-supervised-tokens: 1.298651E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.094307E-02 | lm-token-buffer-mib: 5.818720E+01 | global_load_balancing_loss: 1.000376E+00 | loss scale: 1.0 | grad norm: 0.102 | params norm: 11512.405 | params rms: 0.062336 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 19:10:49.920795 | finish at 2026-10-10 13:47:41 + [2026-10-05 18:38:20] iteration 2025/ 6855 | consumed samples: 259200 | elapsed time per iteration (ms): 88702.2 | throughput per GPU (TFLOP/s/GPU): 41.6 | MFU 13.33% | FLOPs estimator: workload-aware | logical tokens/s: 189139.9 | valid tokens/s: 189139.9 | physical tokens/s: 189623.9 | loss tokens/s: 137329.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.669738E-01 | lm-global-supervised-tokens: 1.218145E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.725020E-02 | lm-token-buffer-mib: 5.820566E+01 | global_load_balancing_loss: 9.998053E-01 | loss scale: 1.0 | grad norm: 0.062 | params norm: 11512.334 | params rms: 0.062336 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 23:00:31.746604 | finish at 2026-10-10 17:38:51 + [2026-10-05 18:39:49] iteration 2026/ 6855 | consumed samples: 259328 | elapsed time per iteration (ms): 89001.0 | throughput per GPU (TFLOP/s/GPU): 41.7 | MFU 13.37% | FLOPs estimator: workload-aware | logical tokens/s: 188505.8 | valid tokens/s: 188505.8 | physical tokens/s: 188959.3 | loss tokens/s: 139447.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.474270E-01 | lm-global-supervised-tokens: 1.241094E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.217641E-02 | lm-token-buffer-mib: 5.803378E+01 | global_load_balancing_loss: 1.000095E+00 | loss scale: 1.0 | grad norm: 0.088 | params norm: 11512.263 | params rms: 0.062335 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 23:23:05.821742 | finish at 2026-10-10 18:02:54 + [2026-10-05 18:41:19] iteration 2027/ 6855 | consumed samples: 259456 | elapsed time per iteration (ms): 90655.6 | throughput per GPU (TFLOP/s/GPU): 41.6 | MFU 13.34% | FLOPs estimator: workload-aware | logical tokens/s: 185065.2 | valid tokens/s: 185065.2 | physical tokens/s: 185401.0 | loss tokens/s: 136803.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.589718E-01 | lm-global-supervised-tokens: 1.240199E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.208767E-02 | lm-token-buffer-mib: 5.809174E+01 | global_load_balancing_loss: 9.997388E-01 | loss scale: 1.0 | grad norm: 0.064 | params norm: 11512.192 | params rms: 0.062335 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 1:34:45.447791 | finish at 2026-10-10 20:16:05 + [2026-10-05 18:42:46] iteration 2028/ 6855 | consumed samples: 259584 | elapsed time per iteration (ms): 86326.8 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.62% | FLOPs estimator: workload-aware | logical tokens/s: 194345.2 | valid tokens/s: 194345.1 | physical tokens/s: 194828.1 | loss tokens/s: 145610.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.810913E-01 | lm-global-supervised-tokens: 1.257012E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.234317E-02 | lm-token-buffer-mib: 5.809460E+01 | global_load_balancing_loss: 9.990775E-01 | loss scale: 1.0 | grad norm: 0.036 | params norm: 11512.121 | params rms: 0.062334 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 19:44:59.702346 | finish at 2026-10-10 14:27:45 + [2026-10-05 18:44:11] iteration 2029/ 6855 | consumed samples: 259712 | elapsed time per iteration (ms): 85299.3 | throughput per GPU (TFLOP/s/GPU): 43.3 | MFU 13.88% | FLOPs estimator: workload-aware | logical tokens/s: 196686.0 | valid tokens/s: 196686.0 | physical tokens/s: 197021.7 | loss tokens/s: 141241.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.586682E-01 | lm-global-supervised-tokens: 1.204780E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.506916E-02 | lm-token-buffer-mib: 5.804881E+01 | global_load_balancing_loss: 9.995325E-01 | loss scale: 1.0 | grad norm: 0.059 | params norm: 11512.049 | params rms: 0.062334 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 18:20:54.219078 | finish at 2026-10-10 13:05:05 + [2026-10-05 18:45:57] iteration 2030/ 6855 | consumed samples: 259840 | elapsed time per iteration (ms): 105557.3 | throughput per GPU (TFLOP/s/GPU): 34.5 | MFU 11.07% | FLOPs estimator: workload-aware | logical tokens/s: 158938.8 | valid tokens/s: 158938.8 | physical tokens/s: 159337.8 | loss tokens/s: 117062.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.692136E-01 | lm-global-supervised-tokens: 1.235679E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.106995E-02 | lm-token-buffer-mib: 5.809135E+01 | global_load_balancing_loss: 9.999593E-01 | loss scale: 1.0 | grad norm: 0.057 | params norm: 11511.978 | params rms: 0.062334 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 21:28:33.785428 | finish at 2026-10-11 16:14:30 + [2026-10-05 18:47:26] iteration 2031/ 6855 | consumed samples: 259968 | elapsed time per iteration (ms): 89850.8 | throughput per GPU (TFLOP/s/GPU): 41.5 | MFU 13.30% | FLOPs estimator: workload-aware | logical tokens/s: 186722.9 | valid tokens/s: 186722.9 | physical tokens/s: 187204.2 | loss tokens/s: 134865.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.803068E-01 | lm-global-supervised-tokens: 1.211776E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.467929E-02 | lm-token-buffer-mib: 5.819564E+01 | global_load_balancing_loss: 1.001172E+00 | loss scale: 1.0 | grad norm: 0.108 | params norm: 11511.908 | params rms: 0.062333 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 0:24:00.220276 | finish at 2026-10-10 19:11:27 + [2026-10-05 18:48:55] iteration 2032/ 6855 | consumed samples: 260096 | elapsed time per iteration (ms): 89093.9 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.49% | FLOPs estimator: workload-aware | logical tokens/s: 188309.0 | valid tokens/s: 188309.0 | physical tokens/s: 188833.1 | loss tokens/s: 141715.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.405694E-01 | lm-global-supervised-tokens: 1.262600E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.625509E-02 | lm-token-buffer-mib: 5.829977E+01 | global_load_balancing_loss: 1.000458E+00 | loss scale: 1.0 | grad norm: 0.052 | params norm: 11511.838 | params rms: 0.062333 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 23:21:39.746145 | finish at 2026-10-10 18:10:35 + [2026-10-05 18:50:20] iteration 2033/ 6855 | consumed samples: 260224 | elapsed time per iteration (ms): 84508.6 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.61% | FLOPs estimator: workload-aware | logical tokens/s: 198526.6 | valid tokens/s: 198526.6 | physical tokens/s: 199102.8 | loss tokens/s: 149621.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.668372E-01 | lm-global-supervised-tokens: 1.264426E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.410230E-02 | lm-token-buffer-mib: 5.818707E+01 | global_load_balancing_loss: 1.001205E+00 | loss scale: 1.0 | grad norm: 0.124 | params norm: 11511.767 | params rms: 0.062332 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 17:11:40.512870 | finish at 2026-10-10 12:02:00 + [2026-10-05 18:51:47] iteration 2034/ 6855 | consumed samples: 260352 | elapsed time per iteration (ms): 87052.4 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.44% | FLOPs estimator: workload-aware | logical tokens/s: 192725.5 | valid tokens/s: 192725.5 | physical tokens/s: 193176.3 | loss tokens/s: 140661.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.319860E-01 | lm-global-supervised-tokens: 1.224490E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.182817E-02 | lm-token-buffer-mib: 5.814153E+01 | global_load_balancing_loss: 1.002172E+00 | loss scale: 1.0 | grad norm: 0.149 | params norm: 11511.696 | params rms: 0.062332 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 20:34:39.388759 | finish at 2026-10-10 15:26:26 + [2026-10-05 18:53:13] iteration 2035/ 6855 | consumed samples: 260480 | elapsed time per iteration (ms): 86422.8 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.67% | FLOPs estimator: workload-aware | logical tokens/s: 194129.5 | valid tokens/s: 194129.5 | physical tokens/s: 194634.2 | loss tokens/s: 139112.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.276229E-01 | lm-global-supervised-tokens: 1.202245E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.217676E-02 | lm-token-buffer-mib: 5.823018E+01 | global_load_balancing_loss: 1.000144E+00 | loss scale: 1.0 | grad norm: 0.072 | params norm: 11511.625 | params rms: 0.062332 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 19:42:37.903204 | finish at 2026-10-10 14:35:51 + [2026-10-05 18:54:43] iteration 2036/ 6855 | consumed samples: 260608 | elapsed time per iteration (ms): 89603.8 | throughput per GPU (TFLOP/s/GPU): 41.6 | MFU 13.33% | FLOPs estimator: workload-aware | logical tokens/s: 187236.8 | valid tokens/s: 187236.8 | physical tokens/s: 187716.8 | loss tokens/s: 140351.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.227864E-01 | lm-global-supervised-tokens: 1.257604E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.571632E-02 | lm-token-buffer-mib: 5.813628E+01 | global_load_balancing_loss: 9.993108E-01 | loss scale: 1.0 | grad norm: 0.093 | params norm: 11511.554 | params rms: 0.062331 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 23:56:40.860694 | finish at 2026-10-10 18:51:24 + [2026-10-05 18:56:07] iteration 2037/ 6855 | consumed samples: 260736 | elapsed time per iteration (ms): 84111.5 | throughput per GPU (TFLOP/s/GPU): 44.8 | MFU 14.35% | FLOPs estimator: workload-aware | logical tokens/s: 199463.0 | valid tokens/s: 199463.0 | physical tokens/s: 199821.2 | loss tokens/s: 149681.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.628255E-01 | lm-global-supervised-tokens: 1.258997E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.515726E-02 | lm-token-buffer-mib: 5.805064E+01 | global_load_balancing_loss: 1.000129E+00 | loss scale: 1.0 | grad norm: 0.057 | params norm: 11511.483 | params rms: 0.062331 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 16:34:09.321990 | finish at 2026-10-10 11:30:16 + [2026-10-05 18:57:34] iteration 2038/ 6855 | consumed samples: 260864 | elapsed time per iteration (ms): 86984.5 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.49% | FLOPs estimator: workload-aware | logical tokens/s: 192875.5 | valid tokens/s: 192875.5 | physical tokens/s: 193277.9 | loss tokens/s: 142565.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.513816E-01 | lm-global-supervised-tokens: 1.240096E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.489371E-02 | lm-token-buffer-mib: 5.805025E+01 | global_load_balancing_loss: 9.983439E-01 | loss scale: 1.0 | grad norm: 0.060 | params norm: 11511.414 | params rms: 0.062331 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 20:23:24.166197 | finish at 2026-10-10 15:20:58 + [2026-10-05 18:59:01] iteration 2039/ 6855 | consumed samples: 260992 | elapsed time per iteration (ms): 86901.9 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.44% | FLOPs estimator: workload-aware | logical tokens/s: 193059.2 | valid tokens/s: 193059.2 | physical tokens/s: 193512.5 | loss tokens/s: 149768.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.929017E-01 | lm-global-supervised-tokens: 1.301514E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.311929E-02 | lm-token-buffer-mib: 5.820570E+01 | global_load_balancing_loss: 9.982470E-01 | loss scale: 1.0 | grad norm: 0.073 | params norm: 11511.342 | params rms: 0.062330 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 20:15:19.591991 | finish at 2026-10-10 15:14:21 + [2026-10-05 19:00:50] iteration 2040/ 6855 | consumed samples: 261120 | elapsed time per iteration (ms): 109141.3 | throughput per GPU (TFLOP/s/GPU): 34.1 | MFU 10.91% | FLOPs estimator: workload-aware | logical tokens/s: 153720.1 | valid tokens/s: 153720.1 | physical tokens/s: 154019.5 | loss tokens/s: 115271.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.628429E-01 | lm-global-supervised-tokens: 1.258085E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.265824E-02 | lm-token-buffer-mib: 5.815955E+01 | global_load_balancing_loss: 9.998206E-01 | loss scale: 1.0 | grad norm: 0.052 | params norm: 11511.270 | params rms: 0.062330 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 1:58:35.337510 | finish at 2026-10-11 20:59:26 +times across ranks (ms): + save-checkpoint: + rank 0: 23695.00 + rank 1: 23694.99 + rank 2: 23695.02 + rank 3: 23695.03 + rank 4: 23695.03 + rank 5: 23695.01 + rank 6: 23695.00 + rank 7: 23694.99 + rank 8: 23695.45 + rank 9: 23695.43 + rank 10: 23695.44 + rank 11: 23695.42 + rank 12: 23695.43 + rank 13: 23695.42 + rank 14: 23695.42 + rank 15: 23695.46 + rank 16: 23694.85 + rank 17: 23694.83 + rank 18: 23694.83 + rank 19: 23694.84 + rank 20: 23694.82 + rank 21: 23694.85 + rank 22: 23694.83 + rank 23: 23694.81 + rank 24: 23695.51 + rank 25: 23695.48 + rank 26: 23695.50 + rank 27: 23695.51 + rank 28: 23695.48 + rank 29: 23695.48 + rank 30: 23695.47 + rank 31: 23695.62 + rank 32: 23695.79 + rank 33: 23695.81 + rank 34: 23695.79 + rank 35: 23695.81 + rank 36: 23695.76 + rank 37: 23695.74 + rank 38: 23695.80 + rank 39: 23695.76 + rank 40: 23696.61 + rank 41: 23696.58 + rank 42: 23696.65 + rank 43: 23696.56 + rank 44: 23696.58 + rank 45: 23696.63 + rank 46: 23696.57 + rank 47: 23696.54 + rank 48: 23695.65 + rank 49: 23695.66 + rank 50: 23695.59 + rank 51: 23695.63 + rank 52: 23695.62 + rank 53: 23695.63 + rank 54: 23695.63 + rank 55: 23695.61 + rank 56: 23695.13 + rank 57: 23695.12 + rank 58: 23695.13 + rank 59: 23695.10 + rank 60: 23695.10 + rank 61: 23695.16 + rank 62: 23695.16 + rank 63: 23695.13 + [2026-10-05 19:02:45] iteration 2041/ 6855 | consumed samples: 261248 | elapsed time per iteration (ms): 90823.1 | throughput per GPU (TFLOP/s/GPU): 41.4 | MFU 13.27% | FLOPs estimator: workload-aware | logical tokens/s: 184723.7 | valid tokens/s: 184723.7 | physical tokens/s: 185128.8 | loss tokens/s: 133136.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.676905E-01 | lm-global-supervised-tokens: 1.209190E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 4.853847E-02 | lm-token-buffer-mib: 5.803226E+01 | global_load_balancing_loss: 1.001232E+00 | loss scale: 1.0 | grad norm: 0.086 | params norm: 11511.199 | params rms: 0.062329 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 1:27:02.365958 | finish at 2026-10-10 20:29:47 + [2026-10-05 19:04:19] iteration 2042/ 6855 | consumed samples: 261376 | elapsed time per iteration (ms): 93679.3 | throughput per GPU (TFLOP/s/GPU): 38.8 | MFU 12.44% | FLOPs estimator: workload-aware | logical tokens/s: 179091.3 | valid tokens/s: 179091.3 | physical tokens/s: 179499.6 | loss tokens/s: 128157.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.633493E-01 | lm-global-supervised-tokens: 1.200566E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.940586E-02 | lm-token-buffer-mib: 5.818629E+01 | global_load_balancing_loss: 1.002108E+00 | loss scale: 1.0 | grad norm: 0.064 | params norm: 11511.127 | params rms: 0.062329 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 5:14:38.433368 | finish at 2026-10-11 00:18:57 + [2026-10-05 19:05:46] iteration 2043/ 6855 | consumed samples: 261504 | elapsed time per iteration (ms): 87188.8 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.61% | FLOPs estimator: workload-aware | logical tokens/s: 192423.9 | valid tokens/s: 192423.9 | physical tokens/s: 192848.2 | loss tokens/s: 140861.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.822352E-01 | lm-global-supervised-tokens: 1.228158E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.170197E-02 | lm-token-buffer-mib: 5.804309E+01 | global_load_balancing_loss: 1.002147E+00 | loss scale: 1.0 | grad norm: 0.059 | params norm: 11511.058 | params rms: 0.062329 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 20:32:32.282347 | finish at 2026-10-10 15:38:18 + [2026-10-05 19:07:11] iteration 2044/ 6855 | consumed samples: 261632 | elapsed time per iteration (ms): 85312.1 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.68% | FLOPs estimator: workload-aware | logical tokens/s: 196656.2 | valid tokens/s: 196656.2 | physical tokens/s: 197029.5 | loss tokens/s: 144832.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.672211E-01 | lm-global-supervised-tokens: 1.235594E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.399397E-02 | lm-token-buffer-mib: 5.828484E+01 | global_load_balancing_loss: 1.000262E+00 | loss scale: 1.0 | grad norm: 0.038 | params norm: 11510.988 | params rms: 0.062328 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 18:00:36.687129 | finish at 2026-10-10 13:07:48 + [2026-10-05 19:08:39] iteration 2045/ 6855 | consumed samples: 261760 | elapsed time per iteration (ms): 87743.7 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.45% | FLOPs estimator: workload-aware | logical tokens/s: 191205.8 | valid tokens/s: 191205.8 | physical tokens/s: 191802.4 | loss tokens/s: 143453.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.834061E-01 | lm-global-supervised-tokens: 1.258715E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.930621E-02 | lm-token-buffer-mib: 5.821869E+01 | global_load_balancing_loss: 9.990829E-01 | loss scale: 1.0 | grad norm: 0.080 | params norm: 11510.919 | params rms: 0.062328 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 21:14:07.331307 | finish at 2026-10-10 16:22:46 + [2026-10-05 19:10:05] iteration 2046/ 6855 | consumed samples: 261888 | elapsed time per iteration (ms): 85758.7 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.67% | FLOPs estimator: workload-aware | logical tokens/s: 195631.4 | valid tokens/s: 195631.4 | physical tokens/s: 196198.3 | loss tokens/s: 147520.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.727355E-01 | lm-global-supervised-tokens: 1.265113E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.417480E-02 | lm-token-buffer-mib: 5.812435E+01 | global_load_balancing_loss: 9.979615E-01 | loss scale: 1.0 | grad norm: 0.075 | params norm: 11510.852 | params rms: 0.062328 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 18:33:33.623333 | finish at 2026-10-10 13:43:38 + [2026-10-05 19:11:34] iteration 2047/ 6855 | consumed samples: 262016 | elapsed time per iteration (ms): 89216.5 | throughput per GPU (TFLOP/s/GPU): 41.6 | MFU 13.34% | FLOPs estimator: workload-aware | logical tokens/s: 188049.9 | valid tokens/s: 188049.8 | physical tokens/s: 188594.5 | loss tokens/s: 148812.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.274378E-01 | lm-global-supervised-tokens: 1.327650E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.152674E-02 | lm-token-buffer-mib: 5.802803E+01 | global_load_balancing_loss: 9.990290E-01 | loss scale: 1.0 | grad norm: 0.081 | params norm: 11510.784 | params rms: 0.062327 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 23:09:12.905846 | finish at 2026-10-10 18:20:47 + [2026-10-05 19:13:03] iteration 2048/ 6855 | consumed samples: 262144 | elapsed time per iteration (ms): 88846.6 | throughput per GPU (TFLOP/s/GPU): 41.3 | MFU 13.24% | FLOPs estimator: workload-aware | logical tokens/s: 188833.2 | valid tokens/s: 188833.2 | physical tokens/s: 189292.7 | loss tokens/s: 137777.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.711822E-01 | lm-global-supervised-tokens: 1.224108E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.635901E-02 | lm-token-buffer-mib: 5.812485E+01 | global_load_balancing_loss: 9.997559E-01 | loss scale: 1.0 | grad norm: 0.093 | params norm: 11510.715 | params rms: 0.062327 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 22:38:05.601883 | finish at 2026-10-10 17:51:08 + [2026-10-05 19:14:32] iteration 2049/ 6855 | consumed samples: 262272 | elapsed time per iteration (ms): 89361.5 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.53% | FLOPs estimator: workload-aware | logical tokens/s: 187744.9 | valid tokens/s: 187744.9 | physical tokens/s: 188101.0 | loss tokens/s: 134590.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.128490E-01 | lm-global-supervised-tokens: 1.202719E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.283742E-02 | lm-token-buffer-mib: 5.815770E+01 | global_load_balancing_loss: 1.000985E+00 | loss scale: 1.0 | grad norm: 0.051 | params norm: 11510.645 | params rms: 0.062326 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 23:17:51.444745 | finish at 2026-10-10 18:32:24 + [2026-10-05 19:16:19] iteration 2050/ 6855 | consumed samples: 262400 | elapsed time per iteration (ms): 107142.2 | throughput per GPU (TFLOP/s/GPU): 34.0 | MFU 10.88% | FLOPs estimator: workload-aware | logical tokens/s: 156588.3 | valid tokens/s: 156588.3 | physical tokens/s: 157019.1 | loss tokens/s: 114170.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.628068E-01 | lm-global-supervised-tokens: 1.223249E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.138877E-02 | lm-token-buffer-mib: 5.823906E+01 | global_load_balancing_loss: 1.001148E+00 | loss scale: 1.0 | grad norm: 0.054 | params norm: 11510.575 | params rms: 0.062326 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 23:00:18.345431 | finish at 2026-10-11 18:16:38 + [2026-10-05 19:17:47] iteration 2051/ 6855 | consumed samples: 262528 | elapsed time per iteration (ms): 87878.7 | throughput per GPU (TFLOP/s/GPU): 41.7 | MFU 13.37% | FLOPs estimator: workload-aware | logical tokens/s: 190912.7 | valid tokens/s: 190912.7 | physical tokens/s: 191420.7 | loss tokens/s: 143353.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.482730E-01 | lm-global-supervised-tokens: 1.259769E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.381898E-02 | lm-token-buffer-mib: 5.826472E+01 | global_load_balancing_loss: 1.002394E+00 | loss scale: 1.0 | grad norm: 0.102 | params norm: 11510.506 | params rms: 0.062326 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 21:16:09.071012 | finish at 2026-10-10 16:33:56 + [2026-10-05 19:19:17] iteration 2052/ 6855 | consumed samples: 262656 | elapsed time per iteration (ms): 89877.9 | throughput per GPU (TFLOP/s/GPU): 41.6 | MFU 13.32% | FLOPs estimator: workload-aware | logical tokens/s: 186665.8 | valid tokens/s: 186665.8 | physical tokens/s: 187013.3 | loss tokens/s: 135424.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.515515E-01 | lm-global-supervised-tokens: 1.217169E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.540484E-02 | lm-token-buffer-mib: 5.809344E+01 | global_load_balancing_loss: 1.001199E+00 | loss scale: 1.0 | grad norm: 0.169 | params norm: 11510.437 | params rms: 0.062325 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 23:54:43.324124 | finish at 2026-10-10 19:14:00 + [2026-10-05 19:20:44] iteration 2053/ 6855 | consumed samples: 262784 | elapsed time per iteration (ms): 86989.6 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.78% | FLOPs estimator: workload-aware | logical tokens/s: 192863.8 | valid tokens/s: 192863.8 | physical tokens/s: 193203.5 | loss tokens/s: 145873.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.845492E-01 | lm-global-supervised-tokens: 1.268951E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.285390E-02 | lm-token-buffer-mib: 5.809504E+01 | global_load_balancing_loss: 9.996938E-01 | loss scale: 1.0 | grad norm: 0.098 | params norm: 11510.368 | params rms: 0.062325 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 20:02:03.932990 | finish at 2026-10-10 15:22:48 + [2026-10-05 19:22:11] iteration 2054/ 6855 | consumed samples: 262912 | elapsed time per iteration (ms): 87355.3 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.39% | FLOPs estimator: workload-aware | logical tokens/s: 192057.1 | valid tokens/s: 192057.1 | physical tokens/s: 192473.7 | loss tokens/s: 142908.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.894464E-01 | lm-global-supervised-tokens: 1.248382E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.354785E-02 | lm-token-buffer-mib: 5.809592E+01 | global_load_balancing_loss: 9.989261E-01 | loss scale: 1.0 | grad norm: 0.090 | params norm: 11510.298 | params rms: 0.062325 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 20:29:52.868316 | finish at 2026-10-10 15:52:04 + [2026-10-05 19:23:38] iteration 2055/ 6855 | consumed samples: 263040 | elapsed time per iteration (ms): 86321.2 | throughput per GPU (TFLOP/s/GPU): 43.3 | MFU 13.89% | FLOPs estimator: workload-aware | logical tokens/s: 194357.9 | valid tokens/s: 194357.9 | physical tokens/s: 194844.3 | loss tokens/s: 134116.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.383780E-01 | lm-global-supervised-tokens: 1.157710E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.514702E-02 | lm-token-buffer-mib: 5.809356E+01 | global_load_balancing_loss: 1.000047E+00 | loss scale: 1.0 | grad norm: 0.066 | params norm: 11510.228 | params rms: 0.062324 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 19:05:41.848755 | finish at 2026-10-10 14:29:20 + [2026-10-05 19:25:03] iteration 2056/ 6855 | consumed samples: 263168 | elapsed time per iteration (ms): 85400.6 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.76% | FLOPs estimator: workload-aware | logical tokens/s: 196452.5 | valid tokens/s: 196452.5 | physical tokens/s: 196872.5 | loss tokens/s: 144705.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.650174E-01 | lm-global-supervised-tokens: 1.235793E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.356063E-02 | lm-token-buffer-mib: 5.813173E+01 | global_load_balancing_loss: 9.997979E-01 | loss scale: 1.0 | grad norm: 0.078 | params norm: 11510.158 | params rms: 0.062324 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 17:50:37.663403 | finish at 2026-10-10 13:15:41 + [2026-10-05 19:26:33] iteration 2057/ 6855 | consumed samples: 263296 | elapsed time per iteration (ms): 89561.1 | throughput per GPU (TFLOP/s/GPU): 41.6 | MFU 13.34% | FLOPs estimator: workload-aware | logical tokens/s: 187326.9 | valid tokens/s: 187326.9 | physical tokens/s: 187721.9 | loss tokens/s: 135997.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.081809E-01 | lm-global-supervised-tokens: 1.218006E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.304323E-02 | lm-token-buffer-mib: 5.823742E+01 | global_load_balancing_loss: 9.992053E-01 | loss scale: 1.0 | grad norm: 0.051 | params norm: 11510.088 | params rms: 0.062323 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 23:21:53.986239 | finish at 2026-10-10 18:48:27 + [2026-10-05 19:28:01] iteration 2058/ 6855 | consumed samples: 263424 | elapsed time per iteration (ms): 88565.4 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.44% | FLOPs estimator: workload-aware | logical tokens/s: 189433.0 | valid tokens/s: 189433.0 | physical tokens/s: 189818.3 | loss tokens/s: 147845.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.948420E-01 | lm-global-supervised-tokens: 1.309398E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.574705E-02 | lm-token-buffer-mib: 5.812639E+01 | global_load_balancing_loss: 9.995127E-01 | loss scale: 1.0 | grad norm: 0.043 | params norm: 11510.017 | params rms: 0.062323 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 22:00:48.190082 | finish at 2026-10-10 17:28:49 + [2026-10-05 19:29:27] iteration 2059/ 6855 | consumed samples: 263552 | elapsed time per iteration (ms): 86157.5 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.60% | FLOPs estimator: workload-aware | logical tokens/s: 194726.2 | valid tokens/s: 194726.2 | physical tokens/s: 195190.0 | loss tokens/s: 143246.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.619071E-01 | lm-global-supervised-tokens: 1.234175E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.434677E-02 | lm-token-buffer-mib: 5.811295E+01 | global_load_balancing_loss: 1.000583E+00 | loss scale: 1.0 | grad norm: 0.098 | params norm: 11509.948 | params rms: 0.062323 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 18:46:51.138016 | finish at 2026-10-10 14:16:19 + [2026-10-05 19:31:17] iteration 2060/ 6855 | consumed samples: 263680 | elapsed time per iteration (ms): 109759.3 | throughput per GPU (TFLOP/s/GPU): 33.4 | MFU 10.70% | FLOPs estimator: workload-aware | logical tokens/s: 152854.5 | valid tokens/s: 152854.5 | physical tokens/s: 153150.9 | loss tokens/s: 114991.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.505337E-01 | lm-global-supervised-tokens: 1.262139E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.241418E-02 | lm-token-buffer-mib: 5.819485E+01 | global_load_balancing_loss: 1.000390E+00 | loss scale: 1.0 | grad norm: 0.101 | params norm: 11509.883 | params rms: 0.062322 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 2:11:36.030957 | finish at 2026-10-11 21:42:53 + [2026-10-05 19:32:43] iteration 2061/ 6855 | consumed samples: 263808 | elapsed time per iteration (ms): 85433.2 | throughput per GPU (TFLOP/s/GPU): 43.2 | MFU 13.85% | FLOPs estimator: workload-aware | logical tokens/s: 196377.6 | valid tokens/s: 196377.6 | physical tokens/s: 196836.5 | loss tokens/s: 142845.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.519756E-01 | lm-global-supervised-tokens: 1.220375E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.168972E-02 | lm-token-buffer-mib: 5.802448E+01 | global_load_balancing_loss: 9.996358E-01 | loss scale: 1.0 | grad norm: 0.086 | params norm: 11509.818 | params rms: 0.062322 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 17:46:06.980832 | finish at 2026-10-10 13:18:50 + [2026-10-05 19:34:13] iteration 2062/ 6855 | consumed samples: 263936 | elapsed time per iteration (ms): 90076.6 | throughput per GPU (TFLOP/s/GPU): 41.5 | MFU 13.30% | FLOPs estimator: workload-aware | logical tokens/s: 186254.5 | valid tokens/s: 186254.5 | physical tokens/s: 186716.7 | loss tokens/s: 133317.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.854334E-01 | lm-global-supervised-tokens: 1.200879E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.355631E-02 | lm-token-buffer-mib: 5.819911E+01 | global_load_balancing_loss: 1.000505E+00 | loss scale: 1.0 | grad norm: 0.093 | params norm: 11509.751 | params rms: 0.062322 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 23:55:37.358994 | finish at 2026-10-10 19:29:50 + [2026-10-05 19:35:39] iteration 2063/ 6855 | consumed samples: 264064 | elapsed time per iteration (ms): 86107.1 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.66% | FLOPs estimator: workload-aware | logical tokens/s: 194840.8 | valid tokens/s: 194840.8 | physical tokens/s: 195228.9 | loss tokens/s: 145074.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.600422E-01 | lm-global-supervised-tokens: 1.249195E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.157478E-02 | lm-token-buffer-mib: 5.808365E+01 | global_load_balancing_loss: 1.000569E+00 | loss scale: 1.0 | grad norm: 0.101 | params norm: 11509.682 | params rms: 0.062321 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 18:37:05.243813 | finish at 2026-10-10 14:12:44 + [2026-10-05 19:37:04] iteration 2064/ 6855 | consumed samples: 264192 | elapsed time per iteration (ms): 84774.0 | throughput per GPU (TFLOP/s/GPU): 43.1 | MFU 13.82% | FLOPs estimator: workload-aware | logical tokens/s: 197904.7 | valid tokens/s: 197904.7 | physical tokens/s: 198437.6 | loss tokens/s: 148482.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.362142E-01 | lm-global-supervised-tokens: 1.258744E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.284274E-02 | lm-token-buffer-mib: 5.803902E+01 | global_load_balancing_loss: 1.001917E+00 | loss scale: 1.0 | grad norm: 0.058 | params norm: 11509.614 | params rms: 0.062321 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 16:49:12.211959 | finish at 2026-10-10 12:26:16 + [2026-10-05 19:38:33] iteration 2065/ 6855 | consumed samples: 264320 | elapsed time per iteration (ms): 89194.9 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.58% | FLOPs estimator: workload-aware | logical tokens/s: 188096.2 | valid tokens/s: 188096.2 | physical tokens/s: 188408.3 | loss tokens/s: 138620.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.088600E-01 | lm-global-supervised-tokens: 1.236423E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.526829E-02 | lm-token-buffer-mib: 5.815661E+01 | global_load_balancing_loss: 1.000744E+00 | loss scale: 1.0 | grad norm: 0.093 | params norm: 11509.547 | params rms: 0.062320 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 22:40:43.464963 | finish at 2026-10-10 18:19:16 + [2026-10-05 19:39:59] iteration 2066/ 6855 | consumed samples: 264448 | elapsed time per iteration (ms): 86109.4 | throughput per GPU (TFLOP/s/GPU): 42.8 | MFU 13.71% | FLOPs estimator: workload-aware | logical tokens/s: 194836.0 | valid tokens/s: 194836.0 | physical tokens/s: 195279.1 | loss tokens/s: 141134.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 5.932589E-01 | lm-global-supervised-tokens: 1.215303E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.241523E-02 | lm-token-buffer-mib: 5.816776E+01 | global_load_balancing_loss: 1.000824E+00 | loss scale: 1.0 | grad norm: 0.155 | params norm: 11509.478 | params rms: 0.062320 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 18:32:58.101732 | finish at 2026-10-10 14:12:57 + [2026-10-05 19:41:27] iteration 2067/ 6855 | consumed samples: 264576 | elapsed time per iteration (ms): 87804.1 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.62% | FLOPs estimator: workload-aware | logical tokens/s: 191075.2 | valid tokens/s: 191075.2 | physical tokens/s: 191465.5 | loss tokens/s: 136471.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.942608E-01 | lm-global-supervised-tokens: 1.198274E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.269784E-02 | lm-token-buffer-mib: 5.812350E+01 | global_load_balancing_loss: 1.000638E+00 | loss scale: 1.0 | grad norm: 0.274 | params norm: 11509.408 | params rms: 0.062320 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 20:46:45.925953 | finish at 2026-10-10 16:28:13 + [2026-10-05 19:42:53] iteration 2068/ 6855 | consumed samples: 264704 | elapsed time per iteration (ms): 86785.7 | throughput per GPU (TFLOP/s/GPU): 41.6 | MFU 13.34% | FLOPs estimator: workload-aware | logical tokens/s: 193317.0 | valid tokens/s: 193317.0 | physical tokens/s: 193914.4 | loss tokens/s: 148177.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.365331E-01 | lm-global-supervised-tokens: 1.285968E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.185904E-02 | lm-token-buffer-mib: 5.824507E+01 | global_load_balancing_loss: 1.000263E+00 | loss scale: 1.0 | grad norm: 0.129 | params norm: 11509.337 | params rms: 0.062319 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 19:24:02.916893 | finish at 2026-10-10 15:06:56 + [2026-10-05 19:44:20] iteration 2069/ 6855 | consumed samples: 264832 | elapsed time per iteration (ms): 86314.4 | throughput per GPU (TFLOP/s/GPU): 42.8 | MFU 13.72% | FLOPs estimator: workload-aware | logical tokens/s: 194373.0 | valid tokens/s: 194373.0 | physical tokens/s: 194853.7 | loss tokens/s: 142113.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.430694E-01 | lm-global-supervised-tokens: 1.226648E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.725932E-02 | lm-token-buffer-mib: 5.814957E+01 | global_load_balancing_loss: 9.989892E-01 | loss scale: 1.0 | grad norm: 0.045 | params norm: 11509.267 | params rms: 0.062319 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 18:45:00.952117 | finish at 2026-10-10 14:29:21 + [2026-10-05 19:46:07] iteration 2070/ 6855 | consumed samples: 264960 | elapsed time per iteration (ms): 106941.8 | throughput per GPU (TFLOP/s/GPU): 34.4 | MFU 11.04% | FLOPs estimator: workload-aware | logical tokens/s: 156881.6 | valid tokens/s: 156881.6 | physical tokens/s: 157236.9 | loss tokens/s: 119134.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.771679E-01 | lm-global-supervised-tokens: 1.274042E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.170238E-02 | lm-token-buffer-mib: 5.813976E+01 | global_load_balancing_loss: 9.986666E-01 | loss scale: 1.0 | grad norm: 0.045 | params norm: 11509.198 | params rms: 0.062319 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 22:08:36.708645 | finish at 2026-10-11 17:54:43 + [2026-10-05 19:47:32] iteration 2071/ 6855 | consumed samples: 265088 | elapsed time per iteration (ms): 85619.3 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.70% | FLOPs estimator: workload-aware | logical tokens/s: 195950.8 | valid tokens/s: 195950.8 | physical tokens/s: 196403.5 | loss tokens/s: 151142.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.850660E-01 | lm-global-supervised-tokens: 1.294070E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.291198E-02 | lm-token-buffer-mib: 5.808235E+01 | global_load_balancing_loss: 9.987823E-01 | loss scale: 1.0 | grad norm: 0.082 | params norm: 11509.129 | params rms: 0.062318 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 17:46:42.825336 | finish at 2026-10-10 13:34:15 + [2026-10-05 19:48:58] iteration 2072/ 6855 | consumed samples: 265216 | elapsed time per iteration (ms): 85292.5 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.61% | FLOPs estimator: workload-aware | logical tokens/s: 196701.3 | valid tokens/s: 196701.3 | physical tokens/s: 197015.1 | loss tokens/s: 150177.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.471024E-01 | lm-global-supervised-tokens: 1.280897E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.893773E-02 | lm-token-buffer-mib: 5.811929E+01 | global_load_balancing_loss: 9.981602E-01 | loss scale: 1.0 | grad norm: 0.037 | params norm: 11509.057 | params rms: 0.062318 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 17:19:14.183820 | finish at 2026-10-10 13:08:12 + [2026-10-05 19:50:25] iteration 2073/ 6855 | consumed samples: 265344 | elapsed time per iteration (ms): 87592.3 | throughput per GPU (TFLOP/s/GPU): 41.4 | MFU 13.28% | FLOPs estimator: workload-aware | logical tokens/s: 191537.6 | valid tokens/s: 191537.5 | physical tokens/s: 191974.4 | loss tokens/s: 139381.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.432925E-01 | lm-global-supervised-tokens: 1.220875E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.575067E-02 | lm-token-buffer-mib: 5.807421E+01 | global_load_balancing_loss: 1.001123E+00 | loss scale: 1.0 | grad norm: 0.229 | params norm: 11508.984 | params rms: 0.062317 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 20:21:06.140020 | finish at 2026-10-10 16:11:31 + [2026-10-05 19:51:53] iteration 2074/ 6855 | consumed samples: 265472 | elapsed time per iteration (ms): 87960.7 | throughput per GPU (TFLOP/s/GPU): 43.2 | MFU 13.84% | FLOPs estimator: workload-aware | logical tokens/s: 190735.4 | valid tokens/s: 190735.4 | physical tokens/s: 191054.1 | loss tokens/s: 141718.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.341621E-01 | lm-global-supervised-tokens: 1.246567E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.125727E-02 | lm-token-buffer-mib: 5.822324E+01 | global_load_balancing_loss: 1.002000E+00 | loss scale: 1.0 | grad norm: 0.144 | params norm: 11508.910 | params rms: 0.062317 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 20:48:59.983761 | finish at 2026-10-10 16:40:53 + [2026-10-05 19:53:23] iteration 2075/ 6855 | consumed samples: 265600 | elapsed time per iteration (ms): 89879.4 | throughput per GPU (TFLOP/s/GPU): 41.3 | MFU 13.25% | FLOPs estimator: workload-aware | logical tokens/s: 186662.8 | valid tokens/s: 186662.8 | physical tokens/s: 187087.2 | loss tokens/s: 141585.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.914772E-01 | lm-global-supervised-tokens: 1.272561E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.127738E-02 | lm-token-buffer-mib: 5.804350E+01 | global_load_balancing_loss: 1.001894E+00 | loss scale: 1.0 | grad norm: 0.236 | params norm: 11508.836 | params rms: 0.062317 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 23:20:23.741765 | finish at 2026-10-10 19:13:47 + [2026-10-05 19:54:48] iteration 2076/ 6855 | consumed samples: 265728 | elapsed time per iteration (ms): 84876.9 | throughput per GPU (TFLOP/s/GPU): 43.1 | MFU 13.81% | FLOPs estimator: workload-aware | logical tokens/s: 197664.9 | valid tokens/s: 197664.9 | physical tokens/s: 198274.5 | loss tokens/s: 151201.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.701556E-01 | lm-global-supervised-tokens: 1.283354E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.438761E-02 | lm-token-buffer-mib: 5.816054E+01 | global_load_balancing_loss: 1.001640E+00 | loss scale: 1.0 | grad norm: 0.229 | params norm: 11508.763 | params rms: 0.062316 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 16:40:26.780098 | finish at 2026-10-10 12:35:15 + [2026-10-05 19:56:18] iteration 2077/ 6855 | consumed samples: 265856 | elapsed time per iteration (ms): 89599.2 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.41% | FLOPs estimator: workload-aware | logical tokens/s: 187246.9 | valid tokens/s: 187246.9 | physical tokens/s: 187926.6 | loss tokens/s: 133352.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.777226E-01 | lm-global-supervised-tokens: 1.194823E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.163139E-02 | lm-token-buffer-mib: 5.820274E+01 | global_load_balancing_loss: 1.001264E+00 | loss scale: 1.0 | grad norm: 0.081 | params norm: 11508.689 | params rms: 0.062316 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 22:55:04.807914 | finish at 2026-10-10 18:51:22 + [2026-10-05 19:57:45] iteration 2078/ 6855 | consumed samples: 265984 | elapsed time per iteration (ms): 87285.4 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.58% | FLOPs estimator: workload-aware | logical tokens/s: 192210.1 | valid tokens/s: 192210.1 | physical tokens/s: 192556.3 | loss tokens/s: 142869.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.224098E-01 | lm-global-supervised-tokens: 1.247042E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.352246E-02 | lm-token-buffer-mib: 5.819063E+01 | global_load_balancing_loss: 9.990099E-01 | loss scale: 1.0 | grad norm: 0.061 | params norm: 11508.615 | params rms: 0.062315 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 19:49:22.444224 | finish at 2026-10-10 15:47:07 + [2026-10-05 19:59:13] iteration 2079/ 6855 | consumed samples: 266112 | elapsed time per iteration (ms): 87810.0 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.60% | FLOPs estimator: workload-aware | logical tokens/s: 191062.6 | valid tokens/s: 191062.6 | physical tokens/s: 191503.1 | loss tokens/s: 132574.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.881278E-01 | lm-global-supervised-tokens: 1.164133E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.323038E-02 | lm-token-buffer-mib: 5.817493E+01 | global_load_balancing_loss: 9.981792E-01 | loss scale: 1.0 | grad norm: 0.081 | params norm: 11508.542 | params rms: 0.062315 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 20:29:40.582500 | finish at 2026-10-10 16:28:53 + [2026-10-05 20:01:02] iteration 2080/ 6855 | consumed samples: 266240 | elapsed time per iteration (ms): 109075.2 | throughput per GPU (TFLOP/s/GPU): 34.2 | MFU 10.96% | FLOPs estimator: workload-aware | logical tokens/s: 153813.2 | valid tokens/s: 153813.2 | physical tokens/s: 154250.9 | loss tokens/s: 112714.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.247350E-01 | lm-global-supervised-tokens: 1.229435E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.235030E-02 | lm-token-buffer-mib: 5.822019E+01 | global_load_balancing_loss: 9.992263E-01 | loss scale: 1.0 | grad norm: 0.083 | params norm: 11508.469 | params rms: 0.062315 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 0:40:33.974510 | finish at 2026-10-11 20:41:36 +times across ranks (ms): + save-checkpoint: + rank 0: 24541.91 + rank 1: 24541.92 + rank 2: 24541.95 + rank 3: 24541.96 + rank 4: 24541.96 + rank 5: 24541.95 + rank 6: 24541.93 + rank 7: 24541.91 + rank 8: 24542.07 + rank 9: 24541.98 + rank 10: 24541.98 + rank 11: 24542.04 + rank 12: 24541.99 + rank 13: 24541.97 + rank 14: 24542.00 + rank 15: 24541.98 + rank 16: 24542.33 + rank 17: 24542.22 + rank 18: 24542.20 + rank 19: 24542.21 + rank 20: 24542.16 + rank 21: 24542.19 + rank 22: 24542.17 + rank 23: 24542.16 + rank 24: 24541.90 + rank 25: 24542.12 + rank 26: 24541.93 + rank 27: 24541.63 + rank 28: 24541.85 + rank 29: 24541.89 + rank 30: 24541.85 + rank 31: 24541.94 + rank 32: 24542.24 + rank 33: 24542.22 + rank 34: 24542.19 + rank 35: 24542.23 + rank 36: 24542.18 + rank 37: 24542.15 + rank 38: 24542.26 + rank 39: 24542.15 + rank 40: 24540.54 + rank 41: 24540.54 + rank 42: 24540.54 + rank 43: 24540.53 + rank 44: 24540.51 + rank 45: 24540.55 + rank 46: 24540.51 + rank 47: 24540.51 + rank 48: 24541.92 + rank 49: 24542.04 + rank 50: 24541.96 + rank 51: 24541.98 + rank 52: 24542.08 + rank 53: 24542.09 + rank 54: 24542.08 + rank 55: 24542.05 + rank 56: 24541.74 + rank 57: 24541.76 + rank 58: 24541.71 + rank 59: 24541.70 + rank 60: 24541.70 + rank 61: 24541.74 + rank 62: 24541.70 + rank 63: 24541.73 + [2026-10-05 20:02:55] iteration 2081/ 6855 | consumed samples: 266368 | elapsed time per iteration (ms): 87912.2 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.74% | FLOPs estimator: workload-aware | logical tokens/s: 190840.0 | valid tokens/s: 190840.0 | physical tokens/s: 191270.4 | loss tokens/s: 145429.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.762525E-01 | lm-global-supervised-tokens: 1.278507E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 4.266666E-01 | lm-token-buffer-mib: 5.818526E+01 | global_load_balancing_loss: 9.989312E-01 | loss scale: 1.0 | grad norm: 0.083 | params norm: 11508.396 | params rms: 0.062314 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 20:34:52.981538 | finish at 2026-10-10 16:37:48 + [2026-10-05 20:04:26] iteration 2082/ 6855 | consumed samples: 266496 | elapsed time per iteration (ms): 91601.0 | throughput per GPU (TFLOP/s/GPU): 39.5 | MFU 12.67% | FLOPs estimator: workload-aware | logical tokens/s: 183154.9 | valid tokens/s: 183154.9 | physical tokens/s: 183613.7 | loss tokens/s: 138317.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.456591E-01 | lm-global-supervised-tokens: 1.267003E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.708540E-02 | lm-token-buffer-mib: 5.811570E+01 | global_load_balancing_loss: 1.000115E+00 | loss scale: 1.0 | grad norm: 0.091 | params norm: 11508.321 | params rms: 0.062314 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 1:26:51.689410 | finish at 2026-10-10 21:31:18 + [2026-10-05 20:05:51] iteration 2083/ 6855 | consumed samples: 266624 | elapsed time per iteration (ms): 84587.9 | throughput per GPU (TFLOP/s/GPU): 43.5 | MFU 13.96% | FLOPs estimator: workload-aware | logical tokens/s: 198339.7 | valid tokens/s: 198339.7 | physical tokens/s: 198882.1 | loss tokens/s: 146252.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.177671E-01 | lm-global-supervised-tokens: 1.237121E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.412005E-02 | lm-token-buffer-mib: 5.809887E+01 | global_load_balancing_loss: 1.000476E+00 | loss scale: 1.0 | grad norm: 0.048 | params norm: 11508.247 | params rms: 0.062313 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 16:07:33.626819 | finish at 2026-10-10 12:13:24 + [2026-10-05 20:07:13] iteration 2084/ 6855 | consumed samples: 266752 | elapsed time per iteration (ms): 81759.9 | throughput per GPU (TFLOP/s/GPU): 44.1 | MFU 14.14% | FLOPs estimator: workload-aware | logical tokens/s: 205200.6 | valid tokens/s: 205200.6 | physical tokens/s: 205776.6 | loss tokens/s: 152196.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.419482E-01 | lm-global-supervised-tokens: 1.244355E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.914059E-02 | lm-token-buffer-mib: 5.826208E+01 | global_load_balancing_loss: 1.001668E+00 | loss scale: 1.0 | grad norm: 0.085 | params norm: 11508.171 | params rms: 0.062313 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 12:21:16.349120 | finish at 2026-10-10 08:28:29 + [2026-10-05 20:08:39] iteration 2085/ 6855 | consumed samples: 266880 | elapsed time per iteration (ms): 86489.5 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.65% | FLOPs estimator: workload-aware | logical tokens/s: 193979.3 | valid tokens/s: 193979.3 | physical tokens/s: 194356.4 | loss tokens/s: 137239.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.258735E-01 | lm-global-supervised-tokens: 1.186978E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.216452E-02 | lm-token-buffer-mib: 5.807423E+01 | global_load_balancing_loss: 1.001666E+00 | loss scale: 1.0 | grad norm: 0.066 | params norm: 11508.095 | params rms: 0.062313 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 18:35:55.042591 | finish at 2026-10-10 14:44:34 + [2026-10-05 20:10:06] iteration 2086/ 6855 | consumed samples: 267008 | elapsed time per iteration (ms): 86471.5 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.63% | FLOPs estimator: workload-aware | logical tokens/s: 194019.9 | valid tokens/s: 194019.9 | physical tokens/s: 194422.2 | loss tokens/s: 151041.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.906479E-01 | lm-global-supervised-tokens: 1.306076E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.186855E-02 | lm-token-buffer-mib: 5.815303E+01 | global_load_balancing_loss: 9.998422E-01 | loss scale: 1.0 | grad norm: 0.031 | params norm: 11508.020 | params rms: 0.062312 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 18:33:02.742301 | finish at 2026-10-10 14:43:08 + [2026-10-05 20:11:31] iteration 2087/ 6855 | consumed samples: 267136 | elapsed time per iteration (ms): 85367.4 | throughput per GPU (TFLOP/s/GPU): 42.8 | MFU 13.71% | FLOPs estimator: workload-aware | logical tokens/s: 196529.5 | valid tokens/s: 196529.5 | physical tokens/s: 197095.7 | loss tokens/s: 148937.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.200475E-01 | lm-global-supervised-tokens: 1.271442E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.612696E-02 | lm-token-buffer-mib: 5.820198E+01 | global_load_balancing_loss: 9.992335E-01 | loss scale: 1.0 | grad norm: 0.138 | params norm: 11507.946 | params rms: 0.062312 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 17:03:51.789017 | finish at 2026-10-10 13:15:23 + [2026-10-05 20:13:02] iteration 2088/ 6855 | consumed samples: 267264 | elapsed time per iteration (ms): 91005.8 | throughput per GPU (TFLOP/s/GPU): 40.8 | MFU 13.09% | FLOPs estimator: workload-aware | logical tokens/s: 184352.9 | valid tokens/s: 184352.9 | physical tokens/s: 184885.6 | loss tokens/s: 130094.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.776446E-01 | lm-global-supervised-tokens: 1.183934E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.171659E-02 | lm-token-buffer-mib: 5.825477E+01 | global_load_balancing_loss: 9.990220E-01 | loss scale: 1.0 | grad norm: 0.124 | params norm: 11507.874 | params rms: 0.062311 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 0:30:24.596361 | finish at 2026-10-10 20:43:26 + [2026-10-05 20:14:31] iteration 2089/ 6855 | consumed samples: 267392 | elapsed time per iteration (ms): 89155.7 | throughput per GPU (TFLOP/s/GPU): 41.4 | MFU 13.27% | FLOPs estimator: workload-aware | logical tokens/s: 188178.3 | valid tokens/s: 188178.3 | physical tokens/s: 188607.3 | loss tokens/s: 135897.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.184328E-01 | lm-global-supervised-tokens: 1.211600E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.206351E-02 | lm-token-buffer-mib: 5.808871E+01 | global_load_balancing_loss: 9.996305E-01 | loss scale: 1.0 | grad norm: 0.060 | params norm: 11507.801 | params rms: 0.062311 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 22:01:56.280810 | finish at 2026-10-10 18:16:27 + [2026-10-05 20:16:19] iteration 2090/ 6855 | consumed samples: 267520 | elapsed time per iteration (ms): 108051.1 | throughput per GPU (TFLOP/s/GPU): 34.7 | MFU 11.11% | FLOPs estimator: workload-aware | logical tokens/s: 155271.1 | valid tokens/s: 155271.0 | physical tokens/s: 155670.7 | loss tokens/s: 112044.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.564965E-01 | lm-global-supervised-tokens: 1.210647E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.381005E-02 | lm-token-buffer-mib: 5.841754E+01 | global_load_balancing_loss: 9.999542E-01 | loss scale: 1.0 | grad norm: 0.091 | params norm: 11507.727 | params rms: 0.062311 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 23:01:03.315485 | finish at 2026-10-11 19:17:22 + [2026-10-05 20:17:47] iteration 2091/ 6855 | consumed samples: 267648 | elapsed time per iteration (ms): 88328.4 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.50% | FLOPs estimator: workload-aware | logical tokens/s: 189941.3 | valid tokens/s: 189941.3 | physical tokens/s: 190494.5 | loss tokens/s: 137419.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.832569E-01 | lm-global-supervised-tokens: 1.213808E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.359905E-02 | lm-token-buffer-mib: 5.815186E+01 | global_load_balancing_loss: 1.002236E+00 | loss scale: 1.0 | grad norm: 0.128 | params norm: 11507.652 | params rms: 0.062310 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 20:53:16.670889 | finish at 2026-10-10 17:11:04 + [2026-10-05 20:19:16] iteration 2092/ 6855 | consumed samples: 267776 | elapsed time per iteration (ms): 88526.1 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.69% | FLOPs estimator: workload-aware | logical tokens/s: 189517.2 | valid tokens/s: 189517.2 | physical tokens/s: 189884.7 | loss tokens/s: 142129.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.609797E-01 | lm-global-supervised-tokens: 1.258215E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.357711E-02 | lm-token-buffer-mib: 5.807165E+01 | global_load_balancing_loss: 1.000187E+00 | loss scale: 1.0 | grad norm: 0.050 | params norm: 11507.577 | params rms: 0.062310 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 21:07:29.645853 | finish at 2026-10-10 17:26:46 + [2026-10-05 20:20:43] iteration 2093/ 6855 | consumed samples: 267904 | elapsed time per iteration (ms): 87402.0 | throughput per GPU (TFLOP/s/GPU): 41.7 | MFU 13.35% | FLOPs estimator: workload-aware | logical tokens/s: 191954.4 | valid tokens/s: 191954.4 | physical tokens/s: 192471.4 | loss tokens/s: 145909.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.379794E-01 | lm-global-supervised-tokens: 1.275275E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.288682E-02 | lm-token-buffer-mib: 5.803234E+01 | global_load_balancing_loss: 1.000975E+00 | loss scale: 1.0 | grad norm: 0.222 | params norm: 11507.503 | params rms: 0.062309 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 19:36:48.256778 | finish at 2026-10-10 15:57:32 + [2026-10-05 20:22:11] iteration 2094/ 6855 | consumed samples: 268032 | elapsed time per iteration (ms): 87557.5 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.46% | FLOPs estimator: workload-aware | logical tokens/s: 191613.6 | valid tokens/s: 191613.5 | physical tokens/s: 192246.7 | loss tokens/s: 141708.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.955191E-01 | lm-global-supervised-tokens: 1.240762E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.278590E-02 | lm-token-buffer-mib: 5.817413E+01 | global_load_balancing_loss: 1.000408E+00 | loss scale: 1.0 | grad norm: 0.055 | params norm: 11507.428 | params rms: 0.062309 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 19:47:41.354575 | finish at 2026-10-10 16:09:52 + [2026-10-05 20:23:44] iteration 2095/ 6855 | consumed samples: 268160 | elapsed time per iteration (ms): 92856.6 | throughput per GPU (TFLOP/s/GPU): 40.9 | MFU 13.12% | FLOPs estimator: workload-aware | logical tokens/s: 180678.8 | valid tokens/s: 180678.8 | physical tokens/s: 180999.7 | loss tokens/s: 132409.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.668456E-01 | lm-global-supervised-tokens: 1.229506E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.462990E-02 | lm-token-buffer-mib: 5.810228E+01 | global_load_balancing_loss: 1.001689E+00 | loss scale: 1.0 | grad norm: 0.120 | params norm: 11507.353 | params rms: 0.062309 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 2:46:37.214212 | finish at 2026-10-10 23:10:21 + [2026-10-05 20:25:11] iteration 2096/ 6855 | consumed samples: 268288 | elapsed time per iteration (ms): 87075.3 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.61% | FLOPs estimator: workload-aware | logical tokens/s: 192674.7 | valid tokens/s: 192674.7 | physical tokens/s: 192927.8 | loss tokens/s: 142407.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.497685E-01 | lm-global-supervised-tokens: 1.240019E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.379014E-02 | lm-token-buffer-mib: 5.817348E+01 | global_load_balancing_loss: 9.985932E-01 | loss scale: 1.0 | grad norm: 0.075 | params norm: 11507.279 | params rms: 0.062308 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 19:06:31.492157 | finish at 2026-10-10 15:31:42 + [2026-10-05 20:26:31] iteration 2097/ 6855 | consumed samples: 268416 | elapsed time per iteration (ms): 80555.2 | throughput per GPU (TFLOP/s/GPU): 44.4 | MFU 14.24% | FLOPs estimator: workload-aware | logical tokens/s: 208268.7 | valid tokens/s: 208268.7 | physical tokens/s: 208858.9 | loss tokens/s: 162703.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.703921E-01 | lm-global-supervised-tokens: 1.310660E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.223999E-02 | lm-token-buffer-mib: 5.810363E+01 | global_load_balancing_loss: 9.987491E-01 | loss scale: 1.0 | grad norm: 0.063 | params norm: 11507.206 | params rms: 0.062308 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 10:28:01.703333 | finish at 2026-10-10 06:54:33 + [2026-10-05 20:27:59] iteration 2098/ 6855 | consumed samples: 268544 | elapsed time per iteration (ms): 87458.2 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.58% | FLOPs estimator: workload-aware | logical tokens/s: 191830.3 | valid tokens/s: 191830.3 | physical tokens/s: 192265.6 | loss tokens/s: 140179.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.829544E-01 | lm-global-supervised-tokens: 1.225981E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.360254E-02 | lm-token-buffer-mib: 5.814494E+01 | global_load_balancing_loss: 9.992279E-01 | loss scale: 1.0 | grad norm: 0.108 | params norm: 11507.134 | params rms: 0.062307 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 19:33:58.586977 | finish at 2026-10-10 16:01:57 + [2026-10-05 20:29:27] iteration 2099/ 6855 | consumed samples: 268672 | elapsed time per iteration (ms): 87919.0 | throughput per GPU (TFLOP/s/GPU): 43.5 | MFU 13.93% | FLOPs estimator: workload-aware | logical tokens/s: 190825.6 | valid tokens/s: 190825.6 | physical tokens/s: 191315.9 | loss tokens/s: 134053.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.833270E-01 | lm-global-supervised-tokens: 1.178582E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.300134E-02 | lm-token-buffer-mib: 5.830434E+01 | global_load_balancing_loss: 1.000701E+00 | loss scale: 1.0 | grad norm: 0.051 | params norm: 11507.062 | params rms: 0.062307 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 20:09:02.783984 | finish at 2026-10-10 16:38:30 + [2026-10-05 20:31:16] iteration 2100/ 6855 | consumed samples: 268800 | elapsed time per iteration (ms): 109083.8 | throughput per GPU (TFLOP/s/GPU): 34.5 | MFU 11.05% | FLOPs estimator: workload-aware | logical tokens/s: 153801.1 | valid tokens/s: 153801.1 | physical tokens/s: 154044.3 | loss tokens/s: 111332.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.518750E-01 | lm-global-supervised-tokens: 1.214459E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.307360E-02 | lm-token-buffer-mib: 5.807951E+01 | global_load_balancing_loss: 1.001909E+00 | loss scale: 1.0 | grad norm: 0.053 | params norm: 11506.990 | params rms: 0.062307 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 0:04:53.513582 | finish at 2026-10-11 20:36:09 + [2026-10-05 20:32:42] iteration 2101/ 6855 | consumed samples: 268928 | elapsed time per iteration (ms): 86561.1 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.70% | FLOPs estimator: workload-aware | logical tokens/s: 193818.4 | valid tokens/s: 193818.4 | physical tokens/s: 194248.5 | loss tokens/s: 142510.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.711496E-01 | lm-global-supervised-tokens: 1.233590E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.156662E-02 | lm-token-buffer-mib: 5.805988E+01 | global_load_balancing_loss: 9.992656E-01 | loss scale: 1.0 | grad norm: 0.086 | params norm: 11506.919 | params rms: 0.062306 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 18:18:31.546503 | finish at 2026-10-10 14:51:14 + [2026-10-05 20:34:07] iteration 2102/ 6855 | consumed samples: 269056 | elapsed time per iteration (ms): 84260.3 | throughput per GPU (TFLOP/s/GPU): 43.4 | MFU 13.91% | FLOPs estimator: workload-aware | logical tokens/s: 199111.4 | valid tokens/s: 199111.4 | physical tokens/s: 199605.2 | loss tokens/s: 146237.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.451421E-01 | lm-global-supervised-tokens: 1.232206E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.172441E-02 | lm-token-buffer-mib: 5.807151E+01 | global_load_balancing_loss: 1.000022E+00 | loss scale: 1.0 | grad norm: 0.054 | params norm: 11506.848 | params rms: 0.062306 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 15:14:49.423100 | finish at 2026-10-10 11:48:56 + [2026-10-05 20:35:33] iteration 2103/ 6855 | consumed samples: 269184 | elapsed time per iteration (ms): 86730.5 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.58% | FLOPs estimator: workload-aware | logical tokens/s: 193440.4 | valid tokens/s: 193440.4 | physical tokens/s: 193964.1 | loss tokens/s: 146330.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.587788E-01 | lm-global-supervised-tokens: 1.269130E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.291186E-02 | lm-token-buffer-mib: 5.820858E+01 | global_load_balancing_loss: 9.998720E-01 | loss scale: 1.0 | grad norm: 0.098 | params norm: 11506.779 | params rms: 0.062305 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 18:29:03.442417 | finish at 2026-10-10 15:04:37 + [2026-10-05 20:37:04] iteration 2104/ 6855 | consumed samples: 269312 | elapsed time per iteration (ms): 90629.1 | throughput per GPU (TFLOP/s/GPU): 41.0 | MFU 13.14% | FLOPs estimator: workload-aware | logical tokens/s: 185119.6 | valid tokens/s: 185119.5 | physical tokens/s: 185666.8 | loss tokens/s: 137821.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.860001E-01 | lm-global-supervised-tokens: 1.249067E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.114656E-02 | lm-token-buffer-mib: 5.819305E+01 | global_load_balancing_loss: 1.000610E+00 | loss scale: 1.0 | grad norm: 0.060 | params norm: 11506.710 | params rms: 0.062305 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 23:36:18.776342 | finish at 2026-10-10 20:13:23 + [2026-10-05 20:38:30] iteration 2105/ 6855 | consumed samples: 269440 | elapsed time per iteration (ms): 85488.4 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.47% | FLOPs estimator: workload-aware | logical tokens/s: 196250.6 | valid tokens/s: 196250.6 | physical tokens/s: 196814.8 | loss tokens/s: 149250.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.506439E-01 | lm-global-supervised-tokens: 1.275919E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.162218E-02 | lm-token-buffer-mib: 5.813819E+01 | global_load_balancing_loss: 1.000133E+00 | loss scale: 1.0 | grad norm: 0.062 | params norm: 11506.641 | params rms: 0.062305 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 16:47:49.839895 | finish at 2026-10-10 13:26:19 + [2026-10-05 20:39:52] iteration 2106/ 6855 | consumed samples: 269568 | elapsed time per iteration (ms): 82032.8 | throughput per GPU (TFLOP/s/GPU): 44.4 | MFU 14.23% | FLOPs estimator: workload-aware | logical tokens/s: 204518.4 | valid tokens/s: 204518.4 | physical tokens/s: 204921.8 | loss tokens/s: 157073.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.854995E-01 | lm-global-supervised-tokens: 1.288514E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.388494E-02 | lm-token-buffer-mib: 5.816100E+01 | global_load_balancing_loss: 1.001022E+00 | loss scale: 1.0 | grad norm: 0.041 | params norm: 11506.570 | params rms: 0.062304 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 12:12:53.572259 | finish at 2026-10-10 08:52:45 + [2026-10-05 20:41:18] iteration 2107/ 6855 | consumed samples: 269696 | elapsed time per iteration (ms): 86405.0 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.48% | FLOPs estimator: workload-aware | logical tokens/s: 194168.9 | valid tokens/s: 194168.9 | physical tokens/s: 194706.3 | loss tokens/s: 137138.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.187489E-01 | lm-global-supervised-tokens: 1.184948E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.488128E-02 | lm-token-buffer-mib: 5.812700E+01 | global_load_balancing_loss: 1.002030E+00 | loss scale: 1.0 | grad norm: 0.071 | params norm: 11506.501 | params rms: 0.062304 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 17:57:30.997597 | finish at 2026-10-10 14:38:49 + [2026-10-05 20:42:44] iteration 2108/ 6855 | consumed samples: 269824 | elapsed time per iteration (ms): 86487.0 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.63% | FLOPs estimator: workload-aware | logical tokens/s: 193984.5 | valid tokens/s: 193984.5 | physical tokens/s: 194335.3 | loss tokens/s: 145539.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.120373E-01 | lm-global-supervised-tokens: 1.258728E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.449499E-02 | lm-token-buffer-mib: 5.807252E+01 | global_load_balancing_loss: 1.000862E+00 | loss scale: 1.0 | grad norm: 0.061 | params norm: 11506.430 | params rms: 0.062304 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 18:02:33.589754 | finish at 2026-10-10 14:45:18 + [2026-10-05 20:44:10] iteration 2109/ 6855 | consumed samples: 269952 | elapsed time per iteration (ms): 85060.1 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.65% | FLOPs estimator: workload-aware | logical tokens/s: 197237.8 | valid tokens/s: 197237.8 | physical tokens/s: 197904.4 | loss tokens/s: 147187.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.247110E-01 | lm-global-supervised-tokens: 1.251974E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.556240E-02 | lm-token-buffer-mib: 5.802459E+01 | global_load_balancing_loss: 1.000039E+00 | loss scale: 1.0 | grad norm: 0.054 | params norm: 11506.360 | params rms: 0.062303 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 16:08:15.055059 | finish at 2026-10-10 12:52:25 + [2026-10-05 20:45:58] iteration 2110/ 6855 | consumed samples: 270080 | elapsed time per iteration (ms): 108089.7 | throughput per GPU (TFLOP/s/GPU): 33.9 | MFU 10.86% | FLOPs estimator: workload-aware | logical tokens/s: 155215.2 | valid tokens/s: 155215.2 | physical tokens/s: 155570.1 | loss tokens/s: 113993.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.374162E-01 | lm-global-supervised-tokens: 1.232148E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.175426E-02 | lm-token-buffer-mib: 5.812492E+01 | global_load_balancing_loss: 9.993403E-01 | loss scale: 1.0 | grad norm: 0.135 | params norm: 11506.290 | params rms: 0.062303 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 22:28:05.537050 | finish at 2026-10-11 19:14:03 + [2026-10-05 20:47:22] iteration 2111/ 6855 | consumed samples: 270208 | elapsed time per iteration (ms): 84603.9 | throughput per GPU (TFLOP/s/GPU): 43.1 | MFU 13.81% | FLOPs estimator: workload-aware | logical tokens/s: 198303.0 | valid tokens/s: 198303.0 | physical tokens/s: 198763.5 | loss tokens/s: 145631.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.142519E-01 | lm-global-supervised-tokens: 1.232101E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.393029E-02 | lm-token-buffer-mib: 5.808589E+01 | global_load_balancing_loss: 9.989199E-01 | loss scale: 1.0 | grad norm: 0.113 | params norm: 11506.220 | params rms: 0.062302 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 15:29:21.008841 | finish at 2026-10-10 12:16:43 + [2026-10-05 20:48:49] iteration 2112/ 6855 | consumed samples: 270336 | elapsed time per iteration (ms): 86413.3 | throughput per GPU (TFLOP/s/GPU): 42.8 | MFU 13.71% | FLOPs estimator: workload-aware | logical tokens/s: 194150.9 | valid tokens/s: 194150.9 | physical tokens/s: 194563.3 | loss tokens/s: 154895.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.085437E-01 | lm-global-supervised-tokens: 1.338504E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.184901E-02 | lm-token-buffer-mib: 5.814470E+01 | global_load_balancing_loss: 9.986221E-01 | loss scale: 1.0 | grad norm: 0.068 | params norm: 11506.147 | params rms: 0.062302 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 17:50:58.130548 | finish at 2026-10-10 14:39:47 + [2026-10-05 20:50:17] iteration 2113/ 6855 | consumed samples: 270464 | elapsed time per iteration (ms): 88439.2 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.50% | FLOPs estimator: workload-aware | logical tokens/s: 189702.2 | valid tokens/s: 189702.2 | physical tokens/s: 190076.1 | loss tokens/s: 134956.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.470806E-01 | lm-global-supervised-tokens: 1.193548E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.353779E-02 | lm-token-buffer-mib: 5.807486E+01 | global_load_balancing_loss: 1.000311E+00 | loss scale: 1.0 | grad norm: 0.148 | params norm: 11506.074 | params rms: 0.062302 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 20:29:38.731265 | finish at 2026-10-10 17:19:56 + [2026-10-05 20:51:45] iteration 2114/ 6855 | consumed samples: 270592 | elapsed time per iteration (ms): 87875.1 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.57% | FLOPs estimator: workload-aware | logical tokens/s: 190921.0 | valid tokens/s: 190921.0 | physical tokens/s: 191624.8 | loss tokens/s: 140081.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.437618E-01 | lm-global-supervised-tokens: 1.230964E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.536355E-02 | lm-token-buffer-mib: 5.813853E+01 | global_load_balancing_loss: 1.000461E+00 | loss scale: 1.0 | grad norm: 0.129 | params norm: 11506.000 | params rms: 0.062301 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 19:43:35.743492 | finish at 2026-10-10 16:35:21 + [2026-10-05 20:53:12] iteration 2115/ 6855 | consumed samples: 270720 | elapsed time per iteration (ms): 86878.8 | throughput per GPU (TFLOP/s/GPU): 43.1 | MFU 13.83% | FLOPs estimator: workload-aware | logical tokens/s: 193110.2 | valid tokens/s: 193110.2 | physical tokens/s: 193523.7 | loss tokens/s: 142916.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.877483E-01 | lm-global-supervised-tokens: 1.241639E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.320062E-02 | lm-token-buffer-mib: 5.816922E+01 | global_load_balancing_loss: 1.000231E+00 | loss scale: 1.0 | grad norm: 0.090 | params norm: 11505.928 | params rms: 0.062301 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 18:23:25.409889 | finish at 2026-10-10 15:16:37 + [2026-10-05 20:54:38] iteration 2116/ 6855 | consumed samples: 270848 | elapsed time per iteration (ms): 85694.1 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.55% | FLOPs estimator: workload-aware | logical tokens/s: 195780.0 | valid tokens/s: 195780.0 | physical tokens/s: 196261.9 | loss tokens/s: 150104.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.868365E-01 | lm-global-supervised-tokens: 1.286308E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.543667E-02 | lm-token-buffer-mib: 5.814857E+01 | global_load_balancing_loss: 1.001646E+00 | loss scale: 1.0 | grad norm: 0.205 | params norm: 11505.853 | params rms: 0.062300 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 16:48:24.524739 | finish at 2026-10-10 13:43:02 + [2026-10-05 20:56:02] iteration 2117/ 6855 | consumed samples: 270976 | elapsed time per iteration (ms): 84681.5 | throughput per GPU (TFLOP/s/GPU): 42.8 | MFU 13.70% | FLOPs estimator: workload-aware | logical tokens/s: 198121.0 | valid tokens/s: 198121.0 | physical tokens/s: 198742.4 | loss tokens/s: 148511.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.611344E-01 | lm-global-supervised-tokens: 1.257617E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.164131E-02 | lm-token-buffer-mib: 5.808479E+01 | global_load_balancing_loss: 1.001537E+00 | loss scale: 1.0 | grad norm: 0.175 | params norm: 11505.778 | params rms: 0.062300 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 15:27:01.089134 | finish at 2026-10-10 12:23:03 + [2026-10-05 20:57:31] iteration 2118/ 6855 | consumed samples: 271104 | elapsed time per iteration (ms): 89091.6 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.77% | FLOPs estimator: workload-aware | logical tokens/s: 188314.0 | valid tokens/s: 188314.0 | physical tokens/s: 188686.9 | loss tokens/s: 134682.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.150396E-01 | lm-global-supervised-tokens: 1.199905E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.804804E-02 | lm-token-buffer-mib: 5.807626E+01 | global_load_balancing_loss: 1.000539E+00 | loss scale: 1.0 | grad norm: 0.059 | params norm: 11505.703 | params rms: 0.062300 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 21:13:46.879558 | finish at 2026-10-10 18:11:18 + [2026-10-05 20:59:00] iteration 2119/ 6855 | consumed samples: 271232 | elapsed time per iteration (ms): 88262.7 | throughput per GPU (TFLOP/s/GPU): 41.7 | MFU 13.36% | FLOPs estimator: workload-aware | logical tokens/s: 190082.2 | valid tokens/s: 190082.2 | physical tokens/s: 190570.5 | loss tokens/s: 141376.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.638771E-01 | lm-global-supervised-tokens: 1.247828E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.479075E-02 | lm-token-buffer-mib: 5.819927E+01 | global_load_balancing_loss: 9.997645E-01 | loss scale: 1.0 | grad norm: 0.072 | params norm: 11505.629 | params rms: 0.062299 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 20:06:51.949982 | finish at 2026-10-10 17:05:51 + [2026-10-05 21:00:50] iteration 2120/ 6855 | consumed samples: 271360 | elapsed time per iteration (ms): 110945.1 | throughput per GPU (TFLOP/s/GPU): 33.1 | MFU 10.62% | FLOPs estimator: workload-aware | logical tokens/s: 151220.4 | valid tokens/s: 151220.4 | physical tokens/s: 151650.1 | loss tokens/s: 109519.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.448056E-01 | lm-global-supervised-tokens: 1.215062E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.119675E-02 | lm-token-buffer-mib: 5.803391E+01 | global_load_balancing_loss: 9.993346E-01 | loss scale: 1.0 | grad norm: 0.098 | params norm: 11505.554 | params rms: 0.062299 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 1:55:25.040925 | finish at 2026-10-11 22:56:16 +times across ranks (ms): + save-checkpoint: + rank 0: 25368.41 + rank 1: 25368.40 + rank 2: 25368.46 + rank 3: 25368.41 + rank 4: 25368.42 + rank 5: 25368.39 + rank 6: 25368.38 + rank 7: 25368.41 + rank 8: 25368.68 + rank 9: 25368.64 + rank 10: 25368.65 + rank 11: 25368.71 + rank 12: 25368.66 + rank 13: 25368.65 + rank 14: 25368.67 + rank 15: 25368.65 + rank 16: 25368.58 + rank 17: 25368.60 + rank 18: 25368.59 + rank 19: 25368.58 + rank 20: 25368.56 + rank 21: 25368.59 + rank 22: 25368.56 + rank 23: 25368.54 + rank 24: 25369.13 + rank 25: 25369.08 + rank 26: 25369.13 + rank 27: 25369.13 + rank 28: 25369.09 + rank 29: 25369.10 + rank 30: 25369.07 + rank 31: 25369.12 + rank 32: 25368.77 + rank 33: 25368.90 + rank 34: 25368.73 + rank 35: 25368.73 + rank 36: 25368.74 + rank 37: 25368.72 + rank 38: 25368.71 + rank 39: 25368.68 + rank 40: 25368.68 + rank 41: 25368.65 + rank 42: 25368.68 + rank 43: 25368.69 + rank 44: 25368.67 + rank 45: 25368.73 + rank 46: 25368.66 + rank 47: 25368.64 + rank 48: 25367.32 + rank 49: 25367.48 + rank 50: 25367.51 + rank 51: 25367.52 + rank 52: 25367.54 + rank 53: 25367.57 + rank 54: 25367.53 + rank 55: 25367.54 + rank 56: 25368.60 + rank 57: 25368.58 + rank 58: 25368.55 + rank 59: 25368.62 + rank 60: 25368.54 + rank 61: 25368.57 + rank 62: 25368.56 + rank 63: 25368.66 + [2026-10-05 21:03:02] iteration 2121/ 6855 | consumed samples: 271488 | elapsed time per iteration (ms): 105828.8 | throughput per GPU (TFLOP/s/GPU): 34.3 | MFU 10.99% | FLOPs estimator: workload-aware | logical tokens/s: 158531.3 | valid tokens/s: 158531.3 | physical tokens/s: 158910.7 | loss tokens/s: 122243.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.608812E-01 | lm-global-supervised-tokens: 1.293687E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 5.300214E-01 | lm-token-buffer-mib: 5.805671E+01 | global_load_balancing_loss: 9.993566E-01 | loss scale: 1.0 | grad norm: 0.101 | params norm: 11505.481 | params rms: 0.062298 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 19:09:53.610131 | finish at 2026-10-11 16:12:56 + [2026-10-05 21:04:35] iteration 2122/ 6855 | consumed samples: 271616 | elapsed time per iteration (ms): 92765.4 | throughput per GPU (TFLOP/s/GPU): 38.8 | MFU 12.43% | FLOPs estimator: workload-aware | logical tokens/s: 180855.9 | valid tokens/s: 180855.9 | physical tokens/s: 181249.5 | loss tokens/s: 137184.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.184312E-01 | lm-global-supervised-tokens: 1.272598E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.706867E-02 | lm-token-buffer-mib: 5.814485E+01 | global_load_balancing_loss: 9.998729E-01 | loss scale: 1.0 | grad norm: 0.039 | params norm: 11505.410 | params rms: 0.062298 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 1:57:38.554378 | finish at 2026-10-10 23:02:13 + [2026-10-05 21:06:04] iteration 2123/ 6855 | consumed samples: 271744 | elapsed time per iteration (ms): 89409.6 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.41% | FLOPs estimator: workload-aware | logical tokens/s: 187643.8 | valid tokens/s: 187643.8 | physical tokens/s: 188044.1 | loss tokens/s: 138007.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.640327E-01 | lm-global-supervised-tokens: 1.233916E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.150226E-02 | lm-token-buffer-mib: 5.805354E+01 | global_load_balancing_loss: 1.001206E+00 | loss scale: 1.0 | grad norm: 0.082 | params norm: 11505.339 | params rms: 0.062298 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 21:31:26.076114 | finish at 2026-10-10 18:37:30 + [2026-10-05 21:07:32] iteration 2124/ 6855 | consumed samples: 271872 | elapsed time per iteration (ms): 88163.6 | throughput per GPU (TFLOP/s/GPU): 43.1 | MFU 13.80% | FLOPs estimator: workload-aware | logical tokens/s: 190295.7 | valid tokens/s: 190295.7 | physical tokens/s: 190530.1 | loss tokens/s: 139445.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.612700E-01 | lm-global-supervised-tokens: 1.229401E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.270124E-02 | lm-token-buffer-mib: 5.803228E+01 | global_load_balancing_loss: 9.999408E-01 | loss scale: 1.0 | grad norm: 0.037 | params norm: 11505.267 | params rms: 0.062297 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 19:51:42.026415 | finish at 2026-10-10 16:59:15 + [2026-10-05 21:08:57] iteration 2125/ 6855 | consumed samples: 272000 | elapsed time per iteration (ms): 84417.6 | throughput per GPU (TFLOP/s/GPU): 43.1 | MFU 13.80% | FLOPs estimator: workload-aware | logical tokens/s: 198740.1 | valid tokens/s: 198740.1 | physical tokens/s: 199307.4 | loss tokens/s: 147211.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.616575E-01 | lm-global-supervised-tokens: 1.242726E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.810901E-02 | lm-token-buffer-mib: 5.804395E+01 | global_load_balancing_loss: 1.000338E+00 | loss scale: 1.0 | grad norm: 0.100 | params norm: 11505.194 | params rms: 0.062297 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 14:54:55.343461 | finish at 2026-10-10 12:03:52 + [2026-10-05 21:10:21] iteration 2126/ 6855 | consumed samples: 272128 | elapsed time per iteration (ms): 84403.7 | throughput per GPU (TFLOP/s/GPU): 43.9 | MFU 14.08% | FLOPs estimator: workload-aware | logical tokens/s: 198773.2 | valid tokens/s: 198773.2 | physical tokens/s: 199358.6 | loss tokens/s: 148146.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.762244E-01 | lm-global-supervised-tokens: 1.250408E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.274184E-02 | lm-token-buffer-mib: 5.827625E+01 | global_load_balancing_loss: 9.995430E-01 | loss scale: 1.0 | grad norm: 0.055 | params norm: 11505.122 | params rms: 0.062296 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 14:52:25.290635 | finish at 2026-10-10 12:02:47 + [2026-10-05 21:11:47] iteration 2127/ 6855 | consumed samples: 272256 | elapsed time per iteration (ms): 85763.2 | throughput per GPU (TFLOP/s/GPU): 42.8 | MFU 13.70% | FLOPs estimator: workload-aware | logical tokens/s: 195622.3 | valid tokens/s: 195622.2 | physical tokens/s: 196046.0 | loss tokens/s: 144792.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.103736E-01 | lm-global-supervised-tokens: 1.241791E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.139020E-02 | lm-token-buffer-mib: 5.811170E+01 | global_load_balancing_loss: 1.000580E+00 | loss scale: 1.0 | grad norm: 0.098 | params norm: 11505.053 | params rms: 0.062296 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 16:38:08.601442 | finish at 2026-10-10 13:49:56 + [2026-10-05 21:13:12] iteration 2128/ 6855 | consumed samples: 272384 | elapsed time per iteration (ms): 84641.6 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.58% | FLOPs estimator: workload-aware | logical tokens/s: 198214.8 | valid tokens/s: 198214.8 | physical tokens/s: 198768.5 | loss tokens/s: 146842.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.825120E-01 | lm-global-supervised-tokens: 1.242894E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.216724E-02 | lm-token-buffer-mib: 5.829891E+01 | global_load_balancing_loss: 1.000856E+00 | loss scale: 1.0 | grad norm: 0.218 | params norm: 11504.983 | params rms: 0.062296 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 15:08:20.851713 | finish at 2026-10-10 12:21:33 + [2026-10-05 21:14:38] iteration 2129/ 6855 | consumed samples: 272512 | elapsed time per iteration (ms): 86294.0 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.75% | FLOPs estimator: workload-aware | logical tokens/s: 194419.1 | valid tokens/s: 194419.1 | physical tokens/s: 194815.8 | loss tokens/s: 142737.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.989913E-01 | lm-global-supervised-tokens: 1.231738E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.174194E-02 | lm-token-buffer-mib: 5.808091E+01 | global_load_balancing_loss: 1.000813E+00 | loss scale: 1.0 | grad norm: 0.162 | params norm: 11504.911 | params rms: 0.062295 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 17:17:05.470619 | finish at 2026-10-10 14:31:43 + [2026-10-05 21:16:26] iteration 2130/ 6855 | consumed samples: 272640 | elapsed time per iteration (ms): 107730.4 | throughput per GPU (TFLOP/s/GPU): 34.3 | MFU 10.99% | FLOPs estimator: workload-aware | logical tokens/s: 155732.7 | valid tokens/s: 155732.7 | physical tokens/s: 156171.9 | loss tokens/s: 116965.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.740247E-01 | lm-global-supervised-tokens: 1.260069E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.573508E-02 | lm-token-buffer-mib: 5.812761E+01 | global_load_balancing_loss: 1.000398E+00 | loss scale: 1.0 | grad norm: 0.143 | params norm: 11504.841 | params rms: 0.062295 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 21:23:46.245171 | finish at 2026-10-11 18:40:12 + [2026-10-05 21:17:55] iteration 2131/ 6855 | consumed samples: 272768 | elapsed time per iteration (ms): 89474.1 | throughput per GPU (TFLOP/s/GPU): 41.6 | MFU 13.32% | FLOPs estimator: workload-aware | logical tokens/s: 187509.2 | valid tokens/s: 187509.2 | physical tokens/s: 187992.8 | loss tokens/s: 133273.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.231519E-01 | lm-global-supervised-tokens: 1.192452E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.528670E-02 | lm-token-buffer-mib: 5.804942E+01 | global_load_balancing_loss: 1.000764E+00 | loss scale: 1.0 | grad norm: 0.209 | params norm: 11504.771 | params rms: 0.062295 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 21:24:35.557704 | finish at 2026-10-10 18:42:31 + [2026-10-05 21:19:19] iteration 2132/ 6855 | consumed samples: 272896 | elapsed time per iteration (ms): 84025.8 | throughput per GPU (TFLOP/s/GPU): 42.8 | MFU 13.72% | FLOPs estimator: workload-aware | logical tokens/s: 199667.2 | valid tokens/s: 199667.2 | physical tokens/s: 200175.8 | loss tokens/s: 149540.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 5.964687E-01 | lm-global-supervised-tokens: 1.256526E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.408251E-02 | lm-token-buffer-mib: 5.808852E+01 | global_load_balancing_loss: 9.994737E-01 | loss scale: 1.0 | grad norm: 0.052 | params norm: 11504.699 | params rms: 0.062294 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 14:14:13.978343 | finish at 2026-10-10 11:33:33 + [2026-10-05 21:20:45] iteration 2133/ 6855 | consumed samples: 273024 | elapsed time per iteration (ms): 85449.1 | throughput per GPU (TFLOP/s/GPU): 43.1 | MFU 13.80% | FLOPs estimator: workload-aware | logical tokens/s: 196341.6 | valid tokens/s: 196341.6 | physical tokens/s: 196922.5 | loss tokens/s: 150577.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.478201E-01 | lm-global-supervised-tokens: 1.286671E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.291172E-02 | lm-token-buffer-mib: 5.806695E+01 | global_load_balancing_loss: 9.993987E-01 | loss scale: 1.0 | grad norm: 0.131 | params norm: 11504.629 | params rms: 0.062294 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 16:04:50.490417 | finish at 2026-10-10 13:25:35 + [2026-10-05 21:22:14] iteration 2134/ 6855 | consumed samples: 273152 | elapsed time per iteration (ms): 89784.8 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.48% | FLOPs estimator: workload-aware | logical tokens/s: 186859.9 | valid tokens/s: 186859.9 | physical tokens/s: 187152.9 | loss tokens/s: 139004.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.468298E-01 | lm-global-supervised-tokens: 1.248044E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.161242E-02 | lm-token-buffer-mib: 5.808939E+01 | global_load_balancing_loss: 9.994631E-01 | loss scale: 1.0 | grad norm: 0.071 | params norm: 11504.560 | params rms: 0.062293 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 21:44:33.985895 | finish at 2026-10-10 19:06:48 + [2026-10-05 21:23:39] iteration 2135/ 6855 | consumed samples: 273280 | elapsed time per iteration (ms): 84995.5 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.70% | FLOPs estimator: workload-aware | logical tokens/s: 197389.4 | valid tokens/s: 197389.4 | physical tokens/s: 197786.4 | loss tokens/s: 151337.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.934724E-01 | lm-global-supervised-tokens: 1.286300E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.442036E-02 | lm-token-buffer-mib: 5.822753E+01 | global_load_balancing_loss: 9.986857E-01 | loss scale: 1.0 | grad norm: 0.112 | params norm: 11504.491 | params rms: 0.062293 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 15:26:18.841438 | finish at 2026-10-10 12:49:58 + [2026-10-05 21:25:07] iteration 2136/ 6855 | consumed samples: 273408 | elapsed time per iteration (ms): 87589.2 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.79% | FLOPs estimator: workload-aware | logical tokens/s: 191543.4 | valid tokens/s: 191543.4 | physical tokens/s: 192076.8 | loss tokens/s: 142069.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.206736E-01 | lm-global-supervised-tokens: 1.244374E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.232675E-02 | lm-token-buffer-mib: 5.826520E+01 | global_load_balancing_loss: 1.000290E+00 | loss scale: 1.0 | grad norm: 0.053 | params norm: 11504.421 | params rms: 0.062293 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 18:48:53.477647 | finish at 2026-10-10 16:14:01 + [2026-10-05 21:26:32] iteration 2137/ 6855 | consumed samples: 273536 | elapsed time per iteration (ms): 84979.4 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.70% | FLOPs estimator: workload-aware | logical tokens/s: 197426.1 | valid tokens/s: 197426.1 | physical tokens/s: 197833.9 | loss tokens/s: 146287.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.311769E-01 | lm-global-supervised-tokens: 1.243140E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.317739E-02 | lm-token-buffer-mib: 5.830835E+01 | global_load_balancing_loss: 1.001408E+00 | loss scale: 1.0 | grad norm: 0.188 | params norm: 11504.348 | params rms: 0.062292 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 15:22:12.938179 | finish at 2026-10-10 12:48:45 + [2026-10-05 21:27:58] iteration 2138/ 6855 | consumed samples: 273664 | elapsed time per iteration (ms): 86317.3 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.56% | FLOPs estimator: workload-aware | logical tokens/s: 194366.6 | valid tokens/s: 194366.6 | physical tokens/s: 195019.0 | loss tokens/s: 144104.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.079847E-01 | lm-global-supervised-tokens: 1.243868E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.317847E-02 | lm-token-buffer-mib: 5.827238E+01 | global_load_balancing_loss: 1.001207E+00 | loss scale: 1.0 | grad norm: 0.117 | params norm: 11504.276 | params rms: 0.062292 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 17:05:58.532416 | finish at 2026-10-10 14:33:57 + [2026-10-05 21:29:25] iteration 2139/ 6855 | consumed samples: 273792 | elapsed time per iteration (ms): 86507.4 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.64% | FLOPs estimator: workload-aware | logical tokens/s: 193939.4 | valid tokens/s: 193939.4 | physical tokens/s: 194280.1 | loss tokens/s: 142635.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.501492E-01 | lm-global-supervised-tokens: 1.233906E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.878054E-02 | lm-token-buffer-mib: 5.807614E+01 | global_load_balancing_loss: 1.000619E+00 | loss scale: 1.0 | grad norm: 0.132 | params norm: 11504.206 | params rms: 0.062292 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 17:19:29.069475 | finish at 2026-10-10 14:48:54 + [2026-10-05 21:31:13] iteration 2140/ 6855 | consumed samples: 273920 | elapsed time per iteration (ms): 107839.4 | throughput per GPU (TFLOP/s/GPU): 34.0 | MFU 10.90% | FLOPs estimator: workload-aware | logical tokens/s: 155575.6 | valid tokens/s: 155575.6 | physical tokens/s: 156000.9 | loss tokens/s: 116852.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.444963E-01 | lm-global-supervised-tokens: 1.260126E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.146824E-02 | lm-token-buffer-mib: 5.807478E+01 | global_load_balancing_loss: 1.000754E+00 | loss scale: 1.0 | grad norm: 0.103 | params norm: 11504.134 | params rms: 0.062291 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 21:14:23.006196 | finish at 2026-10-11 18:45:36 + [2026-10-05 21:32:39] iteration 2141/ 6855 | consumed samples: 274048 | elapsed time per iteration (ms): 86033.5 | throughput per GPU (TFLOP/s/GPU): 44.1 | MFU 14.14% | FLOPs estimator: workload-aware | logical tokens/s: 195007.1 | valid tokens/s: 195007.1 | physical tokens/s: 195384.9 | loss tokens/s: 151747.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.850316E-01 | lm-global-supervised-tokens: 1.305534E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.157151E-02 | lm-token-buffer-mib: 5.813213E+01 | global_load_balancing_loss: 1.000021E+00 | loss scale: 1.0 | grad norm: 0.153 | params norm: 11504.063 | params rms: 0.062291 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 16:39:21.978360 | finish at 2026-10-10 14:12:01 + [2026-10-05 21:34:03] iteration 2142/ 6855 | consumed samples: 274176 | elapsed time per iteration (ms): 84145.9 | throughput per GPU (TFLOP/s/GPU): 43.3 | MFU 13.86% | FLOPs estimator: workload-aware | logical tokens/s: 199381.7 | valid tokens/s: 199381.7 | physical tokens/s: 199800.2 | loss tokens/s: 156069.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.101718E-01 | lm-global-supervised-tokens: 1.313260E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.562564E-02 | lm-token-buffer-mib: 5.809386E+01 | global_load_balancing_loss: 9.985703E-01 | loss scale: 1.0 | grad norm: 0.068 | params norm: 11503.991 | params rms: 0.062290 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 14:09:39.654844 | finish at 2026-10-10 11:43:43 + [2026-10-05 21:35:27] iteration 2143/ 6855 | consumed samples: 274304 | elapsed time per iteration (ms): 84029.4 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.78% | FLOPs estimator: workload-aware | logical tokens/s: 199658.1 | valid tokens/s: 199658.1 | physical tokens/s: 200069.8 | loss tokens/s: 151851.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.547524E-01 | lm-global-supervised-tokens: 1.275998E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.458213E-02 | lm-token-buffer-mib: 5.810389E+01 | global_load_balancing_loss: 1.000299E+00 | loss scale: 1.0 | grad norm: 0.219 | params norm: 11503.920 | params rms: 0.062290 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 13:59:06.565899 | finish at 2026-10-10 11:34:33 + [2026-10-05 21:36:53] iteration 2144/ 6855 | consumed samples: 274432 | elapsed time per iteration (ms): 86358.9 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.60% | FLOPs estimator: workload-aware | logical tokens/s: 194273.1 | valid tokens/s: 194273.1 | physical tokens/s: 194826.9 | loss tokens/s: 146631.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.147461E-01 | lm-global-supervised-tokens: 1.266296E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.493027E-02 | lm-token-buffer-mib: 5.817290E+01 | global_load_balancing_loss: 9.993272E-01 | loss scale: 1.0 | grad norm: 0.086 | params norm: 11503.849 | params rms: 0.062290 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 17:00:36.840006 | finish at 2026-10-10 14:37:30 + [2026-10-05 21:38:22] iteration 2145/ 6855 | consumed samples: 274560 | elapsed time per iteration (ms): 89070.7 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.42% | FLOPs estimator: workload-aware | logical tokens/s: 188357.3 | valid tokens/s: 188357.2 | physical tokens/s: 188892.6 | loss tokens/s: 137556.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.667098E-01 | lm-global-supervised-tokens: 1.225220E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.175153E-02 | lm-token-buffer-mib: 5.819411E+01 | global_load_balancing_loss: 9.995979E-01 | loss scale: 1.0 | grad norm: 0.321 | params norm: 11503.776 | params rms: 0.062289 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 20:32:02.983196 | finish at 2026-10-10 18:10:25 + [2026-10-05 21:39:50] iteration 2146/ 6855 | consumed samples: 274688 | elapsed time per iteration (ms): 87975.3 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.41% | FLOPs estimator: workload-aware | logical tokens/s: 190703.5 | valid tokens/s: 190703.5 | physical tokens/s: 191251.6 | loss tokens/s: 141007.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.073755E-01 | lm-global-supervised-tokens: 1.240516E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.316124E-02 | lm-token-buffer-mib: 5.811107E+01 | global_load_balancing_loss: 1.000575E+00 | loss scale: 1.0 | grad norm: 0.204 | params norm: 11503.704 | params rms: 0.062289 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 19:04:35.858699 | finish at 2026-10-10 16:44:26 + [2026-10-05 21:41:16] iteration 2147/ 6855 | consumed samples: 274816 | elapsed time per iteration (ms): 85727.2 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.79% | FLOPs estimator: workload-aware | logical tokens/s: 195704.6 | valid tokens/s: 195704.6 | physical tokens/s: 196112.7 | loss tokens/s: 142664.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.497008E-01 | lm-global-supervised-tokens: 1.223020E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.578900E-02 | lm-token-buffer-mib: 5.810364E+01 | global_load_balancing_loss: 1.000479E+00 | loss scale: 1.0 | grad norm: 0.192 | params norm: 11503.631 | params rms: 0.062288 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 16:06:43.682442 | finish at 2026-10-10 13:48:00 + [2026-10-05 21:42:42] iteration 2148/ 6855 | consumed samples: 274944 | elapsed time per iteration (ms): 86389.7 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.47% | FLOPs estimator: workload-aware | logical tokens/s: 194203.1 | valid tokens/s: 194203.1 | physical tokens/s: 194729.6 | loss tokens/s: 147647.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.544005E-01 | lm-global-supervised-tokens: 1.275520E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.365001E-02 | lm-token-buffer-mib: 5.830987E+01 | global_load_balancing_loss: 1.000614E+00 | loss scale: 1.0 | grad norm: 0.075 | params norm: 11503.557 | params rms: 0.062288 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 16:57:16.463489 | finish at 2026-10-10 14:39:59 + [2026-10-05 21:44:09] iteration 2149/ 6855 | consumed samples: 275072 | elapsed time per iteration (ms): 86648.1 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.51% | FLOPs estimator: workload-aware | logical tokens/s: 193624.6 | valid tokens/s: 193624.6 | physical tokens/s: 194116.0 | loss tokens/s: 137794.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.661045E-01 | lm-global-supervised-tokens: 1.193961E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.531283E-02 | lm-token-buffer-mib: 5.804404E+01 | global_load_balancing_loss: 1.001359E+00 | loss scale: 1.0 | grad norm: 0.094 | params norm: 11503.482 | params rms: 0.062288 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 17:16:05.746068 | finish at 2026-10-10 15:00:15 + [2026-10-05 21:45:58] iteration 2150/ 6855 | consumed samples: 275200 | elapsed time per iteration (ms): 109182.7 | throughput per GPU (TFLOP/s/GPU): 33.5 | MFU 10.75% | FLOPs estimator: workload-aware | logical tokens/s: 153661.7 | valid tokens/s: 153661.7 | physical tokens/s: 154033.6 | loss tokens/s: 118832.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.930172E-01 | lm-global-supervised-tokens: 1.297449E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.310065E-02 | lm-token-buffer-mib: 5.815300E+01 | global_load_balancing_loss: 9.999736E-01 | loss scale: 1.0 | grad norm: 0.220 | params norm: 11503.407 | params rms: 0.062287 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 22:41:44.594144 | finish at 2026-10-11 20:27:43 + [2026-10-05 21:47:24] iteration 2151/ 6855 | consumed samples: 275328 | elapsed time per iteration (ms): 85968.3 | throughput per GPU (TFLOP/s/GPU): 42.8 | MFU 13.72% | FLOPs estimator: workload-aware | logical tokens/s: 195155.5 | valid tokens/s: 195155.5 | physical tokens/s: 195758.0 | loss tokens/s: 139122.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 5.975920E-01 | lm-global-supervised-tokens: 1.196011E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.142079E-02 | lm-token-buffer-mib: 5.811434E+01 | global_load_balancing_loss: 1.001651E+00 | loss scale: 1.0 | grad norm: 0.120 | params norm: 11503.334 | params rms: 0.062287 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 16:19:54.682938 | finish at 2026-10-10 14:07:19 + [2026-10-05 21:48:53] iteration 2152/ 6855 | consumed samples: 275456 | elapsed time per iteration (ms): 88611.8 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.51% | FLOPs estimator: workload-aware | logical tokens/s: 189333.3 | valid tokens/s: 189333.2 | physical tokens/s: 189703.7 | loss tokens/s: 137255.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.696930E-01 | lm-global-supervised-tokens: 1.216241E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.114698E-02 | lm-token-buffer-mib: 5.811656E+01 | global_load_balancing_loss: 9.982845E-01 | loss scale: 1.0 | grad norm: 0.236 | params norm: 11503.261 | params rms: 0.062286 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 19:45:41.280613 | finish at 2026-10-10 17:34:34 + [2026-10-05 21:50:18] iteration 2153/ 6855 | consumed samples: 275584 | elapsed time per iteration (ms): 85222.7 | throughput per GPU (TFLOP/s/GPU): 43.4 | MFU 13.92% | FLOPs estimator: workload-aware | logical tokens/s: 196862.3 | valid tokens/s: 196862.3 | physical tokens/s: 197289.1 | loss tokens/s: 143699.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.969597E-01 | lm-global-supervised-tokens: 1.224643E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.445482E-02 | lm-token-buffer-mib: 5.807499E+01 | global_load_balancing_loss: 9.995441E-01 | loss scale: 1.0 | grad norm: 0.047 | params norm: 11503.189 | params rms: 0.062286 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 15:18:37.338869 | finish at 2026-10-10 13:08:55 + [2026-10-05 21:51:47] iteration 2154/ 6855 | consumed samples: 275712 | elapsed time per iteration (ms): 89269.0 | throughput per GPU (TFLOP/s/GPU): 41.6 | MFU 13.34% | FLOPs estimator: workload-aware | logical tokens/s: 187939.4 | valid tokens/s: 187939.4 | physical tokens/s: 188568.3 | loss tokens/s: 140108.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.186526E-01 | lm-global-supervised-tokens: 1.250735E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.405867E-02 | lm-token-buffer-mib: 5.818529E+01 | global_load_balancing_loss: 9.986407E-01 | loss scale: 1.0 | grad norm: 0.184 | params norm: 11503.118 | params rms: 0.062286 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 20:34:13.374231 | finish at 2026-10-10 18:26:01 + [2026-10-05 21:53:10] iteration 2155/ 6855 | consumed samples: 275840 | elapsed time per iteration (ms): 82516.1 | throughput per GPU (TFLOP/s/GPU): 43.6 | MFU 13.97% | FLOPs estimator: workload-aware | logical tokens/s: 203320.4 | valid tokens/s: 203320.4 | physical tokens/s: 203885.1 | loss tokens/s: 155691.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.790071E-01 | lm-global-supervised-tokens: 1.284707E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.336787E-02 | lm-token-buffer-mib: 5.823754E+01 | global_load_balancing_loss: 9.992275E-01 | loss scale: 1.0 | grad norm: 0.130 | params norm: 11503.048 | params rms: 0.062285 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 11:43:45.725698 | finish at 2026-10-10 09:36:56 + [2026-10-05 21:54:38] iteration 2156/ 6855 | consumed samples: 275968 | elapsed time per iteration (ms): 88105.9 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.46% | FLOPs estimator: workload-aware | logical tokens/s: 190419.4 | valid tokens/s: 190419.4 | physical tokens/s: 191067.5 | loss tokens/s: 143172.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.594138E-01 | lm-global-supervised-tokens: 1.261435E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.137149E-02 | lm-token-buffer-mib: 5.815165E+01 | global_load_balancing_loss: 1.000721E+00 | loss scale: 1.0 | grad norm: 0.055 | params norm: 11502.977 | params rms: 0.062285 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 19:00:09.722920 | finish at 2026-10-10 16:54:48 + [2026-10-05 21:56:05] iteration 2157/ 6855 | consumed samples: 276096 | elapsed time per iteration (ms): 86667.5 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.76% | FLOPs estimator: workload-aware | logical tokens/s: 193581.3 | valid tokens/s: 193581.3 | physical tokens/s: 193959.2 | loss tokens/s: 146484.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.608158E-01 | lm-global-supervised-tokens: 1.269546E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.530879E-02 | lm-token-buffer-mib: 5.827253E+01 | global_load_balancing_loss: 1.001287E+00 | loss scale: 1.0 | grad norm: 0.175 | params norm: 11502.907 | params rms: 0.062284 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 17:06:04.131267 | finish at 2026-10-10 15:02:09 + [2026-10-05 21:57:31] iteration 2158/ 6855 | consumed samples: 276224 | elapsed time per iteration (ms): 86662.9 | throughput per GPU (TFLOP/s/GPU): 42.8 | MFU 13.72% | FLOPs estimator: workload-aware | logical tokens/s: 193591.2 | valid tokens/s: 193591.2 | physical tokens/s: 194057.1 | loss tokens/s: 138356.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.611041E-01 | lm-global-supervised-tokens: 1.199040E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.587911E-02 | lm-token-buffer-mib: 5.815637E+01 | global_load_balancing_loss: 1.002360E+00 | loss scale: 1.0 | grad norm: 0.077 | params norm: 11502.838 | params rms: 0.062284 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 17:04:15.689317 | finish at 2026-10-10 15:01:47 + [2026-10-05 21:59:00] iteration 2159/ 6855 | consumed samples: 276352 | elapsed time per iteration (ms): 89074.3 | throughput per GPU (TFLOP/s/GPU): 41.1 | MFU 13.19% | FLOPs estimator: workload-aware | logical tokens/s: 188350.8 | valid tokens/s: 188350.8 | physical tokens/s: 188720.9 | loss tokens/s: 135264.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.345240E-01 | lm-global-supervised-tokens: 1.204859E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.224919E-02 | lm-token-buffer-mib: 5.809583E+01 | global_load_balancing_loss: 1.001465E+00 | loss scale: 1.0 | grad norm: 0.047 | params norm: 11502.769 | params rms: 0.062284 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 20:11:32.827948 | finish at 2026-10-10 18:10:33 + [2026-10-05 22:00:49] iteration 2160/ 6855 | consumed samples: 276480 | elapsed time per iteration (ms): 108714.4 | throughput per GPU (TFLOP/s/GPU): 34.2 | MFU 10.96% | FLOPs estimator: workload-aware | logical tokens/s: 154323.4 | valid tokens/s: 154323.4 | physical tokens/s: 154711.9 | loss tokens/s: 109046.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.170092E-01 | lm-global-supervised-tokens: 1.185491E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.183916E-02 | lm-token-buffer-mib: 5.810079E+01 | global_load_balancing_loss: 1.000265E+00 | loss scale: 1.0 | grad norm: 0.173 | params norm: 11502.699 | params rms: 0.062283 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 21:46:53.959372 | finish at 2026-10-11 19:47:43 +times across ranks (ms): + save-checkpoint: + rank 0: 24888.59 + rank 1: 24888.60 + rank 2: 24888.65 + rank 3: 24888.64 + rank 4: 24888.63 + rank 5: 24888.61 + rank 6: 24888.57 + rank 7: 24888.59 + rank 8: 24888.74 + rank 9: 24888.72 + rank 10: 24888.73 + rank 11: 24888.71 + rank 12: 24888.72 + rank 13: 24888.73 + rank 14: 24888.73 + rank 15: 24888.72 + rank 16: 24888.61 + rank 17: 24888.69 + rank 18: 24888.60 + rank 19: 24888.88 + rank 20: 24888.58 + rank 21: 24888.64 + rank 22: 24888.59 + rank 23: 24888.75 + rank 24: 24888.64 + rank 25: 24888.63 + rank 26: 24888.64 + rank 27: 24888.69 + rank 28: 24888.60 + rank 29: 24888.67 + rank 30: 24888.61 + rank 31: 24888.63 + rank 32: 24889.28 + rank 33: 24889.28 + rank 34: 24889.29 + rank 35: 24889.26 + rank 36: 24889.24 + rank 37: 24889.24 + rank 38: 24889.28 + rank 39: 24889.26 + rank 40: 24889.20 + rank 41: 24889.19 + rank 42: 24889.17 + rank 43: 24889.21 + rank 44: 24889.16 + rank 45: 24889.16 + rank 46: 24889.16 + rank 47: 24889.16 + rank 48: 24889.55 + rank 49: 24889.54 + rank 50: 24889.53 + rank 51: 24889.58 + rank 52: 24889.58 + rank 53: 24889.57 + rank 54: 24889.54 + rank 55: 24889.54 + rank 56: 24888.99 + rank 57: 24888.99 + rank 58: 24888.94 + rank 59: 24888.95 + rank 60: 24888.96 + rank 61: 24888.96 + rank 62: 24888.97 + rank 63: 24888.98 + [2026-10-05 22:03:02] iteration 2161/ 6855 | consumed samples: 276608 | elapsed time per iteration (ms): 108068.6 | throughput per GPU (TFLOP/s/GPU): 34.3 | MFU 10.99% | FLOPs estimator: workload-aware | logical tokens/s: 155245.9 | valid tokens/s: 155245.9 | physical tokens/s: 155590.3 | loss tokens/s: 112521.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.393511E-01 | lm-global-supervised-tokens: 1.216002E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 9.993928E-01 | lm-token-buffer-mib: 5.814845E+01 | global_load_balancing_loss: 9.985291E-01 | loss scale: 1.0 | grad norm: 0.063 | params norm: 11502.629 | params rms: 0.062283 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 20:54:34.052881 | finish at 2026-10-11 18:57:36 + [2026-10-05 22:04:34] iteration 2162/ 6855 | consumed samples: 276736 | elapsed time per iteration (ms): 92071.9 | throughput per GPU (TFLOP/s/GPU): 40.5 | MFU 12.99% | FLOPs estimator: workload-aware | logical tokens/s: 182218.3 | valid tokens/s: 182218.3 | physical tokens/s: 182686.8 | loss tokens/s: 132892.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.228887E-01 | lm-global-supervised-tokens: 1.223564E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.193282E-02 | lm-token-buffer-mib: 5.822732E+01 | global_load_balancing_loss: 9.995700E-01 | loss scale: 1.0 | grad norm: 0.090 | params norm: 11502.559 | params rms: 0.062283 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 0:01:33.369124 | finish at 2026-10-10 22:06:08 + [2026-10-05 22:06:04] iteration 2163/ 6855 | consumed samples: 276864 | elapsed time per iteration (ms): 89747.5 | throughput per GPU (TFLOP/s/GPU): 40.4 | MFU 12.95% | FLOPs estimator: workload-aware | logical tokens/s: 186937.6 | valid tokens/s: 186937.6 | physical tokens/s: 187413.8 | loss tokens/s: 140852.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.379309E-01 | lm-global-supervised-tokens: 1.264116E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.089683E-02 | lm-token-buffer-mib: 5.816329E+01 | global_load_balancing_loss: 9.988981E-01 | loss scale: 1.0 | grad norm: 0.074 | params norm: 11502.491 | params rms: 0.062282 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 20:58:15.345800 | finish at 2026-10-10 19:04:19 + [2026-10-05 22:07:29] iteration 2164/ 6855 | consumed samples: 276992 | elapsed time per iteration (ms): 85039.2 | throughput per GPU (TFLOP/s/GPU): 42.8 | MFU 13.72% | FLOPs estimator: workload-aware | logical tokens/s: 197287.7 | valid tokens/s: 197287.7 | physical tokens/s: 197743.6 | loss tokens/s: 145244.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.064542E-01 | lm-global-supervised-tokens: 1.235145E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.185562E-02 | lm-token-buffer-mib: 5.806464E+01 | global_load_balancing_loss: 9.997814E-01 | loss scale: 1.0 | grad norm: 0.042 | params norm: 11502.423 | params rms: 0.062282 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 14:48:38.819293 | finish at 2026-10-10 12:56:08 + [2026-10-05 22:09:01] iteration 2165/ 6855 | consumed samples: 277120 | elapsed time per iteration (ms): 92086.1 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.64% | FLOPs estimator: workload-aware | logical tokens/s: 182190.2 | valid tokens/s: 182190.2 | physical tokens/s: 182421.2 | loss tokens/s: 132356.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.667904E-01 | lm-global-supervised-tokens: 1.218816E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.470890E-02 | lm-token-buffer-mib: 5.803876E+01 | global_load_balancing_loss: 1.000198E+00 | loss scale: 1.0 | grad norm: 0.050 | params norm: 11502.352 | params rms: 0.062281 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 23:58:03.675294 | finish at 2026-10-10 22:07:05 + [2026-10-05 22:10:26] iteration 2166/ 6855 | consumed samples: 277248 | elapsed time per iteration (ms): 85162.0 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.67% | FLOPs estimator: workload-aware | logical tokens/s: 197002.3 | valid tokens/s: 197002.3 | physical tokens/s: 197467.7 | loss tokens/s: 145298.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.474019E-01 | lm-global-supervised-tokens: 1.237394E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.393704E-02 | lm-token-buffer-mib: 5.810509E+01 | global_load_balancing_loss: 1.000611E+00 | loss scale: 1.0 | grad norm: 0.097 | params norm: 11502.282 | params rms: 0.062281 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 14:55:24.499221 | finish at 2026-10-10 13:05:51 + [2026-10-05 22:11:50] iteration 2167/ 6855 | consumed samples: 277376 | elapsed time per iteration (ms): 83967.3 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.59% | FLOPs estimator: workload-aware | logical tokens/s: 199806.4 | valid tokens/s: 199806.4 | physical tokens/s: 200384.4 | loss tokens/s: 148220.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.389369E-01 | lm-global-supervised-tokens: 1.244570E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.146177E-02 | lm-token-buffer-mib: 5.812686E+01 | global_load_balancing_loss: 1.001263E+00 | loss scale: 1.0 | grad norm: 0.054 | params norm: 11502.213 | params rms: 0.062281 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 13:20:38.559044 | finish at 2026-10-10 11:32:29 + [2026-10-05 22:13:18] iteration 2168/ 6855 | consumed samples: 277504 | elapsed time per iteration (ms): 87451.6 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.49% | FLOPs estimator: workload-aware | logical tokens/s: 191845.5 | valid tokens/s: 191845.5 | physical tokens/s: 192311.7 | loss tokens/s: 141194.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.829150E-01 | lm-global-supervised-tokens: 1.234772E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.269882E-02 | lm-token-buffer-mib: 5.807893E+01 | global_load_balancing_loss: 1.001002E+00 | loss scale: 1.0 | grad norm: 0.067 | params norm: 11502.144 | params rms: 0.062280 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 17:51:25.807113 | finish at 2026-10-10 16:04:44 + [2026-10-05 22:14:46] iteration 2169/ 6855 | consumed samples: 277632 | elapsed time per iteration (ms): 88210.6 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.76% | FLOPs estimator: workload-aware | logical tokens/s: 190194.2 | valid tokens/s: 190194.2 | physical tokens/s: 190497.7 | loss tokens/s: 140249.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.308388E-01 | lm-global-supervised-tokens: 1.237152E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.156277E-02 | lm-token-buffer-mib: 5.808237E+01 | global_load_balancing_loss: 1.000954E+00 | loss scale: 1.0 | grad norm: 0.034 | params norm: 11502.075 | params rms: 0.062280 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 18:49:14.999609 | finish at 2026-10-10 17:04:01 + [2026-10-05 22:16:39] iteration 2170/ 6855 | consumed samples: 277760 | elapsed time per iteration (ms): 112811.5 | throughput per GPU (TFLOP/s/GPU): 33.4 | MFU 10.71% | FLOPs estimator: workload-aware | logical tokens/s: 148719.0 | valid tokens/s: 148719.0 | physical tokens/s: 149084.2 | loss tokens/s: 104371.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.650396E-01 | lm-global-supervised-tokens: 1.177430E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.332894E-02 | lm-token-buffer-mib: 5.814413E+01 | global_load_balancing_loss: 1.001407E+00 | loss scale: 1.0 | grad norm: 0.060 | params norm: 11502.006 | params rms: 0.062280 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 2:48:41.916934 | finish at 2026-10-12 01:05:21 + [2026-10-05 22:18:13] iteration 2171/ 6855 | consumed samples: 277888 | elapsed time per iteration (ms): 94220.5 | throughput per GPU (TFLOP/s/GPU): 39.8 | MFU 12.77% | FLOPs estimator: workload-aware | logical tokens/s: 178062.9 | valid tokens/s: 178062.9 | physical tokens/s: 178545.2 | loss tokens/s: 124879.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.944538E-01 | lm-global-supervised-tokens: 1.176622E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.587038E-02 | lm-token-buffer-mib: 5.805966E+01 | global_load_balancing_loss: 9.996643E-01 | loss scale: 1.0 | grad norm: 0.103 | params norm: 11501.936 | params rms: 0.062279 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 2:35:28.671203 | finish at 2026-10-11 00:53:42 + [2026-10-05 22:19:39] iteration 2172/ 6855 | consumed samples: 278016 | elapsed time per iteration (ms): 86190.3 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.57% | FLOPs estimator: workload-aware | logical tokens/s: 194653.0 | valid tokens/s: 194653.0 | physical tokens/s: 195059.1 | loss tokens/s: 147720.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.601255E-01 | lm-global-supervised-tokens: 1.273208E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.321621E-02 | lm-token-buffer-mib: 5.809235E+01 | global_load_balancing_loss: 9.979118E-01 | loss scale: 1.0 | grad norm: 0.054 | params norm: 11501.864 | params rms: 0.062279 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 16:07:09.328922 | finish at 2026-10-10 14:26:49 + [2026-10-05 22:21:07] iteration 2173/ 6855 | consumed samples: 278144 | elapsed time per iteration (ms): 87743.4 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.48% | FLOPs estimator: workload-aware | logical tokens/s: 191207.3 | valid tokens/s: 191207.3 | physical tokens/s: 191612.2 | loss tokens/s: 145246.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.949651E-01 | lm-global-supervised-tokens: 1.274442E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.157603E-02 | lm-token-buffer-mib: 5.807116E+01 | global_load_balancing_loss: 9.991647E-01 | loss scale: 1.0 | grad norm: 0.068 | params norm: 11501.794 | params rms: 0.062278 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 18:06:54.655067 | finish at 2026-10-10 16:28:02 + [2026-10-05 22:22:34] iteration 2174/ 6855 | consumed samples: 278272 | elapsed time per iteration (ms): 86539.3 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.46% | FLOPs estimator: workload-aware | logical tokens/s: 193867.6 | valid tokens/s: 193867.6 | physical tokens/s: 194400.3 | loss tokens/s: 143150.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.925675E-01 | lm-global-supervised-tokens: 1.238811E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.098820E-02 | lm-token-buffer-mib: 5.831210E+01 | global_load_balancing_loss: 1.000303E+00 | loss scale: 1.0 | grad norm: 0.051 | params norm: 11501.724 | params rms: 0.062278 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 16:31:30.507777 | finish at 2026-10-10 14:54:04 + [2026-10-05 22:24:01] iteration 2175/ 6855 | consumed samples: 278400 | elapsed time per iteration (ms): 87574.8 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.52% | FLOPs estimator: workload-aware | logical tokens/s: 191575.5 | valid tokens/s: 191575.5 | physical tokens/s: 192146.3 | loss tokens/s: 138893.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.435416E-01 | lm-global-supervised-tokens: 1.216353E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.402868E-02 | lm-token-buffer-mib: 5.814520E+01 | global_load_balancing_loss: 1.000659E+00 | loss scale: 1.0 | grad norm: 0.103 | params norm: 11501.657 | params rms: 0.062278 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 17:50:49.837561 | finish at 2026-10-10 16:14:51 + [2026-10-05 22:25:29] iteration 2176/ 6855 | consumed samples: 278528 | elapsed time per iteration (ms): 87423.3 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.70% | FLOPs estimator: workload-aware | logical tokens/s: 191907.4 | valid tokens/s: 191907.4 | physical tokens/s: 192356.2 | loss tokens/s: 147276.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.284319E-01 | lm-global-supervised-tokens: 1.287542E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.357935E-02 | lm-token-buffer-mib: 5.812603E+01 | global_load_balancing_loss: 9.997948E-01 | loss scale: 1.0 | grad norm: 0.050 | params norm: 11501.590 | params rms: 0.062277 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 17:37:33.681071 | finish at 2026-10-10 16:03:02 + [2026-10-05 22:26:56] iteration 2177/ 6855 | consumed samples: 278656 | elapsed time per iteration (ms): 87551.4 | throughput per GPU (TFLOP/s/GPU): 41.6 | MFU 13.34% | FLOPs estimator: workload-aware | logical tokens/s: 191627.0 | valid tokens/s: 191627.0 | physical tokens/s: 192134.7 | loss tokens/s: 139289.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.307486E-01 | lm-global-supervised-tokens: 1.219498E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.164953E-02 | lm-token-buffer-mib: 5.836634E+01 | global_load_balancing_loss: 1.001746E+00 | loss scale: 1.0 | grad norm: 0.109 | params norm: 11501.522 | params rms: 0.062277 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 17:46:05.429988 | finish at 2026-10-10 16:13:02 + [2026-10-05 22:28:19] iteration 2178/ 6855 | consumed samples: 278784 | elapsed time per iteration (ms): 82577.1 | throughput per GPU (TFLOP/s/GPU): 43.7 | MFU 14.00% | FLOPs estimator: workload-aware | logical tokens/s: 203170.2 | valid tokens/s: 203170.2 | physical tokens/s: 203661.6 | loss tokens/s: 156160.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.505243E-01 | lm-global-supervised-tokens: 1.289525E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.445314E-02 | lm-token-buffer-mib: 5.816172E+01 | global_load_balancing_loss: 1.000358E+00 | loss scale: 1.0 | grad norm: 0.062 | params norm: 11501.456 | params rms: 0.062277 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 11:16:52.888360 | finish at 2026-10-10 09:45:12 + [2026-10-05 22:29:42] iteration 2179/ 6855 | consumed samples: 278912 | elapsed time per iteration (ms): 83030.0 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.77% | FLOPs estimator: workload-aware | logical tokens/s: 202062.0 | valid tokens/s: 202062.0 | physical tokens/s: 202456.0 | loss tokens/s: 149814.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.283664E-01 | lm-global-supervised-tokens: 1.243908E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.465564E-02 | lm-token-buffer-mib: 5.809142E+01 | global_load_balancing_loss: 1.001175E+00 | loss scale: 1.0 | grad norm: 0.036 | params norm: 11501.391 | params rms: 0.062276 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 11:50:48.291015 | finish at 2026-10-10 10:20:30 + [2026-10-05 22:31:28] iteration 2180/ 6855 | consumed samples: 279040 | elapsed time per iteration (ms): 106183.4 | throughput per GPU (TFLOP/s/GPU): 33.9 | MFU 10.87% | FLOPs estimator: workload-aware | logical tokens/s: 158001.9 | valid tokens/s: 158001.8 | physical tokens/s: 158417.4 | loss tokens/s: 124771.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.481441E-01 | lm-global-supervised-tokens: 1.324871E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.279571E-02 | lm-token-buffer-mib: 5.821201E+01 | global_load_balancing_loss: 9.994976E-01 | loss scale: 1.0 | grad norm: 0.076 | params norm: 11501.325 | params rms: 0.062276 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 17:53:27.385689 | finish at 2026-10-11 16:24:55 + [2026-10-05 22:32:57] iteration 2181/ 6855 | consumed samples: 279168 | elapsed time per iteration (ms): 89069.3 | throughput per GPU (TFLOP/s/GPU): 41.7 | MFU 13.36% | FLOPs estimator: workload-aware | logical tokens/s: 188361.0 | valid tokens/s: 188361.0 | physical tokens/s: 188732.6 | loss tokens/s: 134587.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.261303E-01 | lm-global-supervised-tokens: 1.198761E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.186801E-02 | lm-token-buffer-mib: 5.813737E+01 | global_load_balancing_loss: 9.992827E-01 | loss scale: 1.0 | grad norm: 0.035 | params norm: 11501.259 | params rms: 0.062276 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 19:38:30.122975 | finish at 2026-10-10 18:11:27 + [2026-10-05 22:34:23] iteration 2182/ 6855 | consumed samples: 279296 | elapsed time per iteration (ms): 86066.4 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.52% | FLOPs estimator: workload-aware | logical tokens/s: 194932.3 | valid tokens/s: 194932.3 | physical tokens/s: 195646.4 | loss tokens/s: 147745.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.001013E-01 | lm-global-supervised-tokens: 1.271595E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.299204E-02 | lm-token-buffer-mib: 5.822029E+01 | global_load_balancing_loss: 1.000453E+00 | loss scale: 1.0 | grad norm: 0.065 | params norm: 11501.191 | params rms: 0.062275 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 15:43:08.459015 | finish at 2026-10-10 14:17:32 + [2026-10-05 22:35:51] iteration 2183/ 6855 | consumed samples: 279424 | elapsed time per iteration (ms): 88454.7 | throughput per GPU (TFLOP/s/GPU): 42.8 | MFU 13.73% | FLOPs estimator: workload-aware | logical tokens/s: 189669.7 | valid tokens/s: 189669.7 | physical tokens/s: 190042.2 | loss tokens/s: 140341.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.281511E-01 | lm-global-supervised-tokens: 1.241388E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.349566E-02 | lm-token-buffer-mib: 5.817097E+01 | global_load_balancing_loss: 1.000758E+00 | loss scale: 1.0 | grad norm: 0.055 | params norm: 11501.121 | params rms: 0.062275 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 18:47:40.326065 | finish at 2026-10-10 17:23:32 + [2026-10-05 22:37:20] iteration 2184/ 6855 | consumed samples: 279552 | elapsed time per iteration (ms): 88731.4 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.63% | FLOPs estimator: workload-aware | logical tokens/s: 189078.5 | valid tokens/s: 189078.5 | physical tokens/s: 189462.5 | loss tokens/s: 134506.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.649555E-01 | lm-global-supervised-tokens: 1.193499E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.304161E-02 | lm-token-buffer-mib: 5.806355E+01 | global_load_balancing_loss: 1.000678E+00 | loss scale: 1.0 | grad norm: 0.050 | params norm: 11501.051 | params rms: 0.062274 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 19:07:44.569918 | finish at 2026-10-10 17:45:05 + [2026-10-05 22:38:48] iteration 2185/ 6855 | consumed samples: 279680 | elapsed time per iteration (ms): 87296.3 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.51% | FLOPs estimator: workload-aware | logical tokens/s: 192186.8 | valid tokens/s: 192186.8 | physical tokens/s: 192625.0 | loss tokens/s: 147727.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.073618E-01 | lm-global-supervised-tokens: 1.289603E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.572431E-02 | lm-token-buffer-mib: 5.819004E+01 | global_load_balancing_loss: 9.991052E-01 | loss scale: 1.0 | grad norm: 0.052 | params norm: 11500.981 | params rms: 0.062274 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 17:14:33.905520 | finish at 2026-10-10 15:53:21 + [2026-10-05 22:40:15] iteration 2186/ 6855 | consumed samples: 279808 | elapsed time per iteration (ms): 87034.5 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.46% | FLOPs estimator: workload-aware | logical tokens/s: 192764.7 | valid tokens/s: 192764.7 | physical tokens/s: 193162.1 | loss tokens/s: 139724.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.813121E-01 | lm-global-supervised-tokens: 1.216085E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.919759E-02 | lm-token-buffer-mib: 5.807660E+01 | global_load_balancing_loss: 9.996669E-01 | loss scale: 1.0 | grad norm: 0.046 | params norm: 11500.911 | params rms: 0.062274 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 16:52:43.945262 | finish at 2026-10-10 15:32:59 + [2026-10-05 22:41:39] iteration 2187/ 6855 | consumed samples: 279936 | elapsed time per iteration (ms): 84742.8 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.69% | FLOPs estimator: workload-aware | logical tokens/s: 197977.6 | valid tokens/s: 197977.6 | physical tokens/s: 198504.5 | loss tokens/s: 151377.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.193814E-01 | lm-global-supervised-tokens: 1.282813E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.371009E-02 | lm-token-buffer-mib: 5.808445E+01 | global_load_balancing_loss: 9.987936E-01 | loss scale: 1.0 | grad norm: 0.068 | params norm: 11500.840 | params rms: 0.062273 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 13:52:59.298014 | finish at 2026-10-10 12:34:39 + [2026-10-05 22:43:05] iteration 2188/ 6855 | consumed samples: 280064 | elapsed time per iteration (ms): 85814.8 | throughput per GPU (TFLOP/s/GPU): 43.1 | MFU 13.81% | FLOPs estimator: workload-aware | logical tokens/s: 195504.6 | valid tokens/s: 195504.6 | physical tokens/s: 195791.2 | loss tokens/s: 141990.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.593103E-01 | lm-global-supervised-tokens: 1.218486E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.148525E-02 | lm-token-buffer-mib: 5.804012E+01 | global_load_balancing_loss: 1.000599E+00 | loss scale: 1.0 | grad norm: 0.050 | params norm: 11500.773 | params rms: 0.062273 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 15:14:57.498131 | finish at 2026-10-10 13:58:03 + [2026-10-05 22:44:33] iteration 2189/ 6855 | consumed samples: 280192 | elapsed time per iteration (ms): 87626.1 | throughput per GPU (TFLOP/s/GPU): 43.8 | MFU 14.05% | FLOPs estimator: workload-aware | logical tokens/s: 191463.5 | valid tokens/s: 191463.5 | physical tokens/s: 191843.9 | loss tokens/s: 133115.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.013851E-01 | lm-global-supervised-tokens: 1.166436E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.413208E-02 | lm-token-buffer-mib: 5.828115E+01 | global_load_balancing_loss: 1.001577E+00 | loss scale: 1.0 | grad norm: 0.046 | params norm: 11500.705 | params rms: 0.062273 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 17:34:23.576464 | finish at 2026-10-10 16:18:56 + [2026-10-05 22:46:19] iteration 2190/ 6855 | consumed samples: 280320 | elapsed time per iteration (ms): 106710.9 | throughput per GPU (TFLOP/s/GPU): 35.1 | MFU 11.25% | FLOPs estimator: workload-aware | logical tokens/s: 157220.6 | valid tokens/s: 157220.6 | physical tokens/s: 157564.5 | loss tokens/s: 118343.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.773877E-01 | lm-global-supervised-tokens: 1.262856E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.290425E-02 | lm-token-buffer-mib: 5.806462E+01 | global_load_balancing_loss: 1.000410E+00 | loss scale: 1.0 | grad norm: 0.047 | params norm: 11500.637 | params rms: 0.062272 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 18:16:46.561253 | finish at 2026-10-11 17:03:06 + [2026-10-05 22:47:50] iteration 2191/ 6855 | consumed samples: 280448 | elapsed time per iteration (ms): 90131.9 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.60% | FLOPs estimator: workload-aware | logical tokens/s: 186140.7 | valid tokens/s: 186140.7 | physical tokens/s: 186530.6 | loss tokens/s: 130529.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.634363E-01 | lm-global-supervised-tokens: 1.176487E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.307523E-02 | lm-token-buffer-mib: 5.805362E+01 | global_load_balancing_loss: 1.000358E+00 | loss scale: 1.0 | grad norm: 0.074 | params norm: 11500.572 | params rms: 0.062272 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 20:46:15.084082 | finish at 2026-10-10 19:34:05 + [2026-10-05 22:49:16] iteration 2192/ 6855 | consumed samples: 280576 | elapsed time per iteration (ms): 86675.9 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.54% | FLOPs estimator: workload-aware | logical tokens/s: 193562.0 | valid tokens/s: 193562.0 | physical tokens/s: 194117.1 | loss tokens/s: 145487.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.170457E-01 | lm-global-supervised-tokens: 1.261024E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.395204E-02 | lm-token-buffer-mib: 5.815683E+01 | global_load_balancing_loss: 9.999828E-01 | loss scale: 1.0 | grad norm: 0.093 | params norm: 11500.505 | params rms: 0.062271 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 16:16:09.498157 | finish at 2026-10-10 15:05:26 + [2026-10-05 22:50:43] iteration 2193/ 6855 | consumed samples: 280704 | elapsed time per iteration (ms): 86430.1 | throughput per GPU (TFLOP/s/GPU): 43.5 | MFU 13.94% | FLOPs estimator: workload-aware | logical tokens/s: 194112.5 | valid tokens/s: 194112.5 | physical tokens/s: 194573.0 | loss tokens/s: 143368.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.247130E-01 | lm-global-supervised-tokens: 1.239134E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.439695E-02 | lm-token-buffer-mib: 5.813513E+01 | global_load_balancing_loss: 9.988210E-01 | loss scale: 1.0 | grad norm: 0.077 | params norm: 11500.437 | params rms: 0.062271 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 15:55:37.044893 | finish at 2026-10-10 14:46:20 + [2026-10-05 22:52:13] iteration 2194/ 6855 | consumed samples: 280832 | elapsed time per iteration (ms): 90057.7 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.57% | FLOPs estimator: workload-aware | logical tokens/s: 186294.1 | valid tokens/s: 186294.1 | physical tokens/s: 186775.8 | loss tokens/s: 135589.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.392849E-01 | lm-global-supervised-tokens: 1.221091E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.546246E-02 | lm-token-buffer-mib: 5.806507E+01 | global_load_balancing_loss: 9.993516E-01 | loss scale: 1.0 | grad norm: 0.060 | params norm: 11500.369 | params rms: 0.062271 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 20:35:58.775965 | finish at 2026-10-10 19:28:12 + [2026-10-05 22:53:38] iteration 2195/ 6855 | consumed samples: 280960 | elapsed time per iteration (ms): 85103.2 | throughput per GPU (TFLOP/s/GPU): 43.7 | MFU 13.99% | FLOPs estimator: workload-aware | logical tokens/s: 197139.3 | valid tokens/s: 197139.3 | physical tokens/s: 197583.4 | loss tokens/s: 152407.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.950936E-01 | lm-global-supervised-tokens: 1.297036E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.515708E-02 | lm-token-buffer-mib: 5.812184E+01 | global_load_balancing_loss: 9.993030E-01 | loss scale: 1.0 | grad norm: 0.167 | params norm: 11500.303 | params rms: 0.062270 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 14:09:41.128459 | finish at 2026-10-10 13:03:19 + [2026-10-05 22:55:04] iteration 2196/ 6855 | consumed samples: 281088 | elapsed time per iteration (ms): 86346.2 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.70% | FLOPs estimator: workload-aware | logical tokens/s: 194301.7 | valid tokens/s: 194301.7 | physical tokens/s: 194781.0 | loss tokens/s: 144526.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.708716E-01 | lm-global-supervised-tokens: 1.247929E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.392593E-02 | lm-token-buffer-mib: 5.830185E+01 | global_load_balancing_loss: 1.000876E+00 | loss scale: 1.0 | grad norm: 0.151 | params norm: 11500.238 | params rms: 0.062270 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 15:44:47.095707 | finish at 2026-10-10 14:39:51 + [2026-10-05 22:56:35] iteration 2197/ 6855 | consumed samples: 281216 | elapsed time per iteration (ms): 90586.1 | throughput per GPU (TFLOP/s/GPU): 40.9 | MFU 13.10% | FLOPs estimator: workload-aware | logical tokens/s: 185207.4 | valid tokens/s: 185207.4 | physical tokens/s: 185931.0 | loss tokens/s: 134286.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.589118E-01 | lm-global-supervised-tokens: 1.216447E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.340868E-02 | lm-token-buffer-mib: 5.810510E+01 | global_load_balancing_loss: 1.000612E+00 | loss scale: 1.0 | grad norm: 0.057 | params norm: 11500.171 | params rms: 0.062270 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 21:12:30.046499 | finish at 2026-10-10 20:09:05 + [2026-10-05 22:58:03] iteration 2198/ 6855 | consumed samples: 281344 | elapsed time per iteration (ms): 88494.9 | throughput per GPU (TFLOP/s/GPU): 41.7 | MFU 13.37% | FLOPs estimator: workload-aware | logical tokens/s: 189583.7 | valid tokens/s: 189583.7 | physical tokens/s: 190007.1 | loss tokens/s: 142557.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.760191E-01 | lm-global-supervised-tokens: 1.261559E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.705437E-02 | lm-token-buffer-mib: 5.815292E+01 | global_load_balancing_loss: 1.001130E+00 | loss scale: 1.0 | grad norm: 0.167 | params norm: 11500.104 | params rms: 0.062269 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 18:28:40.569374 | finish at 2026-10-10 17:26:44 + [2026-10-05 22:59:29] iteration 2199/ 6855 | consumed samples: 281472 | elapsed time per iteration (ms): 86204.0 | throughput per GPU (TFLOP/s/GPU): 43.1 | MFU 13.82% | FLOPs estimator: workload-aware | logical tokens/s: 194621.7 | valid tokens/s: 194621.7 | physical tokens/s: 195167.8 | loss tokens/s: 144640.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.904642E-01 | lm-global-supervised-tokens: 1.246860E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.550438E-02 | lm-token-buffer-mib: 5.828484E+01 | global_load_balancing_loss: 1.000971E+00 | loss scale: 1.0 | grad norm: 0.106 | params norm: 11500.037 | params rms: 0.062269 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 15:29:25.721855 | finish at 2026-10-10 14:28:55 + [2026-10-05 23:01:17] iteration 2200/ 6855 | consumed samples: 281600 | elapsed time per iteration (ms): 107872.0 | throughput per GPU (TFLOP/s/GPU): 34.0 | MFU 10.91% | FLOPs estimator: workload-aware | logical tokens/s: 155528.6 | valid tokens/s: 155528.6 | physical tokens/s: 155984.0 | loss tokens/s: 115238.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.731472E-01 | lm-global-supervised-tokens: 1.243097E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.311383E-02 | lm-token-buffer-mib: 5.804279E+01 | global_load_balancing_loss: 1.001760E+00 | loss scale: 1.0 | grad norm: 0.076 | params norm: 11499.971 | params rms: 0.062269 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 19:29:04.085543 | finish at 2026-10-11 18:30:21 +times across ranks (ms): + save-checkpoint: + rank 0: 24880.36 + rank 1: 24880.35 + rank 2: 24880.37 + rank 3: 24880.34 + rank 4: 24880.36 + rank 5: 24880.36 + rank 6: 24880.36 + rank 7: 24880.37 + rank 8: 24880.67 + rank 9: 24880.67 + rank 10: 24880.66 + rank 11: 24880.65 + rank 12: 24880.76 + rank 13: 24880.65 + rank 14: 24880.67 + rank 15: 24880.65 + rank 16: 24880.75 + rank 17: 24880.73 + rank 18: 24880.73 + rank 19: 24880.75 + rank 20: 24880.72 + rank 21: 24880.75 + rank 22: 24880.72 + rank 23: 24880.76 + rank 24: 24881.29 + rank 25: 24881.19 + rank 26: 24881.22 + rank 27: 24881.24 + rank 28: 24881.18 + rank 29: 24881.21 + rank 30: 24881.21 + rank 31: 24881.20 + rank 32: 24881.52 + rank 33: 24881.52 + rank 34: 24881.51 + rank 35: 24881.51 + rank 36: 24881.51 + rank 37: 24881.51 + rank 38: 24881.50 + rank 39: 24881.50 + rank 40: 24881.46 + rank 41: 24881.46 + rank 42: 24881.43 + rank 43: 24881.46 + rank 44: 24881.44 + rank 45: 24881.44 + rank 46: 24881.44 + rank 47: 24881.43 + rank 48: 24881.14 + rank 49: 24881.17 + rank 50: 24881.12 + rank 51: 24881.12 + rank 52: 24881.13 + rank 53: 24881.11 + rank 54: 24881.20 + rank 55: 24881.13 + rank 56: 24880.62 + rank 57: 24880.62 + rank 58: 24880.61 + rank 59: 24880.59 + rank 60: 24880.59 + rank 61: 24880.61 + rank 62: 24880.60 + rank 63: 24880.61 + [2026-10-05 23:03:44] iteration 2201/ 6855 | consumed samples: 281728 | elapsed time per iteration (ms): 120855.2 | throughput per GPU (TFLOP/s/GPU): 30.4 | MFU 9.73% | FLOPs estimator: workload-aware | logical tokens/s: 138820.2 | valid tokens/s: 138820.2 | physical tokens/s: 139236.3 | loss tokens/s: 104969.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.851919E-01 | lm-global-supervised-tokens: 1.268609E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 6.085235E+00 | lm-token-buffer-mib: 5.812624E+01 | global_load_balancing_loss: 1.000401E+00 | loss scale: 1.0 | grad norm: 0.072 | params norm: 11499.903 | params rms: 0.062268 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6 days, 12:14:19.889110 | finish at 2026-10-12 11:18:03 + [2026-10-05 23:05:18] iteration 2202/ 6855 | consumed samples: 281856 | elapsed time per iteration (ms): 94579.4 | throughput per GPU (TFLOP/s/GPU): 39.1 | MFU 12.53% | FLOPs estimator: workload-aware | logical tokens/s: 177387.0 | valid tokens/s: 177387.0 | physical tokens/s: 177750.9 | loss tokens/s: 133502.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.168759E-01 | lm-global-supervised-tokens: 1.262659E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.071189E-02 | lm-token-buffer-mib: 5.802965E+01 | global_load_balancing_loss: 9.996285E-01 | loss scale: 1.0 | grad norm: 0.047 | params norm: 11499.833 | params rms: 0.062268 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 2:14:37.802055 | finish at 2026-10-11 01:19:56 + [2026-10-05 23:06:44] iteration 2203/ 6855 | consumed samples: 281984 | elapsed time per iteration (ms): 86329.0 | throughput per GPU (TFLOP/s/GPU): 42.8 | MFU 13.71% | FLOPs estimator: workload-aware | logical tokens/s: 194339.6 | valid tokens/s: 194339.6 | physical tokens/s: 194793.7 | loss tokens/s: 143509.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.909443E-01 | lm-global-supervised-tokens: 1.238904E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.274342E-02 | lm-token-buffer-mib: 5.808559E+01 | global_load_balancing_loss: 9.988016E-01 | loss scale: 1.0 | grad norm: 0.073 | params norm: 11499.763 | params rms: 0.062267 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 15:33:22.699861 | finish at 2026-10-10 14:40:07 + [2026-10-05 23:08:10] iteration 2204/ 6855 | consumed samples: 282112 | elapsed time per iteration (ms): 85658.6 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.77% | FLOPs estimator: workload-aware | logical tokens/s: 195861.5 | valid tokens/s: 195861.5 | physical tokens/s: 196186.0 | loss tokens/s: 150266.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.862602E-01 | lm-global-supervised-tokens: 1.287158E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.176985E-02 | lm-token-buffer-mib: 5.803980E+01 | global_load_balancing_loss: 9.983844E-01 | loss scale: 1.0 | grad norm: 0.051 | params norm: 11499.694 | params rms: 0.062267 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 14:39:57.931686 | finish at 2026-10-10 13:48:08 + [2026-10-05 23:09:36] iteration 2205/ 6855 | consumed samples: 282240 | elapsed time per iteration (ms): 86123.5 | throughput per GPU (TFLOP/s/GPU): 43.1 | MFU 13.81% | FLOPs estimator: workload-aware | logical tokens/s: 194803.5 | valid tokens/s: 194803.5 | physical tokens/s: 195242.1 | loss tokens/s: 139911.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.427187E-01 | lm-global-supervised-tokens: 1.204963E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.056348E-02 | lm-token-buffer-mib: 5.821967E+01 | global_load_balancing_loss: 1.000123E+00 | loss scale: 1.0 | grad norm: 0.066 | params norm: 11499.626 | params rms: 0.062267 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 15:14:34.456215 | finish at 2026-10-10 14:24:11 + [2026-10-05 23:11:03] iteration 2206/ 6855 | consumed samples: 282368 | elapsed time per iteration (ms): 86634.8 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.65% | FLOPs estimator: workload-aware | logical tokens/s: 193654.4 | valid tokens/s: 193654.4 | physical tokens/s: 193982.7 | loss tokens/s: 143905.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.575972E-01 | lm-global-supervised-tokens: 1.246726E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.229313E-02 | lm-token-buffer-mib: 5.808627E+01 | global_load_balancing_loss: 1.000962E+00 | loss scale: 1.0 | grad norm: 0.062 | params norm: 11499.556 | params rms: 0.062266 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 15:52:45.367889 | finish at 2026-10-10 15:03:48 + [2026-10-05 23:12:30] iteration 2207/ 6855 | consumed samples: 282496 | elapsed time per iteration (ms): 86783.5 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.41% | FLOPs estimator: workload-aware | logical tokens/s: 193322.4 | valid tokens/s: 193322.4 | physical tokens/s: 193955.3 | loss tokens/s: 149040.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.805241E-01 | lm-global-supervised-tokens: 1.293420E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.292043E-02 | lm-token-buffer-mib: 5.819883E+01 | global_load_balancing_loss: 1.001380E+00 | loss scale: 1.0 | grad norm: 0.121 | params norm: 11499.486 | params rms: 0.062266 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 16:02:49.512758 | finish at 2026-10-10 15:15:19 + [2026-10-05 23:14:00] iteration 2208/ 6855 | consumed samples: 282624 | elapsed time per iteration (ms): 90082.4 | throughput per GPU (TFLOP/s/GPU): 41.1 | MFU 13.17% | FLOPs estimator: workload-aware | logical tokens/s: 186242.7 | valid tokens/s: 186242.7 | physical tokens/s: 186742.9 | loss tokens/s: 140762.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.309909E-01 | lm-global-supervised-tokens: 1.268019E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.551174E-02 | lm-token-buffer-mib: 5.809451E+01 | global_load_balancing_loss: 1.001117E+00 | loss scale: 1.0 | grad norm: 0.071 | params norm: 11499.416 | params rms: 0.062266 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 20:16:53.058327 | finish at 2026-10-10 19:30:53 + [2026-10-05 23:15:27] iteration 2209/ 6855 | consumed samples: 282752 | elapsed time per iteration (ms): 86920.2 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.56% | FLOPs estimator: workload-aware | logical tokens/s: 193018.5 | valid tokens/s: 193018.5 | physical tokens/s: 193470.0 | loss tokens/s: 137857.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.404005E-01 | lm-global-supervised-tokens: 1.198260E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.576087E-02 | lm-token-buffer-mib: 5.804527E+01 | global_load_balancing_loss: 1.000354E+00 | loss scale: 1.0 | grad norm: 0.047 | params norm: 11499.346 | params rms: 0.062265 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 16:10:31.119001 | finish at 2026-10-10 15:25:58 + [2026-10-05 23:17:15] iteration 2210/ 6855 | consumed samples: 282880 | elapsed time per iteration (ms): 108163.1 | throughput per GPU (TFLOP/s/GPU): 34.5 | MFU 11.04% | FLOPs estimator: workload-aware | logical tokens/s: 155110.4 | valid tokens/s: 155110.4 | physical tokens/s: 155435.4 | loss tokens/s: 117907.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.300582E-01 | lm-global-supervised-tokens: 1.275324E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.255382E-02 | lm-token-buffer-mib: 5.807364E+01 | global_load_balancing_loss: 9.991440E-01 | loss scale: 1.0 | grad norm: 0.044 | params norm: 11499.276 | params rms: 0.062265 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 19:33:37.545254 | finish at 2026-10-11 18:50:52 + [2026-10-05 23:18:42] iteration 2211/ 6855 | consumed samples: 283008 | elapsed time per iteration (ms): 86936.0 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.40% | FLOPs estimator: workload-aware | logical tokens/s: 192982.0 | valid tokens/s: 192981.9 | physical tokens/s: 193476.6 | loss tokens/s: 144506.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.581176E-01 | lm-global-supervised-tokens: 1.256284E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.633540E-02 | lm-token-buffer-mib: 5.809634E+01 | global_load_balancing_loss: 9.997773E-01 | loss scale: 1.0 | grad norm: 0.135 | params norm: 11499.207 | params rms: 0.062264 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 16:08:50.683748 | finish at 2026-10-10 15:27:32 + [2026-10-05 23:20:09] iteration 2212/ 6855 | consumed samples: 283136 | elapsed time per iteration (ms): 87307.7 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.52% | FLOPs estimator: workload-aware | logical tokens/s: 192161.6 | valid tokens/s: 192161.6 | physical tokens/s: 192658.7 | loss tokens/s: 142986.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.186994E-01 | lm-global-supervised-tokens: 1.248378E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.850852E-02 | lm-token-buffer-mib: 5.815266E+01 | global_load_balancing_loss: 9.990616E-01 | loss scale: 1.0 | grad norm: 0.272 | params norm: 11499.137 | params rms: 0.062264 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 16:36:09.444825 | finish at 2026-10-10 15:56:19 + [2026-10-05 23:21:36] iteration 2213/ 6855 | consumed samples: 283264 | elapsed time per iteration (ms): 87384.6 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.62% | FLOPs estimator: workload-aware | logical tokens/s: 191992.7 | valid tokens/s: 191992.7 | physical tokens/s: 192354.2 | loss tokens/s: 144025.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.126848E-01 | lm-global-supervised-tokens: 1.258564E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.283098E-02 | lm-token-buffer-mib: 5.805118E+01 | global_load_balancing_loss: 9.990478E-01 | loss scale: 1.0 | grad norm: 0.292 | params norm: 11499.068 | params rms: 0.062264 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 16:40:39.333876 | finish at 2026-10-10 16:02:16 + [2026-10-05 23:23:01] iteration 2214/ 6855 | consumed samples: 283392 | elapsed time per iteration (ms): 85038.5 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.44% | FLOPs estimator: workload-aware | logical tokens/s: 197289.2 | valid tokens/s: 197289.2 | physical tokens/s: 197806.3 | loss tokens/s: 149976.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.340619E-01 | lm-global-supervised-tokens: 1.275380E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 3.514290E-02 | lm-token-buffer-mib: 5.811031E+01 | global_load_balancing_loss: 1.001741E+00 | loss scale: 1.0 | grad norm: 0.175 | params norm: 11498.996 | params rms: 0.062263 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 13:37:43.788371 | finish at 2026-10-10 13:00:45 + [2026-10-05 23:24:31] iteration 2215/ 6855 | consumed samples: 283520 | elapsed time per iteration (ms): 89755.5 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.40% | FLOPs estimator: workload-aware | logical tokens/s: 186920.8 | valid tokens/s: 186920.8 | physical tokens/s: 187438.6 | loss tokens/s: 140306.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.295211E-01 | lm-global-supervised-tokens: 1.259329E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.391454E-02 | lm-token-buffer-mib: 5.810826E+01 | global_load_balancing_loss: 1.001508E+00 | loss scale: 1.0 | grad norm: 0.381 | params norm: 11498.924 | params rms: 0.062263 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 19:41:05.591164 | finish at 2026-10-10 19:05:37 + [2026-10-05 23:25:58] iteration 2216/ 6855 | consumed samples: 283648 | elapsed time per iteration (ms): 86944.7 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.38% | FLOPs estimator: workload-aware | logical tokens/s: 192963.6 | valid tokens/s: 192963.6 | physical tokens/s: 193483.4 | loss tokens/s: 142276.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.139011E-01 | lm-global-supervised-tokens: 1.237016E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.194269E-02 | lm-token-buffer-mib: 5.811154E+01 | global_load_balancing_loss: 1.000670E+00 | loss scale: 1.0 | grad norm: 0.224 | params norm: 11498.852 | params rms: 0.062263 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 16:02:16.336120 | finish at 2026-10-10 15:28:15 + [2026-10-05 23:27:28] iteration 2217/ 6855 | consumed samples: 283776 | elapsed time per iteration (ms): 89693.0 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.43% | FLOPs estimator: workload-aware | logical tokens/s: 187051.4 | valid tokens/s: 187051.4 | physical tokens/s: 187457.5 | loss tokens/s: 135630.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.807393E-01 | lm-global-supervised-tokens: 1.216507E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.471361E-02 | lm-token-buffer-mib: 5.817953E+01 | global_load_balancing_loss: 1.000107E+00 | loss scale: 1.0 | grad norm: 0.297 | params norm: 11498.783 | params rms: 0.062262 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 19:33:16.289173 | finish at 2026-10-10 19:00:44 + [2026-10-05 23:28:57] iteration 2218/ 6855 | consumed samples: 283904 | elapsed time per iteration (ms): 88869.4 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.68% | FLOPs estimator: workload-aware | logical tokens/s: 188784.3 | valid tokens/s: 188784.3 | physical tokens/s: 189180.8 | loss tokens/s: 144554.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.924163E-01 | lm-global-supervised-tokens: 1.284651E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.217688E-02 | lm-token-buffer-mib: 5.818725E+01 | global_load_balancing_loss: 9.986470E-01 | loss scale: 1.0 | grad norm: 0.211 | params norm: 11498.714 | params rms: 0.062262 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 18:28:07.412335 | finish at 2026-10-10 17:57:04 + [2026-10-05 23:30:23] iteration 2219/ 6855 | consumed samples: 284032 | elapsed time per iteration (ms): 86307.2 | throughput per GPU (TFLOP/s/GPU): 42.8 | MFU 13.73% | FLOPs estimator: workload-aware | logical tokens/s: 194389.2 | valid tokens/s: 194389.2 | physical tokens/s: 194930.4 | loss tokens/s: 147726.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.855388E-01 | lm-global-supervised-tokens: 1.274984E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.151175E-02 | lm-token-buffer-mib: 5.809399E+01 | global_load_balancing_loss: 9.990988E-01 | loss scale: 1.0 | grad norm: 0.160 | params norm: 11498.644 | params rms: 0.062261 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 15:08:40.317155 | finish at 2026-10-10 14:39:03 + [2026-10-05 23:32:13] iteration 2220/ 6855 | consumed samples: 284160 | elapsed time per iteration (ms): 109603.3 | throughput per GPU (TFLOP/s/GPU): 33.7 | MFU 10.79% | FLOPs estimator: workload-aware | logical tokens/s: 153071.8 | valid tokens/s: 153071.8 | physical tokens/s: 153448.2 | loss tokens/s: 119038.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.972235E-01 | lm-global-supervised-tokens: 1.304706E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.449619E-02 | lm-token-buffer-mib: 5.818233E+01 | global_load_balancing_loss: 9.999532E-01 | loss scale: 1.0 | grad norm: 0.142 | params norm: 11498.572 | params rms: 0.062261 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 21:06:51.456174 | finish at 2026-10-11 20:39:04 + [2026-10-05 23:33:41] iteration 2221/ 6855 | consumed samples: 284288 | elapsed time per iteration (ms): 88082.9 | throughput per GPU (TFLOP/s/GPU): 42.8 | MFU 13.70% | FLOPs estimator: workload-aware | logical tokens/s: 190470.7 | valid tokens/s: 190470.7 | physical tokens/s: 190849.4 | loss tokens/s: 138685.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.813292E-01 | lm-global-supervised-tokens: 1.221585E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.498176E-02 | lm-token-buffer-mib: 5.823425E+01 | global_load_balancing_loss: 1.002267E+00 | loss scale: 1.0 | grad norm: 0.088 | params norm: 11498.502 | params rms: 0.062261 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 17:22:56.247206 | finish at 2026-10-10 16:56:37 + [2026-10-05 23:35:05] iteration 2222/ 6855 | consumed samples: 284416 | elapsed time per iteration (ms): 84185.1 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.80% | FLOPs estimator: workload-aware | logical tokens/s: 199289.0 | valid tokens/s: 199289.0 | physical tokens/s: 199972.5 | loss tokens/s: 145105.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.529388E-01 | lm-global-supervised-tokens: 1.221574E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.385516E-02 | lm-token-buffer-mib: 5.812618E+01 | global_load_balancing_loss: 1.002547E+00 | loss scale: 1.0 | grad norm: 0.093 | params norm: 11498.430 | params rms: 0.062260 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 12:20:29.624998 | finish at 2026-10-10 11:55:35 + [2026-10-05 23:36:34] iteration 2223/ 6855 | consumed samples: 284544 | elapsed time per iteration (ms): 88713.4 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.43% | FLOPs estimator: workload-aware | logical tokens/s: 189116.8 | valid tokens/s: 189116.8 | physical tokens/s: 189531.9 | loss tokens/s: 138116.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.646255E-01 | lm-global-supervised-tokens: 1.225276E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.143300E-02 | lm-token-buffer-mib: 5.813769E+01 | global_load_balancing_loss: 9.997803E-01 | loss scale: 1.0 | grad norm: 0.096 | params norm: 11498.358 | params rms: 0.062260 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 18:08:40.424103 | finish at 2026-10-10 17:45:14 + [2026-10-05 23:38:05] iteration 2224/ 6855 | consumed samples: 284672 | elapsed time per iteration (ms): 91033.3 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.41% | FLOPs estimator: workload-aware | logical tokens/s: 184297.4 | valid tokens/s: 184297.4 | physical tokens/s: 184638.5 | loss tokens/s: 128762.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.038311E-01 | lm-global-supervised-tokens: 1.172166E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.258996E-02 | lm-token-buffer-mib: 5.805335E+01 | global_load_balancing_loss: 1.000540E+00 | loss scale: 1.0 | grad norm: 0.074 | params norm: 11498.289 | params rms: 0.062259 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 21:06:15.326771 | finish at 2026-10-10 20:44:20 + [2026-10-05 23:39:36] iteration 2225/ 6855 | consumed samples: 284800 | elapsed time per iteration (ms): 91798.9 | throughput per GPU (TFLOP/s/GPU): 41.6 | MFU 13.33% | FLOPs estimator: workload-aware | logical tokens/s: 182760.6 | valid tokens/s: 182760.6 | physical tokens/s: 183245.5 | loss tokens/s: 130090.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.565589E-01 | lm-global-supervised-tokens: 1.194213E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.885059E-02 | lm-token-buffer-mib: 5.806020E+01 | global_load_balancing_loss: 9.989449E-01 | loss scale: 1.0 | grad norm: 0.138 | params norm: 11498.220 | params rms: 0.062259 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 22:03:48.742008 | finish at 2026-10-10 21:43:25 + [2026-10-05 23:41:05] iteration 2226/ 6855 | consumed samples: 284928 | elapsed time per iteration (ms): 88107.3 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.44% | FLOPs estimator: workload-aware | logical tokens/s: 190417.9 | valid tokens/s: 190417.9 | physical tokens/s: 190838.8 | loss tokens/s: 135094.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.577355E-01 | lm-global-supervised-tokens: 1.190277E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.640872E-02 | lm-token-buffer-mib: 5.804722E+01 | global_load_balancing_loss: 9.995384E-01 | loss scale: 1.0 | grad norm: 0.056 | params norm: 11498.149 | params rms: 0.062259 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 17:17:28.694107 | finish at 2026-10-10 16:58:33 + [2026-10-05 23:42:33] iteration 2227/ 6855 | consumed samples: 285056 | elapsed time per iteration (ms): 88510.9 | throughput per GPU (TFLOP/s/GPU): 41.3 | MFU 13.22% | FLOPs estimator: workload-aware | logical tokens/s: 189549.1 | valid tokens/s: 189549.1 | physical tokens/s: 190027.5 | loss tokens/s: 139913.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.188092E-01 | lm-global-supervised-tokens: 1.238389E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.551454E-02 | lm-token-buffer-mib: 5.816848E+01 | global_load_balancing_loss: 9.999609E-01 | loss scale: 1.0 | grad norm: 0.089 | params norm: 11498.079 | params rms: 0.062258 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 17:47:08.504879 | finish at 2026-10-10 17:29:42 + [2026-10-05 23:44:00] iteration 2228/ 6855 | consumed samples: 285184 | elapsed time per iteration (ms): 86439.1 | throughput per GPU (TFLOP/s/GPU): 41.6 | MFU 13.33% | FLOPs estimator: workload-aware | logical tokens/s: 194092.1 | valid tokens/s: 194092.1 | physical tokens/s: 194503.9 | loss tokens/s: 146514.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.531177E-01 | lm-global-supervised-tokens: 1.266459E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.274892E-02 | lm-token-buffer-mib: 5.821677E+01 | global_load_balancing_loss: 1.000176E+00 | loss scale: 1.0 | grad norm: 0.039 | params norm: 11498.010 | params rms: 0.062258 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 15:05:53.855927 | finish at 2026-10-10 14:49:53 + [2026-10-05 23:45:25] iteration 2229/ 6855 | consumed samples: 285312 | elapsed time per iteration (ms): 85136.1 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.78% | FLOPs estimator: workload-aware | logical tokens/s: 197062.6 | valid tokens/s: 197062.6 | physical tokens/s: 197458.1 | loss tokens/s: 144962.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.428027E-01 | lm-global-supervised-tokens: 1.234154E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.208502E-02 | lm-token-buffer-mib: 5.814288E+01 | global_load_balancing_loss: 9.995854E-01 | loss scale: 1.0 | grad norm: 0.047 | params norm: 11497.942 | params rms: 0.062258 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 13:23:59.388190 | finish at 2026-10-10 13:09:24 + [2026-10-05 23:47:07] iteration 2230/ 6855 | consumed samples: 285440 | elapsed time per iteration (ms): 101943.8 | throughput per GPU (TFLOP/s/GPU): 34.8 | MFU 11.14% | FLOPs estimator: workload-aware | logical tokens/s: 164572.8 | valid tokens/s: 164572.7 | physical tokens/s: 165125.6 | loss tokens/s: 123713.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.032155E-01 | lm-global-supervised-tokens: 1.261178E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.182812E-02 | lm-token-buffer-mib: 5.819659E+01 | global_load_balancing_loss: 1.000342E+00 | loss scale: 1.0 | grad norm: 0.055 | params norm: 11497.873 | params rms: 0.062257 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 10:58:10.094721 | finish at 2026-10-11 10:45:17 + [2026-10-05 23:48:34] iteration 2231/ 6855 | consumed samples: 285568 | elapsed time per iteration (ms): 87084.8 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.61% | FLOPs estimator: workload-aware | logical tokens/s: 192653.2 | valid tokens/s: 192653.2 | physical tokens/s: 192998.3 | loss tokens/s: 135945.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.042974E-01 | lm-global-supervised-tokens: 1.183878E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.387402E-02 | lm-token-buffer-mib: 5.816080E+01 | global_load_balancing_loss: 1.001445E+00 | loss scale: 1.0 | grad norm: 0.041 | params norm: 11497.804 | params rms: 0.062257 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 15:51:20.149399 | finish at 2026-10-10 15:39:54 + [2026-10-05 23:49:57] iteration 2232/ 6855 | consumed samples: 285696 | elapsed time per iteration (ms): 83437.8 | throughput per GPU (TFLOP/s/GPU): 44.0 | MFU 14.11% | FLOPs estimator: workload-aware | logical tokens/s: 201073.9 | valid tokens/s: 201073.9 | physical tokens/s: 201439.2 | loss tokens/s: 154275.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.559281E-01 | lm-global-supervised-tokens: 1.287244E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.795733E-02 | lm-token-buffer-mib: 5.805533E+01 | global_load_balancing_loss: 1.000464E+00 | loss scale: 1.0 | grad norm: 0.038 | params norm: 11497.735 | params rms: 0.062256 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 11:08:53.028438 | finish at 2026-10-10 10:58:50 + [2026-10-05 23:51:20] iteration 2233/ 6855 | consumed samples: 285824 | elapsed time per iteration (ms): 82499.8 | throughput per GPU (TFLOP/s/GPU): 43.3 | MFU 13.88% | FLOPs estimator: workload-aware | logical tokens/s: 203359.9 | valid tokens/s: 203359.9 | physical tokens/s: 203833.3 | loss tokens/s: 152446.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.359407E-01 | lm-global-supervised-tokens: 1.257679E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.160908E-02 | lm-token-buffer-mib: 5.810137E+01 | global_load_balancing_loss: 1.000502E+00 | loss scale: 1.0 | grad norm: 0.080 | params norm: 11497.666 | params rms: 0.062256 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 9:55:14.147075 | finish at 2026-10-10 09:46:34 + [2026-10-05 23:52:48] iteration 2234/ 6855 | consumed samples: 285952 | elapsed time per iteration (ms): 88361.4 | throughput per GPU (TFLOP/s/GPU): 42.8 | MFU 13.73% | FLOPs estimator: workload-aware | logical tokens/s: 189870.1 | valid tokens/s: 189870.1 | physical tokens/s: 190250.0 | loss tokens/s: 140903.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.218526E-01 | lm-global-supervised-tokens: 1.245038E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.177240E-02 | lm-token-buffer-mib: 5.811925E+01 | global_load_balancing_loss: 9.995261E-01 | loss scale: 1.0 | grad norm: 0.055 | params norm: 11497.599 | params rms: 0.062256 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 17:25:17.831677 | finish at 2026-10-10 17:18:06 + [2026-10-05 23:54:13] iteration 2235/ 6855 | consumed samples: 286080 | elapsed time per iteration (ms): 85180.6 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.75% | FLOPs estimator: workload-aware | logical tokens/s: 196960.6 | valid tokens/s: 196960.5 | physical tokens/s: 197348.5 | loss tokens/s: 143971.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.331596E-01 | lm-global-supervised-tokens: 1.226354E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.371302E-02 | lm-token-buffer-mib: 5.811410E+01 | global_load_balancing_loss: 1.000625E+00 | loss scale: 1.0 | grad norm: 0.063 | params norm: 11497.529 | params rms: 0.062255 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 13:18:54.225168 | finish at 2026-10-10 13:13:07 + [2026-10-05 23:55:41] iteration 2236/ 6855 | consumed samples: 286208 | elapsed time per iteration (ms): 87446.9 | throughput per GPU (TFLOP/s/GPU): 43.6 | MFU 13.96% | FLOPs estimator: workload-aware | logical tokens/s: 191856.0 | valid tokens/s: 191856.0 | physical tokens/s: 192331.5 | loss tokens/s: 139350.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.465423E-01 | lm-global-supervised-tokens: 1.218577E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.387359E-02 | lm-token-buffer-mib: 5.813368E+01 | global_load_balancing_loss: 1.000096E+00 | loss scale: 1.0 | grad norm: 0.048 | params norm: 11497.459 | params rms: 0.062255 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 16:11:57.310988 | finish at 2026-10-10 16:07:38 + [2026-10-05 23:57:05] iteration 2237/ 6855 | consumed samples: 286336 | elapsed time per iteration (ms): 83999.3 | throughput per GPU (TFLOP/s/GPU): 43.8 | MFU 14.05% | FLOPs estimator: workload-aware | logical tokens/s: 199730.3 | valid tokens/s: 199730.3 | physical tokens/s: 200339.7 | loss tokens/s: 143631.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.733426E-01 | lm-global-supervised-tokens: 1.206495E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.311942E-02 | lm-token-buffer-mib: 5.809602E+01 | global_load_balancing_loss: 1.000414E+00 | loss scale: 1.0 | grad norm: 0.081 | params norm: 11497.386 | params rms: 0.062255 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 11:45:08.606666 | finish at 2026-10-10 11:42:13 + [2026-10-05 23:58:30] iteration 2238/ 6855 | consumed samples: 286464 | elapsed time per iteration (ms): 85846.1 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.59% | FLOPs estimator: workload-aware | logical tokens/s: 195433.3 | valid tokens/s: 195433.3 | physical tokens/s: 195800.2 | loss tokens/s: 145420.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.661233E-01 | lm-global-supervised-tokens: 1.248380E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.803049E-02 | lm-token-buffer-mib: 5.818574E+01 | global_load_balancing_loss: 1.000488E+00 | loss scale: 1.0 | grad norm: 0.035 | params norm: 11497.316 | params rms: 0.062254 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 14:05:51.637861 | finish at 2026-10-10 14:04:22 + [2026-10-06 00:00:01] iteration 2239/ 6855 | consumed samples: 286592 | elapsed time per iteration (ms): 90645.0 | throughput per GPU (TFLOP/s/GPU): 41.5 | MFU 13.29% | FLOPs estimator: workload-aware | logical tokens/s: 185086.9 | valid tokens/s: 185086.9 | physical tokens/s: 185453.1 | loss tokens/s: 134027.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.699874E-01 | lm-global-supervised-tokens: 1.214895E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.588026E-02 | lm-token-buffer-mib: 5.814915E+01 | global_load_balancing_loss: 9.994395E-01 | loss scale: 1.0 | grad norm: 0.049 | params norm: 11497.246 | params rms: 0.062254 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 20:13:37.516909 | finish at 2026-10-10 20:13:39 + [2026-10-06 00:01:49] iteration 2240/ 6855 | consumed samples: 286720 | elapsed time per iteration (ms): 107836.9 | throughput per GPU (TFLOP/s/GPU): 33.4 | MFU 10.71% | FLOPs estimator: workload-aware | logical tokens/s: 155579.1 | valid tokens/s: 155579.1 | physical tokens/s: 156018.0 | loss tokens/s: 116410.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.005520E-01 | lm-global-supervised-tokens: 1.255337E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.358204E-02 | lm-token-buffer-mib: 5.807356E+01 | global_load_balancing_loss: 9.993730E-01 | loss scale: 1.0 | grad norm: 0.111 | params norm: 11497.173 | params rms: 0.062253 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 18:14:27.361699 | finish at 2026-10-11 18:16:16 +times across ranks (ms): + save-checkpoint: + rank 0: 24788.39 + rank 1: 24788.38 + rank 2: 24788.43 + rank 3: 24788.41 + rank 4: 24788.40 + rank 5: 24788.40 + rank 6: 24788.37 + rank 7: 24788.41 + rank 8: 24787.15 + rank 9: 24787.04 + rank 10: 24787.05 + rank 11: 24787.08 + rank 12: 24787.06 + rank 13: 24787.04 + rank 14: 24787.05 + rank 15: 24787.04 + rank 16: 24786.97 + rank 17: 24786.97 + rank 18: 24786.97 + rank 19: 24786.98 + rank 20: 24786.96 + rank 21: 24786.97 + rank 22: 24786.96 + rank 23: 24786.98 + rank 24: 24787.06 + rank 25: 24787.27 + rank 26: 24787.10 + rank 27: 24787.05 + rank 28: 24787.01 + rank 29: 24787.04 + rank 30: 24787.01 + rank 31: 24787.01 + rank 32: 24787.05 + rank 33: 24787.10 + rank 34: 24786.86 + rank 35: 24786.97 + rank 36: 24786.92 + rank 37: 24786.99 + rank 38: 24786.96 + rank 39: 24786.98 + rank 40: 24786.40 + rank 41: 24786.28 + rank 42: 24786.28 + rank 43: 24786.41 + rank 44: 24786.36 + rank 45: 24786.37 + rank 46: 24786.27 + rank 47: 24786.38 + rank 48: 24786.48 + rank 49: 24786.65 + rank 50: 24786.57 + rank 51: 24786.48 + rank 52: 24786.59 + rank 53: 24786.56 + rank 54: 24786.63 + rank 55: 24786.63 + rank 56: 24787.84 + rank 57: 24787.84 + rank 58: 24787.80 + rank 59: 24787.81 + rank 60: 24787.80 + rank 61: 24787.82 + rank 62: 24787.80 + rank 63: 24787.77 + [2026-10-06 00:03:50] iteration 2241/ 6855 | consumed samples: 286848 | elapsed time per iteration (ms): 96077.4 | throughput per GPU (TFLOP/s/GPU): 38.1 | MFU 12.22% | FLOPs estimator: workload-aware | logical tokens/s: 174621.5 | valid tokens/s: 174621.5 | physical tokens/s: 174943.5 | loss tokens/s: 132597.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.687177E-01 | lm-global-supervised-tokens: 1.273965E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.710576E+00 | lm-token-buffer-mib: 5.802518E+01 | global_load_balancing_loss: 1.000065E+00 | loss scale: 1.0 | grad norm: 0.050 | params norm: 11497.102 | params rms: 0.062253 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 3:08:21.340170 | finish at 2026-10-11 03:12:11 + [2026-10-06 00:05:24] iteration 2242/ 6855 | consumed samples: 286976 | elapsed time per iteration (ms): 94408.5 | throughput per GPU (TFLOP/s/GPU): 39.5 | MFU 12.67% | FLOPs estimator: workload-aware | logical tokens/s: 177708.2 | valid tokens/s: 177708.2 | physical tokens/s: 178173.2 | loss tokens/s: 129970.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.886170E-01 | lm-global-supervised-tokens: 1.227028E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.318406E-02 | lm-token-buffer-mib: 5.810753E+01 | global_load_balancing_loss: 1.000678E+00 | loss scale: 1.0 | grad norm: 0.140 | params norm: 11497.033 | params rms: 0.062253 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 0:58:26.579671 | finish at 2026-10-11 01:03:51 + [2026-10-06 00:06:48] iteration 2243/ 6855 | consumed samples: 287104 | elapsed time per iteration (ms): 83532.7 | throughput per GPU (TFLOP/s/GPU): 43.6 | MFU 13.99% | FLOPs estimator: workload-aware | logical tokens/s: 200845.7 | valid tokens/s: 200845.7 | physical tokens/s: 201338.7 | loss tokens/s: 146987.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.874502E-01 | lm-global-supervised-tokens: 1.227825E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.569898E-02 | lm-token-buffer-mib: 5.806927E+01 | global_load_balancing_loss: 1.001143E+00 | loss scale: 1.0 | grad norm: 0.037 | params norm: 11496.963 | params rms: 0.062252 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 11:00:52.869864 | finish at 2026-10-10 11:07:41 + [2026-10-06 00:08:13] iteration 2244/ 6855 | consumed samples: 287232 | elapsed time per iteration (ms): 85382.0 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.63% | FLOPs estimator: workload-aware | logical tokens/s: 196495.3 | valid tokens/s: 196495.3 | physical tokens/s: 196869.6 | loss tokens/s: 148059.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.749162E-01 | lm-global-supervised-tokens: 1.264165E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.164866E-02 | lm-token-buffer-mib: 5.816363E+01 | global_load_balancing_loss: 9.993686E-01 | loss scale: 1.0 | grad norm: 0.067 | params norm: 11496.891 | params rms: 0.062252 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 13:21:36.311713 | finish at 2026-10-10 13:29:50 + [2026-10-06 00:09:41] iteration 2245/ 6855 | consumed samples: 287360 | elapsed time per iteration (ms): 88138.5 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.50% | FLOPs estimator: workload-aware | logical tokens/s: 190350.0 | valid tokens/s: 190350.0 | physical tokens/s: 190765.2 | loss tokens/s: 140716.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.918036E-01 | lm-global-supervised-tokens: 1.240255E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.477920E-02 | lm-token-buffer-mib: 5.807269E+01 | global_load_balancing_loss: 1.001072E+00 | loss scale: 1.0 | grad norm: 0.087 | params norm: 11496.817 | params rms: 0.062252 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 16:51:58.558526 | finish at 2026-10-10 17:01:40 + [2026-10-06 00:11:10] iteration 2246/ 6855 | consumed samples: 287488 | elapsed time per iteration (ms): 88487.0 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.40% | FLOPs estimator: workload-aware | logical tokens/s: 189600.4 | valid tokens/s: 189600.4 | physical tokens/s: 190019.4 | loss tokens/s: 130907.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.259363E-01 | lm-global-supervised-tokens: 1.158359E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.363242E-02 | lm-token-buffer-mib: 5.818453E+01 | global_load_balancing_loss: 1.000276E+00 | loss scale: 1.0 | grad norm: 0.051 | params norm: 11496.744 | params rms: 0.062251 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 17:17:16.480752 | finish at 2026-10-10 17:28:26 + [2026-10-06 00:12:38] iteration 2247/ 6855 | consumed samples: 287616 | elapsed time per iteration (ms): 88363.1 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.61% | FLOPs estimator: workload-aware | logical tokens/s: 189866.0 | valid tokens/s: 189866.0 | physical tokens/s: 190365.3 | loss tokens/s: 140306.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.980070E-01 | lm-global-supervised-tokens: 1.239795E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.422232E-02 | lm-token-buffer-mib: 5.808942E+01 | global_load_balancing_loss: 1.000373E+00 | loss scale: 1.0 | grad norm: 0.072 | params norm: 11496.670 | params rms: 0.062251 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 17:06:17.389160 | finish at 2026-10-10 17:18:56 + [2026-10-06 00:14:05] iteration 2248/ 6855 | consumed samples: 287744 | elapsed time per iteration (ms): 86671.8 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.63% | FLOPs estimator: workload-aware | logical tokens/s: 193570.8 | valid tokens/s: 193570.8 | physical tokens/s: 194188.3 | loss tokens/s: 139163.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.411878E-01 | lm-global-supervised-tokens: 1.206152E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.880185E-02 | lm-token-buffer-mib: 5.834223E+01 | global_load_balancing_loss: 1.000411E+00 | loss scale: 1.0 | grad norm: 0.072 | params norm: 11496.596 | params rms: 0.062250 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 14:54:57.048034 | finish at 2026-10-10 15:09:02 + [2026-10-06 00:15:31] iteration 2249/ 6855 | consumed samples: 287872 | elapsed time per iteration (ms): 86144.4 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.48% | FLOPs estimator: workload-aware | logical tokens/s: 194756.5 | valid tokens/s: 194756.4 | physical tokens/s: 195186.2 | loss tokens/s: 146540.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.043581E-01 | lm-global-supervised-tokens: 1.262367E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.147794E-02 | lm-token-buffer-mib: 5.813390E+01 | global_load_balancing_loss: 9.990715E-01 | loss scale: 1.0 | grad norm: 0.071 | params norm: 11496.523 | params rms: 0.062250 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 14:13:01.270577 | finish at 2026-10-10 14:28:32 + [2026-10-06 00:17:19] iteration 2250/ 6855 | consumed samples: 288000 | elapsed time per iteration (ms): 107459.7 | throughput per GPU (TFLOP/s/GPU): 34.4 | MFU 11.04% | FLOPs estimator: workload-aware | logical tokens/s: 156125.5 | valid tokens/s: 156125.5 | physical tokens/s: 156452.9 | loss tokens/s: 114858.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.511989E-01 | lm-global-supervised-tokens: 1.234269E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.430838E-02 | lm-token-buffer-mib: 5.811861E+01 | global_load_balancing_loss: 1.000201E+00 | loss scale: 1.0 | grad norm: 0.034 | params norm: 11496.452 | params rms: 0.062250 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 17:27:32.123208 | finish at 2026-10-11 17:44:51 + [2026-10-06 00:18:53] iteration 2251/ 6855 | consumed samples: 288128 | elapsed time per iteration (ms): 94322.7 | throughput per GPU (TFLOP/s/GPU): 40.0 | MFU 12.83% | FLOPs estimator: workload-aware | logical tokens/s: 177870.1 | valid tokens/s: 177870.1 | physical tokens/s: 178202.6 | loss tokens/s: 129217.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.346470E-01 | lm-global-supervised-tokens: 1.218813E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.262808E-02 | lm-token-buffer-mib: 5.814127E+01 | global_load_balancing_loss: 9.991064E-01 | loss scale: 1.0 | grad norm: 0.053 | params norm: 11496.381 | params rms: 0.062249 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 0:37:41.538573 | finish at 2026-10-11 00:56:34 + [2026-10-06 00:20:17] iteration 2252/ 6855 | consumed samples: 288256 | elapsed time per iteration (ms): 84215.8 | throughput per GPU (TFLOP/s/GPU): 43.5 | MFU 13.96% | FLOPs estimator: workload-aware | logical tokens/s: 199216.3 | valid tokens/s: 199216.3 | physical tokens/s: 199575.2 | loss tokens/s: 146922.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.600711E-01 | lm-global-supervised-tokens: 1.237317E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.632756E-02 | lm-token-buffer-mib: 5.815085E+01 | global_load_balancing_loss: 1.001028E+00 | loss scale: 1.0 | grad norm: 0.089 | params norm: 11496.311 | params rms: 0.062249 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 11:40:45.166292 | finish at 2026-10-10 12:01:02 + [2026-10-06 00:21:45] iteration 2253/ 6855 | consumed samples: 288384 | elapsed time per iteration (ms): 87712.5 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.65% | FLOPs estimator: workload-aware | logical tokens/s: 191275.0 | valid tokens/s: 191275.0 | physical tokens/s: 191567.6 | loss tokens/s: 140854.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.895381E-01 | lm-global-supervised-tokens: 1.235469E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.242013E-02 | lm-token-buffer-mib: 5.810076E+01 | global_load_balancing_loss: 1.001221E+00 | loss scale: 1.0 | grad norm: 0.050 | params norm: 11496.242 | params rms: 0.062248 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 16:07:32.712582 | finish at 2026-10-10 16:29:18 + [2026-10-06 00:23:14] iteration 2254/ 6855 | consumed samples: 288512 | elapsed time per iteration (ms): 88996.4 | throughput per GPU (TFLOP/s/GPU): 41.2 | MFU 13.21% | FLOPs estimator: workload-aware | logical tokens/s: 188515.0 | valid tokens/s: 188515.0 | physical tokens/s: 188919.6 | loss tokens/s: 137280.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.392056E-01 | lm-global-supervised-tokens: 1.221748E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.468186E-02 | lm-token-buffer-mib: 5.818536E+01 | global_load_balancing_loss: 1.001133E+00 | loss scale: 1.0 | grad norm: 0.051 | params norm: 11496.173 | params rms: 0.062248 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 17:44:32.582839 | finish at 2026-10-10 18:07:46 + [2026-10-06 00:24:41] iteration 2255/ 6855 | consumed samples: 288640 | elapsed time per iteration (ms): 87461.0 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.48% | FLOPs estimator: workload-aware | logical tokens/s: 191824.9 | valid tokens/s: 191824.9 | physical tokens/s: 192322.1 | loss tokens/s: 151443.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.515307E-01 | lm-global-supervised-tokens: 1.324537E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.103595E-02 | lm-token-buffer-mib: 5.826289E+01 | global_load_balancing_loss: 9.993035E-01 | loss scale: 1.0 | grad norm: 0.078 | params norm: 11496.104 | params rms: 0.062248 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 15:45:20.379400 | finish at 2026-10-10 16:10:02 + [2026-10-06 00:26:09] iteration 2256/ 6855 | consumed samples: 288768 | elapsed time per iteration (ms): 87824.6 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.48% | FLOPs estimator: workload-aware | logical tokens/s: 191030.6 | valid tokens/s: 191030.6 | physical tokens/s: 191497.6 | loss tokens/s: 140865.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.342053E-01 | lm-global-supervised-tokens: 1.237141E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.437064E-02 | lm-token-buffer-mib: 5.805222E+01 | global_load_balancing_loss: 9.999676E-01 | loss scale: 1.0 | grad norm: 0.080 | params norm: 11496.036 | params rms: 0.062247 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 16:11:45.177420 | finish at 2026-10-10 16:37:54 + [2026-10-06 00:27:36] iteration 2257/ 6855 | consumed samples: 288896 | elapsed time per iteration (ms): 87365.3 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.55% | FLOPs estimator: workload-aware | logical tokens/s: 192035.1 | valid tokens/s: 192035.1 | physical tokens/s: 192536.3 | loss tokens/s: 149171.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.695870E-01 | lm-global-supervised-tokens: 1.303243E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.611099E-02 | lm-token-buffer-mib: 5.820766E+01 | global_load_balancing_loss: 1.000297E+00 | loss scale: 1.0 | grad norm: 0.081 | params norm: 11495.967 | params rms: 0.062247 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 15:35:05.629392 | finish at 2026-10-10 16:02:42 + [2026-10-06 00:29:02] iteration 2258/ 6855 | consumed samples: 289024 | elapsed time per iteration (ms): 85450.6 | throughput per GPU (TFLOP/s/GPU): 43.3 | MFU 13.86% | FLOPs estimator: workload-aware | logical tokens/s: 196338.1 | valid tokens/s: 196338.0 | physical tokens/s: 196848.4 | loss tokens/s: 146267.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.929269E-01 | lm-global-supervised-tokens: 1.249866E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.300333E-02 | lm-token-buffer-mib: 5.823856E+01 | global_load_balancing_loss: 1.000225E+00 | loss scale: 1.0 | grad norm: 0.037 | params norm: 11495.897 | params rms: 0.062247 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 13:06:56.534918 | finish at 2026-10-10 13:35:58 + [2026-10-06 00:30:31] iteration 2259/ 6855 | consumed samples: 289152 | elapsed time per iteration (ms): 89205.0 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.48% | FLOPs estimator: workload-aware | logical tokens/s: 188073.5 | valid tokens/s: 188073.5 | physical tokens/s: 188479.6 | loss tokens/s: 139163.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.106572E-01 | lm-global-supervised-tokens: 1.241408E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.680428E-02 | lm-token-buffer-mib: 5.802614E+01 | global_load_balancing_loss: 1.000441E+00 | loss scale: 1.0 | grad norm: 0.105 | params norm: 11495.829 | params rms: 0.062246 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 17:53:06.284843 | finish at 2026-10-10 18:23:37 + [2026-10-06 00:32:17] iteration 2260/ 6855 | consumed samples: 289280 | elapsed time per iteration (ms): 105700.6 | throughput per GPU (TFLOP/s/GPU): 34.5 | MFU 11.05% | FLOPs estimator: workload-aware | logical tokens/s: 158723.7 | valid tokens/s: 158723.7 | physical tokens/s: 159264.3 | loss tokens/s: 119992.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.056494E-01 | lm-global-supervised-tokens: 1.268328E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.167168E-02 | lm-token-buffer-mib: 5.828521E+01 | global_load_balancing_loss: 9.995415E-01 | loss scale: 1.0 | grad norm: 0.079 | params norm: 11495.760 | params rms: 0.062246 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 14:54:54.458159 | finish at 2026-10-11 15:27:11 + [2026-10-06 00:33:45] iteration 2261/ 6855 | consumed samples: 289408 | elapsed time per iteration (ms): 88307.7 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.62% | FLOPs estimator: workload-aware | logical tokens/s: 189985.5 | valid tokens/s: 189985.5 | physical tokens/s: 190342.0 | loss tokens/s: 138215.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.294857E-01 | lm-global-supervised-tokens: 1.220548E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.284708E-02 | lm-token-buffer-mib: 5.828554E+01 | global_load_balancing_loss: 1.000609E+00 | loss scale: 1.0 | grad norm: 0.060 | params norm: 11495.691 | params rms: 0.062245 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 16:41:25.693909 | finish at 2026-10-10 17:15:11 + [2026-10-06 00:35:14] iteration 2262/ 6855 | consumed samples: 289536 | elapsed time per iteration (ms): 89118.9 | throughput per GPU (TFLOP/s/GPU): 41.2 | MFU 13.22% | FLOPs estimator: workload-aware | logical tokens/s: 188256.5 | valid tokens/s: 188256.1 | physical tokens/s: 188768.9 | loss tokens/s: 140170.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.019965E-01 | lm-global-supervised-tokens: 1.249185E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.207763E-02 | lm-token-buffer-mib: 5.812644E+01 | global_load_balancing_loss: 9.999354E-01 | loss scale: 1.0 | grad norm: 0.095 | params norm: 11495.624 | params rms: 0.062245 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 17:42:03.267857 | finish at 2026-10-10 18:17:18 + [2026-10-06 00:36:40] iteration 2263/ 6855 | consumed samples: 289664 | elapsed time per iteration (ms): 86055.3 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.54% | FLOPs estimator: workload-aware | logical tokens/s: 194958.6 | valid tokens/s: 194958.6 | physical tokens/s: 195310.5 | loss tokens/s: 138672.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.031713E-01 | lm-global-supervised-tokens: 1.193347E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.066769E-02 | lm-token-buffer-mib: 5.803893E+01 | global_load_balancing_loss: 1.000364E+00 | loss scale: 1.0 | grad norm: 0.075 | params norm: 11495.556 | params rms: 0.062245 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 13:46:05.893795 | finish at 2026-10-10 14:22:46 + [2026-10-06 00:38:07] iteration 2264/ 6855 | consumed samples: 289792 | elapsed time per iteration (ms): 86216.2 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.79% | FLOPs estimator: workload-aware | logical tokens/s: 194593.8 | valid tokens/s: 194593.8 | physical tokens/s: 195230.8 | loss tokens/s: 140021.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.879960E-01 | lm-global-supervised-tokens: 1.207208E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.526212E-02 | lm-token-buffer-mib: 5.819658E+01 | global_load_balancing_loss: 1.000633E+00 | loss scale: 1.0 | grad norm: 0.040 | params norm: 11495.486 | params rms: 0.062244 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 13:56:58.761893 | finish at 2026-10-10 14:35:05 + [2026-10-06 00:39:34] iteration 2265/ 6855 | consumed samples: 289920 | elapsed time per iteration (ms): 87670.4 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.55% | FLOPs estimator: workload-aware | logical tokens/s: 191366.4 | valid tokens/s: 191366.4 | physical tokens/s: 191747.7 | loss tokens/s: 141091.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.548213E-01 | lm-global-supervised-tokens: 1.236954E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.155950E-02 | lm-token-buffer-mib: 5.814636E+01 | global_load_balancing_loss: 9.999198E-01 | loss scale: 1.0 | grad norm: 0.039 | params norm: 11495.419 | params rms: 0.062244 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 15:46:47.293146 | finish at 2026-10-10 16:26:21 + [2026-10-06 00:41:04] iteration 2266/ 6855 | consumed samples: 290048 | elapsed time per iteration (ms): 89530.5 | throughput per GPU (TFLOP/s/GPU): 41.1 | MFU 13.16% | FLOPs estimator: workload-aware | logical tokens/s: 187390.7 | valid tokens/s: 187390.6 | physical tokens/s: 187842.1 | loss tokens/s: 141203.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.640661E-01 | lm-global-supervised-tokens: 1.264204E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.533387E-02 | lm-token-buffer-mib: 5.803017E+01 | global_load_balancing_loss: 1.000225E+00 | loss scale: 1.0 | grad norm: 0.041 | params norm: 11495.351 | params rms: 0.062244 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 18:07:35.558295 | finish at 2026-10-10 18:48:39 + [2026-10-06 00:42:28] iteration 2267/ 6855 | consumed samples: 290176 | elapsed time per iteration (ms): 84652.0 | throughput per GPU (TFLOP/s/GPU): 43.8 | MFU 14.05% | FLOPs estimator: workload-aware | logical tokens/s: 198190.1 | valid tokens/s: 198190.1 | physical tokens/s: 198610.9 | loss tokens/s: 149464.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.409451E-01 | lm-global-supervised-tokens: 1.265245E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.366387E-02 | lm-token-buffer-mib: 5.827081E+01 | global_load_balancing_loss: 9.996854E-01 | loss scale: 1.0 | grad norm: 0.051 | params norm: 11495.284 | params rms: 0.062243 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 11:53:03.496098 | finish at 2026-10-10 12:35:32 + [2026-10-06 00:43:54] iteration 2268/ 6855 | consumed samples: 290304 | elapsed time per iteration (ms): 85236.5 | throughput per GPU (TFLOP/s/GPU): 43.1 | MFU 13.80% | FLOPs estimator: workload-aware | logical tokens/s: 196831.1 | valid tokens/s: 196831.1 | physical tokens/s: 197161.2 | loss tokens/s: 149421.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.159060E-01 | lm-global-supervised-tokens: 1.273620E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.362354E-02 | lm-token-buffer-mib: 5.815227E+01 | global_load_balancing_loss: 9.996026E-01 | loss scale: 1.0 | grad norm: 0.063 | params norm: 11495.216 | params rms: 0.062243 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 12:36:19.860610 | finish at 2026-10-10 13:20:13 + [2026-10-06 00:45:22] iteration 2269/ 6855 | consumed samples: 290432 | elapsed time per iteration (ms): 88645.8 | throughput per GPU (TFLOP/s/GPU): 41.7 | MFU 13.35% | FLOPs estimator: workload-aware | logical tokens/s: 189260.1 | valid tokens/s: 189260.1 | physical tokens/s: 189696.3 | loss tokens/s: 142827.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.770701E-01 | lm-global-supervised-tokens: 1.266110E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.163625E-02 | lm-token-buffer-mib: 5.812555E+01 | global_load_balancing_loss: 1.000760E+00 | loss scale: 1.0 | grad norm: 0.040 | params norm: 11495.149 | params rms: 0.062242 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 16:55:29.829571 | finish at 2026-10-10 17:40:52 + [2026-10-06 00:47:11] iteration 2270/ 6855 | consumed samples: 290560 | elapsed time per iteration (ms): 108865.5 | throughput per GPU (TFLOP/s/GPU): 33.4 | MFU 10.69% | FLOPs estimator: workload-aware | logical tokens/s: 154109.5 | valid tokens/s: 154109.5 | physical tokens/s: 154605.2 | loss tokens/s: 116665.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.349029E-01 | lm-global-supervised-tokens: 1.270085E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.293656E-02 | lm-token-buffer-mib: 5.820673E+01 | global_load_balancing_loss: 1.001474E+00 | loss scale: 1.0 | grad norm: 0.070 | params norm: 11495.081 | params rms: 0.062242 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 18:39:08.092189 | finish at 2026-10-11 19:26:19 + [2026-10-06 00:48:39] iteration 2271/ 6855 | consumed samples: 290688 | elapsed time per iteration (ms): 87502.6 | throughput per GPU (TFLOP/s/GPU): 43.2 | MFU 13.85% | FLOPs estimator: workload-aware | logical tokens/s: 191733.9 | valid tokens/s: 191733.9 | physical tokens/s: 192083.3 | loss tokens/s: 134530.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.277702E-01 | lm-global-supervised-tokens: 1.177176E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.747312E-02 | lm-token-buffer-mib: 5.803083E+01 | global_load_balancing_loss: 1.000924E+00 | loss scale: 1.0 | grad norm: 0.143 | params norm: 11495.012 | params rms: 0.062242 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 15:25:11.921471 | finish at 2026-10-10 16:13:51 + [2026-10-06 00:50:05] iteration 2272/ 6855 | consumed samples: 290816 | elapsed time per iteration (ms): 86819.1 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.48% | FLOPs estimator: workload-aware | logical tokens/s: 193243.4 | valid tokens/s: 193243.4 | physical tokens/s: 193671.5 | loss tokens/s: 140581.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.843021E-01 | lm-global-supervised-tokens: 1.220517E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.144669E-02 | lm-token-buffer-mib: 5.808160E+01 | global_load_balancing_loss: 1.001105E+00 | loss scale: 1.0 | grad norm: 0.273 | params norm: 11494.942 | params rms: 0.062241 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 14:31:31.867110 | finish at 2026-10-10 15:21:37 + [2026-10-06 00:51:32] iteration 2273/ 6855 | consumed samples: 290944 | elapsed time per iteration (ms): 86285.7 | throughput per GPU (TFLOP/s/GPU): 41.7 | MFU 13.37% | FLOPs estimator: workload-aware | logical tokens/s: 194437.6 | valid tokens/s: 194437.6 | physical tokens/s: 194889.1 | loss tokens/s: 144477.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.576930E-01 | lm-global-supervised-tokens: 1.246632E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.155408E-02 | lm-token-buffer-mib: 5.804735E+01 | global_load_balancing_loss: 9.998723E-01 | loss scale: 1.0 | grad norm: 0.347 | params norm: 11494.873 | params rms: 0.062241 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 13:49:21.085426 | finish at 2026-10-10 14:40:53 + [2026-10-06 00:53:04] iteration 2274/ 6855 | consumed samples: 291072 | elapsed time per iteration (ms): 92618.0 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.45% | FLOPs estimator: workload-aware | logical tokens/s: 181143.2 | valid tokens/s: 181143.2 | physical tokens/s: 181499.8 | loss tokens/s: 129615.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.948997E-01 | lm-global-supervised-tokens: 1.200470E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.192833E-02 | lm-token-buffer-mib: 5.809047E+01 | global_load_balancing_loss: 9.992427E-01 | loss scale: 1.0 | grad norm: 0.062 | params norm: 11494.803 | params rms: 0.062241 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 21:51:23.251458 | finish at 2026-10-10 22:44:28 + [2026-10-06 00:54:29] iteration 2275/ 6855 | consumed samples: 291200 | elapsed time per iteration (ms): 84160.4 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.79% | FLOPs estimator: workload-aware | logical tokens/s: 199347.8 | valid tokens/s: 199347.8 | physical tokens/s: 199778.5 | loss tokens/s: 148642.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.643208E-01 | lm-global-supervised-tokens: 1.250981E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.487039E-02 | lm-token-buffer-mib: 5.807665E+01 | global_load_balancing_loss: 9.999270E-01 | loss scale: 1.0 | grad norm: 0.325 | params norm: 11494.732 | params rms: 0.062240 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 11:04:14.701490 | finish at 2026-10-10 11:58:43 + [2026-10-06 00:55:54] iteration 2276/ 6855 | consumed samples: 291328 | elapsed time per iteration (ms): 85914.8 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.59% | FLOPs estimator: workload-aware | logical tokens/s: 195275.9 | valid tokens/s: 195275.9 | physical tokens/s: 195899.1 | loss tokens/s: 144378.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.229753E-01 | lm-global-supervised-tokens: 1.240428E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.846703E-02 | lm-token-buffer-mib: 5.822169E+01 | global_load_balancing_loss: 9.996097E-01 | loss scale: 1.0 | grad norm: 0.186 | params norm: 11494.661 | params rms: 0.062240 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 13:16:44.016255 | finish at 2026-10-10 14:12:38 + [2026-10-06 00:57:23] iteration 2277/ 6855 | consumed samples: 291456 | elapsed time per iteration (ms): 88450.1 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.50% | FLOPs estimator: workload-aware | logical tokens/s: 189680.1 | valid tokens/s: 189680.1 | physical tokens/s: 190219.3 | loss tokens/s: 140701.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.851306E-01 | lm-global-supervised-tokens: 1.244506E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.269518E-02 | lm-token-buffer-mib: 5.810503E+01 | global_load_balancing_loss: 9.999137E-01 | loss scale: 1.0 | grad norm: 0.264 | params norm: 11494.591 | params rms: 0.062239 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 16:28:44.387278 | finish at 2026-10-10 17:26:07 + [2026-10-06 00:58:53] iteration 2278/ 6855 | consumed samples: 291584 | elapsed time per iteration (ms): 90127.2 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.45% | FLOPs estimator: workload-aware | logical tokens/s: 186150.0 | valid tokens/s: 186150.0 | physical tokens/s: 186610.8 | loss tokens/s: 139756.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.052603E-01 | lm-global-supervised-tokens: 1.259584E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.173787E-02 | lm-token-buffer-mib: 5.819997E+01 | global_load_balancing_loss: 9.996135E-01 | loss scale: 1.0 | grad norm: 0.166 | params norm: 11494.521 | params rms: 0.062239 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 18:35:12.390312 | finish at 2026-10-10 19:34:05 + [2026-10-06 01:00:20] iteration 2279/ 6855 | consumed samples: 291712 | elapsed time per iteration (ms): 86593.6 | throughput per GPU (TFLOP/s/GPU): 42.8 | MFU 13.73% | FLOPs estimator: workload-aware | logical tokens/s: 193746.4 | valid tokens/s: 193746.4 | physical tokens/s: 194181.3 | loss tokens/s: 139993.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.949992E-01 | lm-global-supervised-tokens: 1.212257E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.410301E-02 | lm-token-buffer-mib: 5.807173E+01 | global_load_balancing_loss: 1.000936E+00 | loss scale: 1.0 | grad norm: 0.116 | params norm: 11494.450 | params rms: 0.062239 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 14:04:12.153381 | finish at 2026-10-10 15:04:32 + [2026-10-06 01:02:11] iteration 2280/ 6855 | consumed samples: 291840 | elapsed time per iteration (ms): 111452.6 | throughput per GPU (TFLOP/s/GPU): 33.8 | MFU 10.85% | FLOPs estimator: workload-aware | logical tokens/s: 150532.0 | valid tokens/s: 150532.0 | physical tokens/s: 150852.5 | loss tokens/s: 106225.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.831436E-01 | lm-global-supervised-tokens: 1.183906E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.147665E-02 | lm-token-buffer-mib: 5.804870E+01 | global_load_balancing_loss: 9.999580E-01 | loss scale: 1.0 | grad norm: 0.214 | params norm: 11494.380 | params rms: 0.062238 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 21:38:15.762813 | finish at 2026-10-11 22:40:27 +times across ranks (ms): + save-checkpoint: + rank 0: 23581.12 + rank 1: 23581.11 + rank 2: 23581.16 + rank 3: 23581.12 + rank 4: 23581.12 + rank 5: 23581.11 + rank 6: 23581.12 + rank 7: 23581.16 + rank 8: 23581.18 + rank 9: 23581.16 + rank 10: 23581.16 + rank 11: 23581.18 + rank 12: 23581.17 + rank 13: 23581.16 + rank 14: 23581.16 + rank 15: 23581.19 + rank 16: 23581.32 + rank 17: 23581.31 + rank 18: 23581.34 + rank 19: 23581.33 + rank 20: 23581.33 + rank 21: 23581.43 + rank 22: 23581.31 + rank 23: 23581.33 + rank 24: 23581.34 + rank 25: 23581.34 + rank 26: 23581.33 + rank 27: 23581.33 + rank 28: 23581.41 + rank 29: 23581.32 + rank 30: 23581.34 + rank 31: 23581.32 + rank 32: 23581.42 + rank 33: 23581.41 + rank 34: 23581.54 + rank 35: 23581.40 + rank 36: 23581.38 + rank 37: 23581.39 + rank 38: 23581.42 + rank 39: 23581.41 + rank 40: 23581.34 + rank 41: 23581.39 + rank 42: 23581.32 + rank 43: 23581.30 + rank 44: 23581.32 + rank 45: 23581.33 + rank 46: 23581.31 + rank 47: 23581.30 + rank 48: 23580.68 + rank 49: 23580.67 + rank 50: 23580.65 + rank 51: 23580.66 + rank 52: 23580.69 + rank 53: 23580.65 + rank 54: 23580.67 + rank 55: 23580.65 + rank 56: 23581.22 + rank 57: 23581.20 + rank 58: 23581.20 + rank 59: 23581.20 + rank 60: 23581.19 + rank 61: 23581.20 + rank 62: 23581.20 + rank 63: 23581.24 + [2026-10-06 01:04:28] iteration 2281/ 6855 | consumed samples: 291968 | elapsed time per iteration (ms): 113074.6 | throughput per GPU (TFLOP/s/GPU): 32.4 | MFU 10.39% | FLOPs estimator: workload-aware | logical tokens/s: 148372.1 | valid tokens/s: 148372.1 | physical tokens/s: 148696.1 | loss tokens/s: 108305.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.801760E-01 | lm-global-supervised-tokens: 1.224665E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 6.304173E+00 | lm-token-buffer-mib: 5.809586E+01 | global_load_balancing_loss: 1.001299E+00 | loss scale: 1.0 | grad norm: 0.115 | params norm: 11494.309 | params rms: 0.062238 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 23:40:03.046921 | finish at 2026-10-12 00:44:31 + [2026-10-06 01:06:04] iteration 2282/ 6855 | consumed samples: 292096 | elapsed time per iteration (ms): 95804.1 | throughput per GPU (TFLOP/s/GPU): 39.4 | MFU 12.64% | FLOPs estimator: workload-aware | logical tokens/s: 175120.0 | valid tokens/s: 175120.0 | physical tokens/s: 175410.5 | loss tokens/s: 125678.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.775535E-01 | lm-global-supervised-tokens: 1.204055E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.637163E-02 | lm-token-buffer-mib: 5.807693E+01 | global_load_balancing_loss: 1.000560E+00 | loss scale: 1.0 | grad norm: 0.130 | params norm: 11494.239 | params rms: 0.062238 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 1:41:52.129842 | finish at 2026-10-11 02:47:56 + [2026-10-06 01:07:32] iteration 2283/ 6855 | consumed samples: 292224 | elapsed time per iteration (ms): 88371.0 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.62% | FLOPs estimator: workload-aware | logical tokens/s: 189849.3 | valid tokens/s: 189849.3 | physical tokens/s: 190257.9 | loss tokens/s: 140681.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.755238E-01 | lm-global-supervised-tokens: 1.243217E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.822860E-02 | lm-token-buffer-mib: 5.806961E+01 | global_load_balancing_loss: 1.000365E+00 | loss scale: 1.0 | grad norm: 0.081 | params norm: 11494.170 | params rms: 0.062237 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 16:13:52.248207 | finish at 2026-10-10 17:21:24 + [2026-10-06 01:08:56] iteration 2284/ 6855 | consumed samples: 292352 | elapsed time per iteration (ms): 84338.4 | throughput per GPU (TFLOP/s/GPU): 43.4 | MFU 13.92% | FLOPs estimator: workload-aware | logical tokens/s: 198927.3 | valid tokens/s: 198927.3 | physical tokens/s: 199343.3 | loss tokens/s: 151623.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.648040E-01 | lm-global-supervised-tokens: 1.278764E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.898300E-02 | lm-token-buffer-mib: 5.813869E+01 | global_load_balancing_loss: 1.000233E+00 | loss scale: 1.0 | grad norm: 0.088 | params norm: 11494.100 | params rms: 0.062237 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 11:05:10.724531 | finish at 2026-10-10 12:14:07 + [2026-10-06 01:10:22] iteration 2285/ 6855 | consumed samples: 292480 | elapsed time per iteration (ms): 85344.4 | throughput per GPU (TFLOP/s/GPU): 43.3 | MFU 13.87% | FLOPs estimator: workload-aware | logical tokens/s: 196582.0 | valid tokens/s: 196582.0 | physical tokens/s: 196988.2 | loss tokens/s: 143495.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.613141E-01 | lm-global-supervised-tokens: 1.224655E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.487003E-02 | lm-token-buffer-mib: 5.805868E+01 | global_load_balancing_loss: 1.000502E+00 | loss scale: 1.0 | grad norm: 0.079 | params norm: 11494.033 | params rms: 0.062236 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 12:20:23.842301 | finish at 2026-10-10 13:30:46 + [2026-10-06 01:11:46] iteration 2286/ 6855 | consumed samples: 292608 | elapsed time per iteration (ms): 84585.0 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.69% | FLOPs estimator: workload-aware | logical tokens/s: 198345.9 | valid tokens/s: 198345.9 | physical tokens/s: 198951.3 | loss tokens/s: 155633.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.528870E-01 | lm-global-supervised-tokens: 1.316426E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.922214E-02 | lm-token-buffer-mib: 5.815348E+01 | global_load_balancing_loss: 9.996734E-01 | loss scale: 1.0 | grad norm: 0.120 | params norm: 11493.966 | params rms: 0.062236 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 11:21:09.041646 | finish at 2026-10-10 12:32:55 + [2026-10-06 01:13:15] iteration 2287/ 6855 | consumed samples: 292736 | elapsed time per iteration (ms): 88729.2 | throughput per GPU (TFLOP/s/GPU): 41.4 | MFU 13.26% | FLOPs estimator: workload-aware | logical tokens/s: 189083.3 | valid tokens/s: 189083.3 | physical tokens/s: 189427.8 | loss tokens/s: 142381.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.896377E-01 | lm-global-supervised-tokens: 1.263343E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.162531E-02 | lm-token-buffer-mib: 5.810271E+01 | global_load_balancing_loss: 1.000042E+00 | loss scale: 1.0 | grad norm: 0.061 | params norm: 11493.900 | params rms: 0.062236 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 16:35:14.932804 | finish at 2026-10-10 17:48:30 + [2026-10-06 01:14:45] iteration 2288/ 6855 | consumed samples: 292864 | elapsed time per iteration (ms): 89850.9 | throughput per GPU (TFLOP/s/GPU): 40.7 | MFU 13.03% | FLOPs estimator: workload-aware | logical tokens/s: 186722.7 | valid tokens/s: 186722.6 | physical tokens/s: 187288.0 | loss tokens/s: 140842.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.652330E-01 | lm-global-supervised-tokens: 1.265478E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.567968E-02 | lm-token-buffer-mib: 5.807865E+01 | global_load_balancing_loss: 9.998428E-01 | loss scale: 1.0 | grad norm: 0.145 | params norm: 11493.831 | params rms: 0.062235 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 17:59:08.853119 | finish at 2026-10-10 19:13:54 + [2026-10-06 01:16:14] iteration 2289/ 6855 | consumed samples: 292992 | elapsed time per iteration (ms): 88783.2 | throughput per GPU (TFLOP/s/GPU): 41.5 | MFU 13.29% | FLOPs estimator: workload-aware | logical tokens/s: 188968.1 | valid tokens/s: 188968.0 | physical tokens/s: 189453.1 | loss tokens/s: 136124.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.641341E-01 | lm-global-supervised-tokens: 1.208561E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.060389E-02 | lm-token-buffer-mib: 5.814066E+01 | global_load_balancing_loss: 1.000659E+00 | loss scale: 1.0 | grad norm: 0.073 | params norm: 11493.762 | params rms: 0.062235 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 16:36:24.212153 | finish at 2026-10-10 17:52:38 + [2026-10-06 01:18:06] iteration 2290/ 6855 | consumed samples: 293120 | elapsed time per iteration (ms): 111916.5 | throughput per GPU (TFLOP/s/GPU): 33.0 | MFU 10.59% | FLOPs estimator: workload-aware | logical tokens/s: 149908.2 | valid tokens/s: 149908.2 | physical tokens/s: 150296.0 | loss tokens/s: 110752.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.802535E-01 | lm-global-supervised-tokens: 1.239506E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.141445E-02 | lm-token-buffer-mib: 5.806808E+01 | global_load_balancing_loss: 9.998627E-01 | loss scale: 1.0 | grad norm: 0.112 | params norm: 11493.694 | params rms: 0.062235 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 21:54:58.952435 | finish at 2026-10-11 23:13:05 + [2026-10-06 01:19:34] iteration 2291/ 6855 | consumed samples: 293248 | elapsed time per iteration (ms): 88243.1 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.65% | FLOPs estimator: workload-aware | logical tokens/s: 190124.6 | valid tokens/s: 190124.6 | physical tokens/s: 190491.5 | loss tokens/s: 140815.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.937093E-01 | lm-global-supervised-tokens: 1.242595E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.453985E-02 | lm-token-buffer-mib: 5.820265E+01 | global_load_balancing_loss: 9.998350E-01 | loss scale: 1.0 | grad norm: 0.052 | params norm: 11493.624 | params rms: 0.062234 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 15:52:21.377494 | finish at 2026-10-10 17:11:55 + [2026-10-06 01:21:01] iteration 2292/ 6855 | consumed samples: 293376 | elapsed time per iteration (ms): 87102.9 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.59% | FLOPs estimator: workload-aware | logical tokens/s: 192613.1 | valid tokens/s: 192613.1 | physical tokens/s: 193084.8 | loss tokens/s: 142073.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.844073E-01 | lm-global-supervised-tokens: 1.237502E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.223323E-02 | lm-token-buffer-mib: 5.823442E+01 | global_load_balancing_loss: 9.998137E-01 | loss scale: 1.0 | grad norm: 0.184 | params norm: 11493.553 | params rms: 0.062234 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 14:24:10.345709 | finish at 2026-10-10 15:45:11 + [2026-10-06 01:22:30] iteration 2293/ 6855 | consumed samples: 293504 | elapsed time per iteration (ms): 88561.5 | throughput per GPU (TFLOP/s/GPU): 43.1 | MFU 13.83% | FLOPs estimator: workload-aware | logical tokens/s: 189440.2 | valid tokens/s: 189440.2 | physical tokens/s: 189802.8 | loss tokens/s: 135851.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.397652E-01 | lm-global-supervised-tokens: 1.203119E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.814279E-02 | lm-token-buffer-mib: 5.808031E+01 | global_load_balancing_loss: 1.000370E+00 | loss scale: 1.0 | grad norm: 0.049 | params norm: 11493.484 | params rms: 0.062233 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 16:13:37.469791 | finish at 2026-10-10 17:36:07 + [2026-10-06 01:23:58] iteration 2294/ 6855 | consumed samples: 293632 | elapsed time per iteration (ms): 88191.8 | throughput per GPU (TFLOP/s/GPU): 41.6 | MFU 13.33% | FLOPs estimator: workload-aware | logical tokens/s: 190234.7 | valid tokens/s: 190234.7 | physical tokens/s: 190875.3 | loss tokens/s: 141588.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.631573E-01 | lm-global-supervised-tokens: 1.248696E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.171850E-02 | lm-token-buffer-mib: 5.808421E+01 | global_load_balancing_loss: 9.999974E-01 | loss scale: 1.0 | grad norm: 0.111 | params norm: 11493.414 | params rms: 0.062233 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 15:44:02.999335 | finish at 2026-10-10 17:08:01 + [2026-10-06 01:25:25] iteration 2295/ 6855 | consumed samples: 293760 | elapsed time per iteration (ms): 87240.5 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.45% | FLOPs estimator: workload-aware | logical tokens/s: 192309.4 | valid tokens/s: 192309.4 | physical tokens/s: 192895.1 | loss tokens/s: 144463.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.343397E-01 | lm-global-supervised-tokens: 1.260302E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.335816E-02 | lm-token-buffer-mib: 5.841246E+01 | global_load_balancing_loss: 1.001393E+00 | loss scale: 1.0 | grad norm: 0.082 | params norm: 11493.343 | params rms: 0.062233 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 14:30:16.476574 | finish at 2026-10-10 15:55:41 + [2026-10-06 01:26:48] iteration 2296/ 6855 | consumed samples: 293888 | elapsed time per iteration (ms): 83138.6 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.61% | FLOPs estimator: workload-aware | logical tokens/s: 201797.7 | valid tokens/s: 201797.7 | physical tokens/s: 202340.8 | loss tokens/s: 152877.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.469840E-01 | lm-global-supervised-tokens: 1.271001E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.840584E-02 | lm-token-buffer-mib: 5.805811E+01 | global_load_balancing_loss: 1.000006E+00 | loss scale: 1.0 | grad norm: 0.046 | params norm: 11493.272 | params rms: 0.062232 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 9:17:08.937688 | finish at 2026-10-10 10:43:57 + [2026-10-06 01:28:19] iteration 2297/ 6855 | consumed samples: 294016 | elapsed time per iteration (ms): 90722.5 | throughput per GPU (TFLOP/s/GPU): 40.9 | MFU 13.10% | FLOPs estimator: workload-aware | logical tokens/s: 184928.4 | valid tokens/s: 184928.4 | physical tokens/s: 185411.7 | loss tokens/s: 131919.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.592559E-01 | lm-global-supervised-tokens: 1.196806E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.642210E-02 | lm-token-buffer-mib: 5.808044E+01 | global_load_balancing_loss: 1.000849E+00 | loss scale: 1.0 | grad norm: 0.109 | params norm: 11493.201 | params rms: 0.062232 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 18:51:53.032593 | finish at 2026-10-10 20:20:12 + [2026-10-06 01:29:45] iteration 2298/ 6855 | consumed samples: 294144 | elapsed time per iteration (ms): 86011.7 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.77% | FLOPs estimator: workload-aware | logical tokens/s: 195056.9 | valid tokens/s: 195056.9 | physical tokens/s: 195522.2 | loss tokens/s: 140540.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.573025E-01 | lm-global-supervised-tokens: 1.208814E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.159893E-02 | lm-token-buffer-mib: 5.803469E+01 | global_load_balancing_loss: 1.000482E+00 | loss scale: 1.0 | grad norm: 0.042 | params norm: 11493.129 | params rms: 0.062232 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 12:52:35.432765 | finish at 2026-10-10 14:22:20 + [2026-10-06 01:31:14] iteration 2299/ 6855 | consumed samples: 294272 | elapsed time per iteration (ms): 89420.9 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.45% | FLOPs estimator: workload-aware | logical tokens/s: 187620.0 | valid tokens/s: 187620.0 | physical tokens/s: 188260.6 | loss tokens/s: 136072.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.611866E-01 | lm-global-supervised-tokens: 1.216776E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.179164E-02 | lm-token-buffer-mib: 5.846316E+01 | global_load_balancing_loss: 9.998168E-01 | loss scale: 1.0 | grad norm: 0.093 | params norm: 11493.056 | params rms: 0.062231 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 17:10:01.602419 | finish at 2026-10-10 18:41:16 + [2026-10-06 01:33:00] iteration 2300/ 6855 | consumed samples: 294400 | elapsed time per iteration (ms): 105602.6 | throughput per GPU (TFLOP/s/GPU): 34.3 | MFU 10.98% | FLOPs estimator: workload-aware | logical tokens/s: 158870.8 | valid tokens/s: 158870.8 | physical tokens/s: 159191.3 | loss tokens/s: 119017.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.388085E-01 | lm-global-supervised-tokens: 1.256851E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.154442E-02 | lm-token-buffer-mib: 5.819731E+01 | global_load_balancing_loss: 9.993919E-01 | loss scale: 1.0 | grad norm: 0.058 | params norm: 11492.982 | params rms: 0.062231 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 13:37:00.049784 | finish at 2026-10-11 15:10:00 + [2026-10-06 01:34:29] iteration 2301/ 6855 | consumed samples: 294528 | elapsed time per iteration (ms): 89244.5 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.38% | FLOPs estimator: workload-aware | logical tokens/s: 187991.1 | valid tokens/s: 187991.1 | physical tokens/s: 188321.0 | loss tokens/s: 144306.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.728055E-01 | lm-global-supervised-tokens: 1.287859E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.431905E-02 | lm-token-buffer-mib: 5.810153E+01 | global_load_balancing_loss: 9.984956E-01 | loss scale: 1.0 | grad norm: 0.076 | params norm: 11492.909 | params rms: 0.062230 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 16:53:39.227892 | finish at 2026-10-10 18:28:08 + [2026-10-06 01:35:53] iteration 2302/ 6855 | consumed samples: 294656 | elapsed time per iteration (ms): 84053.2 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.70% | FLOPs estimator: workload-aware | logical tokens/s: 199602.4 | valid tokens/s: 199602.4 | physical tokens/s: 200037.8 | loss tokens/s: 152148.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.446182E-01 | lm-global-supervised-tokens: 1.278853E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.321300E-02 | lm-token-buffer-mib: 5.808296E+01 | global_load_balancing_loss: 9.997740E-01 | loss scale: 1.0 | grad norm: 0.044 | params norm: 11492.836 | params rms: 0.062230 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 10:18:14.081769 | finish at 2026-10-10 11:54:07 + [2026-10-06 01:37:20] iteration 2303/ 6855 | consumed samples: 294784 | elapsed time per iteration (ms): 87236.5 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.59% | FLOPs estimator: workload-aware | logical tokens/s: 192318.3 | valid tokens/s: 192318.3 | physical tokens/s: 192648.3 | loss tokens/s: 140733.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.737879E-01 | lm-global-supervised-tokens: 1.227710E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.332167E-02 | lm-token-buffer-mib: 5.818278E+01 | global_load_balancing_loss: 1.000557E+00 | loss scale: 1.0 | grad norm: 0.058 | params norm: 11492.764 | params rms: 0.062230 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 14:18:20.746719 | finish at 2026-10-10 15:55:41 + [2026-10-06 01:38:47] iteration 2304/ 6855 | consumed samples: 294912 | elapsed time per iteration (ms): 86475.3 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.77% | FLOPs estimator: workload-aware | logical tokens/s: 194011.2 | valid tokens/s: 194011.2 | physical tokens/s: 194530.6 | loss tokens/s: 138788.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 5.956838E-01 | lm-global-supervised-tokens: 1.200175E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.154858E-02 | lm-token-buffer-mib: 5.824252E+01 | global_load_balancing_loss: 1.001607E+00 | loss scale: 1.0 | grad norm: 0.057 | params norm: 11492.691 | params rms: 0.062229 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 13:19:09.254477 | finish at 2026-10-10 14:57:56 + [2026-10-06 01:40:14] iteration 2305/ 6855 | consumed samples: 295040 | elapsed time per iteration (ms): 87102.8 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.67% | FLOPs estimator: workload-aware | logical tokens/s: 192614.0 | valid tokens/s: 192614.0 | physical tokens/s: 193096.6 | loss tokens/s: 139897.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.598085E-01 | lm-global-supervised-tokens: 1.218544E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.226108E-02 | lm-token-buffer-mib: 5.819437E+01 | global_load_balancing_loss: 1.001484E+00 | loss scale: 1.0 | grad norm: 0.045 | params norm: 11492.619 | params rms: 0.062229 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 14:05:17.683101 | finish at 2026-10-10 15:45:32 + [2026-10-06 01:41:45] iteration 2306/ 6855 | consumed samples: 295168 | elapsed time per iteration (ms): 90641.1 | throughput per GPU (TFLOP/s/GPU): 40.4 | MFU 12.94% | FLOPs estimator: workload-aware | logical tokens/s: 185094.6 | valid tokens/s: 185094.6 | physical tokens/s: 185554.7 | loss tokens/s: 139940.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.932220E-01 | lm-global-supervised-tokens: 1.268434E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.151349E-02 | lm-token-buffer-mib: 5.825682E+01 | global_load_balancing_loss: 1.000564E+00 | loss scale: 1.0 | grad norm: 0.055 | params norm: 11492.545 | params rms: 0.062228 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 18:32:06.375511 | finish at 2026-10-10 20:13:51 + [2026-10-06 01:43:09] iteration 2307/ 6855 | consumed samples: 295296 | elapsed time per iteration (ms): 84083.0 | throughput per GPU (TFLOP/s/GPU): 43.6 | MFU 13.97% | FLOPs estimator: workload-aware | logical tokens/s: 199530.9 | valid tokens/s: 199530.9 | physical tokens/s: 199990.4 | loss tokens/s: 142878.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.370081E-01 | lm-global-supervised-tokens: 1.201365E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.393338E-02 | lm-token-buffer-mib: 5.805688E+01 | global_load_balancing_loss: 9.996961E-01 | loss scale: 1.0 | grad norm: 0.048 | params norm: 11492.473 | params rms: 0.062228 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 10:13:29.431701 | finish at 2026-10-10 11:56:38 + [2026-10-06 01:44:37] iteration 2308/ 6855 | consumed samples: 295424 | elapsed time per iteration (ms): 88187.7 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.62% | FLOPs estimator: workload-aware | logical tokens/s: 190243.7 | valid tokens/s: 190243.7 | physical tokens/s: 190838.2 | loss tokens/s: 142393.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.145360E-01 | lm-global-supervised-tokens: 1.255733E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.147994E-02 | lm-token-buffer-mib: 5.808789E+01 | global_load_balancing_loss: 9.993706E-01 | loss scale: 1.0 | grad norm: 0.044 | params norm: 11492.401 | params rms: 0.062228 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 15:23:09.479640 | finish at 2026-10-10 17:07:46 + [2026-10-06 01:46:06] iteration 2309/ 6855 | consumed samples: 295552 | elapsed time per iteration (ms): 89493.8 | throughput per GPU (TFLOP/s/GPU): 41.5 | MFU 13.29% | FLOPs estimator: workload-aware | logical tokens/s: 187467.0 | valid tokens/s: 187467.0 | physical tokens/s: 187912.9 | loss tokens/s: 142934.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.201250E-01 | lm-global-supervised-tokens: 1.279178E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.398849E-02 | lm-token-buffer-mib: 5.831207E+01 | global_load_balancing_loss: 9.990361E-01 | loss scale: 1.0 | grad norm: 0.040 | params norm: 11492.328 | params rms: 0.062227 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 17:00:38.868651 | finish at 2026-10-10 18:46:45 + [2026-10-06 01:47:55] iteration 2310/ 6855 | consumed samples: 295680 | elapsed time per iteration (ms): 108210.9 | throughput per GPU (TFLOP/s/GPU): 33.4 | MFU 10.70% | FLOPs estimator: workload-aware | logical tokens/s: 155041.5 | valid tokens/s: 155041.5 | physical tokens/s: 155369.0 | loss tokens/s: 115609.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.743931E-01 | lm-global-supervised-tokens: 1.251021E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.147557E-02 | lm-token-buffer-mib: 5.813599E+01 | global_load_balancing_loss: 9.995276E-01 | loss scale: 1.0 | grad norm: 0.060 | params norm: 11492.254 | params rms: 0.062227 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 16:36:58.650255 | finish at 2026-10-11 18:24:53 + [2026-10-06 01:49:23] iteration 2311/ 6855 | consumed samples: 295808 | elapsed time per iteration (ms): 88031.9 | throughput per GPU (TFLOP/s/GPU): 41.5 | MFU 13.31% | FLOPs estimator: workload-aware | logical tokens/s: 190580.3 | valid tokens/s: 190580.3 | physical tokens/s: 190996.0 | loss tokens/s: 145453.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.728237E-01 | lm-global-supervised-tokens: 1.280459E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.974608E-02 | lm-token-buffer-mib: 5.814207E+01 | global_load_balancing_loss: 1.001194E+00 | loss scale: 1.0 | grad norm: 0.037 | params norm: 11492.179 | params rms: 0.062226 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 15:06:56.947861 | finish at 2026-10-10 16:56:20 + [2026-10-06 01:50:50] iteration 2312/ 6855 | consumed samples: 295936 | elapsed time per iteration (ms): 87471.4 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.46% | FLOPs estimator: workload-aware | logical tokens/s: 191801.4 | valid tokens/s: 191801.4 | physical tokens/s: 192210.8 | loss tokens/s: 150207.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.867933E-01 | lm-global-supervised-tokens: 1.313886E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.139177E-02 | lm-token-buffer-mib: 5.807859E+01 | global_load_balancing_loss: 9.995308E-01 | loss scale: 1.0 | grad norm: 0.030 | params norm: 11492.106 | params rms: 0.062226 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 14:23:02.790179 | finish at 2026-10-10 16:13:53 + [2026-10-06 01:52:17] iteration 2313/ 6855 | consumed samples: 296064 | elapsed time per iteration (ms): 87348.6 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.66% | FLOPs estimator: workload-aware | logical tokens/s: 192071.4 | valid tokens/s: 192071.4 | physical tokens/s: 192652.1 | loss tokens/s: 142485.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.705873E-01 | lm-global-supervised-tokens: 1.244594E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.789456E-02 | lm-token-buffer-mib: 5.813794E+01 | global_load_balancing_loss: 1.001261E+00 | loss scale: 1.0 | grad norm: 0.046 | params norm: 11492.034 | params rms: 0.062226 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 14:12:17.292064 | finish at 2026-10-10 16:04:35 + [2026-10-06 01:53:43] iteration 2314/ 6855 | consumed samples: 296192 | elapsed time per iteration (ms): 85115.5 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.63% | FLOPs estimator: workload-aware | logical tokens/s: 197110.7 | valid tokens/s: 197110.7 | physical tokens/s: 197740.3 | loss tokens/s: 142823.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.729506E-01 | lm-global-supervised-tokens: 1.215652E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.202116E-02 | lm-token-buffer-mib: 5.810706E+01 | global_load_balancing_loss: 1.000810E+00 | loss scale: 1.0 | grad norm: 0.092 | params norm: 11491.960 | params rms: 0.062225 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 11:21:49.638034 | finish at 2026-10-10 13:15:32 + [2026-10-06 01:55:09] iteration 2315/ 6855 | consumed samples: 296320 | elapsed time per iteration (ms): 86903.3 | throughput per GPU (TFLOP/s/GPU): 41.7 | MFU 13.35% | FLOPs estimator: workload-aware | logical tokens/s: 193055.7 | valid tokens/s: 193055.7 | physical tokens/s: 193624.4 | loss tokens/s: 142956.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.500467E-01 | lm-global-supervised-tokens: 1.242337E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.161200E-02 | lm-token-buffer-mib: 5.815900E+01 | global_load_balancing_loss: 1.000015E+00 | loss scale: 1.0 | grad norm: 0.166 | params norm: 11491.887 | params rms: 0.062225 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 13:35:40.881548 | finish at 2026-10-10 15:30:50 + [2026-10-06 01:56:34] iteration 2316/ 6855 | consumed samples: 296448 | elapsed time per iteration (ms): 84702.3 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.61% | FLOPs estimator: workload-aware | logical tokens/s: 198072.4 | valid tokens/s: 198072.4 | physical tokens/s: 198404.1 | loss tokens/s: 148459.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.451169E-01 | lm-global-supervised-tokens: 1.257488E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.849233E-02 | lm-token-buffer-mib: 5.803759E+01 | global_load_balancing_loss: 9.999712E-01 | loss scale: 1.0 | grad norm: 0.183 | params norm: 11491.815 | params rms: 0.062224 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 10:47:43.616657 | finish at 2026-10-10 12:44:18 + [2026-10-06 01:58:01] iteration 2317/ 6855 | consumed samples: 296576 | elapsed time per iteration (ms): 87049.0 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.63% | FLOPs estimator: workload-aware | logical tokens/s: 192732.6 | valid tokens/s: 192732.6 | physical tokens/s: 193241.5 | loss tokens/s: 144017.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.719277E-01 | lm-global-supervised-tokens: 1.253653E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.156481E-02 | lm-token-buffer-mib: 5.806067E+01 | global_load_balancing_loss: 1.000931E+00 | loss scale: 1.0 | grad norm: 0.043 | params norm: 11491.744 | params rms: 0.062224 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 13:43:48.174936 | finish at 2026-10-10 15:41:49 + [2026-10-06 01:59:28] iteration 2318/ 6855 | consumed samples: 296704 | elapsed time per iteration (ms): 87095.5 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.44% | FLOPs estimator: workload-aware | logical tokens/s: 192629.7 | valid tokens/s: 192629.7 | physical tokens/s: 193148.1 | loss tokens/s: 141805.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.673428E-01 | lm-global-supervised-tokens: 1.235062E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.567264E-02 | lm-token-buffer-mib: 5.812464E+01 | global_load_balancing_loss: 9.998862E-01 | loss scale: 1.0 | grad norm: 0.320 | params norm: 11491.674 | params rms: 0.062224 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 13:45:52.115801 | finish at 2026-10-10 15:45:20 + [2026-10-06 02:00:54] iteration 2319/ 6855 | consumed samples: 296832 | elapsed time per iteration (ms): 85989.2 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.46% | FLOPs estimator: workload-aware | logical tokens/s: 195108.3 | valid tokens/s: 195108.3 | physical tokens/s: 195711.1 | loss tokens/s: 144995.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.729835E-01 | lm-global-supervised-tokens: 1.246806E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.255640E-02 | lm-token-buffer-mib: 5.813386E+01 | global_load_balancing_loss: 9.992067E-01 | loss scale: 1.0 | grad norm: 0.517 | params norm: 11491.602 | params rms: 0.062223 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 12:20:47.001989 | finish at 2026-10-10 14:21:41 + [2026-10-06 02:02:41] iteration 2320/ 6855 | consumed samples: 296960 | elapsed time per iteration (ms): 107133.7 | throughput per GPU (TFLOP/s/GPU): 34.6 | MFU 11.10% | FLOPs estimator: workload-aware | logical tokens/s: 156600.7 | valid tokens/s: 156600.7 | physical tokens/s: 156886.5 | loss tokens/s: 115179.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.495608E-01 | lm-global-supervised-tokens: 1.233964E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.668379E-02 | lm-token-buffer-mib: 5.808014E+01 | global_load_balancing_loss: 1.000181E+00 | loss scale: 1.0 | grad norm: 0.068 | params norm: 11491.531 | params rms: 0.062223 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 14:57:31.455523 | finish at 2026-10-11 17:00:13 +times across ranks (ms): + save-checkpoint: + rank 0: 26129.39 + rank 1: 26129.35 + rank 2: 26129.41 + rank 3: 26129.38 + rank 4: 26129.40 + rank 5: 26129.34 + rank 6: 26129.33 + rank 7: 26129.38 + rank 8: 26129.62 + rank 9: 26129.56 + rank 10: 26129.62 + rank 11: 26129.56 + rank 12: 26129.59 + rank 13: 26129.57 + rank 14: 26129.57 + rank 15: 26129.55 + rank 16: 26129.59 + rank 17: 26129.56 + rank 18: 26129.58 + rank 19: 26129.60 + rank 20: 26129.54 + rank 21: 26129.59 + rank 22: 26129.55 + rank 23: 26129.53 + rank 24: 26129.04 + rank 25: 26128.99 + rank 26: 26129.05 + rank 27: 26129.05 + rank 28: 26129.01 + rank 29: 26128.98 + rank 30: 26129.13 + rank 31: 26128.99 + rank 32: 26128.98 + rank 33: 26128.98 + rank 34: 26129.01 + rank 35: 26129.00 + rank 36: 26129.03 + rank 37: 26129.00 + rank 38: 26129.01 + rank 39: 26129.04 + rank 40: 26129.76 + rank 41: 26129.74 + rank 42: 26129.74 + rank 43: 26129.77 + rank 44: 26129.73 + rank 45: 26129.75 + rank 46: 26129.71 + rank 47: 26129.59 + rank 48: 26128.97 + rank 49: 26129.03 + rank 50: 26129.06 + rank 51: 26129.00 + rank 52: 26129.08 + rank 53: 26129.08 + rank 54: 26128.93 + rank 55: 26129.09 + rank 56: 26129.50 + rank 57: 26129.49 + rank 58: 26129.42 + rank 59: 26129.40 + rank 60: 26129.39 + rank 61: 26129.43 + rank 62: 26129.41 + rank 63: 26129.43 + [2026-10-06 02:04:46] iteration 2321/ 6855 | consumed samples: 297088 | elapsed time per iteration (ms): 98183.0 | throughput per GPU (TFLOP/s/GPU): 38.0 | MFU 12.17% | FLOPs estimator: workload-aware | logical tokens/s: 170876.6 | valid tokens/s: 170876.6 | physical tokens/s: 171323.6 | loss tokens/s: 123442.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.946355E-01 | lm-global-supervised-tokens: 1.211998E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 7.681363E-01 | lm-token-buffer-mib: 5.812608E+01 | global_load_balancing_loss: 1.000645E+00 | loss scale: 1.0 | grad norm: 0.373 | params norm: 11491.460 | params rms: 0.062223 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 3:39:21.566735 | finish at 2026-10-11 05:44:07 + [2026-10-06 02:06:16] iteration 2322/ 6855 | consumed samples: 297216 | elapsed time per iteration (ms): 89945.1 | throughput per GPU (TFLOP/s/GPU): 41.2 | MFU 13.22% | FLOPs estimator: workload-aware | logical tokens/s: 186526.1 | valid tokens/s: 186526.1 | physical tokens/s: 186792.6 | loss tokens/s: 145929.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.229602E-01 | lm-global-supervised-tokens: 1.312566E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.836821E-02 | lm-token-buffer-mib: 5.818753E+01 | global_load_balancing_loss: 9.998233E-01 | loss scale: 1.0 | grad norm: 0.081 | params norm: 11491.388 | params rms: 0.062222 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 17:15:21.354763 | finish at 2026-10-10 19:21:37 + [2026-10-06 02:07:42] iteration 2323/ 6855 | consumed samples: 297344 | elapsed time per iteration (ms): 86071.1 | throughput per GPU (TFLOP/s/GPU): 41.7 | MFU 13.37% | FLOPs estimator: workload-aware | logical tokens/s: 194922.7 | valid tokens/s: 194922.7 | physical tokens/s: 195510.6 | loss tokens/s: 146257.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.581006E-01 | lm-global-supervised-tokens: 1.258857E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.867400E-02 | lm-token-buffer-mib: 5.824139E+01 | global_load_balancing_loss: 1.000358E+00 | loss scale: 1.0 | grad norm: 0.229 | params norm: 11491.314 | params rms: 0.062222 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 12:21:14.028531 | finish at 2026-10-10 14:28:56 + [2026-10-06 02:09:08] iteration 2324/ 6855 | consumed samples: 297472 | elapsed time per iteration (ms): 86298.8 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.78% | FLOPs estimator: workload-aware | logical tokens/s: 194407.4 | valid tokens/s: 194407.4 | physical tokens/s: 194873.7 | loss tokens/s: 144485.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.573145E-01 | lm-global-supervised-tokens: 1.246889E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.335447E-02 | lm-token-buffer-mib: 5.819844E+01 | global_load_balancing_loss: 1.000675E+00 | loss scale: 1.0 | grad norm: 0.358 | params norm: 11491.242 | params rms: 0.062221 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 12:37:00.070175 | finish at 2026-10-10 14:46:08 + [2026-10-06 02:10:37] iteration 2325/ 6855 | consumed samples: 297600 | elapsed time per iteration (ms): 88513.1 | throughput per GPU (TFLOP/s/GPU): 41.5 | MFU 13.30% | FLOPs estimator: workload-aware | logical tokens/s: 189544.6 | valid tokens/s: 189544.6 | physical tokens/s: 189885.8 | loss tokens/s: 136665.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.157837E-01 | lm-global-supervised-tokens: 1.209669E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.283451E-02 | lm-token-buffer-mib: 5.817983E+01 | global_load_balancing_loss: 9.998173E-01 | loss scale: 1.0 | grad norm: 0.173 | params norm: 11491.170 | params rms: 0.062221 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 15:22:44.528353 | finish at 2026-10-10 17:33:21 + [2026-10-06 02:12:04] iteration 2326/ 6855 | consumed samples: 297728 | elapsed time per iteration (ms): 87328.1 | throughput per GPU (TFLOP/s/GPU): 41.0 | MFU 13.14% | FLOPs estimator: workload-aware | logical tokens/s: 192115.6 | valid tokens/s: 192115.6 | physical tokens/s: 192587.0 | loss tokens/s: 142048.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.613052E-01 | lm-global-supervised-tokens: 1.240485E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.162699E-02 | lm-token-buffer-mib: 5.839410E+01 | global_load_balancing_loss: 1.000781E+00 | loss scale: 1.0 | grad norm: 0.146 | params norm: 11491.099 | params rms: 0.062221 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 13:51:48.857846 | finish at 2026-10-10 16:03:53 + [2026-10-06 02:13:31] iteration 2327/ 6855 | consumed samples: 297856 | elapsed time per iteration (ms): 86684.9 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.62% | FLOPs estimator: workload-aware | logical tokens/s: 193542.3 | valid tokens/s: 193542.3 | physical tokens/s: 193857.9 | loss tokens/s: 145025.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.572447E-01 | lm-global-supervised-tokens: 1.257154E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.192346E-02 | lm-token-buffer-mib: 5.802481E+01 | global_load_balancing_loss: 1.000713E+00 | loss scale: 1.0 | grad norm: 0.056 | params norm: 11491.028 | params rms: 0.062220 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 13:01:49.035244 | finish at 2026-10-10 15:15:20 + [2026-10-06 02:14:55] iteration 2328/ 6855 | consumed samples: 297984 | elapsed time per iteration (ms): 84633.1 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.79% | FLOPs estimator: workload-aware | logical tokens/s: 198234.6 | valid tokens/s: 198234.5 | physical tokens/s: 198679.4 | loss tokens/s: 149378.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.689022E-01 | lm-global-supervised-tokens: 1.264237E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.192111E-02 | lm-token-buffer-mib: 5.816209E+01 | global_load_balancing_loss: 1.000244E+00 | loss scale: 1.0 | grad norm: 0.118 | params norm: 11490.956 | params rms: 0.062220 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 10:25:33.959661 | finish at 2026-10-10 12:40:29 + [2026-10-06 02:16:21] iteration 2329/ 6855 | consumed samples: 298112 | elapsed time per iteration (ms): 85743.1 | throughput per GPU (TFLOP/s/GPU): 42.8 | MFU 13.72% | FLOPs estimator: workload-aware | logical tokens/s: 195668.1 | valid tokens/s: 195668.1 | physical tokens/s: 196069.5 | loss tokens/s: 142688.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.697389E-01 | lm-global-supervised-tokens: 1.223455E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.941187E-02 | lm-token-buffer-mib: 5.814172E+01 | global_load_balancing_loss: 9.996796E-01 | loss scale: 1.0 | grad norm: 0.086 | params norm: 11490.882 | params rms: 0.062219 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 11:47:53.132151 | finish at 2026-10-10 14:04:14 + [2026-10-06 02:18:08] iteration 2330/ 6855 | consumed samples: 298240 | elapsed time per iteration (ms): 107185.9 | throughput per GPU (TFLOP/s/GPU): 33.5 | MFU 10.73% | FLOPs estimator: workload-aware | logical tokens/s: 156523.6 | valid tokens/s: 156523.6 | physical tokens/s: 156946.8 | loss tokens/s: 120066.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.610380E-01 | lm-global-supervised-tokens: 1.286943E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.143894E-02 | lm-token-buffer-mib: 5.824760E+01 | global_load_balancing_loss: 9.999455E-01 | loss scale: 1.0 | grad norm: 0.119 | params norm: 11490.811 | params rms: 0.062219 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 14:43:35.981609 | finish at 2026-10-11 17:01:44 + [2026-10-06 02:19:39] iteration 2331/ 6855 | consumed samples: 298368 | elapsed time per iteration (ms): 90880.8 | throughput per GPU (TFLOP/s/GPU): 41.7 | MFU 13.37% | FLOPs estimator: workload-aware | logical tokens/s: 184606.4 | valid tokens/s: 184606.4 | physical tokens/s: 184894.9 | loss tokens/s: 136598.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.479825E-01 | lm-global-supervised-tokens: 1.241422E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.165129E-02 | lm-token-buffer-mib: 5.812992E+01 | global_load_balancing_loss: 9.989337E-01 | loss scale: 1.0 | grad norm: 0.033 | params norm: 11490.741 | params rms: 0.062219 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 18:12:24.687467 | finish at 2026-10-10 20:32:04 + [2026-10-06 02:21:09] iteration 2332/ 6855 | consumed samples: 298496 | elapsed time per iteration (ms): 89424.7 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.41% | FLOPs estimator: workload-aware | logical tokens/s: 187612.7 | valid tokens/s: 187612.6 | physical tokens/s: 188031.0 | loss tokens/s: 139990.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.024766E-01 | lm-global-supervised-tokens: 1.251861E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.125144E-02 | lm-token-buffer-mib: 5.833681E+01 | global_load_balancing_loss: 1.000089E+00 | loss scale: 1.0 | grad norm: 0.057 | params norm: 11490.671 | params rms: 0.062218 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 16:21:08.036819 | finish at 2026-10-10 18:42:17 + [2026-10-06 02:22:33] iteration 2333/ 6855 | consumed samples: 298624 | elapsed time per iteration (ms): 84900.7 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.78% | FLOPs estimator: workload-aware | logical tokens/s: 197609.1 | valid tokens/s: 197609.1 | physical tokens/s: 198091.9 | loss tokens/s: 148878.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.857206E-01 | lm-global-supervised-tokens: 1.263991E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.111871E-02 | lm-token-buffer-mib: 5.820254E+01 | global_load_balancing_loss: 9.999211E-01 | loss scale: 1.0 | grad norm: 0.048 | params norm: 11490.600 | params rms: 0.062218 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 10:38:41.017568 | finish at 2026-10-10 13:01:14 + [2026-10-06 02:24:05] iteration 2334/ 6855 | consumed samples: 298752 | elapsed time per iteration (ms): 91068.6 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.51% | FLOPs estimator: workload-aware | logical tokens/s: 184226.1 | valid tokens/s: 184226.1 | physical tokens/s: 184618.3 | loss tokens/s: 132634.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.653660E-01 | lm-global-supervised-tokens: 1.207880E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.153478E-02 | lm-token-buffer-mib: 5.812426E+01 | global_load_balancing_loss: 1.000824E+00 | loss scale: 1.0 | grad norm: 0.054 | params norm: 11490.529 | params rms: 0.062217 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 18:22:00.992655 | finish at 2026-10-10 20:46:06 + [2026-10-06 02:25:33] iteration 2335/ 6855 | consumed samples: 298880 | elapsed time per iteration (ms): 88731.6 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.65% | FLOPs estimator: workload-aware | logical tokens/s: 189078.1 | valid tokens/s: 189078.1 | physical tokens/s: 189486.6 | loss tokens/s: 139070.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.903592E-01 | lm-global-supervised-tokens: 1.233999E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.315719E-02 | lm-token-buffer-mib: 5.820348E+01 | global_load_balancing_loss: 1.000733E+00 | loss scale: 1.0 | grad norm: 0.047 | params norm: 11490.458 | params rms: 0.062217 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 15:24:26.988468 | finish at 2026-10-10 17:50:00 + [2026-10-06 02:27:02] iteration 2336/ 6855 | consumed samples: 299008 | elapsed time per iteration (ms): 89069.9 | throughput per GPU (TFLOP/s/GPU): 41.3 | MFU 13.23% | FLOPs estimator: workload-aware | logical tokens/s: 188359.8 | valid tokens/s: 188359.8 | physical tokens/s: 188863.2 | loss tokens/s: 143263.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 5.685034E-01 | lm-global-supervised-tokens: 1.276050E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.579736E-02 | lm-token-buffer-mib: 5.818741E+01 | global_load_balancing_loss: 1.001741E+00 | loss scale: 1.0 | grad norm: 0.046 | params norm: 11490.387 | params rms: 0.062217 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 15:48:26.660625 | finish at 2026-10-10 18:15:29 + [2026-10-06 02:28:27] iteration 2337/ 6855 | consumed samples: 299136 | elapsed time per iteration (ms): 85145.6 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.79% | FLOPs estimator: workload-aware | logical tokens/s: 197040.4 | valid tokens/s: 197040.4 | physical tokens/s: 197610.8 | loss tokens/s: 147329.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.513428E-01 | lm-global-supervised-tokens: 1.254444E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.846742E-02 | lm-token-buffer-mib: 5.813533E+01 | global_load_balancing_loss: 9.999288E-01 | loss scale: 1.0 | grad norm: 0.080 | params norm: 11490.317 | params rms: 0.062216 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 10:51:27.980586 | finish at 2026-10-10 13:19:55 + [2026-10-06 02:29:58] iteration 2338/ 6855 | consumed samples: 299264 | elapsed time per iteration (ms): 90692.3 | throughput per GPU (TFLOP/s/GPU): 41.2 | MFU 13.21% | FLOPs estimator: workload-aware | logical tokens/s: 184989.3 | valid tokens/s: 184989.3 | physical tokens/s: 185384.0 | loss tokens/s: 129473.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.533552E-01 | lm-global-supervised-tokens: 1.174221E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.330388E-02 | lm-token-buffer-mib: 5.823776E+01 | global_load_balancing_loss: 1.000342E+00 | loss scale: 1.0 | grad norm: 0.058 | params norm: 11490.247 | params rms: 0.062216 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 17:47:37.069928 | finish at 2026-10-10 20:17:35 + [2026-10-06 02:31:24] iteration 2339/ 6855 | consumed samples: 299392 | elapsed time per iteration (ms): 85337.8 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.65% | FLOPs estimator: workload-aware | logical tokens/s: 196597.2 | valid tokens/s: 196597.1 | physical tokens/s: 197192.2 | loss tokens/s: 147302.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.244287E-01 | lm-global-supervised-tokens: 1.257046E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.152640E-02 | lm-token-buffer-mib: 5.813372E+01 | global_load_balancing_loss: 9.995726E-01 | loss scale: 1.0 | grad norm: 0.061 | params norm: 11490.176 | params rms: 0.062216 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 11:03:05.539371 | finish at 2026-10-10 13:34:29 + [2026-10-06 02:33:14] iteration 2340/ 6855 | consumed samples: 299520 | elapsed time per iteration (ms): 110933.1 | throughput per GPU (TFLOP/s/GPU): 33.5 | MFU 10.72% | FLOPs estimator: workload-aware | logical tokens/s: 151237.1 | valid tokens/s: 151237.1 | physical tokens/s: 151635.1 | loss tokens/s: 116053.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.696430E-01 | lm-global-supervised-tokens: 1.287417E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.770568E-02 | lm-token-buffer-mib: 5.812322E+01 | global_load_balancing_loss: 9.995230E-01 | loss scale: 1.0 | grad norm: 0.044 | params norm: 11490.103 | params rms: 0.062215 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 19:07:42.752670 | finish at 2026-10-11 21:40:57 + [2026-10-06 02:34:42] iteration 2341/ 6855 | consumed samples: 299648 | elapsed time per iteration (ms): 87481.2 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.45% | FLOPs estimator: workload-aware | logical tokens/s: 191780.7 | valid tokens/s: 191780.6 | physical tokens/s: 192363.1 | loss tokens/s: 145827.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.588023E-01 | lm-global-supervised-tokens: 1.275714E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.331649E-02 | lm-token-buffer-mib: 5.809997E+01 | global_load_balancing_loss: 9.996346E-01 | loss scale: 1.0 | grad norm: 0.049 | params norm: 11490.030 | params rms: 0.062215 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 13:41:30.322895 | finish at 2026-10-10 16:16:12 + [2026-10-06 02:36:09] iteration 2342/ 6855 | consumed samples: 299776 | elapsed time per iteration (ms): 87124.0 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.55% | FLOPs estimator: workload-aware | logical tokens/s: 192567.2 | valid tokens/s: 192567.2 | physical tokens/s: 193072.0 | loss tokens/s: 134198.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 5.701352E-01 | lm-global-supervised-tokens: 1.169187E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.325241E-02 | lm-token-buffer-mib: 5.828406E+01 | global_load_balancing_loss: 9.999729E-01 | loss scale: 1.0 | grad norm: 0.069 | params norm: 11489.958 | params rms: 0.062214 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 13:13:10.400358 | finish at 2026-10-10 15:49:19 + [2026-10-06 02:37:33] iteration 2343/ 6855 | consumed samples: 299904 | elapsed time per iteration (ms): 83615.8 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.79% | FLOPs estimator: workload-aware | logical tokens/s: 200645.4 | valid tokens/s: 200645.4 | physical tokens/s: 201224.3 | loss tokens/s: 148709.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.437669E-01 | lm-global-supervised-tokens: 1.243445E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.651571E-02 | lm-token-buffer-mib: 5.822916E+01 | global_load_balancing_loss: 1.000015E+00 | loss scale: 1.0 | grad norm: 0.122 | params norm: 11489.887 | params rms: 0.062214 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 8:47:54.549408 | finish at 2026-10-10 11:25:27 + [2026-10-06 02:39:00] iteration 2344/ 6855 | consumed samples: 300032 | elapsed time per iteration (ms): 87719.0 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.56% | FLOPs estimator: workload-aware | logical tokens/s: 191260.8 | valid tokens/s: 191260.8 | physical tokens/s: 191749.2 | loss tokens/s: 140188.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.617657E-01 | lm-global-supervised-tokens: 1.229715E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 3.070548E-01 | lm-token-buffer-mib: 5.817224E+01 | global_load_balancing_loss: 1.000659E+00 | loss scale: 1.0 | grad norm: 0.106 | params norm: 11489.816 | params rms: 0.062214 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 13:55:00.336322 | finish at 2026-10-10 16:34:01 + [2026-10-06 02:40:32] iteration 2345/ 6855 | consumed samples: 300160 | elapsed time per iteration (ms): 91388.7 | throughput per GPU (TFLOP/s/GPU): 41.3 | MFU 13.23% | FLOPs estimator: workload-aware | logical tokens/s: 183580.7 | valid tokens/s: 183580.7 | physical tokens/s: 183929.0 | loss tokens/s: 135385.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.162099E-01 | lm-global-supervised-tokens: 1.237271E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.151286E-02 | lm-token-buffer-mib: 5.804242E+01 | global_load_balancing_loss: 1.000901E+00 | loss scale: 1.0 | grad norm: 0.041 | params norm: 11489.745 | params rms: 0.062213 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 18:29:22.913382 | finish at 2026-10-10 21:09:55 + [2026-10-06 02:42:00] iteration 2346/ 6855 | consumed samples: 300288 | elapsed time per iteration (ms): 88464.9 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.65% | FLOPs estimator: workload-aware | logical tokens/s: 189647.7 | valid tokens/s: 189647.7 | physical tokens/s: 189906.0 | loss tokens/s: 140326.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.716005E-01 | lm-global-supervised-tokens: 1.241392E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.650778E-02 | lm-token-buffer-mib: 5.809127E+01 | global_load_balancing_loss: 1.001066E+00 | loss scale: 1.0 | grad norm: 0.042 | params norm: 11489.672 | params rms: 0.062213 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 14:48:08.322328 | finish at 2026-10-10 17:30:09 + [2026-10-06 02:43:27] iteration 2347/ 6855 | consumed samples: 300416 | elapsed time per iteration (ms): 86344.4 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.57% | FLOPs estimator: workload-aware | logical tokens/s: 194305.3 | valid tokens/s: 194305.3 | physical tokens/s: 194969.8 | loss tokens/s: 145846.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.949681E-01 | lm-global-supervised-tokens: 1.259304E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.140891E-02 | lm-token-buffer-mib: 5.824852E+01 | global_load_balancing_loss: 9.998875E-01 | loss scale: 1.0 | grad norm: 0.094 | params norm: 11489.599 | params rms: 0.062212 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 12:07:20.680631 | finish at 2026-10-10 14:50:47 + [2026-10-06 02:44:54] iteration 2348/ 6855 | consumed samples: 300544 | elapsed time per iteration (ms): 87183.7 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.65% | FLOPs estimator: workload-aware | logical tokens/s: 192434.2 | valid tokens/s: 192434.2 | physical tokens/s: 192804.3 | loss tokens/s: 140806.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.887046E-01 | lm-global-supervised-tokens: 1.227601E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.847832E-02 | lm-token-buffer-mib: 5.808197E+01 | global_load_balancing_loss: 1.000775E+00 | loss scale: 1.0 | grad norm: 0.292 | params norm: 11489.529 | params rms: 0.062212 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 13:08:56.937356 | finish at 2026-10-10 15:53:51 + [2026-10-06 02:46:25] iteration 2349/ 6855 | consumed samples: 300672 | elapsed time per iteration (ms): 90811.6 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.41% | FLOPs estimator: workload-aware | logical tokens/s: 184747.0 | valid tokens/s: 184747.0 | physical tokens/s: 185079.1 | loss tokens/s: 131031.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.597612E-01 | lm-global-supervised-tokens: 1.189917E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.163855E-02 | lm-token-buffer-mib: 5.803711E+01 | global_load_balancing_loss: 1.001191E+00 | loss scale: 1.0 | grad norm: 0.287 | params norm: 11489.459 | params rms: 0.062212 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 17:39:57.025417 | finish at 2026-10-10 20:26:22 + [2026-10-06 02:48:13] iteration 2350/ 6855 | consumed samples: 300800 | elapsed time per iteration (ms): 108136.2 | throughput per GPU (TFLOP/s/GPU): 33.9 | MFU 10.88% | FLOPs estimator: workload-aware | logical tokens/s: 155147.9 | valid tokens/s: 155147.9 | physical tokens/s: 155577.7 | loss tokens/s: 115865.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.726816E-01 | lm-global-supervised-tokens: 1.252929E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.716844E-02 | lm-token-buffer-mib: 5.811131E+01 | global_load_balancing_loss: 1.000977E+00 | loss scale: 1.0 | grad norm: 0.054 | params norm: 11489.389 | params rms: 0.062211 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 15:19:13.517410 | finish at 2026-10-11 18:07:26 + [2026-10-06 02:49:45] iteration 2351/ 6855 | consumed samples: 300928 | elapsed time per iteration (ms): 91954.4 | throughput per GPU (TFLOP/s/GPU): 40.1 | MFU 12.85% | FLOPs estimator: workload-aware | logical tokens/s: 182450.7 | valid tokens/s: 182450.7 | physical tokens/s: 182909.8 | loss tokens/s: 138302.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.476214E-01 | lm-global-supervised-tokens: 1.271753E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.138144E-02 | lm-token-buffer-mib: 5.814979E+01 | global_load_balancing_loss: 9.988785E-01 | loss scale: 1.0 | grad norm: 0.258 | params norm: 11489.322 | params rms: 0.062211 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 19:02:42.502981 | finish at 2026-10-10 21:52:27 + [2026-10-06 02:51:09] iteration 2352/ 6855 | consumed samples: 301056 | elapsed time per iteration (ms): 84174.5 | throughput per GPU (TFLOP/s/GPU): 43.2 | MFU 13.84% | FLOPs estimator: workload-aware | logical tokens/s: 199314.0 | valid tokens/s: 199314.0 | physical tokens/s: 200129.0 | loss tokens/s: 148890.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.736839E-01 | lm-global-supervised-tokens: 1.253276E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.188744E-02 | lm-token-buffer-mib: 5.813908E+01 | global_load_balancing_loss: 9.994102E-01 | loss scale: 1.0 | grad norm: 0.077 | params norm: 11489.255 | params rms: 0.062211 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 9:17:17.778816 | finish at 2026-10-10 12:08:27 + [2026-10-06 02:52:39] iteration 2353/ 6855 | consumed samples: 301184 | elapsed time per iteration (ms): 90266.6 | throughput per GPU (TFLOP/s/GPU): 41.5 | MFU 13.29% | FLOPs estimator: workload-aware | logical tokens/s: 185862.5 | valid tokens/s: 185862.5 | physical tokens/s: 186341.8 | loss tokens/s: 134788.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.586536E-01 | lm-global-supervised-tokens: 1.216689E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.244123E-02 | lm-token-buffer-mib: 5.820116E+01 | global_load_balancing_loss: 9.994418E-01 | loss scale: 1.0 | grad norm: 0.277 | params norm: 11489.186 | params rms: 0.062210 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 16:53:00.344350 | finish at 2026-10-10 19:45:39 + [2026-10-06 02:54:07] iteration 2354/ 6855 | consumed samples: 301312 | elapsed time per iteration (ms): 87932.8 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.52% | FLOPs estimator: workload-aware | logical tokens/s: 190795.3 | valid tokens/s: 190795.3 | physical tokens/s: 191229.0 | loss tokens/s: 142704.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.674290E-01 | lm-global-supervised-tokens: 1.254844E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.369533E-02 | lm-token-buffer-mib: 5.808868E+01 | global_load_balancing_loss: 9.998971E-01 | loss scale: 1.0 | grad norm: 0.082 | params norm: 11489.117 | params rms: 0.062210 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 13:56:25.465439 | finish at 2026-10-10 16:50:33 + [2026-10-06 02:55:33] iteration 2355/ 6855 | consumed samples: 301440 | elapsed time per iteration (ms): 86236.6 | throughput per GPU (TFLOP/s/GPU): 43.8 | MFU 14.02% | FLOPs estimator: workload-aware | logical tokens/s: 194548.0 | valid tokens/s: 194548.0 | physical tokens/s: 194848.1 | loss tokens/s: 142057.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.200548E-01 | lm-global-supervised-tokens: 1.225059E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.513284E-02 | lm-token-buffer-mib: 5.803978E+01 | global_load_balancing_loss: 9.990153E-01 | loss scale: 1.0 | grad norm: 0.147 | params norm: 11489.051 | params rms: 0.062209 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 11:47:44.846635 | finish at 2026-10-10 14:43:18 + [2026-10-06 02:57:03] iteration 2356/ 6855 | consumed samples: 301568 | elapsed time per iteration (ms): 90023.1 | throughput per GPU (TFLOP/s/GPU): 41.1 | MFU 13.18% | FLOPs estimator: workload-aware | logical tokens/s: 186365.3 | valid tokens/s: 186365.2 | physical tokens/s: 186861.9 | loss tokens/s: 132861.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.282994E-01 | lm-global-supervised-tokens: 1.196058E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.204766E-02 | lm-token-buffer-mib: 5.810381E+01 | global_load_balancing_loss: 9.998139E-01 | loss scale: 1.0 | grad norm: 0.191 | params norm: 11488.986 | params rms: 0.062209 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 16:30:13.998314 | finish at 2026-10-10 19:27:17 + [2026-10-06 02:58:30] iteration 2357/ 6855 | consumed samples: 301696 | elapsed time per iteration (ms): 86769.7 | throughput per GPU (TFLOP/s/GPU): 42.8 | MFU 13.73% | FLOPs estimator: workload-aware | logical tokens/s: 193352.7 | valid tokens/s: 193352.7 | physical tokens/s: 193653.8 | loss tokens/s: 140501.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.035899E-01 | lm-global-supervised-tokens: 1.219128E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.342502E-02 | lm-token-buffer-mib: 5.806861E+01 | global_load_balancing_loss: 1.000651E+00 | loss scale: 1.0 | grad norm: 0.066 | params norm: 11488.918 | params rms: 0.062209 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 12:24:50.044337 | finish at 2026-10-10 15:23:20 + [2026-10-06 02:59:57] iteration 2358/ 6855 | consumed samples: 301824 | elapsed time per iteration (ms): 86807.8 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.66% | FLOPs estimator: workload-aware | logical tokens/s: 193267.8 | valid tokens/s: 193267.8 | physical tokens/s: 193706.3 | loss tokens/s: 148135.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.173814E-01 | lm-global-supervised-tokens: 1.285935E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.360943E-02 | lm-token-buffer-mib: 5.812103E+01 | global_load_balancing_loss: 1.000586E+00 | loss scale: 1.0 | grad norm: 0.374 | params norm: 11488.850 | params rms: 0.062208 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 12:26:14.661835 | finish at 2026-10-10 15:26:12 + [2026-10-06 03:01:26] iteration 2359/ 6855 | consumed samples: 301952 | elapsed time per iteration (ms): 88758.1 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.49% | FLOPs estimator: workload-aware | logical tokens/s: 189021.5 | valid tokens/s: 189021.5 | physical tokens/s: 189353.5 | loss tokens/s: 143545.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.387319E-01 | lm-global-supervised-tokens: 1.274086E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.350952E-02 | lm-token-buffer-mib: 5.807893E+01 | global_load_balancing_loss: 1.001114E+00 | loss scale: 1.0 | grad norm: 0.066 | params norm: 11488.784 | params rms: 0.062208 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 14:50:56.349144 | finish at 2026-10-10 17:52:22 + [2026-10-06 03:03:16] iteration 2360/ 6855 | consumed samples: 302080 | elapsed time per iteration (ms): 110598.8 | throughput per GPU (TFLOP/s/GPU): 33.7 | MFU 10.81% | FLOPs estimator: workload-aware | logical tokens/s: 151693.7 | valid tokens/s: 151693.7 | physical tokens/s: 151978.9 | loss tokens/s: 115251.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.676429E-01 | lm-global-supervised-tokens: 1.274671E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.141114E-02 | lm-token-buffer-mib: 5.805337E+01 | global_load_balancing_loss: 1.001588E+00 | loss scale: 1.0 | grad norm: 0.280 | params norm: 11488.718 | params rms: 0.062208 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 18:05:41.802939 | finish at 2026-10-11 21:08:58 +times across ranks (ms): + save-checkpoint: + rank 0: 23968.03 + rank 1: 23968.01 + rank 2: 23968.07 + rank 3: 23968.06 + rank 4: 23968.03 + rank 5: 23968.02 + rank 6: 23968.01 + rank 7: 23968.02 + rank 8: 23968.21 + rank 9: 23968.19 + rank 10: 23968.19 + rank 11: 23968.18 + rank 12: 23968.19 + rank 13: 23968.19 + rank 14: 23968.20 + rank 15: 23968.18 + rank 16: 23968.04 + rank 17: 23968.04 + rank 18: 23968.04 + rank 19: 23968.05 + rank 20: 23968.04 + rank 21: 23968.04 + rank 22: 23968.03 + rank 23: 23968.04 + rank 24: 23968.73 + rank 25: 23968.72 + rank 26: 23968.74 + rank 27: 23968.74 + rank 28: 23968.88 + rank 29: 23968.70 + rank 30: 23968.81 + rank 31: 23968.73 + rank 32: 23968.14 + rank 33: 23968.12 + rank 34: 23968.12 + rank 35: 23968.11 + rank 36: 23968.13 + rank 37: 23968.09 + rank 38: 23968.08 + rank 39: 23968.07 + rank 40: 23968.80 + rank 41: 23968.78 + rank 42: 23968.77 + rank 43: 23968.79 + rank 44: 23968.75 + rank 45: 23968.76 + rank 46: 23968.76 + rank 47: 23968.79 + rank 48: 23967.92 + rank 49: 23967.89 + rank 50: 23967.88 + rank 51: 23967.88 + rank 52: 23967.88 + rank 53: 23967.87 + rank 54: 23967.88 + rank 55: 23967.90 + rank 56: 23967.91 + rank 57: 23967.90 + rank 58: 23967.89 + rank 59: 23967.90 + rank 60: 23967.88 + rank 61: 23967.91 + rank 62: 23967.89 + rank 63: 23967.91 + [2026-10-06 03:05:15] iteration 2361/ 6855 | consumed samples: 302208 | elapsed time per iteration (ms): 94571.3 | throughput per GPU (TFLOP/s/GPU): 40.3 | MFU 12.93% | FLOPs estimator: workload-aware | logical tokens/s: 177402.4 | valid tokens/s: 177402.4 | physical tokens/s: 177760.7 | loss tokens/s: 127216.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.678495E-01 | lm-global-supervised-tokens: 1.203099E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 6.293274E-01 | lm-token-buffer-mib: 5.826566E+01 | global_load_balancing_loss: 1.002947E+00 | loss scale: 1.0 | grad norm: 0.085 | params norm: 11488.650 | params rms: 0.062207 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 22:03:23.216615 | finish at 2026-10-11 01:08:38 + [2026-10-06 03:06:51] iteration 2362/ 6855 | consumed samples: 302336 | elapsed time per iteration (ms): 95649.7 | throughput per GPU (TFLOP/s/GPU): 38.6 | MFU 12.38% | FLOPs estimator: workload-aware | logical tokens/s: 175402.7 | valid tokens/s: 175402.7 | physical tokens/s: 175764.2 | loss tokens/s: 128869.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.447793E-01 | lm-global-supervised-tokens: 1.232633E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.020806E-02 | lm-token-buffer-mib: 5.817352E+01 | global_load_balancing_loss: 1.000025E+00 | loss scale: 1.0 | grad norm: 0.111 | params norm: 11488.582 | params rms: 0.062207 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 23:22:34.103912 | finish at 2026-10-11 02:29:25 + [2026-10-06 03:08:15] iteration 2363/ 6855 | consumed samples: 302464 | elapsed time per iteration (ms): 84546.9 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.79% | FLOPs estimator: workload-aware | logical tokens/s: 198436.9 | valid tokens/s: 198436.8 | physical tokens/s: 198792.5 | loss tokens/s: 144273.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.608851E-01 | lm-global-supervised-tokens: 1.219789E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.173579E-02 | lm-token-buffer-mib: 5.820388E+01 | global_load_balancing_loss: 9.998953E-01 | loss scale: 1.0 | grad norm: 0.108 | params norm: 11488.516 | params rms: 0.062207 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 9:29:44.518590 | finish at 2026-10-10 12:38:00 + [2026-10-06 03:09:43] iteration 2364/ 6855 | consumed samples: 302592 | elapsed time per iteration (ms): 87855.0 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.75% | FLOPs estimator: workload-aware | logical tokens/s: 190964.9 | valid tokens/s: 190964.8 | physical tokens/s: 191465.7 | loss tokens/s: 141453.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.562465E-01 | lm-global-supervised-tokens: 1.242736E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.167865E-02 | lm-token-buffer-mib: 5.821077E+01 | global_load_balancing_loss: 1.000122E+00 | loss scale: 1.0 | grad norm: 0.088 | params norm: 11488.450 | params rms: 0.062206 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 13:35:56.750478 | finish at 2026-10-10 16:45:40 + [2026-10-06 03:11:08] iteration 2365/ 6855 | consumed samples: 302720 | elapsed time per iteration (ms): 84609.2 | throughput per GPU (TFLOP/s/GPU): 43.8 | MFU 14.03% | FLOPs estimator: workload-aware | logical tokens/s: 198290.0 | valid tokens/s: 198290.0 | physical tokens/s: 198775.3 | loss tokens/s: 143959.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.232335E-01 | lm-global-supervised-tokens: 1.218029E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.555721E-02 | lm-token-buffer-mib: 5.817274E+01 | global_load_balancing_loss: 9.978530E-01 | loss scale: 1.0 | grad norm: 0.088 | params norm: 11488.382 | params rms: 0.062206 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 9:31:35.100327 | finish at 2026-10-10 12:42:43 + [2026-10-06 03:12:33] iteration 2366/ 6855 | consumed samples: 302848 | elapsed time per iteration (ms): 84907.3 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.61% | FLOPs estimator: workload-aware | logical tokens/s: 197592.6 | valid tokens/s: 197592.6 | physical tokens/s: 198191.1 | loss tokens/s: 146089.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.954089E-01 | lm-global-supervised-tokens: 1.240404E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.496515E-02 | lm-token-buffer-mib: 5.826241E+01 | global_load_balancing_loss: 9.988641E-01 | loss scale: 1.0 | grad norm: 0.104 | params norm: 11488.315 | params rms: 0.062205 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 9:52:28.882523 | finish at 2026-10-10 13:05:02 + [2026-10-06 03:13:59] iteration 2367/ 6855 | consumed samples: 302976 | elapsed time per iteration (ms): 86730.7 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.48% | FLOPs estimator: workload-aware | logical tokens/s: 193439.7 | valid tokens/s: 193439.7 | physical tokens/s: 193919.7 | loss tokens/s: 145600.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.535056E-01 | lm-global-supervised-tokens: 1.262802E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.364098E-02 | lm-token-buffer-mib: 5.816138E+01 | global_load_balancing_loss: 1.000119E+00 | loss scale: 1.0 | grad norm: 0.116 | params norm: 11488.248 | params rms: 0.062205 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 12:07:27.278950 | finish at 2026-10-10 15:21:27 + [2026-10-06 03:15:28] iteration 2368/ 6855 | consumed samples: 303104 | elapsed time per iteration (ms): 88294.4 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.46% | FLOPs estimator: workload-aware | logical tokens/s: 190014.3 | valid tokens/s: 190014.3 | physical tokens/s: 190495.2 | loss tokens/s: 141537.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.794332E-01 | lm-global-supervised-tokens: 1.249698E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.181480E-02 | lm-token-buffer-mib: 5.833875E+01 | global_load_balancing_loss: 1.000908E+00 | loss scale: 1.0 | grad norm: 0.052 | params norm: 11488.181 | params rms: 0.062205 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 14:02:57.150280 | finish at 2026-10-10 17:18:25 + [2026-10-06 03:16:52] iteration 2369/ 6855 | consumed samples: 303232 | elapsed time per iteration (ms): 84529.3 | throughput per GPU (TFLOP/s/GPU): 43.5 | MFU 13.95% | FLOPs estimator: workload-aware | logical tokens/s: 198477.6 | valid tokens/s: 198477.6 | physical tokens/s: 198934.7 | loss tokens/s: 142097.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.207790E-01 | lm-global-supervised-tokens: 1.201141E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.569108E-02 | lm-token-buffer-mib: 5.814134E+01 | global_load_balancing_loss: 9.999790E-01 | loss scale: 1.0 | grad norm: 0.051 | params norm: 11488.115 | params rms: 0.062204 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 9:19:58.643969 | finish at 2026-10-10 12:36:51 + [2026-10-06 03:18:44] iteration 2370/ 6855 | consumed samples: 303360 | elapsed time per iteration (ms): 111783.5 | throughput per GPU (TFLOP/s/GPU): 33.2 | MFU 10.64% | FLOPs estimator: workload-aware | logical tokens/s: 150086.0 | valid tokens/s: 150086.0 | physical tokens/s: 150533.8 | loss tokens/s: 112654.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.080565E-01 | lm-global-supervised-tokens: 1.259289E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.170551E-02 | lm-token-buffer-mib: 5.806354E+01 | global_load_balancing_loss: 1.000485E+00 | loss scale: 1.0 | grad norm: 0.080 | params norm: 11488.046 | params rms: 0.062204 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 19:15:49.218650 | finish at 2026-10-11 22:34:33 + [2026-10-06 03:20:11] iteration 2371/ 6855 | consumed samples: 303488 | elapsed time per iteration (ms): 86876.4 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.57% | FLOPs estimator: workload-aware | logical tokens/s: 193115.2 | valid tokens/s: 193115.2 | physical tokens/s: 193539.1 | loss tokens/s: 145093.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.551451E-01 | lm-global-supervised-tokens: 1.260516E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.314692E-02 | lm-token-buffer-mib: 5.813395E+01 | global_load_balancing_loss: 1.001067E+00 | loss scale: 1.0 | grad norm: 0.048 | params norm: 11487.978 | params rms: 0.062204 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 12:12:33.879134 | finish at 2026-10-10 15:32:45 + [2026-10-06 03:21:43] iteration 2372/ 6855 | consumed samples: 303616 | elapsed time per iteration (ms): 91702.9 | throughput per GPU (TFLOP/s/GPU): 41.1 | MFU 13.17% | FLOPs estimator: workload-aware | logical tokens/s: 182951.3 | valid tokens/s: 182951.3 | physical tokens/s: 183359.6 | loss tokens/s: 130597.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.856149E-01 | lm-global-supervised-tokens: 1.197617E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.340254E-02 | lm-token-buffer-mib: 5.833598E+01 | global_load_balancing_loss: 1.000217E+00 | loss scale: 1.0 | grad norm: 0.057 | params norm: 11487.911 | params rms: 0.062203 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 18:11:44.161322 | finish at 2026-10-10 21:33:27 + [2026-10-06 03:23:11] iteration 2373/ 6855 | consumed samples: 303744 | elapsed time per iteration (ms): 88601.2 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.66% | FLOPs estimator: workload-aware | logical tokens/s: 189356.1 | valid tokens/s: 189356.1 | physical tokens/s: 189826.5 | loss tokens/s: 132739.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.852821E-01 | lm-global-supervised-tokens: 1.176090E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.139877E-02 | lm-token-buffer-mib: 5.811569E+01 | global_load_balancing_loss: 1.000429E+00 | loss scale: 1.0 | grad norm: 0.068 | params norm: 11487.843 | params rms: 0.062203 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 14:18:30.795789 | finish at 2026-10-10 17:41:42 + [2026-10-06 03:24:40] iteration 2374/ 6855 | consumed samples: 303872 | elapsed time per iteration (ms): 88413.7 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.53% | FLOPs estimator: workload-aware | logical tokens/s: 189756.7 | valid tokens/s: 189756.7 | physical tokens/s: 190224.4 | loss tokens/s: 141440.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.112535E-01 | lm-global-supervised-tokens: 1.250530E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.105488E-02 | lm-token-buffer-mib: 5.830275E+01 | global_load_balancing_loss: 9.998539E-01 | loss scale: 1.0 | grad norm: 0.056 | params norm: 11487.776 | params rms: 0.062203 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 14:03:01.863881 | finish at 2026-10-10 17:27:41 + [2026-10-06 03:26:08] iteration 2375/ 6855 | consumed samples: 304000 | elapsed time per iteration (ms): 88030.2 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.45% | FLOPs estimator: workload-aware | logical tokens/s: 190583.8 | valid tokens/s: 190583.8 | physical tokens/s: 191198.2 | loss tokens/s: 135080.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.583890E-01 | lm-global-supervised-tokens: 1.189116E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.764437E-02 | lm-token-buffer-mib: 5.813590E+01 | global_load_balancing_loss: 1.000770E+00 | loss scale: 1.0 | grad norm: 0.040 | params norm: 11487.709 | params rms: 0.062202 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 13:32:55.206299 | finish at 2026-10-10 16:59:03 + [2026-10-06 03:27:37] iteration 2376/ 6855 | consumed samples: 304128 | elapsed time per iteration (ms): 89384.4 | throughput per GPU (TFLOP/s/GPU): 41.1 | MFU 13.19% | FLOPs estimator: workload-aware | logical tokens/s: 187696.7 | valid tokens/s: 187696.7 | physical tokens/s: 188153.2 | loss tokens/s: 142325.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.132706E-01 | lm-global-supervised-tokens: 1.272169E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.246177E-02 | lm-token-buffer-mib: 5.824422E+01 | global_load_balancing_loss: 9.994616E-01 | loss scale: 1.0 | grad norm: 0.042 | params norm: 11487.643 | params rms: 0.062202 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 15:12:32.821084 | finish at 2026-10-10 18:40:10 + [2026-10-06 03:29:03] iteration 2377/ 6855 | consumed samples: 304256 | elapsed time per iteration (ms): 86297.2 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.51% | FLOPs estimator: workload-aware | logical tokens/s: 194411.5 | valid tokens/s: 194411.5 | physical tokens/s: 194974.4 | loss tokens/s: 143766.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.483166E-01 | lm-global-supervised-tokens: 1.240668E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.216487E-02 | lm-token-buffer-mib: 5.817408E+01 | global_load_balancing_loss: 1.000077E+00 | loss scale: 1.0 | grad norm: 0.086 | params norm: 11487.577 | params rms: 0.062201 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 11:20:38.790977 | finish at 2026-10-10 14:49:42 + [2026-10-06 03:30:29] iteration 2378/ 6855 | consumed samples: 304384 | elapsed time per iteration (ms): 85775.5 | throughput per GPU (TFLOP/s/GPU): 43.8 | MFU 14.03% | FLOPs estimator: workload-aware | logical tokens/s: 195593.6 | valid tokens/s: 195593.6 | physical tokens/s: 195998.9 | loss tokens/s: 144621.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.860602E-01 | lm-global-supervised-tokens: 1.240498E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.441291E-02 | lm-token-buffer-mib: 5.814384E+01 | global_load_balancing_loss: 1.000714E+00 | loss scale: 1.0 | grad norm: 0.036 | params norm: 11487.510 | params rms: 0.062201 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 10:40:16.846519 | finish at 2026-10-10 14:10:46 + [2026-10-06 03:31:56] iteration 2379/ 6855 | consumed samples: 304512 | elapsed time per iteration (ms): 86476.4 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.39% | FLOPs estimator: workload-aware | logical tokens/s: 194009.1 | valid tokens/s: 194009.1 | physical tokens/s: 194468.5 | loss tokens/s: 148642.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.671554E-01 | lm-global-supervised-tokens: 1.285407E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.144391E-02 | lm-token-buffer-mib: 5.811773E+01 | global_load_balancing_loss: 1.000497E+00 | loss scale: 1.0 | grad norm: 0.057 | params norm: 11487.444 | params rms: 0.062201 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 11:31:08.211207 | finish at 2026-10-10 15:03:04 + [2026-10-06 03:33:44] iteration 2380/ 6855 | consumed samples: 304640 | elapsed time per iteration (ms): 108476.3 | throughput per GPU (TFLOP/s/GPU): 34.3 | MFU 10.98% | FLOPs estimator: workload-aware | logical tokens/s: 154662.2 | valid tokens/s: 154662.2 | physical tokens/s: 155017.6 | loss tokens/s: 115880.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.812636E-01 | lm-global-supervised-tokens: 1.257027E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.175430E-02 | lm-token-buffer-mib: 5.801920E+01 | global_load_balancing_loss: 9.999058E-01 | loss scale: 1.0 | grad norm: 0.099 | params norm: 11487.378 | params rms: 0.062200 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 14:50:31.272864 | finish at 2026-10-11 18:24:15 + [2026-10-06 03:35:09] iteration 2381/ 6855 | consumed samples: 304768 | elapsed time per iteration (ms): 84892.9 | throughput per GPU (TFLOP/s/GPU): 43.2 | MFU 13.84% | FLOPs estimator: workload-aware | logical tokens/s: 197628.0 | valid tokens/s: 197628.0 | physical tokens/s: 197999.0 | loss tokens/s: 141128.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.038586E-01 | lm-global-supervised-tokens: 1.198081E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.299842E-02 | lm-token-buffer-mib: 5.806856E+01 | global_load_balancing_loss: 1.000052E+00 | loss scale: 1.0 | grad norm: 0.120 | params norm: 11487.312 | params rms: 0.062200 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 9:30:10.694820 | finish at 2026-10-10 13:05:20 + [2026-10-06 03:36:34] iteration 2382/ 6855 | consumed samples: 304896 | elapsed time per iteration (ms): 85190.7 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.74% | FLOPs estimator: workload-aware | logical tokens/s: 196937.1 | valid tokens/s: 196937.1 | physical tokens/s: 197471.4 | loss tokens/s: 144143.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.280236E-01 | lm-global-supervised-tokens: 1.227972E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.494904E-02 | lm-token-buffer-mib: 5.817225E+01 | global_load_balancing_loss: 1.000660E+00 | loss scale: 1.0 | grad norm: 0.051 | params norm: 11487.245 | params rms: 0.062200 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 9:50:58.143180 | finish at 2026-10-10 13:27:32 + [2026-10-06 03:38:04] iteration 2383/ 6855 | consumed samples: 305024 | elapsed time per iteration (ms): 89912.7 | throughput per GPU (TFLOP/s/GPU): 41.4 | MFU 13.26% | FLOPs estimator: workload-aware | logical tokens/s: 186594.5 | valid tokens/s: 186594.5 | physical tokens/s: 187093.5 | loss tokens/s: 137510.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.843686E-01 | lm-global-supervised-tokens: 1.236395E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.848970E-02 | lm-token-buffer-mib: 5.815458E+01 | global_load_balancing_loss: 1.001167E+00 | loss scale: 1.0 | grad norm: 0.079 | params norm: 11487.176 | params rms: 0.062199 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 15:41:29.552540 | finish at 2026-10-10 19:19:34 + [2026-10-06 03:39:31] iteration 2384/ 6855 | consumed samples: 305152 | elapsed time per iteration (ms): 87017.1 | throughput per GPU (TFLOP/s/GPU): 42.8 | MFU 13.71% | FLOPs estimator: workload-aware | logical tokens/s: 192802.9 | valid tokens/s: 192802.9 | physical tokens/s: 193332.5 | loss tokens/s: 139743.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.765631E-01 | lm-global-supervised-tokens: 1.216009E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.670020E-02 | lm-token-buffer-mib: 5.820794E+01 | global_load_balancing_loss: 1.000598E+00 | loss scale: 1.0 | grad norm: 0.247 | params norm: 11487.107 | params rms: 0.062199 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 12:04:13.385240 | finish at 2026-10-10 15:43:44 + [2026-10-06 03:40:58] iteration 2385/ 6855 | consumed samples: 305280 | elapsed time per iteration (ms): 87008.2 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.39% | FLOPs estimator: workload-aware | logical tokens/s: 192823.4 | valid tokens/s: 192823.4 | physical tokens/s: 193248.6 | loss tokens/s: 148036.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.934762E-01 | lm-global-supervised-tokens: 1.288039E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.501730E-02 | lm-token-buffer-mib: 5.818388E+01 | global_load_balancing_loss: 9.979892E-01 | loss scale: 1.0 | grad norm: 0.422 | params norm: 11487.039 | params rms: 0.062199 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 12:02:06.600401 | finish at 2026-10-10 15:43:05 + [2026-10-06 03:42:28] iteration 2386/ 6855 | consumed samples: 305408 | elapsed time per iteration (ms): 89483.5 | throughput per GPU (TFLOP/s/GPU): 41.5 | MFU 13.30% | FLOPs estimator: workload-aware | logical tokens/s: 187489.5 | valid tokens/s: 187489.5 | physical tokens/s: 187878.8 | loss tokens/s: 135892.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.561268E-01 | lm-global-supervised-tokens: 1.216016E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.147028E-02 | lm-token-buffer-mib: 5.812423E+01 | global_load_balancing_loss: 9.996806E-01 | loss scale: 1.0 | grad norm: 0.050 | params norm: 11486.971 | params rms: 0.062198 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 15:05:01.674147 | finish at 2026-10-10 18:47:29 + [2026-10-06 03:43:59] iteration 2387/ 6855 | consumed samples: 305536 | elapsed time per iteration (ms): 91380.7 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.42% | FLOPs estimator: workload-aware | logical tokens/s: 183596.4 | valid tokens/s: 183596.4 | physical tokens/s: 183862.6 | loss tokens/s: 132061.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.892713E-01 | lm-global-supervised-tokens: 1.206785E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.165493E-02 | lm-token-buffer-mib: 5.826192E+01 | global_load_balancing_loss: 9.994546E-01 | loss scale: 1.0 | grad norm: 0.349 | params norm: 11486.902 | params rms: 0.062198 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 17:24:48.825354 | finish at 2026-10-10 21:08:48 + [2026-10-06 03:45:25] iteration 2388/ 6855 | consumed samples: 305664 | elapsed time per iteration (ms): 86194.1 | throughput per GPU (TFLOP/s/GPU): 43.1 | MFU 13.82% | FLOPs estimator: workload-aware | logical tokens/s: 194644.4 | valid tokens/s: 194644.3 | physical tokens/s: 195087.4 | loss tokens/s: 145105.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.750434E-01 | lm-global-supervised-tokens: 1.250724E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.207854E-02 | lm-token-buffer-mib: 5.813254E+01 | global_load_balancing_loss: 9.998000E-01 | loss scale: 1.0 | grad norm: 0.073 | params norm: 11486.831 | params rms: 0.062197 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 10:57:08.985691 | finish at 2026-10-10 14:42:34 + [2026-10-06 03:46:50] iteration 2389/ 6855 | consumed samples: 305792 | elapsed time per iteration (ms): 85391.5 | throughput per GPU (TFLOP/s/GPU): 42.8 | MFU 13.72% | FLOPs estimator: workload-aware | logical tokens/s: 196473.8 | valid tokens/s: 196473.8 | physical tokens/s: 197197.7 | loss tokens/s: 139816.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.532698E-01 | lm-global-supervised-tokens: 1.193912E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.466385E-02 | lm-token-buffer-mib: 5.813681E+01 | global_load_balancing_loss: 1.001785E+00 | loss scale: 1.0 | grad norm: 0.229 | params norm: 11486.763 | params rms: 0.062197 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 9:55:58.438983 | finish at 2026-10-10 13:42:49 + [2026-10-06 03:48:39] iteration 2390/ 6855 | consumed samples: 305920 | elapsed time per iteration (ms): 108833.0 | throughput per GPU (TFLOP/s/GPU): 33.9 | MFU 10.87% | FLOPs estimator: workload-aware | logical tokens/s: 154155.1 | valid tokens/s: 154155.1 | physical tokens/s: 154498.5 | loss tokens/s: 114256.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.704446E-01 | lm-global-supervised-tokens: 1.243483E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.313844E-02 | lm-token-buffer-mib: 5.809482E+01 | global_load_balancing_loss: 1.000530E+00 | loss scale: 1.0 | grad norm: 0.099 | params norm: 11486.697 | params rms: 0.062197 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 14:58:59.204234 | finish at 2026-10-11 18:47:39 + [2026-10-06 03:50:08] iteration 2391/ 6855 | consumed samples: 306048 | elapsed time per iteration (ms): 89108.0 | throughput per GPU (TFLOP/s/GPU): 43.4 | MFU 13.92% | FLOPs estimator: workload-aware | logical tokens/s: 188279.2 | valid tokens/s: 188279.2 | physical tokens/s: 188601.9 | loss tokens/s: 127534.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.685668E-01 | lm-global-supervised-tokens: 1.136436E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.936790E-02 | lm-token-buffer-mib: 5.803693E+01 | global_load_balancing_loss: 1.000532E+00 | loss scale: 1.0 | grad norm: 0.058 | params norm: 11486.632 | params rms: 0.062196 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 14:29:38.075993 | finish at 2026-10-10 18:19:46 + [2026-10-06 03:51:33] iteration 2392/ 6855 | consumed samples: 306176 | elapsed time per iteration (ms): 84326.3 | throughput per GPU (TFLOP/s/GPU): 43.4 | MFU 13.92% | FLOPs estimator: workload-aware | logical tokens/s: 198954.9 | valid tokens/s: 198954.9 | physical tokens/s: 199415.8 | loss tokens/s: 148188.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.960933E-01 | lm-global-supervised-tokens: 1.249618E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.390657E-02 | lm-token-buffer-mib: 5.814680E+01 | global_load_balancing_loss: 1.000848E+00 | loss scale: 1.0 | grad norm: 0.149 | params norm: 11486.565 | params rms: 0.062196 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 8:32:28.387142 | finish at 2026-10-10 12:24:01 + [2026-10-06 03:53:01] iteration 2393/ 6855 | consumed samples: 306304 | elapsed time per iteration (ms): 88522.1 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.75% | FLOPs estimator: workload-aware | logical tokens/s: 189525.4 | valid tokens/s: 189525.4 | physical tokens/s: 189901.6 | loss tokens/s: 139674.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.507768E-01 | lm-global-supervised-tokens: 1.236424E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.290622E-02 | lm-token-buffer-mib: 5.826347E+01 | global_load_balancing_loss: 1.002097E+00 | loss scale: 1.0 | grad norm: 0.101 | params norm: 11486.497 | params rms: 0.062196 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 13:43:05.409011 | finish at 2026-10-10 17:36:07 + [2026-10-06 03:54:28] iteration 2394/ 6855 | consumed samples: 306432 | elapsed time per iteration (ms): 86726.5 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.78% | FLOPs estimator: workload-aware | logical tokens/s: 193449.4 | valid tokens/s: 193449.3 | physical tokens/s: 193970.6 | loss tokens/s: 137890.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.674695E-01 | lm-global-supervised-tokens: 1.195872E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.153917E-02 | lm-token-buffer-mib: 5.805238E+01 | global_load_balancing_loss: 1.000638E+00 | loss scale: 1.0 | grad norm: 0.072 | params norm: 11486.431 | params rms: 0.062195 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 11:28:07.074064 | finish at 2026-10-10 15:22:35 + [2026-10-06 03:55:52] iteration 2395/ 6855 | consumed samples: 306560 | elapsed time per iteration (ms): 83760.3 | throughput per GPU (TFLOP/s/GPU): 43.7 | MFU 14.02% | FLOPs estimator: workload-aware | logical tokens/s: 200298.8 | valid tokens/s: 200298.7 | physical tokens/s: 200796.2 | loss tokens/s: 146105.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.484439E-01 | lm-global-supervised-tokens: 1.223780E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.138355E-02 | lm-token-buffer-mib: 5.814379E+01 | global_load_balancing_loss: 9.985886E-01 | loss scale: 1.0 | grad norm: 0.130 | params norm: 11486.365 | params rms: 0.062195 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 7:46:10.970612 | finish at 2026-10-10 11:42:03 + [2026-10-06 03:57:19] iteration 2396/ 6855 | consumed samples: 306688 | elapsed time per iteration (ms): 87468.6 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.55% | FLOPs estimator: workload-aware | logical tokens/s: 191807.9 | valid tokens/s: 191807.9 | physical tokens/s: 192163.4 | loss tokens/s: 138270.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.499478E-01 | lm-global-supervised-tokens: 1.209432E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.595557E-02 | lm-token-buffer-mib: 5.818737E+01 | global_load_balancing_loss: 9.985949E-01 | loss scale: 1.0 | grad norm: 0.094 | params norm: 11486.299 | params rms: 0.062195 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 12:20:22.689545 | finish at 2026-10-10 16:17:42 + [2026-10-06 03:58:49] iteration 2397/ 6855 | consumed samples: 306816 | elapsed time per iteration (ms): 89801.9 | throughput per GPU (TFLOP/s/GPU): 41.5 | MFU 13.29% | FLOPs estimator: workload-aware | logical tokens/s: 186824.0 | valid tokens/s: 186824.0 | physical tokens/s: 187380.2 | loss tokens/s: 131033.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.714270E-01 | lm-global-supervised-tokens: 1.176707E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.261898E-02 | lm-token-buffer-mib: 5.824172E+01 | global_load_balancing_loss: 9.986264E-01 | loss scale: 1.0 | grad norm: 0.118 | params norm: 11486.231 | params rms: 0.062194 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 15:12:16.914439 | finish at 2026-10-10 19:11:06 + [2026-10-06 04:00:19] iteration 2398/ 6855 | consumed samples: 306944 | elapsed time per iteration (ms): 89811.8 | throughput per GPU (TFLOP/s/GPU): 41.5 | MFU 13.31% | FLOPs estimator: workload-aware | logical tokens/s: 186804.2 | valid tokens/s: 186804.2 | physical tokens/s: 187238.3 | loss tokens/s: 133612.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.569425E-01 | lm-global-supervised-tokens: 1.199995E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.611260E-02 | lm-token-buffer-mib: 5.803320E+01 | global_load_balancing_loss: 9.993121E-01 | loss scale: 1.0 | grad norm: 0.116 | params norm: 11486.163 | params rms: 0.062194 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 15:11:31.133106 | finish at 2026-10-10 19:11:50 + [2026-10-06 04:01:46] iteration 2399/ 6855 | consumed samples: 307072 | elapsed time per iteration (ms): 87148.0 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.51% | FLOPs estimator: workload-aware | logical tokens/s: 192513.9 | valid tokens/s: 192513.9 | physical tokens/s: 192951.0 | loss tokens/s: 140268.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.941431E-01 | lm-global-supervised-tokens: 1.222409E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.143042E-02 | lm-token-buffer-mib: 5.805838E+01 | global_load_balancing_loss: 1.000008E+00 | loss scale: 1.0 | grad norm: 0.049 | params norm: 11486.091 | params rms: 0.062193 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 11:52:11.578312 | finish at 2026-10-10 15:53:58 + [2026-10-06 04:03:36] iteration 2400/ 6855 | consumed samples: 307200 | elapsed time per iteration (ms): 109958.2 | throughput per GPU (TFLOP/s/GPU): 33.5 | MFU 10.72% | FLOPs estimator: workload-aware | logical tokens/s: 152577.2 | valid tokens/s: 152577.2 | physical tokens/s: 152994.9 | loss tokens/s: 119372.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.444916E-01 | lm-global-supervised-tokens: 1.312602E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.523888E-02 | lm-token-buffer-mib: 5.805158E+01 | global_load_balancing_loss: 1.000124E+00 | loss scale: 1.0 | grad norm: 0.102 | params norm: 11486.025 | params rms: 0.062193 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 16:04:23.684245 | finish at 2026-10-11 20:08:00 +times across ranks (ms): + save-checkpoint: + rank 0: 23675.70 + rank 1: 23675.71 + rank 2: 23675.72 + rank 3: 23675.70 + rank 4: 23675.72 + rank 5: 23675.69 + rank 6: 23675.68 + rank 7: 23675.70 + rank 8: 23675.76 + rank 9: 23675.71 + rank 10: 23675.74 + rank 11: 23675.76 + rank 12: 23675.75 + rank 13: 23675.74 + rank 14: 23675.75 + rank 15: 23675.72 + rank 16: 23676.28 + rank 17: 23676.32 + rank 18: 23676.27 + rank 19: 23676.28 + rank 20: 23676.21 + rank 21: 23676.27 + rank 22: 23676.26 + rank 23: 23676.25 + rank 24: 23674.65 + rank 25: 23674.62 + rank 26: 23674.67 + rank 27: 23674.69 + rank 28: 23674.65 + rank 29: 23674.67 + rank 30: 23674.67 + rank 31: 23674.62 + rank 32: 23675.94 + rank 33: 23675.94 + rank 34: 23675.93 + rank 35: 23675.91 + rank 36: 23675.91 + rank 37: 23675.91 + rank 38: 23675.91 + rank 39: 23675.92 + rank 40: 23675.93 + rank 41: 23675.93 + rank 42: 23675.95 + rank 43: 23675.95 + rank 44: 23675.93 + rank 45: 23675.94 + rank 46: 23675.93 + rank 47: 23675.92 + rank 48: 23675.51 + rank 49: 23675.47 + rank 50: 23675.43 + rank 51: 23675.44 + rank 52: 23675.45 + rank 53: 23675.44 + rank 54: 23675.46 + rank 55: 23675.57 + rank 56: 23676.31 + rank 57: 23676.29 + rank 58: 23676.29 + rank 59: 23676.27 + rank 60: 23676.28 + rank 61: 23676.30 + rank 62: 23676.27 + rank 63: 23676.29 + [2026-10-06 04:05:40] iteration 2401/ 6855 | consumed samples: 307328 | elapsed time per iteration (ms): 99558.6 | throughput per GPU (TFLOP/s/GPU): 37.3 | MFU 11.95% | FLOPs estimator: workload-aware | logical tokens/s: 168515.9 | valid tokens/s: 168515.9 | physical tokens/s: 168901.8 | loss tokens/s: 124844.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.769331E-01 | lm-global-supervised-tokens: 1.242936E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 4.528889E-01 | lm-token-buffer-mib: 5.806656E+01 | global_load_balancing_loss: 1.000774E+00 | loss scale: 1.0 | grad norm: 0.042 | params norm: 11485.962 | params rms: 0.062193 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 3:10:33.790727 | finish at 2026-10-11 07:16:13 + [2026-10-06 04:07:15] iteration 2402/ 6855 | consumed samples: 307456 | elapsed time per iteration (ms): 95411.3 | throughput per GPU (TFLOP/s/GPU): 39.5 | MFU 12.67% | FLOPs estimator: workload-aware | logical tokens/s: 175840.9 | valid tokens/s: 175840.9 | physical tokens/s: 176177.3 | loss tokens/s: 132317.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.148662E-01 | lm-global-supervised-tokens: 1.262458E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.211222E-02 | lm-token-buffer-mib: 5.813186E+01 | global_load_balancing_loss: 1.000501E+00 | loss scale: 1.0 | grad norm: 0.057 | params norm: 11485.901 | params rms: 0.062192 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 22:01:06.340288 | finish at 2026-10-11 02:08:21 + [2026-10-06 04:08:41] iteration 2403/ 6855 | consumed samples: 307584 | elapsed time per iteration (ms): 86496.1 | throughput per GPU (TFLOP/s/GPU): 43.1 | MFU 13.81% | FLOPs estimator: workload-aware | logical tokens/s: 193964.9 | valid tokens/s: 193964.9 | physical tokens/s: 194324.7 | loss tokens/s: 140247.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.598508E-01 | lm-global-supervised-tokens: 1.213083E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.132669E-02 | lm-token-buffer-mib: 5.821662E+01 | global_load_balancing_loss: 1.001562E+00 | loss scale: 1.0 | grad norm: 0.124 | params norm: 11485.843 | params rms: 0.062192 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 10:58:00.441668 | finish at 2026-10-10 15:06:42 + [2026-10-06 04:10:07] iteration 2404/ 6855 | consumed samples: 307712 | elapsed time per iteration (ms): 85221.9 | throughput per GPU (TFLOP/s/GPU): 44.3 | MFU 14.19% | FLOPs estimator: workload-aware | logical tokens/s: 196865.0 | valid tokens/s: 196865.0 | physical tokens/s: 197139.9 | loss tokens/s: 134865.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.265696E-01 | lm-global-supervised-tokens: 1.149346E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.399394E-02 | lm-token-buffer-mib: 5.814180E+01 | global_load_balancing_loss: 1.001929E+00 | loss scale: 1.0 | grad norm: 0.035 | params norm: 11485.785 | params rms: 0.062192 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 9:22:02.749001 | finish at 2026-10-10 13:32:09 + [2026-10-06 04:11:35] iteration 2405/ 6855 | consumed samples: 307840 | elapsed time per iteration (ms): 88487.8 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.48% | FLOPs estimator: workload-aware | logical tokens/s: 189598.0 | valid tokens/s: 189598.0 | physical tokens/s: 190190.1 | loss tokens/s: 139651.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.085805E-01 | lm-global-supervised-tokens: 1.235749E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.151092E-02 | lm-token-buffer-mib: 5.817864E+01 | global_load_balancing_loss: 1.000849E+00 | loss scale: 1.0 | grad norm: 0.173 | params norm: 11485.728 | params rms: 0.062191 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 13:22:50.743430 | finish at 2026-10-10 17:34:26 + [2026-10-06 04:13:01] iteration 2406/ 6855 | consumed samples: 307968 | elapsed time per iteration (ms): 85578.1 | throughput per GPU (TFLOP/s/GPU): 43.1 | MFU 13.82% | FLOPs estimator: workload-aware | logical tokens/s: 196045.2 | valid tokens/s: 196045.2 | physical tokens/s: 196396.1 | loss tokens/s: 145567.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.728551E-01 | lm-global-supervised-tokens: 1.245740E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.710958E-02 | lm-token-buffer-mib: 5.804773E+01 | global_load_balancing_loss: 9.995222E-01 | loss scale: 1.0 | grad norm: 0.053 | params norm: 11485.671 | params rms: 0.062191 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 9:45:37.141768 | finish at 2026-10-10 13:58:38 + [2026-10-06 04:14:29] iteration 2407/ 6855 | consumed samples: 308096 | elapsed time per iteration (ms): 88573.5 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.54% | FLOPs estimator: workload-aware | logical tokens/s: 189415.6 | valid tokens/s: 189415.6 | physical tokens/s: 189801.1 | loss tokens/s: 138113.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.534970E-01 | lm-global-supervised-tokens: 1.223321E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.188845E-02 | lm-token-buffer-mib: 5.803226E+01 | global_load_balancing_loss: 1.000506E+00 | loss scale: 1.0 | grad norm: 0.143 | params norm: 11485.612 | params rms: 0.062191 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 13:26:14.790833 | finish at 2026-10-10 17:40:44 + [2026-10-06 04:16:00] iteration 2408/ 6855 | consumed samples: 308224 | elapsed time per iteration (ms): 90228.7 | throughput per GPU (TFLOP/s/GPU): 41.4 | MFU 13.28% | FLOPs estimator: workload-aware | logical tokens/s: 185940.6 | valid tokens/s: 185940.6 | physical tokens/s: 186425.8 | loss tokens/s: 136811.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.835900E-01 | lm-global-supervised-tokens: 1.234428E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.248406E-02 | lm-token-buffer-mib: 5.808125E+01 | global_load_balancing_loss: 1.000468E+00 | loss scale: 1.0 | grad norm: 0.046 | params norm: 11485.555 | params rms: 0.062191 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 15:27:27.096411 | finish at 2026-10-10 19:43:27 + [2026-10-06 04:17:26] iteration 2409/ 6855 | consumed samples: 308352 | elapsed time per iteration (ms): 86139.9 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.62% | FLOPs estimator: workload-aware | logical tokens/s: 194766.7 | valid tokens/s: 194766.7 | physical tokens/s: 195208.2 | loss tokens/s: 142124.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.404886E-01 | lm-global-supervised-tokens: 1.224258E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.157730E-02 | lm-token-buffer-mib: 5.811359E+01 | global_load_balancing_loss: 1.000781E+00 | loss scale: 1.0 | grad norm: 0.108 | params norm: 11485.495 | params rms: 0.062190 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 10:22:58.041903 | finish at 2026-10-10 14:40:24 + [2026-10-06 04:19:18] iteration 2410/ 6855 | consumed samples: 308480 | elapsed time per iteration (ms): 112647.0 | throughput per GPU (TFLOP/s/GPU): 33.1 | MFU 10.61% | FLOPs estimator: workload-aware | logical tokens/s: 148936.1 | valid tokens/s: 148936.1 | physical tokens/s: 149280.3 | loss tokens/s: 110899.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.404002E-01 | lm-global-supervised-tokens: 1.249250E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.580574E-02 | lm-token-buffer-mib: 5.805704E+01 | global_load_balancing_loss: 9.992985E-01 | loss scale: 1.0 | grad norm: 0.030 | params norm: 11485.437 | params rms: 0.062190 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 19:05:15.964080 | finish at 2026-10-11 23:24:34 + [2026-10-06 04:20:44] iteration 2411/ 6855 | consumed samples: 308608 | elapsed time per iteration (ms): 85300.4 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.57% | FLOPs estimator: workload-aware | logical tokens/s: 196683.8 | valid tokens/s: 196683.8 | physical tokens/s: 197155.0 | loss tokens/s: 148874.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.065767E-01 | lm-global-supervised-tokens: 1.269902E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.506691E-02 | lm-token-buffer-mib: 5.813497E+01 | global_load_balancing_loss: 9.994825E-01 | loss scale: 1.0 | grad norm: 0.061 | params norm: 11485.378 | params rms: 0.062190 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 9:17:55.124958 | finish at 2026-10-10 13:38:39 + [2026-10-06 04:22:12] iteration 2412/ 6855 | consumed samples: 308736 | elapsed time per iteration (ms): 87966.0 | throughput per GPU (TFLOP/s/GPU): 42.8 | MFU 13.72% | FLOPs estimator: workload-aware | logical tokens/s: 190723.5 | valid tokens/s: 190723.5 | physical tokens/s: 191218.3 | loss tokens/s: 137195.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.243473E-01 | lm-global-supervised-tokens: 1.206850E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.383354E-02 | lm-token-buffer-mib: 5.811920E+01 | global_load_balancing_loss: 9.998853E-01 | loss scale: 1.0 | grad norm: 0.044 | params norm: 11485.318 | params rms: 0.062189 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 12:33:53.109962 | finish at 2026-10-10 16:56:05 + [2026-10-06 04:23:40] iteration 2413/ 6855 | consumed samples: 308864 | elapsed time per iteration (ms): 87904.3 | throughput per GPU (TFLOP/s/GPU): 41.5 | MFU 13.30% | FLOPs estimator: workload-aware | logical tokens/s: 190857.4 | valid tokens/s: 190857.4 | physical tokens/s: 191318.2 | loss tokens/s: 138104.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.601762E-01 | lm-global-supervised-tokens: 1.214000E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 5.359148E-02 | lm-token-buffer-mib: 5.826693E+01 | global_load_balancing_loss: 1.000398E+00 | loss scale: 1.0 | grad norm: 0.043 | params norm: 11485.256 | params rms: 0.062189 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 12:27:50.946026 | finish at 2026-10-10 16:51:30 + [2026-10-06 04:25:07] iteration 2414/ 6855 | consumed samples: 308992 | elapsed time per iteration (ms): 87759.3 | throughput per GPU (TFLOP/s/GPU): 41.6 | MFU 13.34% | FLOPs estimator: workload-aware | logical tokens/s: 191172.4 | valid tokens/s: 191172.4 | physical tokens/s: 191730.5 | loss tokens/s: 143648.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.734639E-01 | lm-global-supervised-tokens: 1.260651E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.274499E-02 | lm-token-buffer-mib: 5.814980E+01 | global_load_balancing_loss: 9.991775E-01 | loss scale: 1.0 | grad norm: 0.075 | params norm: 11485.192 | params rms: 0.062189 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 12:15:38.960213 | finish at 2026-10-10 16:40:46 + [2026-10-06 04:26:37] iteration 2415/ 6855 | consumed samples: 309120 | elapsed time per iteration (ms): 89222.5 | throughput per GPU (TFLOP/s/GPU): 41.5 | MFU 13.29% | FLOPs estimator: workload-aware | logical tokens/s: 188037.4 | valid tokens/s: 188037.4 | physical tokens/s: 188468.8 | loss tokens/s: 140225.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.582133E-01 | lm-global-supervised-tokens: 1.251124E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.297753E-02 | lm-token-buffer-mib: 5.823865E+01 | global_load_balancing_loss: 9.997123E-01 | loss scale: 1.0 | grad norm: 0.086 | params norm: 11485.128 | params rms: 0.062188 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 14:02:27.700310 | finish at 2026-10-10 18:29:04 + [2026-10-06 04:28:04] iteration 2416/ 6855 | consumed samples: 309248 | elapsed time per iteration (ms): 87028.9 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.66% | FLOPs estimator: workload-aware | logical tokens/s: 192777.2 | valid tokens/s: 192777.2 | physical tokens/s: 193327.6 | loss tokens/s: 141563.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.709997E-01 | lm-global-supervised-tokens: 1.232013E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.143527E-02 | lm-token-buffer-mib: 5.814853E+01 | global_load_balancing_loss: 1.001786E+00 | loss scale: 1.0 | grad norm: 0.051 | params norm: 11485.062 | params rms: 0.062188 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 11:18:41.192500 | finish at 2026-10-10 15:46:45 + [2026-10-06 04:29:30] iteration 2417/ 6855 | consumed samples: 309376 | elapsed time per iteration (ms): 86124.4 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.78% | FLOPs estimator: workload-aware | logical tokens/s: 194802.1 | valid tokens/s: 194802.1 | physical tokens/s: 195504.3 | loss tokens/s: 141667.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.978594E-01 | lm-global-supervised-tokens: 1.220099E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.140830E-02 | lm-token-buffer-mib: 5.821702E+01 | global_load_balancing_loss: 1.000655E+00 | loss scale: 1.0 | grad norm: 0.089 | params norm: 11484.995 | params rms: 0.062188 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 10:10:20.190452 | finish at 2026-10-10 14:39:50 + [2026-10-06 04:30:53] iteration 2418/ 6855 | consumed samples: 309504 | elapsed time per iteration (ms): 83800.3 | throughput per GPU (TFLOP/s/GPU): 43.5 | MFU 13.95% | FLOPs estimator: workload-aware | logical tokens/s: 200204.4 | valid tokens/s: 200204.4 | physical tokens/s: 200725.4 | loss tokens/s: 147384.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.827756E-01 | lm-global-supervised-tokens: 1.235090E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.399088E-02 | lm-token-buffer-mib: 5.838354E+01 | global_load_balancing_loss: 1.000387E+00 | loss scale: 1.0 | grad norm: 0.056 | params norm: 11484.925 | params rms: 0.062187 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 7:17:01.857588 | finish at 2026-10-10 11:47:55 + [2026-10-06 04:32:20] iteration 2419/ 6855 | consumed samples: 309632 | elapsed time per iteration (ms): 86384.0 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.57% | FLOPs estimator: workload-aware | logical tokens/s: 194215.7 | valid tokens/s: 194215.7 | physical tokens/s: 194663.4 | loss tokens/s: 142519.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.442276E-01 | lm-global-supervised-tokens: 1.231137E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.567014E-02 | lm-token-buffer-mib: 5.821318E+01 | global_load_balancing_loss: 1.000757E+00 | loss scale: 1.0 | grad norm: 0.100 | params norm: 11484.855 | params rms: 0.062187 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 10:26:39.522638 | finish at 2026-10-10 14:58:59 + [2026-10-06 04:34:12] iteration 2420/ 6855 | consumed samples: 309760 | elapsed time per iteration (ms): 112008.5 | throughput per GPU (TFLOP/s/GPU): 33.2 | MFU 10.65% | FLOPs estimator: workload-aware | logical tokens/s: 149784.9 | valid tokens/s: 149784.9 | physical tokens/s: 150162.0 | loss tokens/s: 110588.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.181000E-01 | lm-global-supervised-tokens: 1.238682E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.626989E-02 | lm-token-buffer-mib: 5.815977E+01 | global_load_balancing_loss: 9.991196E-01 | loss scale: 1.0 | grad norm: 0.051 | params norm: 11484.784 | params rms: 0.062186 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 17:59:17.656702 | finish at 2026-10-11 22:33:30 + [2026-10-06 04:35:39] iteration 2421/ 6855 | consumed samples: 309888 | elapsed time per iteration (ms): 86692.4 | throughput per GPU (TFLOP/s/GPU): 43.1 | MFU 13.81% | FLOPs estimator: workload-aware | logical tokens/s: 193525.8 | valid tokens/s: 193525.8 | physical tokens/s: 193942.5 | loss tokens/s: 148903.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.069364E-01 | lm-global-supervised-tokens: 1.290883E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.382063E-02 | lm-token-buffer-mib: 5.807954E+01 | global_load_balancing_loss: 1.000207E+00 | loss scale: 1.0 | grad norm: 0.081 | params norm: 11484.712 | params rms: 0.062186 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 10:46:34.135334 | finish at 2026-10-10 15:22:13 + [2026-10-06 04:37:04] iteration 2422/ 6855 | consumed samples: 310016 | elapsed time per iteration (ms): 85292.4 | throughput per GPU (TFLOP/s/GPU): 43.4 | MFU 13.91% | FLOPs estimator: workload-aware | logical tokens/s: 196701.5 | valid tokens/s: 196701.5 | physical tokens/s: 197269.7 | loss tokens/s: 147952.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.462749E-01 | lm-global-supervised-tokens: 1.261922E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.158359E-02 | lm-token-buffer-mib: 5.809260E+01 | global_load_balancing_loss: 1.000619E+00 | loss scale: 1.0 | grad norm: 0.072 | params norm: 11484.640 | params rms: 0.062186 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 9:01:41.428520 | finish at 2026-10-10 13:38:45 + [2026-10-06 04:38:32] iteration 2423/ 6855 | consumed samples: 310144 | elapsed time per iteration (ms): 88364.3 | throughput per GPU (TFLOP/s/GPU): 43.4 | MFU 13.92% | FLOPs estimator: workload-aware | logical tokens/s: 189864.0 | valid tokens/s: 189864.0 | physical tokens/s: 190232.6 | loss tokens/s: 130722.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.492940E-01 | lm-global-supervised-tokens: 1.155120E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.158595E-02 | lm-token-buffer-mib: 5.820889E+01 | global_load_balancing_loss: 1.000688E+00 | loss scale: 1.0 | grad norm: 0.053 | params norm: 11484.568 | params rms: 0.062185 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 12:47:10.384285 | finish at 2026-10-10 17:25:43 + [2026-10-06 04:39:56] iteration 2424/ 6855 | consumed samples: 310272 | elapsed time per iteration (ms): 83800.9 | throughput per GPU (TFLOP/s/GPU): 43.4 | MFU 13.90% | FLOPs estimator: workload-aware | logical tokens/s: 200203.2 | valid tokens/s: 200203.2 | physical tokens/s: 200607.0 | loss tokens/s: 148243.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.813727E-01 | lm-global-supervised-tokens: 1.242295E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.183986E-02 | lm-token-buffer-mib: 5.808423E+01 | global_load_balancing_loss: 9.994086E-01 | loss scale: 1.0 | grad norm: 0.099 | params norm: 11484.494 | params rms: 0.062185 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 7:08:41.669502 | finish at 2026-10-10 11:48:38 + [2026-10-06 04:41:26] iteration 2425/ 6855 | consumed samples: 310400 | elapsed time per iteration (ms): 90425.1 | throughput per GPU (TFLOP/s/GPU): 41.5 | MFU 13.31% | FLOPs estimator: workload-aware | logical tokens/s: 185536.7 | valid tokens/s: 185536.7 | physical tokens/s: 186215.6 | loss tokens/s: 135371.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.740677E-01 | lm-global-supervised-tokens: 1.224097E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.008338E-02 | lm-token-buffer-mib: 5.808714E+01 | global_load_balancing_loss: 9.990550E-01 | loss scale: 1.0 | grad norm: 0.056 | params norm: 11484.422 | params rms: 0.062184 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 15:16:22.986376 | finish at 2026-10-10 19:57:49 + [2026-10-06 04:42:53] iteration 2426/ 6855 | consumed samples: 310528 | elapsed time per iteration (ms): 86859.3 | throughput per GPU (TFLOP/s/GPU): 41.7 | MFU 13.38% | FLOPs estimator: workload-aware | logical tokens/s: 193153.9 | valid tokens/s: 193153.9 | physical tokens/s: 193622.4 | loss tokens/s: 143138.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.634244E-01 | lm-global-supervised-tokens: 1.243288E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.592669E-02 | lm-token-buffer-mib: 5.811476E+01 | global_load_balancing_loss: 9.997834E-01 | loss scale: 1.0 | grad norm: 0.070 | params norm: 11484.348 | params rms: 0.062184 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 10:51:39.798067 | finish at 2026-10-10 15:34:33 + [2026-10-06 04:44:20] iteration 2427/ 6855 | consumed samples: 310656 | elapsed time per iteration (ms): 86933.3 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.60% | FLOPs estimator: workload-aware | logical tokens/s: 192988.8 | valid tokens/s: 192988.8 | physical tokens/s: 193500.7 | loss tokens/s: 140850.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.514593E-01 | lm-global-supervised-tokens: 1.224464E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.854033E-02 | lm-token-buffer-mib: 5.812132E+01 | global_load_balancing_loss: 1.000664E+00 | loss scale: 1.0 | grad norm: 0.085 | params norm: 11484.274 | params rms: 0.062184 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 10:55:40.856235 | finish at 2026-10-10 15:40:01 + [2026-10-06 04:45:48] iteration 2428/ 6855 | consumed samples: 310784 | elapsed time per iteration (ms): 88244.5 | throughput per GPU (TFLOP/s/GPU): 41.6 | MFU 13.32% | FLOPs estimator: workload-aware | logical tokens/s: 190121.6 | valid tokens/s: 190121.6 | physical tokens/s: 190543.4 | loss tokens/s: 145932.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.577640E-01 | lm-global-supervised-tokens: 1.287775E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.536692E-02 | lm-token-buffer-mib: 5.804163E+01 | global_load_balancing_loss: 1.000641E+00 | loss scale: 1.0 | grad norm: 0.065 | params norm: 11484.200 | params rms: 0.062183 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 12:30:58.234388 | finish at 2026-10-10 17:16:47 + [2026-10-06 04:47:18] iteration 2429/ 6855 | consumed samples: 310912 | elapsed time per iteration (ms): 89367.1 | throughput per GPU (TFLOP/s/GPU): 41.2 | MFU 13.20% | FLOPs estimator: workload-aware | logical tokens/s: 187733.3 | valid tokens/s: 187733.3 | physical tokens/s: 188289.5 | loss tokens/s: 139132.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.649926E-01 | lm-global-supervised-tokens: 1.243384E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.749735E-02 | lm-token-buffer-mib: 5.809471E+01 | global_load_balancing_loss: 1.000483E+00 | loss scale: 1.0 | grad norm: 0.075 | params norm: 11484.126 | params rms: 0.062183 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 13:52:18.955551 | finish at 2026-10-10 18:39:37 + [2026-10-06 04:49:07] iteration 2430/ 6855 | consumed samples: 311040 | elapsed time per iteration (ms): 109577.1 | throughput per GPU (TFLOP/s/GPU): 33.9 | MFU 10.88% | FLOPs estimator: workload-aware | logical tokens/s: 153108.8 | valid tokens/s: 153108.8 | physical tokens/s: 153442.4 | loss tokens/s: 112003.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.413656E-01 | lm-global-supervised-tokens: 1.227304E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.152294E-02 | lm-token-buffer-mib: 5.815676E+01 | global_load_balancing_loss: 1.001606E+00 | loss scale: 1.0 | grad norm: 0.038 | params norm: 11484.051 | params rms: 0.062182 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 14:41:18.614920 | finish at 2026-10-11 19:30:26 + [2026-10-06 04:50:32] iteration 2431/ 6855 | consumed samples: 311168 | elapsed time per iteration (ms): 84919.4 | throughput per GPU (TFLOP/s/GPU): 43.5 | MFU 13.94% | FLOPs estimator: workload-aware | logical tokens/s: 197566.4 | valid tokens/s: 197566.4 | physical tokens/s: 198076.3 | loss tokens/s: 140258.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.166298E-01 | lm-global-supervised-tokens: 1.191070E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.164891E-02 | lm-token-buffer-mib: 5.811754E+01 | global_load_balancing_loss: 1.001534E+00 | loss scale: 1.0 | grad norm: 0.209 | params norm: 11483.975 | params rms: 0.062182 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 8:21:23.212435 | finish at 2026-10-10 13:11:56 + [2026-10-06 04:51:57] iteration 2432/ 6855 | consumed samples: 311296 | elapsed time per iteration (ms): 84735.2 | throughput per GPU (TFLOP/s/GPU): 42.8 | MFU 13.73% | FLOPs estimator: workload-aware | logical tokens/s: 197995.6 | valid tokens/s: 197995.6 | physical tokens/s: 198408.6 | loss tokens/s: 149107.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.684318E-01 | lm-global-supervised-tokens: 1.263466E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.068334E-02 | lm-token-buffer-mib: 5.813206E+01 | global_load_balancing_loss: 9.991124E-01 | loss scale: 1.0 | grad norm: 0.226 | params norm: 11483.900 | params rms: 0.062182 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 8:06:23.763974 | finish at 2026-10-10 12:58:21 + [2026-10-06 04:53:25] iteration 2433/ 6855 | consumed samples: 311424 | elapsed time per iteration (ms): 87915.2 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.60% | FLOPs estimator: workload-aware | logical tokens/s: 190833.8 | valid tokens/s: 190833.8 | physical tokens/s: 191197.6 | loss tokens/s: 136567.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.711967E-01 | lm-global-supervised-tokens: 1.200632E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.417000E-02 | lm-token-buffer-mib: 5.802791E+01 | global_load_balancing_loss: 9.987540E-01 | loss scale: 1.0 | grad norm: 0.116 | params norm: 11483.825 | params rms: 0.062181 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 11:59:21.182010 | finish at 2026-10-10 16:52:46 + [2026-10-06 04:54:54] iteration 2434/ 6855 | consumed samples: 311552 | elapsed time per iteration (ms): 89299.5 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.42% | FLOPs estimator: workload-aware | logical tokens/s: 187875.2 | valid tokens/s: 187875.2 | physical tokens/s: 188291.2 | loss tokens/s: 140044.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.577175E-01 | lm-global-supervised-tokens: 1.250590E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.409222E-02 | lm-token-buffer-mib: 5.810059E+01 | global_load_balancing_loss: 9.989353E-01 | loss scale: 1.0 | grad norm: 0.304 | params norm: 11483.752 | params rms: 0.062181 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 13:39:52.992477 | finish at 2026-10-10 18:34:47 + [2026-10-06 04:56:22] iteration 2435/ 6855 | consumed samples: 311680 | elapsed time per iteration (ms): 87570.2 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.46% | FLOPs estimator: workload-aware | logical tokens/s: 191584.9 | valid tokens/s: 191584.9 | physical tokens/s: 192313.0 | loss tokens/s: 143161.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.973359E-01 | lm-global-supervised-tokens: 1.253668E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.856847E-02 | lm-token-buffer-mib: 5.813824E+01 | global_load_balancing_loss: 9.990485E-01 | loss scale: 1.0 | grad norm: 0.714 | params norm: 11483.677 | params rms: 0.062180 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 11:31:00.178471 | finish at 2026-10-10 16:27:22 + [2026-10-06 04:57:48] iteration 2436/ 6855 | consumed samples: 311808 | elapsed time per iteration (ms): 85980.4 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.60% | FLOPs estimator: workload-aware | logical tokens/s: 195128.1 | valid tokens/s: 195128.1 | physical tokens/s: 195521.8 | loss tokens/s: 148861.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.288629E-01 | lm-global-supervised-tokens: 1.279917E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.289612E-02 | lm-token-buffer-mib: 5.806102E+01 | global_load_balancing_loss: 1.000489E+00 | loss scale: 1.0 | grad norm: 0.092 | params norm: 11483.601 | params rms: 0.062180 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 9:32:27.257335 | finish at 2026-10-10 14:30:15 + [2026-10-06 04:59:14] iteration 2437/ 6855 | consumed samples: 311936 | elapsed time per iteration (ms): 85918.7 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.59% | FLOPs estimator: workload-aware | logical tokens/s: 195268.2 | valid tokens/s: 195268.2 | physical tokens/s: 195817.8 | loss tokens/s: 148443.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.452308E-01 | lm-global-supervised-tokens: 1.275406E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.166106E-02 | lm-token-buffer-mib: 5.812392E+01 | global_load_balancing_loss: 1.001703E+00 | loss scale: 1.0 | grad norm: 0.377 | params norm: 11483.526 | params rms: 0.062180 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 9:26:29.025071 | finish at 2026-10-10 14:25:43 + [2026-10-06 05:00:40] iteration 2438/ 6855 | consumed samples: 312064 | elapsed time per iteration (ms): 86626.8 | throughput per GPU (TFLOP/s/GPU): 43.2 | MFU 13.86% | FLOPs estimator: workload-aware | logical tokens/s: 193672.1 | valid tokens/s: 193672.1 | physical tokens/s: 194022.1 | loss tokens/s: 142787.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.533490E-01 | lm-global-supervised-tokens: 1.236925E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.177937E-02 | lm-token-buffer-mib: 5.808427E+01 | global_load_balancing_loss: 1.002652E+00 | loss scale: 1.0 | grad norm: 0.197 | params norm: 11483.450 | params rms: 0.062179 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 10:17:10.441070 | finish at 2026-10-10 15:17:51 + [2026-10-06 05:02:05] iteration 2439/ 6855 | consumed samples: 312192 | elapsed time per iteration (ms): 84621.4 | throughput per GPU (TFLOP/s/GPU): 43.5 | MFU 13.93% | FLOPs estimator: workload-aware | logical tokens/s: 198262.0 | valid tokens/s: 198262.0 | physical tokens/s: 198843.4 | loss tokens/s: 148718.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.106646E-01 | lm-global-supervised-tokens: 1.258472E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.171821E-02 | lm-token-buffer-mib: 5.828542E+01 | global_load_balancing_loss: 1.000533E+00 | loss scale: 1.0 | grad norm: 0.161 | params norm: 11483.374 | params rms: 0.062179 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 7:48:07.958679 | finish at 2026-10-10 12:50:13 + [2026-10-06 05:03:56] iteration 2440/ 6855 | consumed samples: 312320 | elapsed time per iteration (ms): 111165.4 | throughput per GPU (TFLOP/s/GPU): 33.6 | MFU 10.75% | FLOPs estimator: workload-aware | logical tokens/s: 150921.0 | valid tokens/s: 150921.0 | physical tokens/s: 151239.5 | loss tokens/s: 107457.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.087156E-01 | lm-global-supervised-tokens: 1.194560E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.144722E-02 | lm-token-buffer-mib: 5.809347E+01 | global_load_balancing_loss: 1.000622E+00 | loss scale: 1.0 | grad norm: 0.136 | params norm: 11483.297 | params rms: 0.062178 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 16:19:55.460069 | finish at 2026-10-11 21:23:52 +times across ranks (ms): + save-checkpoint: + rank 0: 23902.03 + rank 1: 23902.02 + rank 2: 23902.05 + rank 3: 23902.04 + rank 4: 23902.05 + rank 5: 23902.01 + rank 6: 23902.00 + rank 7: 23902.04 + rank 8: 23901.93 + rank 9: 23901.92 + rank 10: 23901.93 + rank 11: 23901.92 + rank 12: 23901.95 + rank 13: 23901.92 + rank 14: 23901.94 + rank 15: 23901.92 + rank 16: 23902.22 + rank 17: 23902.22 + rank 18: 23902.20 + rank 19: 23902.22 + rank 20: 23902.20 + rank 21: 23902.21 + rank 22: 23902.22 + rank 23: 23902.19 + rank 24: 23902.18 + rank 25: 23902.23 + rank 26: 23902.24 + rank 27: 23902.16 + rank 28: 23902.15 + rank 29: 23902.17 + rank 30: 23902.19 + rank 31: 23902.15 + rank 32: 23902.05 + rank 33: 23902.06 + rank 34: 23902.06 + rank 35: 23902.03 + rank 36: 23902.01 + rank 37: 23902.00 + rank 38: 23902.01 + rank 39: 23902.01 + rank 40: 23902.02 + rank 41: 23902.03 + rank 42: 23902.00 + rank 43: 23902.03 + rank 44: 23902.01 + rank 45: 23902.03 + rank 46: 23902.04 + rank 47: 23902.00 + rank 48: 23902.25 + rank 49: 23902.24 + rank 50: 23902.22 + rank 51: 23902.23 + rank 52: 23902.24 + rank 53: 23902.23 + rank 54: 23902.23 + rank 55: 23902.25 + rank 56: 23902.11 + rank 57: 23902.12 + rank 58: 23902.09 + rank 59: 23902.09 + rank 60: 23902.10 + rank 61: 23902.10 + rank 62: 23902.11 + rank 63: 23902.09 + [2026-10-06 05:06:05] iteration 2441/ 6855 | consumed samples: 312448 | elapsed time per iteration (ms): 104714.0 | throughput per GPU (TFLOP/s/GPU): 35.3 | MFU 11.32% | FLOPs estimator: workload-aware | logical tokens/s: 160218.9 | valid tokens/s: 160218.9 | physical tokens/s: 160648.3 | loss tokens/s: 119570.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.689287E-01 | lm-global-supervised-tokens: 1.252066E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.085807E-01 | lm-token-buffer-mib: 5.812255E+01 | global_load_balancing_loss: 9.991375E-01 | loss scale: 1.0 | grad norm: 0.223 | params norm: 11483.221 | params rms: 0.062178 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 8:23:27.629617 | finish at 2026-10-11 13:29:33 + [2026-10-06 05:07:37] iteration 2442/ 6855 | consumed samples: 312576 | elapsed time per iteration (ms): 91927.3 | throughput per GPU (TFLOP/s/GPU): 40.0 | MFU 12.83% | FLOPs estimator: workload-aware | logical tokens/s: 182505.2 | valid tokens/s: 182505.2 | physical tokens/s: 182896.0 | loss tokens/s: 134861.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.697012E-01 | lm-global-supervised-tokens: 1.239746E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 6.427278E-02 | lm-token-buffer-mib: 5.812943E+01 | global_load_balancing_loss: 9.987381E-01 | loss scale: 1.0 | grad norm: 0.196 | params norm: 11483.147 | params rms: 0.062178 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 16:41:15.242133 | finish at 2026-10-10 21:48:52 + [2026-10-06 05:09:03] iteration 2443/ 6855 | consumed samples: 312704 | elapsed time per iteration (ms): 86252.5 | throughput per GPU (TFLOP/s/GPU): 42.8 | MFU 13.71% | FLOPs estimator: workload-aware | logical tokens/s: 194512.6 | valid tokens/s: 194512.6 | physical tokens/s: 194944.8 | loss tokens/s: 147505.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.210822E-01 | lm-global-supervised-tokens: 1.272271E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.358160E-02 | lm-token-buffer-mib: 5.810706E+01 | global_load_balancing_loss: 9.984286E-01 | loss scale: 1.0 | grad norm: 0.092 | params norm: 11483.071 | params rms: 0.062177 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 9:42:26.156481 | finish at 2026-10-10 14:51:29 + [2026-10-06 05:10:31] iteration 2444/ 6855 | consumed samples: 312832 | elapsed time per iteration (ms): 87862.9 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.48% | FLOPs estimator: workload-aware | logical tokens/s: 190947.3 | valid tokens/s: 190947.3 | physical tokens/s: 191424.7 | loss tokens/s: 144890.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.891391E-01 | lm-global-supervised-tokens: 1.273049E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.357587E-02 | lm-token-buffer-mib: 5.815099E+01 | global_load_balancing_loss: 9.995040E-01 | loss scale: 1.0 | grad norm: 0.180 | params norm: 11482.992 | params rms: 0.062177 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 11:39:23.359252 | finish at 2026-10-10 16:49:54 + [2026-10-06 05:12:00] iteration 2445/ 6855 | consumed samples: 312960 | elapsed time per iteration (ms): 89356.0 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.40% | FLOPs estimator: workload-aware | logical tokens/s: 187756.1 | valid tokens/s: 187756.1 | physical tokens/s: 188074.8 | loss tokens/s: 136105.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.458045E-01 | lm-global-supervised-tokens: 1.216186E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.244011E-02 | lm-token-buffer-mib: 5.808044E+01 | global_load_balancing_loss: 1.000188E+00 | loss scale: 1.0 | grad norm: 0.052 | params norm: 11482.915 | params rms: 0.062176 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 13:27:40.039673 | finish at 2026-10-10 18:39:41 + [2026-10-06 05:13:29] iteration 2446/ 6855 | consumed samples: 313088 | elapsed time per iteration (ms): 88377.6 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.61% | FLOPs estimator: workload-aware | logical tokens/s: 189834.7 | valid tokens/s: 189834.6 | physical tokens/s: 190224.1 | loss tokens/s: 138529.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.815673E-01 | lm-global-supervised-tokens: 1.224292E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.168529E-02 | lm-token-buffer-mib: 5.803010E+01 | global_load_balancing_loss: 1.002476E+00 | loss scale: 1.0 | grad norm: 0.143 | params norm: 11482.839 | params rms: 0.062176 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 12:14:16.929653 | finish at 2026-10-10 17:27:46 + [2026-10-06 05:14:51] iteration 2447/ 6855 | consumed samples: 313216 | elapsed time per iteration (ms): 82225.5 | throughput per GPU (TFLOP/s/GPU): 43.7 | MFU 14.01% | FLOPs estimator: workload-aware | logical tokens/s: 204038.5 | valid tokens/s: 204038.5 | physical tokens/s: 204550.0 | loss tokens/s: 154066.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.605350E-01 | lm-global-supervised-tokens: 1.266817E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.160008E-02 | lm-token-buffer-mib: 5.807853E+01 | global_load_balancing_loss: 1.001546E+00 | loss scale: 1.0 | grad norm: 0.080 | params norm: 11482.763 | params rms: 0.062175 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 4:40:49.865746 | finish at 2026-10-10 09:55:41 + [2026-10-06 05:16:19] iteration 2448/ 6855 | consumed samples: 313344 | elapsed time per iteration (ms): 87749.1 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.54% | FLOPs estimator: workload-aware | logical tokens/s: 191195.1 | valid tokens/s: 191195.1 | physical tokens/s: 191595.2 | loss tokens/s: 138159.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.409815E-01 | lm-global-supervised-tokens: 1.212336E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.348265E-02 | lm-token-buffer-mib: 5.811624E+01 | global_load_balancing_loss: 1.002179E+00 | loss scale: 1.0 | grad norm: 0.117 | params norm: 11482.687 | params rms: 0.062175 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 11:25:10.328748 | finish at 2026-10-10 16:41:29 + [2026-10-06 05:17:43] iteration 2449/ 6855 | consumed samples: 313472 | elapsed time per iteration (ms): 84074.6 | throughput per GPU (TFLOP/s/GPU): 44.1 | MFU 14.12% | FLOPs estimator: workload-aware | logical tokens/s: 199551.5 | valid tokens/s: 199551.5 | physical tokens/s: 199883.8 | loss tokens/s: 148628.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.956831E-01 | lm-global-supervised-tokens: 1.249585E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.600632E-02 | lm-token-buffer-mib: 5.808726E+01 | global_load_balancing_loss: 1.000527E+00 | loss scale: 1.0 | grad norm: 0.045 | params norm: 11482.610 | params rms: 0.062175 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 6:53:52.723234 | finish at 2026-10-10 12:11:36 + [2026-10-06 05:19:31] iteration 2450/ 6855 | consumed samples: 313600 | elapsed time per iteration (ms): 107922.1 | throughput per GPU (TFLOP/s/GPU): 34.1 | MFU 10.93% | FLOPs estimator: workload-aware | logical tokens/s: 155456.7 | valid tokens/s: 155456.7 | physical tokens/s: 155921.3 | loss tokens/s: 112591.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.827042E-01 | lm-global-supervised-tokens: 1.215115E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.139512E-02 | lm-token-buffer-mib: 5.849255E+01 | global_load_balancing_loss: 9.988803E-01 | loss scale: 1.0 | grad norm: 0.127 | params norm: 11482.535 | params rms: 0.062174 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 12:03:17.021984 | finish at 2026-10-11 17:22:48 + [2026-10-06 05:20:58] iteration 2451/ 6855 | consumed samples: 313728 | elapsed time per iteration (ms): 87239.2 | throughput per GPU (TFLOP/s/GPU): 43.1 | MFU 13.81% | FLOPs estimator: workload-aware | logical tokens/s: 192312.1 | valid tokens/s: 192312.1 | physical tokens/s: 192634.6 | loss tokens/s: 143547.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.978676E-01 | lm-global-supervised-tokens: 1.252298E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.486220E-02 | lm-token-buffer-mib: 5.817664E+01 | global_load_balancing_loss: 9.988263E-01 | loss scale: 1.0 | grad norm: 0.040 | params norm: 11482.461 | params rms: 0.062174 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 10:43:21.576957 | finish at 2026-10-10 16:04:20 + [2026-10-06 05:22:23] iteration 2452/ 6855 | consumed samples: 313856 | elapsed time per iteration (ms): 84726.3 | throughput per GPU (TFLOP/s/GPU): 43.1 | MFU 13.80% | FLOPs estimator: workload-aware | logical tokens/s: 198016.5 | valid tokens/s: 198016.5 | physical tokens/s: 198428.6 | loss tokens/s: 147139.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.587220E-01 | lm-global-supervised-tokens: 1.246659E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.147881E-02 | lm-token-buffer-mib: 5.815217E+01 | global_load_balancing_loss: 9.985253E-01 | loss scale: 1.0 | grad norm: 0.056 | params norm: 11482.387 | params rms: 0.062173 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 7:37:30.066866 | finish at 2026-10-10 12:59:53 + [2026-10-06 05:23:53] iteration 2453/ 6855 | consumed samples: 313984 | elapsed time per iteration (ms): 89993.0 | throughput per GPU (TFLOP/s/GPU): 41.7 | MFU 13.37% | FLOPs estimator: workload-aware | logical tokens/s: 186427.8 | valid tokens/s: 186427.8 | physical tokens/s: 186742.3 | loss tokens/s: 133877.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.519797E-01 | lm-global-supervised-tokens: 1.204802E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.172399E-02 | lm-token-buffer-mib: 5.807642E+01 | global_load_balancing_loss: 9.999405E-01 | loss scale: 1.0 | grad norm: 0.064 | params norm: 11482.311 | params rms: 0.062173 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 14:02:29.201877 | finish at 2026-10-10 19:26:22 + [2026-10-06 05:25:17] iteration 2454/ 6855 | consumed samples: 314112 | elapsed time per iteration (ms): 83874.7 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.67% | FLOPs estimator: workload-aware | logical tokens/s: 200026.0 | valid tokens/s: 200026.0 | physical tokens/s: 200551.6 | loss tokens/s: 150824.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.501518E-01 | lm-global-supervised-tokens: 1.265040E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.464975E-02 | lm-token-buffer-mib: 5.828173E+01 | global_load_balancing_loss: 1.001003E+00 | loss scale: 1.0 | grad norm: 0.038 | params norm: 11482.235 | params rms: 0.062173 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 6:32:12.752270 | finish at 2026-10-10 11:57:29 + [2026-10-06 05:26:48] iteration 2455/ 6855 | consumed samples: 314240 | elapsed time per iteration (ms): 91450.6 | throughput per GPU (TFLOP/s/GPU): 40.2 | MFU 12.87% | FLOPs estimator: workload-aware | logical tokens/s: 183456.4 | valid tokens/s: 183456.4 | physical tokens/s: 183924.8 | loss tokens/s: 139519.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.745642E-01 | lm-global-supervised-tokens: 1.275912E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.492646E-02 | lm-token-buffer-mib: 5.807494E+01 | global_load_balancing_loss: 1.000280E+00 | loss scale: 1.0 | grad norm: 0.084 | params norm: 11482.161 | params rms: 0.062172 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 15:46:22.529449 | finish at 2026-10-10 21:13:11 + [2026-10-06 05:28:18] iteration 2456/ 6855 | consumed samples: 314368 | elapsed time per iteration (ms): 89776.1 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.42% | FLOPs estimator: workload-aware | logical tokens/s: 186877.9 | valid tokens/s: 186877.9 | physical tokens/s: 187447.0 | loss tokens/s: 137805.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.956277E-01 | lm-global-supervised-tokens: 1.237163E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.796322E-02 | lm-token-buffer-mib: 5.809504E+01 | global_load_balancing_loss: 1.001020E+00 | loss scale: 1.0 | grad norm: 0.079 | params norm: 11482.086 | params rms: 0.062172 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 13:42:04.992231 | finish at 2026-10-10 19:10:23 + [2026-10-06 05:29:47] iteration 2457/ 6855 | consumed samples: 314496 | elapsed time per iteration (ms): 89022.2 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.43% | FLOPs estimator: workload-aware | logical tokens/s: 188460.4 | valid tokens/s: 188460.4 | physical tokens/s: 188785.2 | loss tokens/s: 144461.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.756266E-01 | lm-global-supervised-tokens: 1.286024E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.356507E-02 | lm-token-buffer-mib: 5.815501E+01 | global_load_balancing_loss: 1.000985E+00 | loss scale: 1.0 | grad norm: 0.033 | params norm: 11482.010 | params rms: 0.062171 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 12:45:19.772387 | finish at 2026-10-10 18:15:07 + [2026-10-06 05:31:14] iteration 2458/ 6855 | consumed samples: 314624 | elapsed time per iteration (ms): 86624.7 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.76% | FLOPs estimator: workload-aware | logical tokens/s: 193676.1 | valid tokens/s: 193676.1 | physical tokens/s: 194101.1 | loss tokens/s: 141515.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.068288E-01 | lm-global-supervised-tokens: 1.225876E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.170832E-02 | lm-token-buffer-mib: 5.808556E+01 | global_load_balancing_loss: 1.000701E+00 | loss scale: 1.0 | grad norm: 0.046 | params norm: 11481.934 | params rms: 0.062171 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 9:48:08.789432 | finish at 2026-10-10 15:19:22 + [2026-10-06 05:32:41] iteration 2459/ 6855 | consumed samples: 314752 | elapsed time per iteration (ms): 87384.4 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.68% | FLOPs estimator: workload-aware | logical tokens/s: 191992.3 | valid tokens/s: 191992.3 | physical tokens/s: 192333.6 | loss tokens/s: 138756.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.272731E-01 | lm-global-supervised-tokens: 1.212518E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.311924E-02 | lm-token-buffer-mib: 5.821094E+01 | global_load_balancing_loss: 9.999186E-01 | loss scale: 1.0 | grad norm: 0.041 | params norm: 11481.859 | params rms: 0.062171 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 10:42:21.964460 | finish at 2026-10-10 16:15:03 + [2026-10-06 05:34:30] iteration 2460/ 6855 | consumed samples: 314880 | elapsed time per iteration (ms): 108953.9 | throughput per GPU (TFLOP/s/GPU): 34.4 | MFU 11.02% | FLOPs estimator: workload-aware | logical tokens/s: 153984.1 | valid tokens/s: 153984.1 | physical tokens/s: 154323.1 | loss tokens/s: 110403.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.800465E-01 | lm-global-supervised-tokens: 1.202885E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.123534E-02 | lm-token-buffer-mib: 5.806198E+01 | global_load_balancing_loss: 9.990053E-01 | loss scale: 1.0 | grad norm: 0.034 | params norm: 11481.783 | params rms: 0.062170 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 13:00:52.245283 | finish at 2026-10-11 18:35:22 + [2026-10-06 05:35:55] iteration 2461/ 6855 | consumed samples: 315008 | elapsed time per iteration (ms): 84813.7 | throughput per GPU (TFLOP/s/GPU): 43.4 | MFU 13.92% | FLOPs estimator: workload-aware | logical tokens/s: 197812.4 | valid tokens/s: 197812.4 | physical tokens/s: 198282.4 | loss tokens/s: 148500.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.061278E-01 | lm-global-supervised-tokens: 1.259487E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.519639E-02 | lm-token-buffer-mib: 5.805281E+01 | global_load_balancing_loss: 9.991992E-01 | loss scale: 1.0 | grad norm: 0.042 | params norm: 11481.707 | params rms: 0.062170 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 7:31:11.586197 | finish at 2026-10-10 13:07:06 + [2026-10-06 05:37:21] iteration 2462/ 6855 | consumed samples: 315136 | elapsed time per iteration (ms): 86214.3 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.75% | FLOPs estimator: workload-aware | logical tokens/s: 194598.1 | valid tokens/s: 194598.1 | physical tokens/s: 195322.1 | loss tokens/s: 137548.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.141248E-01 | lm-global-supervised-tokens: 1.185866E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.119285E-02 | lm-token-buffer-mib: 5.821867E+01 | global_load_balancing_loss: 9.994724E-01 | loss scale: 1.0 | grad norm: 0.049 | params norm: 11481.632 | params rms: 0.062169 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 9:12:19.639485 | finish at 2026-10-10 14:49:41 + [2026-10-06 05:38:49] iteration 2463/ 6855 | consumed samples: 315264 | elapsed time per iteration (ms): 87648.6 | throughput per GPU (TFLOP/s/GPU): 42.8 | MFU 13.71% | FLOPs estimator: workload-aware | logical tokens/s: 191414.2 | valid tokens/s: 191414.1 | physical tokens/s: 191802.4 | loss tokens/s: 143561.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.710048E-01 | lm-global-supervised-tokens: 1.258295E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.195448E-02 | lm-token-buffer-mib: 5.805293E+01 | global_load_balancing_loss: 1.000736E+00 | loss scale: 1.0 | grad norm: 0.038 | params norm: 11481.559 | params rms: 0.062169 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 10:55:52.479916 | finish at 2026-10-10 16:34:41 + [2026-10-06 05:40:20] iteration 2464/ 6855 | consumed samples: 315392 | elapsed time per iteration (ms): 91383.7 | throughput per GPU (TFLOP/s/GPU): 41.6 | MFU 13.34% | FLOPs estimator: workload-aware | logical tokens/s: 183590.3 | valid tokens/s: 183590.3 | physical tokens/s: 184068.2 | loss tokens/s: 129611.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.938399E-01 | lm-global-supervised-tokens: 1.184437E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.161513E-02 | lm-token-buffer-mib: 5.817008E+01 | global_load_balancing_loss: 1.000700E+00 | loss scale: 1.0 | grad norm: 0.065 | params norm: 11481.487 | params rms: 0.062169 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 15:27:45.708982 | finish at 2026-10-10 21:08:06 + [2026-10-06 05:41:51] iteration 2465/ 6855 | consumed samples: 315520 | elapsed time per iteration (ms): 91108.9 | throughput per GPU (TFLOP/s/GPU): 41.4 | MFU 13.28% | FLOPs estimator: workload-aware | logical tokens/s: 184143.8 | valid tokens/s: 184143.8 | physical tokens/s: 184520.4 | loss tokens/s: 135433.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.103994E-01 | lm-global-supervised-tokens: 1.233923E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.166442E-02 | lm-token-buffer-mib: 5.829677E+01 | global_load_balancing_loss: 1.000706E+00 | loss scale: 1.0 | grad norm: 0.080 | params norm: 11481.416 | params rms: 0.062168 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 15:06:08.232493 | finish at 2026-10-10 20:47:59 + [2026-10-06 05:43:17] iteration 2466/ 6855 | consumed samples: 315648 | elapsed time per iteration (ms): 86312.7 | throughput per GPU (TFLOP/s/GPU): 43.9 | MFU 14.05% | FLOPs estimator: workload-aware | logical tokens/s: 194377.0 | valid tokens/s: 194377.0 | physical tokens/s: 194739.4 | loss tokens/s: 138610.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.936469E-01 | lm-global-supervised-tokens: 1.196383E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.164533E-02 | lm-token-buffer-mib: 5.807628E+01 | global_load_balancing_loss: 1.000746E+00 | loss scale: 1.0 | grad norm: 0.035 | params norm: 11481.346 | params rms: 0.062168 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 9:13:46.644497 | finish at 2026-10-10 14:57:04 + [2026-10-06 05:44:46] iteration 2467/ 6855 | consumed samples: 315776 | elapsed time per iteration (ms): 88285.4 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.42% | FLOPs estimator: workload-aware | logical tokens/s: 190032.6 | valid tokens/s: 190032.6 | physical tokens/s: 190543.3 | loss tokens/s: 135959.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.296477E-01 | lm-global-supervised-tokens: 1.200324E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.264933E-02 | lm-token-buffer-mib: 5.809994E+01 | global_load_balancing_loss: 1.001705E+00 | loss scale: 1.0 | grad norm: 0.095 | params norm: 11481.277 | params rms: 0.062167 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 11:36:36.277282 | finish at 2026-10-10 17:21:22 + [2026-10-06 05:46:14] iteration 2468/ 6855 | consumed samples: 315904 | elapsed time per iteration (ms): 88554.3 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.46% | FLOPs estimator: workload-aware | logical tokens/s: 189456.9 | valid tokens/s: 189456.9 | physical tokens/s: 189924.3 | loss tokens/s: 142520.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.127145E-01 | lm-global-supervised-tokens: 1.262080E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.578899E-02 | lm-token-buffer-mib: 5.824661E+01 | global_load_balancing_loss: 1.000615E+00 | loss scale: 1.0 | grad norm: 0.112 | params norm: 11481.208 | params rms: 0.062167 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 11:54:47.509402 | finish at 2026-10-10 17:41:02 + [2026-10-06 05:47:40] iteration 2469/ 6855 | consumed samples: 316032 | elapsed time per iteration (ms): 86168.2 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.66% | FLOPs estimator: workload-aware | logical tokens/s: 194702.5 | valid tokens/s: 194702.5 | physical tokens/s: 195158.4 | loss tokens/s: 145873.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.474335E-01 | lm-global-supervised-tokens: 1.256963E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.304538E-02 | lm-token-buffer-mib: 5.809386E+01 | global_load_balancing_loss: 9.987618E-01 | loss scale: 1.0 | grad norm: 0.034 | params norm: 11481.139 | params rms: 0.062167 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 8:58:53.701219 | finish at 2026-10-10 14:46:34 + [2026-10-06 05:49:32] iteration 2470/ 6855 | consumed samples: 316160 | elapsed time per iteration (ms): 111391.1 | throughput per GPU (TFLOP/s/GPU): 34.1 | MFU 10.95% | FLOPs estimator: workload-aware | logical tokens/s: 150615.1 | valid tokens/s: 150615.1 | physical tokens/s: 150992.9 | loss tokens/s: 108273.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.320807E-01 | lm-global-supervised-tokens: 1.206074E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.954313E-02 | lm-token-buffer-mib: 5.806960E+01 | global_load_balancing_loss: 1.000424E+00 | loss scale: 1.0 | grad norm: 0.150 | params norm: 11481.069 | params rms: 0.062166 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 15:40:49.789190 | finish at 2026-10-11 21:30:22 + [2026-10-06 05:51:00] iteration 2471/ 6855 | consumed samples: 316288 | elapsed time per iteration (ms): 88437.8 | throughput per GPU (TFLOP/s/GPU): 41.5 | MFU 13.30% | FLOPs estimator: workload-aware | logical tokens/s: 189705.3 | valid tokens/s: 189705.3 | physical tokens/s: 190155.5 | loss tokens/s: 140981.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.524975E-01 | lm-global-supervised-tokens: 1.246808E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.199889E-02 | lm-token-buffer-mib: 5.826703E+01 | global_load_balancing_loss: 9.993717E-01 | loss scale: 1.0 | grad norm: 0.186 | params norm: 11481.002 | params rms: 0.062166 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 11:41:51.453911 | finish at 2026-10-10 17:32:52 + [2026-10-06 05:52:30] iteration 2472/ 6855 | consumed samples: 316416 | elapsed time per iteration (ms): 89623.0 | throughput per GPU (TFLOP/s/GPU): 41.7 | MFU 13.38% | FLOPs estimator: workload-aware | logical tokens/s: 187197.3 | valid tokens/s: 187197.3 | physical tokens/s: 187633.9 | loss tokens/s: 135323.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.380607E-01 | lm-global-supervised-tokens: 1.212810E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.168520E-02 | lm-token-buffer-mib: 5.823912E+01 | global_load_balancing_loss: 1.000069E+00 | loss scale: 1.0 | grad norm: 0.043 | params norm: 11480.935 | params rms: 0.062166 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 13:06:57.702639 | finish at 2026-10-10 18:59:28 + [2026-10-06 05:53:57] iteration 2473/ 6855 | consumed samples: 316544 | elapsed time per iteration (ms): 86932.3 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.54% | FLOPs estimator: workload-aware | logical tokens/s: 192991.0 | valid tokens/s: 192991.0 | physical tokens/s: 193483.8 | loss tokens/s: 146842.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.882374E-01 | lm-global-supervised-tokens: 1.276538E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.883354E-02 | lm-token-buffer-mib: 5.811131E+01 | global_load_balancing_loss: 9.993375E-01 | loss scale: 1.0 | grad norm: 0.241 | params norm: 11480.868 | params rms: 0.062165 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 9:48:57.484218 | finish at 2026-10-10 15:42:54 + [2026-10-06 05:55:21] iteration 2474/ 6855 | consumed samples: 316672 | elapsed time per iteration (ms): 84391.2 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.54% | FLOPs estimator: workload-aware | logical tokens/s: 198802.8 | valid tokens/s: 198802.8 | physical tokens/s: 199320.9 | loss tokens/s: 150863.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.854146E-01 | lm-global-supervised-tokens: 1.273152E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.296324E-02 | lm-token-buffer-mib: 5.809241E+01 | global_load_balancing_loss: 1.000584E+00 | loss scale: 1.0 | grad norm: 0.293 | params norm: 11480.800 | params rms: 0.062165 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 6:41:57.791084 | finish at 2026-10-10 12:37:19 + [2026-10-06 05:56:49] iteration 2475/ 6855 | consumed samples: 316800 | elapsed time per iteration (ms): 87616.4 | throughput per GPU (TFLOP/s/GPU): 41.7 | MFU 13.37% | FLOPs estimator: workload-aware | logical tokens/s: 191483.9 | valid tokens/s: 191483.9 | physical tokens/s: 192345.6 | loss tokens/s: 147188.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.745995E-01 | lm-global-supervised-tokens: 1.289609E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.485126E-02 | lm-token-buffer-mib: 5.828544E+01 | global_load_balancing_loss: 9.994366E-01 | loss scale: 1.0 | grad norm: 0.036 | params norm: 11480.732 | params rms: 0.062164 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 10:35:59.630470 | finish at 2026-10-10 16:32:48 + [2026-10-06 05:58:15] iteration 2476/ 6855 | consumed samples: 316928 | elapsed time per iteration (ms): 86089.6 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.75% | FLOPs estimator: workload-aware | logical tokens/s: 194880.4 | valid tokens/s: 194880.4 | physical tokens/s: 195402.1 | loss tokens/s: 141030.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.937855E-01 | lm-global-supervised-tokens: 1.214126E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.317241E-02 | lm-token-buffer-mib: 5.822241E+01 | global_load_balancing_loss: 9.999751E-01 | loss scale: 1.0 | grad norm: 0.180 | params norm: 11480.662 | params rms: 0.062164 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 8:43:06.478842 | finish at 2026-10-10 14:41:21 + [2026-10-06 05:59:44] iteration 2477/ 6855 | consumed samples: 317056 | elapsed time per iteration (ms): 89027.9 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.40% | FLOPs estimator: workload-aware | logical tokens/s: 188448.7 | valid tokens/s: 188448.7 | physical tokens/s: 188850.5 | loss tokens/s: 134316.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.888292E-01 | lm-global-supervised-tokens: 1.195788E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.198223E-02 | lm-token-buffer-mib: 5.805956E+01 | global_load_balancing_loss: 1.001351E+00 | loss scale: 1.0 | grad norm: 0.074 | params norm: 11480.591 | params rms: 0.062164 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 12:16:04.086618 | finish at 2026-10-10 18:15:48 + [2026-10-06 06:01:12] iteration 2478/ 6855 | consumed samples: 317184 | elapsed time per iteration (ms): 88224.0 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.57% | FLOPs estimator: workload-aware | logical tokens/s: 190165.2 | valid tokens/s: 190165.2 | physical tokens/s: 190901.5 | loss tokens/s: 135223.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.119769E-01 | lm-global-supervised-tokens: 1.192998E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.237119E-02 | lm-token-buffer-mib: 5.811773E+01 | global_load_balancing_loss: 1.001250E+00 | loss scale: 1.0 | grad norm: 0.099 | params norm: 11480.519 | params rms: 0.062163 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 11:15:56.515731 | finish at 2026-10-10 17:17:09 + [2026-10-06 06:02:40] iteration 2479/ 6855 | consumed samples: 317312 | elapsed time per iteration (ms): 87603.0 | throughput per GPU (TFLOP/s/GPU): 41.5 | MFU 13.31% | FLOPs estimator: workload-aware | logical tokens/s: 191513.4 | valid tokens/s: 191513.4 | physical tokens/s: 192010.4 | loss tokens/s: 142475.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.378155E-01 | lm-global-supervised-tokens: 1.248126E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.209863E-02 | lm-token-buffer-mib: 5.810779E+01 | global_load_balancing_loss: 1.001030E+00 | loss scale: 1.0 | grad norm: 0.108 | params norm: 11480.448 | params rms: 0.062163 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 10:29:10.780884 | finish at 2026-10-10 16:31:50 + [2026-10-06 06:04:29] iteration 2480/ 6855 | consumed samples: 317440 | elapsed time per iteration (ms): 109176.9 | throughput per GPU (TFLOP/s/GPU): 33.5 | MFU 10.75% | FLOPs estimator: workload-aware | logical tokens/s: 153669.6 | valid tokens/s: 153669.6 | physical tokens/s: 153994.0 | loss tokens/s: 109971.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.134120E-01 | lm-global-supervised-tokens: 1.200634E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.044830E-02 | lm-token-buffer-mib: 5.813138E+01 | global_load_balancing_loss: 9.994119E-01 | loss scale: 1.0 | grad norm: 0.055 | params norm: 11480.377 | params rms: 0.062163 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 12:40:49.105042 | finish at 2026-10-11 18:45:18 +times across ranks (ms): + save-checkpoint: + rank 0: 24832.04 + rank 1: 24832.07 + rank 2: 24832.09 + rank 3: 24832.05 + rank 4: 24832.10 + rank 5: 24832.08 + rank 6: 24832.03 + rank 7: 24832.05 + rank 8: 24831.37 + rank 9: 24831.36 + rank 10: 24831.37 + rank 11: 24831.36 + rank 12: 24831.37 + rank 13: 24831.35 + rank 14: 24831.36 + rank 15: 24831.36 + rank 16: 24831.91 + rank 17: 24831.94 + rank 18: 24831.90 + rank 19: 24831.93 + rank 20: 24831.93 + rank 21: 24831.93 + rank 22: 24831.90 + rank 23: 24831.91 + rank 24: 24832.39 + rank 25: 24832.35 + rank 26: 24832.35 + rank 27: 24832.35 + rank 28: 24832.33 + rank 29: 24832.46 + rank 30: 24832.35 + rank 31: 24832.36 + rank 32: 24831.84 + rank 33: 24831.85 + rank 34: 24831.83 + rank 35: 24831.81 + rank 36: 24831.76 + rank 37: 24831.78 + rank 38: 24831.82 + rank 39: 24831.78 + rank 40: 24832.30 + rank 41: 24832.31 + rank 42: 24832.28 + rank 43: 24832.28 + rank 44: 24832.27 + rank 45: 24832.30 + rank 46: 24832.27 + rank 47: 24832.27 + rank 48: 24832.04 + rank 49: 24832.00 + rank 50: 24831.98 + rank 51: 24832.00 + rank 52: 24832.02 + rank 53: 24831.99 + rank 54: 24832.01 + rank 55: 24832.01 + rank 56: 24832.01 + rank 57: 24832.02 + rank 58: 24832.01 + rank 59: 24831.99 + rank 60: 24831.99 + rank 61: 24832.00 + rank 62: 24832.00 + rank 63: 24832.00 + [2026-10-06 06:06:21] iteration 2481/ 6855 | consumed samples: 317568 | elapsed time per iteration (ms): 86623.2 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.79% | FLOPs estimator: workload-aware | logical tokens/s: 193679.0 | valid tokens/s: 193679.0 | physical tokens/s: 194010.2 | loss tokens/s: 143069.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.612502E-01 | lm-global-supervised-tokens: 1.239315E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 4.263409E-01 | lm-token-buffer-mib: 5.805585E+01 | global_load_balancing_loss: 9.992039E-01 | loss scale: 1.0 | grad norm: 0.078 | params norm: 11480.305 | params rms: 0.062162 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 9:14:49.662754 | finish at 2026-10-10 15:21:10 + [2026-10-06 06:07:50] iteration 2482/ 6855 | consumed samples: 317696 | elapsed time per iteration (ms): 89693.1 | throughput per GPU (TFLOP/s/GPU): 40.7 | MFU 13.05% | FLOPs estimator: workload-aware | logical tokens/s: 187051.4 | valid tokens/s: 187051.4 | physical tokens/s: 187508.1 | loss tokens/s: 145702.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.638688E-01 | lm-global-supervised-tokens: 1.306848E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.666695E-02 | lm-token-buffer-mib: 5.812669E+01 | global_load_balancing_loss: 9.994715E-01 | loss scale: 1.0 | grad norm: 0.050 | params norm: 11480.235 | params rms: 0.062162 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 12:57:07.861552 | finish at 2026-10-10 19:04:58 + [2026-10-06 06:09:15] iteration 2483/ 6855 | consumed samples: 317824 | elapsed time per iteration (ms): 84700.0 | throughput per GPU (TFLOP/s/GPU): 43.6 | MFU 13.96% | FLOPs estimator: workload-aware | logical tokens/s: 198077.8 | valid tokens/s: 198077.8 | physical tokens/s: 198586.0 | loss tokens/s: 142204.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.533781E-01 | lm-global-supervised-tokens: 1.204472E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.259038E-02 | lm-token-buffer-mib: 5.835398E+01 | global_load_balancing_loss: 1.000497E+00 | loss scale: 1.0 | grad norm: 0.054 | params norm: 11480.165 | params rms: 0.062161 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 6:51:48.436691 | finish at 2026-10-10 13:01:03 + [2026-10-06 06:10:39] iteration 2484/ 6855 | consumed samples: 317952 | elapsed time per iteration (ms): 84456.5 | throughput per GPU (TFLOP/s/GPU): 43.9 | MFU 14.07% | FLOPs estimator: workload-aware | logical tokens/s: 198648.5 | valid tokens/s: 198648.5 | physical tokens/s: 199017.8 | loss tokens/s: 154964.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.927125E-01 | lm-global-supervised-tokens: 1.308772E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.104293E-02 | lm-token-buffer-mib: 5.817136E+01 | global_load_balancing_loss: 9.993969E-01 | loss scale: 1.0 | grad norm: 0.084 | params norm: 11480.094 | params rms: 0.062161 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 6:32:39.185614 | finish at 2026-10-10 12:43:19 + [2026-10-06 06:12:07] iteration 2485/ 6855 | consumed samples: 318080 | elapsed time per iteration (ms): 88004.9 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.49% | FLOPs estimator: workload-aware | logical tokens/s: 190638.9 | valid tokens/s: 190638.9 | physical tokens/s: 191143.2 | loss tokens/s: 139224.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.757923E-01 | lm-global-supervised-tokens: 1.225245E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.268683E-02 | lm-token-buffer-mib: 5.803450E+01 | global_load_balancing_loss: 1.000977E+00 | loss scale: 1.0 | grad norm: 0.055 | params norm: 11480.024 | params rms: 0.062161 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 10:49:41.262875 | finish at 2026-10-10 17:01:49 + [2026-10-06 06:13:34] iteration 2486/ 6855 | consumed samples: 318208 | elapsed time per iteration (ms): 86367.6 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.46% | FLOPs estimator: workload-aware | logical tokens/s: 194253.4 | valid tokens/s: 194253.3 | physical tokens/s: 194749.1 | loss tokens/s: 146250.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.566815E-01 | lm-global-supervised-tokens: 1.263129E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.387792E-02 | lm-token-buffer-mib: 5.809254E+01 | global_load_balancing_loss: 1.001140E+00 | loss scale: 1.0 | grad norm: 0.078 | params norm: 11479.955 | params rms: 0.062160 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 8:49:00.071326 | finish at 2026-10-10 15:02:34 + [2026-10-06 06:14:58] iteration 2487/ 6855 | consumed samples: 318336 | elapsed time per iteration (ms): 83812.4 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.75% | FLOPs estimator: workload-aware | logical tokens/s: 200175.8 | valid tokens/s: 200175.8 | physical tokens/s: 200876.9 | loss tokens/s: 151886.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.474656E-01 | lm-global-supervised-tokens: 1.272999E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.244435E-02 | lm-token-buffer-mib: 5.831136E+01 | global_load_balancing_loss: 1.000619E+00 | loss scale: 1.0 | grad norm: 0.040 | params norm: 11479.884 | params rms: 0.062160 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 5:41:32.496998 | finish at 2026-10-10 11:56:30 + [2026-10-06 06:16:25] iteration 2488/ 6855 | consumed samples: 318464 | elapsed time per iteration (ms): 87393.7 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.49% | FLOPs estimator: workload-aware | logical tokens/s: 191972.7 | valid tokens/s: 191972.7 | physical tokens/s: 192395.4 | loss tokens/s: 149797.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.133831E-01 | lm-global-supervised-tokens: 1.309132E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.523748E-02 | lm-token-buffer-mib: 5.807787E+01 | global_load_balancing_loss: 9.987473E-01 | loss scale: 1.0 | grad norm: 0.076 | params norm: 11479.812 | params rms: 0.062159 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 10:00:48.250954 | finish at 2026-10-10 16:17:13 + [2026-10-06 06:17:49] iteration 2489/ 6855 | consumed samples: 318592 | elapsed time per iteration (ms): 83639.2 | throughput per GPU (TFLOP/s/GPU): 43.4 | MFU 13.90% | FLOPs estimator: workload-aware | logical tokens/s: 200589.6 | valid tokens/s: 200589.6 | physical tokens/s: 201010.8 | loss tokens/s: 153854.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.088376E-01 | lm-global-supervised-tokens: 1.286828E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.538110E-01 | lm-token-buffer-mib: 5.811268E+01 | global_load_balancing_loss: 1.000365E+00 | loss scale: 1.0 | grad norm: 0.042 | params norm: 11479.738 | params rms: 0.062159 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 5:26:08.730423 | finish at 2026-10-10 11:43:57 + [2026-10-06 06:19:40] iteration 2490/ 6855 | consumed samples: 318720 | elapsed time per iteration (ms): 111049.1 | throughput per GPU (TFLOP/s/GPU): 33.2 | MFU 10.63% | FLOPs estimator: workload-aware | logical tokens/s: 151079.1 | valid tokens/s: 151079.1 | physical tokens/s: 151420.7 | loss tokens/s: 111922.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.356910E-01 | lm-global-supervised-tokens: 1.242889E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.148069E-02 | lm-token-buffer-mib: 5.830835E+01 | global_load_balancing_loss: 9.993346E-01 | loss scale: 1.0 | grad norm: 0.122 | params norm: 11479.665 | params rms: 0.062159 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 14:38:49.117184 | finish at 2026-10-11 20:58:29 + [2026-10-06 06:21:10] iteration 2491/ 6855 | consumed samples: 318848 | elapsed time per iteration (ms): 90102.1 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.55% | FLOPs estimator: workload-aware | logical tokens/s: 186202.3 | valid tokens/s: 186202.3 | physical tokens/s: 186580.9 | loss tokens/s: 131128.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.007680E-01 | lm-global-supervised-tokens: 1.181493E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.334496E-02 | lm-token-buffer-mib: 5.809164E+01 | global_load_balancing_loss: 9.998122E-01 | loss scale: 1.0 | grad norm: 0.084 | params norm: 11479.593 | params rms: 0.062158 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 13:13:25.522326 | finish at 2026-10-10 19:34:35 + [2026-10-06 06:22:38] iteration 2492/ 6855 | consumed samples: 318976 | elapsed time per iteration (ms): 88083.6 | throughput per GPU (TFLOP/s/GPU): 41.2 | MFU 13.22% | FLOPs estimator: workload-aware | logical tokens/s: 190469.0 | valid tokens/s: 190469.0 | physical tokens/s: 190890.7 | loss tokens/s: 142479.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.357190E-01 | lm-global-supervised-tokens: 1.255011E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.468230E-02 | lm-token-buffer-mib: 5.810781E+01 | global_load_balancing_loss: 1.001327E+00 | loss scale: 1.0 | grad norm: 0.121 | params norm: 11479.519 | params rms: 0.062158 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 10:45:08.583678 | finish at 2026-10-10 17:07:46 + [2026-10-06 06:24:04] iteration 2493/ 6855 | consumed samples: 319104 | elapsed time per iteration (ms): 86429.2 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.67% | FLOPs estimator: workload-aware | logical tokens/s: 194115.1 | valid tokens/s: 194115.1 | physical tokens/s: 194503.6 | loss tokens/s: 145599.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.653206E-01 | lm-global-supervised-tokens: 1.258403E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.166145E-02 | lm-token-buffer-mib: 5.810635E+01 | global_load_balancing_loss: 1.001382E+00 | loss scale: 1.0 | grad norm: 0.105 | params norm: 11479.446 | params rms: 0.062157 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 8:43:24.114993 | finish at 2026-10-10 15:07:28 + [2026-10-06 06:25:33] iteration 2494/ 6855 | consumed samples: 319232 | elapsed time per iteration (ms): 88842.6 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.43% | FLOPs estimator: workload-aware | logical tokens/s: 188841.0 | valid tokens/s: 188841.0 | physical tokens/s: 189259.4 | loss tokens/s: 140578.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.826426E-01 | lm-global-supervised-tokens: 1.248935E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.971724E-02 | lm-token-buffer-mib: 5.826362E+01 | global_load_balancing_loss: 1.001495E+00 | loss scale: 1.0 | grad norm: 0.093 | params norm: 11479.374 | params rms: 0.062157 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 11:37:22.773499 | finish at 2026-10-10 18:02:56 + [2026-10-06 06:27:00] iteration 2495/ 6855 | consumed samples: 319360 | elapsed time per iteration (ms): 86823.5 | throughput per GPU (TFLOP/s/GPU): 41.6 | MFU 13.35% | FLOPs estimator: workload-aware | logical tokens/s: 193233.1 | valid tokens/s: 193233.1 | physical tokens/s: 193813.4 | loss tokens/s: 144658.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.641658E-01 | lm-global-supervised-tokens: 1.255973E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.098680E-02 | lm-token-buffer-mib: 5.806158E+01 | global_load_balancing_loss: 9.992683E-01 | loss scale: 1.0 | grad norm: 0.050 | params norm: 11479.302 | params rms: 0.062157 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 9:09:10.365047 | finish at 2026-10-10 15:36:10 + [2026-10-06 06:28:29] iteration 2496/ 6855 | consumed samples: 319488 | elapsed time per iteration (ms): 88733.6 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.61% | FLOPs estimator: workload-aware | logical tokens/s: 189073.6 | valid tokens/s: 189073.6 | physical tokens/s: 189669.3 | loss tokens/s: 135743.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.650662E-01 | lm-global-supervised-tokens: 1.204497E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.423585E-02 | lm-token-buffer-mib: 5.812997E+01 | global_load_balancing_loss: 9.991760E-01 | loss scale: 1.0 | grad norm: 0.090 | params norm: 11479.229 | params rms: 0.062156 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 11:26:29.549959 | finish at 2026-10-10 17:54:58 + [2026-10-06 06:29:57] iteration 2497/ 6855 | consumed samples: 319616 | elapsed time per iteration (ms): 88558.3 | throughput per GPU (TFLOP/s/GPU): 41.2 | MFU 13.20% | FLOPs estimator: workload-aware | logical tokens/s: 189447.7 | valid tokens/s: 189447.7 | physical tokens/s: 189740.0 | loss tokens/s: 137437.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.515198E-01 | lm-global-supervised-tokens: 1.217127E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.217840E-02 | lm-token-buffer-mib: 5.810504E+01 | global_load_balancing_loss: 1.000090E+00 | loss scale: 1.0 | grad norm: 0.036 | params norm: 11479.156 | params rms: 0.062156 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 11:12:17.116158 | finish at 2026-10-10 17:42:14 + [2026-10-06 06:31:26] iteration 2498/ 6855 | consumed samples: 319744 | elapsed time per iteration (ms): 88429.7 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.42% | FLOPs estimator: workload-aware | logical tokens/s: 189723.7 | valid tokens/s: 189723.7 | physical tokens/s: 190185.1 | loss tokens/s: 140488.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.630261E-01 | lm-global-supervised-tokens: 1.242338E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.252738E-02 | lm-token-buffer-mib: 5.808189E+01 | global_load_balancing_loss: 9.994043E-01 | loss scale: 1.0 | grad norm: 0.106 | params norm: 11479.085 | params rms: 0.062156 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 11:01:28.039365 | finish at 2026-10-10 17:32:54 + [2026-10-06 06:32:52] iteration 2499/ 6855 | consumed samples: 319872 | elapsed time per iteration (ms): 86714.2 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.67% | FLOPs estimator: workload-aware | logical tokens/s: 193476.9 | valid tokens/s: 193476.9 | physical tokens/s: 193986.2 | loss tokens/s: 146415.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.657901E-01 | lm-global-supervised-tokens: 1.269631E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.139446E-02 | lm-token-buffer-mib: 5.822375E+01 | global_load_balancing_loss: 9.996163E-01 | loss scale: 1.0 | grad norm: 0.104 | params norm: 11479.012 | params rms: 0.062155 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 8:55:27.000243 | finish at 2026-10-10 15:28:19 + [2026-10-06 06:34:41] iteration 2500/ 6855 | consumed samples: 320000 | elapsed time per iteration (ms): 108489.9 | throughput per GPU (TFLOP/s/GPU): 34.1 | MFU 10.94% | FLOPs estimator: workload-aware | logical tokens/s: 154642.3 | valid tokens/s: 154642.3 | physical tokens/s: 155098.3 | loss tokens/s: 111946.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.139136E-01 | lm-global-supervised-tokens: 1.214504E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.297483E-02 | lm-token-buffer-mib: 5.828312E+01 | global_load_balancing_loss: 1.000758E+00 | loss scale: 1.0 | grad norm: 0.071 | params norm: 11478.938 | params rms: 0.062155 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 11:14:33.642701 | finish at 2026-10-11 17:49:15 + [2026-10-06 06:36:08] iteration 2501/ 6855 | consumed samples: 320128 | elapsed time per iteration (ms): 86599.2 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.55% | FLOPs estimator: workload-aware | logical tokens/s: 193734.0 | valid tokens/s: 193734.0 | physical tokens/s: 194161.6 | loss tokens/s: 138450.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.424433E-01 | lm-global-supervised-tokens: 1.198970E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.234654E-02 | lm-token-buffer-mib: 5.815818E+01 | global_load_balancing_loss: 1.001039E+00 | loss scale: 1.0 | grad norm: 0.064 | params norm: 11478.866 | params rms: 0.062154 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 8:44:12.967710 | finish at 2026-10-10 15:20:20 + [2026-10-06 06:37:35] iteration 2502/ 6855 | consumed samples: 320256 | elapsed time per iteration (ms): 87924.0 | throughput per GPU (TFLOP/s/GPU): 43.2 | MFU 13.84% | FLOPs estimator: workload-aware | logical tokens/s: 190813.8 | valid tokens/s: 190813.8 | physical tokens/s: 191186.9 | loss tokens/s: 138334.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.174856E-01 | lm-global-supervised-tokens: 1.216296E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.157788E-02 | lm-token-buffer-mib: 5.811809E+01 | global_load_balancing_loss: 1.000753E+00 | loss scale: 1.0 | grad norm: 0.087 | params norm: 11478.794 | params rms: 0.062154 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 10:18:53.291459 | finish at 2026-10-10 16:56:29 + [2026-10-06 06:39:02] iteration 2503/ 6855 | consumed samples: 320384 | elapsed time per iteration (ms): 86947.9 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.40% | FLOPs estimator: workload-aware | logical tokens/s: 192957.1 | valid tokens/s: 192957.0 | physical tokens/s: 193354.9 | loss tokens/s: 137773.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.242449E-01 | lm-global-supervised-tokens: 1.197908E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.265991E-02 | lm-token-buffer-mib: 5.811804E+01 | global_load_balancing_loss: 1.000636E+00 | loss scale: 1.0 | grad norm: 0.088 | params norm: 11478.724 | params rms: 0.062154 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 9:06:37.105408 | finish at 2026-10-10 15:45:39 + [2026-10-06 06:40:28] iteration 2504/ 6855 | consumed samples: 320512 | elapsed time per iteration (ms): 85149.0 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.54% | FLOPs estimator: workload-aware | logical tokens/s: 197032.9 | valid tokens/s: 197032.9 | physical tokens/s: 197497.7 | loss tokens/s: 154426.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.197275E-01 | lm-global-supervised-tokens: 1.314926E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.279486E-02 | lm-token-buffer-mib: 5.822800E+01 | global_load_balancing_loss: 9.993846E-01 | loss scale: 1.0 | grad norm: 0.077 | params norm: 11478.651 | params rms: 0.062153 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 6:54:43.352636 | finish at 2026-10-10 13:35:11 + [2026-10-06 06:41:53] iteration 2505/ 6855 | consumed samples: 320640 | elapsed time per iteration (ms): 85713.9 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.51% | FLOPs estimator: workload-aware | logical tokens/s: 195734.4 | valid tokens/s: 195734.4 | physical tokens/s: 196214.4 | loss tokens/s: 146888.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.009200E-01 | lm-global-supervised-tokens: 1.259042E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.200470E-02 | lm-token-buffer-mib: 5.832936E+01 | global_load_balancing_loss: 1.001021E+00 | loss scale: 1.0 | grad norm: 0.125 | params norm: 11478.578 | params rms: 0.062153 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 7:34:15.522430 | finish at 2026-10-10 14:16:09 + [2026-10-06 06:43:22] iteration 2506/ 6855 | consumed samples: 320768 | elapsed time per iteration (ms): 88868.3 | throughput per GPU (TFLOP/s/GPU): 41.5 | MFU 13.31% | FLOPs estimator: workload-aware | logical tokens/s: 188787.1 | valid tokens/s: 188787.1 | physical tokens/s: 189115.6 | loss tokens/s: 137533.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.326876E-01 | lm-global-supervised-tokens: 1.222235E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.185102E-02 | lm-token-buffer-mib: 5.810878E+01 | global_load_balancing_loss: 9.996989E-01 | loss scale: 1.0 | grad norm: 0.038 | params norm: 11478.509 | params rms: 0.062152 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 11:21:28.206461 | finish at 2026-10-10 18:04:50 + [2026-10-06 06:44:53] iteration 2507/ 6855 | consumed samples: 320896 | elapsed time per iteration (ms): 90540.8 | throughput per GPU (TFLOP/s/GPU): 41.2 | MFU 13.22% | FLOPs estimator: workload-aware | logical tokens/s: 185299.2 | valid tokens/s: 185299.2 | physical tokens/s: 185908.2 | loss tokens/s: 134564.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.855215E-01 | lm-global-supervised-tokens: 1.218354E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.016714E-02 | lm-token-buffer-mib: 5.813264E+01 | global_load_balancing_loss: 9.996777E-01 | loss scale: 1.0 | grad norm: 0.186 | params norm: 11478.441 | params rms: 0.062152 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 13:21:11.339723 | finish at 2026-10-10 20:06:04 + [2026-10-06 06:46:21] iteration 2508/ 6855 | consumed samples: 321024 | elapsed time per iteration (ms): 88345.2 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.53% | FLOPs estimator: workload-aware | logical tokens/s: 189904.7 | valid tokens/s: 189904.7 | physical tokens/s: 190333.1 | loss tokens/s: 136923.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.664129E-01 | lm-global-supervised-tokens: 1.209656E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.633796E-02 | lm-token-buffer-mib: 5.809137E+01 | global_load_balancing_loss: 1.000712E+00 | loss scale: 1.0 | grad norm: 0.127 | params norm: 11478.373 | params rms: 0.062152 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 10:40:36.733911 | finish at 2026-10-10 17:26:58 + [2026-10-06 06:47:44] iteration 2509/ 6855 | consumed samples: 321152 | elapsed time per iteration (ms): 83254.7 | throughput per GPU (TFLOP/s/GPU): 43.2 | MFU 13.83% | FLOPs estimator: workload-aware | logical tokens/s: 201515.4 | valid tokens/s: 201515.4 | physical tokens/s: 201962.1 | loss tokens/s: 155932.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 5.935811E-01 | lm-global-supervised-tokens: 1.298209E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.156275E-02 | lm-token-buffer-mib: 5.811649E+01 | global_load_balancing_loss: 1.000646E+00 | loss scale: 1.0 | grad norm: 0.167 | params norm: 11478.303 | params rms: 0.062151 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 4:30:25.074135 | finish at 2026-10-10 11:18:09 + [2026-10-06 06:49:34] iteration 2510/ 6855 | consumed samples: 321280 | elapsed time per iteration (ms): 109243.9 | throughput per GPU (TFLOP/s/GPU): 33.1 | MFU 10.61% | FLOPs estimator: workload-aware | logical tokens/s: 153575.4 | valid tokens/s: 153575.4 | physical tokens/s: 154092.4 | loss tokens/s: 113040.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.600934E-01 | lm-global-supervised-tokens: 1.234903E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.358078E-02 | lm-token-buffer-mib: 5.810674E+01 | global_load_balancing_loss: 9.999676E-01 | loss scale: 1.0 | grad norm: 0.219 | params norm: 11478.235 | params rms: 0.062151 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 11:51:04.677392 | finish at 2026-10-11 18:40:38 + [2026-10-06 06:51:02] iteration 2511/ 6855 | consumed samples: 321408 | elapsed time per iteration (ms): 88875.0 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.57% | FLOPs estimator: workload-aware | logical tokens/s: 188773.1 | valid tokens/s: 188773.1 | physical tokens/s: 189122.3 | loss tokens/s: 137391.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.725813E-01 | lm-global-supervised-tokens: 1.221065E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.454929E-02 | lm-token-buffer-mib: 5.811086E+01 | global_load_balancing_loss: 1.000365E+00 | loss scale: 1.0 | grad norm: 0.106 | params norm: 11478.167 | params rms: 0.062151 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 11:14:33.196781 | finish at 2026-10-10 18:05:36 + [2026-10-06 06:52:28] iteration 2512/ 6855 | consumed samples: 321536 | elapsed time per iteration (ms): 85475.4 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.77% | FLOPs estimator: workload-aware | logical tokens/s: 196280.7 | valid tokens/s: 196280.7 | physical tokens/s: 196711.4 | loss tokens/s: 140271.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.301349E-01 | lm-global-supervised-tokens: 1.198978E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.150630E-02 | lm-token-buffer-mib: 5.809106E+01 | global_load_balancing_loss: 9.995233E-01 | loss scale: 1.0 | grad norm: 0.089 | params norm: 11478.096 | params rms: 0.062150 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 7:06:59.680712 | finish at 2026-10-10 13:59:28 + [2026-10-06 06:53:56] iteration 2513/ 6855 | consumed samples: 321664 | elapsed time per iteration (ms): 88149.3 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.51% | FLOPs estimator: workload-aware | logical tokens/s: 190327.0 | valid tokens/s: 190327.0 | physical tokens/s: 190792.2 | loss tokens/s: 141314.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.688949E-01 | lm-global-supervised-tokens: 1.245680E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.251005E-02 | lm-token-buffer-mib: 5.812920E+01 | global_load_balancing_loss: 9.999571E-01 | loss scale: 1.0 | grad norm: 0.135 | params norm: 11478.027 | params rms: 0.062150 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 10:19:04.392499 | finish at 2026-10-10 17:13:00 + [2026-10-06 06:55:24] iteration 2514/ 6855 | consumed samples: 321792 | elapsed time per iteration (ms): 87809.3 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.53% | FLOPs estimator: workload-aware | logical tokens/s: 191064.2 | valid tokens/s: 191064.2 | physical tokens/s: 191563.5 | loss tokens/s: 141956.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.621085E-01 | lm-global-supervised-tokens: 1.246510E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.943947E-02 | lm-token-buffer-mib: 5.813039E+01 | global_load_balancing_loss: 9.996680E-01 | loss scale: 1.0 | grad norm: 0.098 | params norm: 11477.959 | params rms: 0.062149 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 9:53:00.167960 | finish at 2026-10-10 16:48:24 + [2026-10-06 06:56:52] iteration 2515/ 6855 | consumed samples: 321920 | elapsed time per iteration (ms): 88495.4 | throughput per GPU (TFLOP/s/GPU): 41.5 | MFU 13.31% | FLOPs estimator: workload-aware | logical tokens/s: 189582.7 | valid tokens/s: 189582.7 | physical tokens/s: 190166.4 | loss tokens/s: 139569.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.258026E-01 | lm-global-supervised-tokens: 1.235123E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.848320E-02 | lm-token-buffer-mib: 5.836617E+01 | global_load_balancing_loss: 9.993134E-01 | loss scale: 1.0 | grad norm: 0.234 | params norm: 11477.891 | params rms: 0.062149 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 10:41:10.015097 | finish at 2026-10-10 17:38:02 + [2026-10-06 06:58:18] iteration 2516/ 6855 | consumed samples: 322048 | elapsed time per iteration (ms): 85379.4 | throughput per GPU (TFLOP/s/GPU): 43.7 | MFU 13.99% | FLOPs estimator: workload-aware | logical tokens/s: 196501.8 | valid tokens/s: 196501.7 | physical tokens/s: 196937.9 | loss tokens/s: 148177.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.681471E-01 | lm-global-supervised-tokens: 1.265132E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.142521E-02 | lm-token-buffer-mib: 5.809449E+01 | global_load_balancing_loss: 1.000164E+00 | loss scale: 1.0 | grad norm: 0.082 | params norm: 11477.824 | params rms: 0.062149 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 6:54:21.402874 | finish at 2026-10-10 13:52:39 + [2026-10-06 06:59:43] iteration 2517/ 6855 | consumed samples: 322176 | elapsed time per iteration (ms): 85086.4 | throughput per GPU (TFLOP/s/GPU): 43.6 | MFU 13.97% | FLOPs estimator: workload-aware | logical tokens/s: 197178.4 | valid tokens/s: 197178.4 | physical tokens/s: 197540.8 | loss tokens/s: 147971.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.332646E-01 | lm-global-supervised-tokens: 1.259038E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.359970E-02 | lm-token-buffer-mib: 5.822180E+01 | global_load_balancing_loss: 1.000956E+00 | loss scale: 1.0 | grad norm: 0.143 | params norm: 11477.757 | params rms: 0.062148 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 6:31:44.799202 | finish at 2026-10-10 13:31:28 + [2026-10-06 07:01:08] iteration 2518/ 6855 | consumed samples: 322304 | elapsed time per iteration (ms): 85284.2 | throughput per GPU (TFLOP/s/GPU): 43.1 | MFU 13.80% | FLOPs estimator: workload-aware | logical tokens/s: 196721.3 | valid tokens/s: 196721.3 | physical tokens/s: 197100.1 | loss tokens/s: 145806.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.288304E-01 | lm-global-supervised-tokens: 1.243500E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.460859E-02 | lm-token-buffer-mib: 5.810905E+01 | global_load_balancing_loss: 1.001988E+00 | loss scale: 1.0 | grad norm: 0.059 | params norm: 11477.689 | params rms: 0.062148 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 6:44:37.466624 | finish at 2026-10-10 13:45:46 + [2026-10-06 07:02:33] iteration 2519/ 6855 | consumed samples: 322432 | elapsed time per iteration (ms): 85119.9 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.42% | FLOPs estimator: workload-aware | logical tokens/s: 197100.9 | valid tokens/s: 197100.8 | physical tokens/s: 197691.3 | loss tokens/s: 146924.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.515908E-01 | lm-global-supervised-tokens: 1.250617E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.421375E-02 | lm-token-buffer-mib: 5.808598E+01 | global_load_balancing_loss: 1.001055E+00 | loss scale: 1.0 | grad norm: 0.064 | params norm: 11477.621 | params rms: 0.062148 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 6:31:19.773666 | finish at 2026-10-10 13:33:53 + [2026-10-06 07:04:22] iteration 2520/ 6855 | consumed samples: 322560 | elapsed time per iteration (ms): 108684.8 | throughput per GPU (TFLOP/s/GPU): 34.5 | MFU 11.05% | FLOPs estimator: workload-aware | logical tokens/s: 154365.7 | valid tokens/s: 154365.7 | physical tokens/s: 154699.0 | loss tokens/s: 109924.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.805022E-01 | lm-global-supervised-tokens: 1.194708E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.259685E-02 | lm-token-buffer-mib: 5.812914E+01 | global_load_balancing_loss: 9.997988E-01 | loss scale: 1.0 | grad norm: 0.083 | params norm: 11477.553 | params rms: 0.062147 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 10:52:28.515623 | finish at 2026-10-11 17:56:50 +times across ranks (ms): + save-checkpoint: + rank 0: 24935.13 + rank 1: 24935.13 + rank 2: 24935.16 + rank 3: 24935.16 + rank 4: 24935.11 + rank 5: 24935.16 + rank 6: 24935.17 + rank 7: 24935.16 + rank 8: 24935.46 + rank 9: 24935.44 + rank 10: 24935.44 + rank 11: 24935.48 + rank 12: 24935.45 + rank 13: 24935.46 + rank 14: 24935.45 + rank 15: 24935.43 + rank 16: 24935.42 + rank 17: 24935.46 + rank 18: 24935.42 + rank 19: 24935.46 + rank 20: 24935.42 + rank 21: 24935.44 + rank 22: 24935.41 + rank 23: 24935.41 + rank 24: 24935.50 + rank 25: 24935.48 + rank 26: 24935.52 + rank 27: 24935.52 + rank 28: 24935.47 + rank 29: 24935.53 + rank 30: 24935.52 + rank 31: 24935.49 + rank 32: 24935.09 + rank 33: 24935.09 + rank 34: 24935.10 + rank 35: 24935.05 + rank 36: 24935.05 + rank 37: 24935.04 + rank 38: 24935.07 + rank 39: 24935.05 + rank 40: 24934.92 + rank 41: 24934.92 + rank 42: 24934.92 + rank 43: 24935.00 + rank 44: 24934.90 + rank 45: 24934.94 + rank 46: 24934.90 + rank 47: 24934.90 + rank 48: 24935.29 + rank 49: 24935.27 + rank 50: 24935.24 + rank 51: 24935.24 + rank 52: 24935.25 + rank 53: 24935.22 + rank 54: 24935.25 + rank 55: 24935.22 + rank 56: 24935.54 + rank 57: 24935.51 + rank 58: 24935.51 + rank 59: 24935.50 + rank 60: 24935.50 + rank 61: 24935.52 + rank 62: 24935.50 + rank 63: 24935.51 + [2026-10-06 07:06:13] iteration 2521/ 6855 | consumed samples: 322688 | elapsed time per iteration (ms): 85760.1 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.64% | FLOPs estimator: workload-aware | logical tokens/s: 195629.1 | valid tokens/s: 195629.1 | physical tokens/s: 196207.9 | loss tokens/s: 147197.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.812945E-01 | lm-global-supervised-tokens: 1.262367E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 4.772122E-01 | lm-token-buffer-mib: 5.822635E+01 | global_load_balancing_loss: 9.993485E-01 | loss scale: 1.0 | grad norm: 0.072 | params norm: 11477.485 | params rms: 0.062147 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 7:14:44.086919 | finish at 2026-10-10 14:20:57 + [2026-10-06 07:07:44] iteration 2522/ 6855 | consumed samples: 322816 | elapsed time per iteration (ms): 90560.1 | throughput per GPU (TFLOP/s/GPU): 39.8 | MFU 12.76% | FLOPs estimator: workload-aware | logical tokens/s: 185260.2 | valid tokens/s: 185260.2 | physical tokens/s: 185871.6 | loss tokens/s: 144746.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.668667E-01 | lm-global-supervised-tokens: 1.310825E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.408762E-02 | lm-token-buffer-mib: 5.840132E+01 | global_load_balancing_loss: 1.000935E+00 | loss scale: 1.0 | grad norm: 0.058 | params norm: 11477.418 | params rms: 0.062146 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 12:59:56.850623 | finish at 2026-10-10 20:07:40 + [2026-10-06 07:09:09] iteration 2523/ 6855 | consumed samples: 322944 | elapsed time per iteration (ms): 85134.9 | throughput per GPU (TFLOP/s/GPU): 43.1 | MFU 13.81% | FLOPs estimator: workload-aware | logical tokens/s: 197065.5 | valid tokens/s: 197065.5 | physical tokens/s: 197472.5 | loss tokens/s: 144485.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.844391E-01 | lm-global-supervised-tokens: 1.230074E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.139327E-02 | lm-token-buffer-mib: 5.811475E+01 | global_load_balancing_loss: 9.996651E-01 | loss scale: 1.0 | grad norm: 0.047 | params norm: 11477.352 | params rms: 0.062146 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 6:26:44.186834 | finish at 2026-10-10 13:35:53 + [2026-10-06 07:10:30] iteration 2524/ 6855 | consumed samples: 323072 | elapsed time per iteration (ms): 81528.7 | throughput per GPU (TFLOP/s/GPU): 43.6 | MFU 13.96% | FLOPs estimator: workload-aware | logical tokens/s: 205782.9 | valid tokens/s: 205782.9 | physical tokens/s: 206232.0 | loss tokens/s: 153300.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.379799E-01 | lm-global-supervised-tokens: 1.249834E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.605771E-02 | lm-token-buffer-mib: 5.806005E+01 | global_load_balancing_loss: 9.997974E-01 | loss scale: 1.0 | grad norm: 0.072 | params norm: 11477.285 | params rms: 0.062146 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 2:05:00.708290 | finish at 2026-10-10 09:15:31 + [2026-10-06 07:11:58] iteration 2525/ 6855 | consumed samples: 323200 | elapsed time per iteration (ms): 87524.1 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.64% | FLOPs estimator: workload-aware | logical tokens/s: 191686.6 | valid tokens/s: 191686.6 | physical tokens/s: 192487.3 | loss tokens/s: 140125.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.963568E-01 | lm-global-supervised-tokens: 1.226432E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.169685E-02 | lm-token-buffer-mib: 5.828937E+01 | global_load_balancing_loss: 1.000671E+00 | loss scale: 1.0 | grad norm: 0.085 | params norm: 11477.218 | params rms: 0.062145 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 9:16:19.313021 | finish at 2026-10-10 16:28:17 + [2026-10-06 07:13:27] iteration 2526/ 6855 | consumed samples: 323328 | elapsed time per iteration (ms): 88931.1 | throughput per GPU (TFLOP/s/GPU): 43.2 | MFU 13.86% | FLOPs estimator: workload-aware | logical tokens/s: 188654.1 | valid tokens/s: 188654.1 | physical tokens/s: 188916.8 | loss tokens/s: 137570.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.322759E-01 | lm-global-supervised-tokens: 1.223426E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.502596E-02 | lm-token-buffer-mib: 5.802896E+01 | global_load_balancing_loss: 1.000481E+00 | loss scale: 1.0 | grad norm: 0.063 | params norm: 11477.151 | params rms: 0.062145 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 10:56:22.727303 | finish at 2026-10-10 18:09:49 + [2026-10-06 07:14:56] iteration 2527/ 6855 | consumed samples: 323456 | elapsed time per iteration (ms): 88866.7 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.53% | FLOPs estimator: workload-aware | logical tokens/s: 188790.0 | valid tokens/s: 188790.0 | physical tokens/s: 189339.9 | loss tokens/s: 139129.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.697521E-01 | lm-global-supervised-tokens: 1.236394E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.246921E-02 | lm-token-buffer-mib: 5.814357E+01 | global_load_balancing_loss: 9.999741E-01 | loss scale: 1.0 | grad norm: 0.112 | params norm: 11477.082 | params rms: 0.062145 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 10:50:15.069059 | finish at 2026-10-10 18:05:11 + [2026-10-06 07:16:23] iteration 2528/ 6855 | consumed samples: 323584 | elapsed time per iteration (ms): 86943.0 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.53% | FLOPs estimator: workload-aware | logical tokens/s: 192967.8 | valid tokens/s: 192967.8 | physical tokens/s: 193394.7 | loss tokens/s: 144829.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.690659E-01 | lm-global-supervised-tokens: 1.259194E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.156861E-02 | lm-token-buffer-mib: 5.826725E+01 | global_load_balancing_loss: 9.992751E-01 | loss scale: 1.0 | grad norm: 0.111 | params norm: 11477.014 | params rms: 0.062144 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 8:30:02.532590 | finish at 2026-10-10 15:46:25 + [2026-10-06 07:17:49] iteration 2529/ 6855 | consumed samples: 323712 | elapsed time per iteration (ms): 86790.7 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.60% | FLOPs estimator: workload-aware | logical tokens/s: 193306.5 | valid tokens/s: 193306.5 | physical tokens/s: 193766.5 | loss tokens/s: 145077.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.755896E-01 | lm-global-supervised-tokens: 1.259136E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.153736E-02 | lm-token-buffer-mib: 5.810045E+01 | global_load_balancing_loss: 1.001048E+00 | loss scale: 1.0 | grad norm: 0.054 | params norm: 11476.948 | params rms: 0.062144 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 8:17:36.642282 | finish at 2026-10-10 15:35:26 + [2026-10-06 07:19:40] iteration 2530/ 6855 | consumed samples: 323840 | elapsed time per iteration (ms): 110689.8 | throughput per GPU (TFLOP/s/GPU): 33.3 | MFU 10.66% | FLOPs estimator: workload-aware | logical tokens/s: 151569.2 | valid tokens/s: 151569.2 | physical tokens/s: 151926.9 | loss tokens/s: 118054.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.130963E-01 | lm-global-supervised-tokens: 1.306741E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.213114E-02 | lm-token-buffer-mib: 5.805331E+01 | global_load_balancing_loss: 9.990832E-01 | loss scale: 1.0 | grad norm: 0.074 | params norm: 11476.881 | params rms: 0.062144 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 12:58:53.590305 | finish at 2026-10-11 20:18:34 + [2026-10-06 07:21:10] iteration 2531/ 6855 | consumed samples: 323968 | elapsed time per iteration (ms): 89639.7 | throughput per GPU (TFLOP/s/GPU): 41.5 | MFU 13.31% | FLOPs estimator: workload-aware | logical tokens/s: 187162.6 | valid tokens/s: 187162.6 | physical tokens/s: 187646.7 | loss tokens/s: 137163.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.789400E-01 | lm-global-supervised-tokens: 1.229532E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.522140E-02 | lm-token-buffer-mib: 5.810064E+01 | global_load_balancing_loss: 1.000204E+00 | loss scale: 1.0 | grad norm: 0.084 | params norm: 11476.813 | params rms: 0.062143 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 11:40:02.014070 | finish at 2026-10-10 19:01:12 + [2026-10-06 07:22:37] iteration 2532/ 6855 | consumed samples: 324096 | elapsed time per iteration (ms): 87827.5 | throughput per GPU (TFLOP/s/GPU): 41.6 | MFU 13.32% | FLOPs estimator: workload-aware | logical tokens/s: 191024.4 | valid tokens/s: 191024.4 | physical tokens/s: 191499.8 | loss tokens/s: 143896.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.476630E-01 | lm-global-supervised-tokens: 1.263808E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.081450E-02 | lm-token-buffer-mib: 5.823386E+01 | global_load_balancing_loss: 1.000324E+00 | loss scale: 1.0 | grad norm: 0.042 | params norm: 11476.743 | params rms: 0.062143 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 9:27:58.093308 | finish at 2026-10-10 16:50:36 + [2026-10-06 07:24:08] iteration 2533/ 6855 | consumed samples: 324224 | elapsed time per iteration (ms): 90298.9 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.58% | FLOPs estimator: workload-aware | logical tokens/s: 185795.9 | valid tokens/s: 185795.9 | physical tokens/s: 186055.8 | loss tokens/s: 129031.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.037018E-01 | lm-global-supervised-tokens: 1.165137E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.524668E-02 | lm-token-buffer-mib: 5.805247E+01 | global_load_balancing_loss: 1.000426E+00 | loss scale: 1.0 | grad norm: 0.064 | params norm: 11476.674 | params rms: 0.062142 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 12:24:31.630988 | finish at 2026-10-10 19:48:39 + [2026-10-06 07:25:34] iteration 2534/ 6855 | consumed samples: 324352 | elapsed time per iteration (ms): 86522.6 | throughput per GPU (TFLOP/s/GPU): 43.1 | MFU 13.82% | FLOPs estimator: workload-aware | logical tokens/s: 193904.5 | valid tokens/s: 193904.5 | physical tokens/s: 194447.0 | loss tokens/s: 136903.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.601823E-01 | lm-global-supervised-tokens: 1.184526E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.297795E-02 | lm-token-buffer-mib: 5.831773E+01 | global_load_balancing_loss: 1.001344E+00 | loss scale: 1.0 | grad norm: 0.111 | params norm: 11476.606 | params rms: 0.062142 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 7:51:04.151231 | finish at 2026-10-10 15:16:38 + [2026-10-06 07:27:05] iteration 2535/ 6855 | consumed samples: 324480 | elapsed time per iteration (ms): 90587.3 | throughput per GPU (TFLOP/s/GPU): 41.3 | MFU 13.22% | FLOPs estimator: workload-aware | logical tokens/s: 185203.8 | valid tokens/s: 185203.8 | physical tokens/s: 185547.5 | loss tokens/s: 142083.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.457458E-01 | lm-global-supervised-tokens: 1.287092E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.448415E-02 | lm-token-buffer-mib: 5.834788E+01 | global_load_balancing_loss: 9.996914E-01 | loss scale: 1.0 | grad norm: 0.064 | params norm: 11476.539 | params rms: 0.062142 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 12:42:17.067261 | finish at 2026-10-10 20:09:22 + [2026-10-06 07:28:32] iteration 2536/ 6855 | consumed samples: 324608 | elapsed time per iteration (ms): 87236.0 | throughput per GPU (TFLOP/s/GPU): 40.9 | MFU 13.09% | FLOPs estimator: workload-aware | logical tokens/s: 192319.7 | valid tokens/s: 192319.7 | physical tokens/s: 192749.3 | loss tokens/s: 153122.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.761789E-01 | lm-global-supervised-tokens: 1.335780E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.267485E-02 | lm-token-buffer-mib: 5.810247E+01 | global_load_balancing_loss: 9.992654E-01 | loss scale: 1.0 | grad norm: 0.195 | params norm: 11476.472 | params rms: 0.062141 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 8:39:32.491239 | finish at 2026-10-10 16:08:05 + [2026-10-06 07:29:58] iteration 2537/ 6855 | consumed samples: 324736 | elapsed time per iteration (ms): 86033.8 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.54% | FLOPs estimator: workload-aware | logical tokens/s: 195006.4 | valid tokens/s: 195006.4 | physical tokens/s: 195560.1 | loss tokens/s: 153038.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.368890E-01 | lm-global-supervised-tokens: 1.316653E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.211257E-02 | lm-token-buffer-mib: 5.808144E+01 | global_load_balancing_loss: 9.996380E-01 | loss scale: 1.0 | grad norm: 0.121 | params norm: 11476.406 | params rms: 0.062141 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 7:11:34.105423 | finish at 2026-10-10 14:41:32 + [2026-10-06 07:31:23] iteration 2538/ 6855 | consumed samples: 324864 | elapsed time per iteration (ms): 84591.0 | throughput per GPU (TFLOP/s/GPU): 43.5 | MFU 13.95% | FLOPs estimator: workload-aware | logical tokens/s: 198332.9 | valid tokens/s: 198332.9 | physical tokens/s: 198829.5 | loss tokens/s: 145472.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.926848E-01 | lm-global-supervised-tokens: 1.230568E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.500452E-02 | lm-token-buffer-mib: 5.821159E+01 | global_load_balancing_loss: 1.001246E+00 | loss scale: 1.0 | grad norm: 0.198 | params norm: 11476.342 | params rms: 0.062141 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 5:26:19.228982 | finish at 2026-10-10 12:57:42 + [2026-10-06 07:32:50] iteration 2539/ 6855 | consumed samples: 324992 | elapsed time per iteration (ms): 87256.3 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.61% | FLOPs estimator: workload-aware | logical tokens/s: 192275.0 | valid tokens/s: 192275.0 | physical tokens/s: 192698.6 | loss tokens/s: 138180.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.338413E-01 | lm-global-supervised-tokens: 1.205712E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.510862E-02 | lm-token-buffer-mib: 5.831136E+01 | global_load_balancing_loss: 9.999654E-01 | loss scale: 1.0 | grad norm: 0.086 | params norm: 11476.277 | params rms: 0.062140 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 8:36:38.100128 | finish at 2026-10-10 16:09:28 + [2026-10-06 07:34:39] iteration 2540/ 6855 | consumed samples: 325120 | elapsed time per iteration (ms): 108675.3 | throughput per GPU (TFLOP/s/GPU): 33.9 | MFU 10.87% | FLOPs estimator: workload-aware | logical tokens/s: 154378.9 | valid tokens/s: 154378.9 | physical tokens/s: 154837.2 | loss tokens/s: 113606.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.740572E-01 | lm-global-supervised-tokens: 1.234624E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.034585E-02 | lm-token-buffer-mib: 5.822550E+01 | global_load_balancing_loss: 9.999887E-01 | loss scale: 1.0 | grad norm: 0.379 | params norm: 11476.212 | params rms: 0.062140 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 10:15:33.895333 | finish at 2026-10-11 17:50:13 + [2026-10-06 07:36:06] iteration 2541/ 6855 | consumed samples: 325248 | elapsed time per iteration (ms): 87637.5 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.44% | FLOPs estimator: workload-aware | logical tokens/s: 191438.2 | valid tokens/s: 191438.2 | physical tokens/s: 191897.7 | loss tokens/s: 143559.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.647731E-01 | lm-global-supervised-tokens: 1.258122E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.161189E-02 | lm-token-buffer-mib: 5.814100E+01 | global_load_balancing_loss: 9.998319E-01 | loss scale: 1.0 | grad norm: 0.081 | params norm: 11476.144 | params rms: 0.062140 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 9:01:07.993155 | finish at 2026-10-10 16:37:14 + [2026-10-06 07:37:34] iteration 2542/ 6855 | consumed samples: 325376 | elapsed time per iteration (ms): 87177.0 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.62% | FLOPs estimator: workload-aware | logical tokens/s: 192449.5 | valid tokens/s: 192449.5 | physical tokens/s: 193019.1 | loss tokens/s: 149318.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.864943E-01 | lm-global-supervised-tokens: 1.301714E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.192274E-02 | lm-token-buffer-mib: 5.814343E+01 | global_load_balancing_loss: 1.000214E+00 | loss scale: 1.0 | grad norm: 0.481 | params norm: 11476.077 | params rms: 0.062139 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 8:26:34.270603 | finish at 2026-10-10 16:04:08 + [2026-10-06 07:39:01] iteration 2543/ 6855 | consumed samples: 325504 | elapsed time per iteration (ms): 87843.5 | throughput per GPU (TFLOP/s/GPU): 43.7 | MFU 14.02% | FLOPs estimator: workload-aware | logical tokens/s: 190989.8 | valid tokens/s: 190989.8 | physical tokens/s: 191315.3 | loss tokens/s: 136425.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.517805E-01 | lm-global-supervised-tokens: 1.198406E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 3.114539E-02 | lm-token-buffer-mib: 5.814037E+01 | global_load_balancing_loss: 1.001027E+00 | loss scale: 1.0 | grad norm: 0.062 | params norm: 11476.012 | params rms: 0.062139 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 9:13:01.269230 | finish at 2026-10-10 16:52:03 + [2026-10-06 07:40:27] iteration 2544/ 6855 | consumed samples: 325632 | elapsed time per iteration (ms): 85571.5 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.74% | FLOPs estimator: workload-aware | logical tokens/s: 196060.5 | valid tokens/s: 196060.5 | physical tokens/s: 196526.6 | loss tokens/s: 145529.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.540021E-01 | lm-global-supervised-tokens: 1.245320E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.512196E-02 | lm-token-buffer-mib: 5.817047E+01 | global_load_balancing_loss: 1.000694E+00 | loss scale: 1.0 | grad norm: 0.294 | params norm: 11475.946 | params rms: 0.062139 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 6:28:18.687436 | finish at 2026-10-10 14:08:46 + [2026-10-06 07:41:58] iteration 2545/ 6855 | consumed samples: 325760 | elapsed time per iteration (ms): 90849.6 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.50% | FLOPs estimator: workload-aware | logical tokens/s: 184670.2 | valid tokens/s: 184670.2 | physical tokens/s: 185110.2 | loss tokens/s: 139137.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.570243E-01 | lm-global-supervised-tokens: 1.264060E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.235171E-02 | lm-token-buffer-mib: 5.820944E+01 | global_load_balancing_loss: 9.997268E-01 | loss scale: 1.0 | grad norm: 0.234 | params norm: 11475.879 | params rms: 0.062138 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 12:46:01.792772 | finish at 2026-10-10 20:28:00 + [2026-10-06 07:43:27] iteration 2546/ 6855 | consumed samples: 325888 | elapsed time per iteration (ms): 89425.1 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.46% | FLOPs estimator: workload-aware | logical tokens/s: 187611.9 | valid tokens/s: 187611.9 | physical tokens/s: 188052.0 | loss tokens/s: 137586.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.663250E-01 | lm-global-supervised-tokens: 1.230368E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.150697E-02 | lm-token-buffer-mib: 5.812085E+01 | global_load_balancing_loss: 1.000588E+00 | loss scale: 1.0 | grad norm: 0.221 | params norm: 11475.811 | params rms: 0.062138 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 11:02:12.572385 | finish at 2026-10-10 18:45:40 + [2026-10-06 07:44:55] iteration 2547/ 6855 | consumed samples: 326016 | elapsed time per iteration (ms): 87529.0 | throughput per GPU (TFLOP/s/GPU): 41.3 | MFU 13.22% | FLOPs estimator: workload-aware | logical tokens/s: 191675.7 | valid tokens/s: 191675.7 | physical tokens/s: 192233.2 | loss tokens/s: 144187.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.383874E-01 | lm-global-supervised-tokens: 1.262054E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.337092E-02 | lm-token-buffer-mib: 5.808657E+01 | global_load_balancing_loss: 1.000865E+00 | loss scale: 1.0 | grad norm: 0.253 | params norm: 11475.745 | params rms: 0.062137 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 8:44:34.765798 | finish at 2026-10-10 16:29:29 + [2026-10-06 07:46:23] iteration 2548/ 6855 | consumed samples: 326144 | elapsed time per iteration (ms): 88347.1 | throughput per GPU (TFLOP/s/GPU): 41.2 | MFU 13.21% | FLOPs estimator: workload-aware | logical tokens/s: 189900.2 | valid tokens/s: 189900.2 | physical tokens/s: 190298.3 | loss tokens/s: 140252.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.865885E-01 | lm-global-supervised-tokens: 1.239093E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.355387E-02 | lm-token-buffer-mib: 5.813894E+01 | global_load_balancing_loss: 1.000507E+00 | loss scale: 1.0 | grad norm: 0.199 | params norm: 11475.680 | params rms: 0.062137 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 9:41:50.887903 | finish at 2026-10-10 17:28:14 + [2026-10-06 07:47:49] iteration 2549/ 6855 | consumed samples: 326272 | elapsed time per iteration (ms): 86045.7 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.66% | FLOPs estimator: workload-aware | logical tokens/s: 194979.7 | valid tokens/s: 194979.7 | physical tokens/s: 195491.3 | loss tokens/s: 147171.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.236475E-01 | lm-global-supervised-tokens: 1.266343E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.454044E-02 | lm-token-buffer-mib: 5.833162E+01 | global_load_balancing_loss: 1.000209E+00 | loss scale: 1.0 | grad norm: 0.071 | params norm: 11475.612 | params rms: 0.062137 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 6:55:12.704438 | finish at 2026-10-10 14:43:02 + [2026-10-06 07:49:39] iteration 2550/ 6855 | consumed samples: 326400 | elapsed time per iteration (ms): 109992.2 | throughput per GPU (TFLOP/s/GPU): 33.5 | MFU 10.74% | FLOPs estimator: workload-aware | logical tokens/s: 152530.4 | valid tokens/s: 152530.4 | physical tokens/s: 152889.1 | loss tokens/s: 109005.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.475667E-01 | lm-global-supervised-tokens: 1.198972E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.346723E-02 | lm-token-buffer-mib: 5.822651E+01 | global_load_balancing_loss: 9.997051E-01 | loss scale: 1.0 | grad norm: 0.139 | params norm: 11475.545 | params rms: 0.062136 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 11:31:56.355897 | finish at 2026-10-11 19:21:35 + [2026-10-06 07:51:08] iteration 2551/ 6855 | consumed samples: 326528 | elapsed time per iteration (ms): 88544.1 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.45% | FLOPs estimator: workload-aware | logical tokens/s: 189477.7 | valid tokens/s: 189477.7 | physical tokens/s: 190380.4 | loss tokens/s: 134331.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.591883E-01 | lm-global-supervised-tokens: 1.189424E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.742510E-02 | lm-token-buffer-mib: 5.821907E+01 | global_load_balancing_loss: 9.990265E-01 | loss scale: 1.0 | grad norm: 0.093 | params norm: 11475.480 | params rms: 0.062136 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 9:51:33.649597 | finish at 2026-10-10 17:42:41 + [2026-10-06 07:52:37] iteration 2552/ 6855 | consumed samples: 326656 | elapsed time per iteration (ms): 89820.8 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.47% | FLOPs estimator: workload-aware | logical tokens/s: 186785.0 | valid tokens/s: 186785.0 | physical tokens/s: 187225.9 | loss tokens/s: 133418.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.031590E-01 | lm-global-supervised-tokens: 1.198378E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.289696E-02 | lm-token-buffer-mib: 5.809079E+01 | global_load_balancing_loss: 9.991372E-01 | loss scale: 1.0 | grad norm: 0.053 | params norm: 11475.414 | params rms: 0.062136 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 11:21:38.690320 | finish at 2026-10-10 19:14:16 + [2026-10-06 07:54:06] iteration 2553/ 6855 | consumed samples: 326784 | elapsed time per iteration (ms): 88178.9 | throughput per GPU (TFLOP/s/GPU): 41.7 | MFU 13.36% | FLOPs estimator: workload-aware | logical tokens/s: 190263.4 | valid tokens/s: 190263.2 | physical tokens/s: 190673.6 | loss tokens/s: 141082.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.757239E-01 | lm-global-supervised-tokens: 1.244052E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.303833E-02 | lm-token-buffer-mib: 5.805219E+01 | global_load_balancing_loss: 1.000404E+00 | loss scale: 1.0 | grad norm: 0.078 | params norm: 11475.345 | params rms: 0.062135 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 9:22:25.662688 | finish at 2026-10-10 17:16:31 + [2026-10-06 07:55:32] iteration 2554/ 6855 | consumed samples: 326912 | elapsed time per iteration (ms): 86037.6 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.78% | FLOPs estimator: workload-aware | logical tokens/s: 194998.4 | valid tokens/s: 194998.4 | physical tokens/s: 195414.2 | loss tokens/s: 148534.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.271889E-01 | lm-global-supervised-tokens: 1.277957E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.704060E-02 | lm-token-buffer-mib: 5.809073E+01 | global_load_balancing_loss: 9.999226E-01 | loss scale: 1.0 | grad norm: 0.060 | params norm: 11475.277 | params rms: 0.062135 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 6:47:27.549602 | finish at 2026-10-10 14:42:59 + [2026-10-06 07:56:56] iteration 2555/ 6855 | consumed samples: 327040 | elapsed time per iteration (ms): 84266.6 | throughput per GPU (TFLOP/s/GPU): 43.3 | MFU 13.89% | FLOPs estimator: workload-aware | logical tokens/s: 199096.8 | valid tokens/s: 199096.8 | physical tokens/s: 199725.5 | loss tokens/s: 152537.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.863745E-01 | lm-global-supervised-tokens: 1.285382E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.178890E-02 | lm-token-buffer-mib: 5.823444E+01 | global_load_balancing_loss: 1.000645E+00 | loss scale: 1.0 | grad norm: 0.076 | params norm: 11475.209 | params rms: 0.062135 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 4:39:06.433878 | finish at 2026-10-10 12:36:02 + [2026-10-06 07:58:23] iteration 2556/ 6855 | consumed samples: 327168 | elapsed time per iteration (ms): 87251.1 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.48% | FLOPs estimator: workload-aware | logical tokens/s: 192286.4 | valid tokens/s: 192286.3 | physical tokens/s: 192743.7 | loss tokens/s: 143605.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.933945E-01 | lm-global-supervised-tokens: 1.252972E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.527099E-02 | lm-token-buffer-mib: 5.809387E+01 | global_load_balancing_loss: 1.000278E+00 | loss scale: 1.0 | grad norm: 0.075 | params norm: 11475.142 | params rms: 0.062134 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 8:11:32.591918 | finish at 2026-10-10 16:09:56 + [2026-10-06 07:59:48] iteration 2557/ 6855 | consumed samples: 327296 | elapsed time per iteration (ms): 84303.0 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.64% | FLOPs estimator: workload-aware | logical tokens/s: 199010.6 | valid tokens/s: 199010.6 | physical tokens/s: 199531.0 | loss tokens/s: 146264.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 5.745462E-01 | lm-global-supervised-tokens: 1.233049E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.585408E-02 | lm-token-buffer-mib: 5.811328E+01 | global_load_balancing_loss: 1.001624E+00 | loss scale: 1.0 | grad norm: 0.043 | params norm: 11475.073 | params rms: 0.062134 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 4:38:54.120707 | finish at 2026-10-10 12:38:42 + [2026-10-06 08:01:16] iteration 2558/ 6855 | consumed samples: 327424 | elapsed time per iteration (ms): 88302.6 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.55% | FLOPs estimator: workload-aware | logical tokens/s: 189996.7 | valid tokens/s: 189996.7 | physical tokens/s: 190506.8 | loss tokens/s: 139611.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.506697E-01 | lm-global-supervised-tokens: 1.232808E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.621485E-02 | lm-token-buffer-mib: 5.805933E+01 | global_load_balancing_loss: 1.000941E+00 | loss scale: 1.0 | grad norm: 0.056 | params norm: 11475.006 | params rms: 0.062133 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 9:23:56.107882 | finish at 2026-10-10 17:25:12 + [2026-10-06 08:02:45] iteration 2559/ 6855 | consumed samples: 327552 | elapsed time per iteration (ms): 89521.0 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.51% | FLOPs estimator: workload-aware | logical tokens/s: 187410.9 | valid tokens/s: 187410.8 | physical tokens/s: 187672.0 | loss tokens/s: 133711.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.086966E-01 | lm-global-supervised-tokens: 1.196999E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.277813E-02 | lm-token-buffer-mib: 5.806136E+01 | global_load_balancing_loss: 1.000025E+00 | loss scale: 1.0 | grad norm: 0.032 | params norm: 11474.939 | params rms: 0.062133 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 10:49:42.350807 | finish at 2026-10-10 18:52:28 + [2026-10-06 08:04:38] iteration 2560/ 6855 | consumed samples: 327680 | elapsed time per iteration (ms): 112847.5 | throughput per GPU (TFLOP/s/GPU): 32.8 | MFU 10.50% | FLOPs estimator: workload-aware | logical tokens/s: 148671.1 | valid tokens/s: 148671.1 | physical tokens/s: 149186.8 | loss tokens/s: 107955.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.736606E-01 | lm-global-supervised-tokens: 1.218252E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.168847E-02 | lm-token-buffer-mib: 5.820823E+01 | global_load_balancing_loss: 1.001056E+00 | loss scale: 1.0 | grad norm: 0.126 | params norm: 11474.871 | params rms: 0.062133 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 14:38:00.047685 | finish at 2026-10-11 22:42:38 +times across ranks (ms): + save-checkpoint: + rank 0: 23424.72 + rank 1: 23424.75 + rank 2: 23424.79 + rank 3: 23424.75 + rank 4: 23424.74 + rank 5: 23424.77 + rank 6: 23424.72 + rank 7: 23424.75 + rank 8: 23425.00 + rank 9: 23424.96 + rank 10: 23424.96 + rank 11: 23424.95 + rank 12: 23424.99 + rank 13: 23424.98 + rank 14: 23424.95 + rank 15: 23424.96 + rank 16: 23425.02 + rank 17: 23425.10 + rank 18: 23425.04 + rank 19: 23425.06 + rank 20: 23425.03 + rank 21: 23425.05 + rank 22: 23425.02 + rank 23: 23425.03 + rank 24: 23425.15 + rank 25: 23425.15 + rank 26: 23425.18 + rank 27: 23425.21 + rank 28: 23425.12 + rank 29: 23425.19 + rank 30: 23425.16 + rank 31: 23425.18 + rank 32: 23426.42 + rank 33: 23426.44 + rank 34: 23426.43 + rank 35: 23426.41 + rank 36: 23426.40 + rank 37: 23426.40 + rank 38: 23426.40 + rank 39: 23426.39 + rank 40: 23425.15 + rank 41: 23425.02 + rank 42: 23425.03 + rank 43: 23425.04 + rank 44: 23424.98 + rank 45: 23424.98 + rank 46: 23425.01 + rank 47: 23424.98 + rank 48: 23424.76 + rank 49: 23424.79 + rank 50: 23424.77 + rank 51: 23424.74 + rank 52: 23424.77 + rank 53: 23424.73 + rank 54: 23424.72 + rank 55: 23424.78 + rank 56: 23424.76 + rank 57: 23424.76 + rank 58: 23424.75 + rank 59: 23424.74 + rank 60: 23424.76 + rank 61: 23424.73 + rank 62: 23424.74 + rank 63: 23424.74 + [2026-10-06 08:06:37] iteration 2561/ 6855 | consumed samples: 327808 | elapsed time per iteration (ms): 95354.0 | throughput per GPU (TFLOP/s/GPU): 39.0 | MFU 12.49% | FLOPs estimator: workload-aware | logical tokens/s: 175946.0 | valid tokens/s: 175946.0 | physical tokens/s: 176393.6 | loss tokens/s: 132103.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.804618E-01 | lm-global-supervised-tokens: 1.259661E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.374822E+00 | lm-token-buffer-mib: 5.813082E+01 | global_load_balancing_loss: 9.988893E-01 | loss scale: 1.0 | grad norm: 0.128 | params norm: 11474.803 | params rms: 0.062132 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 17:44:10.260672 | finish at 2026-10-11 01:50:47 + [2026-10-06 08:08:10] iteration 2562/ 6855 | consumed samples: 327936 | elapsed time per iteration (ms): 93066.4 | throughput per GPU (TFLOP/s/GPU): 40.6 | MFU 13.03% | FLOPs estimator: workload-aware | logical tokens/s: 180271.2 | valid tokens/s: 180271.2 | physical tokens/s: 180653.9 | loss tokens/s: 135700.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.749535E-01 | lm-global-supervised-tokens: 1.262911E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.762733E-02 | lm-token-buffer-mib: 5.801239E+01 | global_load_balancing_loss: 1.000678E+00 | loss scale: 1.0 | grad norm: 0.043 | params norm: 11474.736 | params rms: 0.062132 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 14:58:54.245796 | finish at 2026-10-10 23:07:04 + [2026-10-06 08:09:35] iteration 2563/ 6855 | consumed samples: 328064 | elapsed time per iteration (ms): 85289.4 | throughput per GPU (TFLOP/s/GPU): 42.8 | MFU 13.73% | FLOPs estimator: workload-aware | logical tokens/s: 196709.3 | valid tokens/s: 196709.3 | physical tokens/s: 197137.4 | loss tokens/s: 145019.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.405448E-01 | lm-global-supervised-tokens: 1.236866E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.244546E-02 | lm-token-buffer-mib: 5.819421E+01 | global_load_balancing_loss: 9.996491E-01 | loss scale: 1.0 | grad norm: 0.088 | params norm: 11474.669 | params rms: 0.062132 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 5:41:02.065324 | finish at 2026-10-10 13:50:38 + [2026-10-06 08:11:03] iteration 2564/ 6855 | consumed samples: 328192 | elapsed time per iteration (ms): 87385.8 | throughput per GPU (TFLOP/s/GPU): 43.3 | MFU 13.87% | FLOPs estimator: workload-aware | logical tokens/s: 191988.9 | valid tokens/s: 191988.9 | physical tokens/s: 192398.1 | loss tokens/s: 141176.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.831514E-01 | lm-global-supervised-tokens: 1.233684E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.183887E-02 | lm-token-buffer-mib: 5.804133E+01 | global_load_balancing_loss: 9.999986E-01 | loss scale: 1.0 | grad norm: 0.042 | params norm: 11474.602 | params rms: 0.062131 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 8:09:32.496974 | finish at 2026-10-10 16:20:35 + [2026-10-06 08:12:30] iteration 2565/ 6855 | consumed samples: 328320 | elapsed time per iteration (ms): 87348.1 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.48% | FLOPs estimator: workload-aware | logical tokens/s: 192072.6 | valid tokens/s: 192072.6 | physical tokens/s: 192516.6 | loss tokens/s: 144909.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 5.971470E-01 | lm-global-supervised-tokens: 1.265757E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.080760E-02 | lm-token-buffer-mib: 5.813763E+01 | global_load_balancing_loss: 1.001027E+00 | loss scale: 1.0 | grad norm: 0.065 | params norm: 11474.535 | params rms: 0.062131 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 8:05:23.453100 | finish at 2026-10-10 16:17:54 + [2026-10-06 08:13:53] iteration 2566/ 6855 | consumed samples: 328448 | elapsed time per iteration (ms): 82706.4 | throughput per GPU (TFLOP/s/GPU): 43.8 | MFU 14.04% | FLOPs estimator: workload-aware | logical tokens/s: 202852.5 | valid tokens/s: 202852.4 | physical tokens/s: 203391.2 | loss tokens/s: 153595.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.770516E-01 | lm-global-supervised-tokens: 1.270333E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.275970E-02 | lm-token-buffer-mib: 5.817496E+01 | global_load_balancing_loss: 9.996255E-01 | loss scale: 1.0 | grad norm: 0.048 | params norm: 11474.470 | params rms: 0.062131 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 2:32:07.566205 | finish at 2026-10-10 10:46:00 + [2026-10-06 08:15:20] iteration 2567/ 6855 | consumed samples: 328576 | elapsed time per iteration (ms): 86939.9 | throughput per GPU (TFLOP/s/GPU): 41.7 | MFU 13.38% | FLOPs estimator: workload-aware | logical tokens/s: 192974.6 | valid tokens/s: 192974.6 | physical tokens/s: 193546.2 | loss tokens/s: 148953.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.212624E-01 | lm-global-supervised-tokens: 1.295002E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.211358E-02 | lm-token-buffer-mib: 5.814313E+01 | global_load_balancing_loss: 9.994068E-01 | loss scale: 1.0 | grad norm: 0.038 | params norm: 11474.404 | params rms: 0.062130 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 7:33:18.471817 | finish at 2026-10-10 15:48:38 + [2026-10-06 08:16:48] iteration 2568/ 6855 | consumed samples: 328704 | elapsed time per iteration (ms): 88356.5 | throughput per GPU (TFLOP/s/GPU): 41.1 | MFU 13.18% | FLOPs estimator: workload-aware | logical tokens/s: 189879.9 | valid tokens/s: 189879.9 | physical tokens/s: 190591.7 | loss tokens/s: 141387.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.890776E-01 | lm-global-supervised-tokens: 1.249248E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.388663E-02 | lm-token-buffer-mib: 5.839409E+01 | global_load_balancing_loss: 9.983195E-01 | loss scale: 1.0 | grad norm: 0.058 | params norm: 11474.336 | params rms: 0.062130 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 9:13:04.293652 | finish at 2026-10-10 17:29:52 + [2026-10-06 08:18:13] iteration 2569/ 6855 | consumed samples: 328832 | elapsed time per iteration (ms): 85285.0 | throughput per GPU (TFLOP/s/GPU): 41.6 | MFU 13.33% | FLOPs estimator: workload-aware | logical tokens/s: 196718.5 | valid tokens/s: 196718.5 | physical tokens/s: 197468.3 | loss tokens/s: 151291.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.565937E-01 | lm-global-supervised-tokens: 1.290290E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.810146E-02 | lm-token-buffer-mib: 5.824924E+01 | global_load_balancing_loss: 9.999927E-01 | loss scale: 1.0 | grad norm: 0.051 | params norm: 11474.268 | params rms: 0.062129 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 5:32:11.337673 | finish at 2026-10-10 13:50:25 + [2026-10-06 08:19:59] iteration 2570/ 6855 | consumed samples: 328960 | elapsed time per iteration (ms): 105748.0 | throughput per GPU (TFLOP/s/GPU): 34.2 | MFU 10.96% | FLOPs estimator: workload-aware | logical tokens/s: 158652.7 | valid tokens/s: 158652.7 | physical tokens/s: 159127.3 | loss tokens/s: 118689.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.459444E-01 | lm-global-supervised-tokens: 1.255118E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.251881E-02 | lm-token-buffer-mib: 5.818618E+01 | global_load_balancing_loss: 1.001231E+00 | loss scale: 1.0 | grad norm: 0.041 | params norm: 11474.201 | params rms: 0.062129 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 5:52:10.051897 | finish at 2026-10-11 14:12:09 + [2026-10-06 08:21:28] iteration 2571/ 6855 | consumed samples: 329088 | elapsed time per iteration (ms): 88637.2 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.41% | FLOPs estimator: workload-aware | logical tokens/s: 189279.1 | valid tokens/s: 189279.1 | physical tokens/s: 189806.1 | loss tokens/s: 145983.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.361550E-01 | lm-global-supervised-tokens: 1.293954E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.485678E-02 | lm-token-buffer-mib: 5.825591E+01 | global_load_balancing_loss: 1.000874E+00 | loss scale: 1.0 | grad norm: 0.067 | params norm: 11474.134 | params rms: 0.062129 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 9:28:41.650907 | finish at 2026-10-10 17:50:10 + [2026-10-06 08:22:59] iteration 2572/ 6855 | consumed samples: 329216 | elapsed time per iteration (ms): 90934.7 | throughput per GPU (TFLOP/s/GPU): 40.7 | MFU 13.04% | FLOPs estimator: workload-aware | logical tokens/s: 184497.5 | valid tokens/s: 184497.5 | physical tokens/s: 185326.9 | loss tokens/s: 133135.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.453522E-01 | lm-global-supervised-tokens: 1.210667E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.722147E-02 | lm-token-buffer-mib: 5.875178E+01 | global_load_balancing_loss: 1.000993E+00 | loss scale: 1.0 | grad norm: 0.067 | params norm: 11474.069 | params rms: 0.062128 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 12:11:13.110817 | finish at 2026-10-10 20:34:12 + [2026-10-06 08:24:25] iteration 2573/ 6855 | consumed samples: 329344 | elapsed time per iteration (ms): 86234.9 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.45% | FLOPs estimator: workload-aware | logical tokens/s: 194552.3 | valid tokens/s: 194552.3 | physical tokens/s: 195146.0 | loss tokens/s: 146998.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.657891E-01 | lm-global-supervised-tokens: 1.267643E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.158659E-02 | lm-token-buffer-mib: 5.820110E+01 | global_load_balancing_loss: 1.000896E+00 | loss scale: 1.0 | grad norm: 0.053 | params norm: 11474.001 | params rms: 0.062128 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 6:34:17.957153 | finish at 2026-10-10 14:58:43 + [2026-10-06 08:25:55] iteration 2574/ 6855 | consumed samples: 329472 | elapsed time per iteration (ms): 89513.6 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.39% | FLOPs estimator: workload-aware | logical tokens/s: 187426.3 | valid tokens/s: 187426.3 | physical tokens/s: 187838.1 | loss tokens/s: 129946.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.673040E-01 | lm-global-supervised-tokens: 1.163196E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.482646E-02 | lm-token-buffer-mib: 5.811751E+01 | global_load_balancing_loss: 1.001256E+00 | loss scale: 1.0 | grad norm: 0.049 | params norm: 11473.932 | params rms: 0.062128 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 10:26:47.692476 | finish at 2026-10-10 18:52:42 + [2026-10-06 08:27:22] iteration 2575/ 6855 | consumed samples: 329600 | elapsed time per iteration (ms): 87918.9 | throughput per GPU (TFLOP/s/GPU): 41.6 | MFU 13.34% | FLOPs estimator: workload-aware | logical tokens/s: 190825.4 | valid tokens/s: 190825.3 | physical tokens/s: 191190.2 | loss tokens/s: 138739.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.654241E-01 | lm-global-supervised-tokens: 1.219782E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.156927E-02 | lm-token-buffer-mib: 5.813020E+01 | global_load_balancing_loss: 9.996078E-01 | loss scale: 1.0 | grad norm: 0.051 | params norm: 11473.864 | params rms: 0.062127 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 8:31:32.871647 | finish at 2026-10-10 16:58:55 + [2026-10-06 08:28:48] iteration 2576/ 6855 | consumed samples: 329728 | elapsed time per iteration (ms): 85063.8 | throughput per GPU (TFLOP/s/GPU): 43.8 | MFU 14.04% | FLOPs estimator: workload-aware | logical tokens/s: 197230.7 | valid tokens/s: 197230.7 | physical tokens/s: 197621.5 | loss tokens/s: 144775.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.778706E-01 | lm-global-supervised-tokens: 1.231517E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.559635E-02 | lm-token-buffer-mib: 5.818607E+01 | global_load_balancing_loss: 9.989744E-01 | loss scale: 1.0 | grad norm: 0.055 | params norm: 11473.798 | params rms: 0.062127 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 5:06:27.957765 | finish at 2026-10-10 13:35:15 + [2026-10-06 08:30:16] iteration 2577/ 6855 | consumed samples: 329856 | elapsed time per iteration (ms): 88865.7 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.39% | FLOPs estimator: workload-aware | logical tokens/s: 188792.7 | valid tokens/s: 188792.7 | physical tokens/s: 189225.0 | loss tokens/s: 141138.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.458309E-01 | lm-global-supervised-tokens: 1.254234E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.509049E-02 | lm-token-buffer-mib: 5.817202E+01 | global_load_balancing_loss: 9.991355E-01 | loss scale: 1.0 | grad norm: 0.055 | params norm: 11473.729 | params rms: 0.062127 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 9:36:07.536025 | finish at 2026-10-10 18:06:24 + [2026-10-06 08:31:43] iteration 2578/ 6855 | consumed samples: 329984 | elapsed time per iteration (ms): 86400.4 | throughput per GPU (TFLOP/s/GPU): 42.8 | MFU 13.72% | FLOPs estimator: workload-aware | logical tokens/s: 194179.5 | valid tokens/s: 194179.5 | physical tokens/s: 194610.6 | loss tokens/s: 141843.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.565335E-01 | lm-global-supervised-tokens: 1.225534E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.319125E-02 | lm-token-buffer-mib: 5.811324E+01 | global_load_balancing_loss: 9.997255E-01 | loss scale: 1.0 | grad norm: 0.040 | params norm: 11473.662 | params rms: 0.062126 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 6:38:54.441131 | finish at 2026-10-10 15:10:37 + [2026-10-06 08:33:07] iteration 2579/ 6855 | consumed samples: 330112 | elapsed time per iteration (ms): 84501.1 | throughput per GPU (TFLOP/s/GPU): 43.4 | MFU 13.90% | FLOPs estimator: workload-aware | logical tokens/s: 198543.6 | valid tokens/s: 198543.6 | physical tokens/s: 199091.5 | loss tokens/s: 150453.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.433226E-01 | lm-global-supervised-tokens: 1.271345E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.170303E-02 | lm-token-buffer-mib: 5.802952E+01 | global_load_balancing_loss: 1.000021E+00 | loss scale: 1.0 | grad norm: 0.066 | params norm: 11473.596 | params rms: 0.062126 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 4:22:06.562163 | finish at 2026-10-10 12:55:14 + [2026-10-06 08:34:58] iteration 2580/ 6855 | consumed samples: 330240 | elapsed time per iteration (ms): 110313.8 | throughput per GPU (TFLOP/s/GPU): 34.1 | MFU 10.92% | FLOPs estimator: workload-aware | logical tokens/s: 152086.3 | valid tokens/s: 152086.3 | physical tokens/s: 152549.0 | loss tokens/s: 108303.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.604680E-01 | lm-global-supervised-tokens: 1.194739E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.180780E-02 | lm-token-buffer-mib: 5.816114E+01 | global_load_balancing_loss: 1.000842E+00 | loss scale: 1.0 | grad norm: 0.085 | params norm: 11473.531 | params rms: 0.062125 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 10:59:51.281372 | finish at 2026-10-11 19:34:49 + [2026-10-06 08:36:28] iteration 2581/ 6855 | consumed samples: 330368 | elapsed time per iteration (ms): 90130.3 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.65% | FLOPs estimator: workload-aware | logical tokens/s: 186143.7 | valid tokens/s: 186143.7 | physical tokens/s: 186756.0 | loss tokens/s: 137454.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.103649E-01 | lm-global-supervised-tokens: 1.238882E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.306185E-02 | lm-token-buffer-mib: 5.824582E+01 | global_load_balancing_loss: 1.000998E+00 | loss scale: 1.0 | grad norm: 0.110 | params norm: 11473.463 | params rms: 0.062125 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 11:00:16.717953 | finish at 2026-10-10 19:36:44 + [2026-10-06 08:37:53] iteration 2582/ 6855 | consumed samples: 330496 | elapsed time per iteration (ms): 84886.9 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.56% | FLOPs estimator: workload-aware | logical tokens/s: 197641.4 | valid tokens/s: 197641.4 | physical tokens/s: 198184.9 | loss tokens/s: 149227.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.923461E-01 | lm-global-supervised-tokens: 1.266744E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.298358E-02 | lm-token-buffer-mib: 5.806041E+01 | global_load_balancing_loss: 1.001329E+00 | loss scale: 1.0 | grad norm: 0.059 | params norm: 11473.395 | params rms: 0.062125 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 4:45:21.541120 | finish at 2026-10-10 13:23:14 + [2026-10-06 08:39:21] iteration 2583/ 6855 | consumed samples: 330624 | elapsed time per iteration (ms): 87885.3 | throughput per GPU (TFLOP/s/GPU): 43.1 | MFU 13.81% | FLOPs estimator: workload-aware | logical tokens/s: 190898.9 | valid tokens/s: 190898.9 | physical tokens/s: 191416.1 | loss tokens/s: 142164.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.545434E-01 | lm-global-supervised-tokens: 1.249417E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.645191E-02 | lm-token-buffer-mib: 5.804357E+01 | global_load_balancing_loss: 9.993795E-01 | loss scale: 1.0 | grad norm: 0.058 | params norm: 11473.330 | params rms: 0.062124 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 8:17:26.163208 | finish at 2026-10-10 16:56:47 + [2026-10-06 08:40:48] iteration 2584/ 6855 | consumed samples: 330752 | elapsed time per iteration (ms): 87942.8 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.39% | FLOPs estimator: workload-aware | logical tokens/s: 190774.0 | valid tokens/s: 190774.0 | physical tokens/s: 191191.8 | loss tokens/s: 138795.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.776040E-01 | lm-global-supervised-tokens: 1.220603E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.407694E-02 | lm-token-buffer-mib: 5.807191E+01 | global_load_balancing_loss: 1.000216E+00 | loss scale: 1.0 | grad norm: 0.035 | params norm: 11473.265 | params rms: 0.062124 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 8:20:03.511628 | finish at 2026-10-10 17:00:52 + [2026-10-06 08:42:15] iteration 2585/ 6855 | consumed samples: 330880 | elapsed time per iteration (ms): 86135.8 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.48% | FLOPs estimator: workload-aware | logical tokens/s: 194775.8 | valid tokens/s: 194775.8 | physical tokens/s: 195228.9 | loss tokens/s: 147526.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.644799E-01 | lm-global-supervised-tokens: 1.270734E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.356277E-02 | lm-token-buffer-mib: 5.817286E+01 | global_load_balancing_loss: 1.000792E+00 | loss scale: 1.0 | grad norm: 0.081 | params norm: 11473.199 | params rms: 0.062124 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 6:10:00.002944 | finish at 2026-10-10 14:52:15 + [2026-10-06 08:43:41] iteration 2586/ 6855 | consumed samples: 331008 | elapsed time per iteration (ms): 86116.0 | throughput per GPU (TFLOP/s/GPU): 43.1 | MFU 13.81% | FLOPs estimator: workload-aware | logical tokens/s: 194820.6 | valid tokens/s: 194820.6 | physical tokens/s: 195060.4 | loss tokens/s: 145575.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.168826E-01 | lm-global-supervised-tokens: 1.253642E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.212325E-02 | lm-token-buffer-mib: 5.809093E+01 | global_load_balancing_loss: 9.997487E-01 | loss scale: 1.0 | grad norm: 0.070 | params norm: 11473.133 | params rms: 0.062123 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 6:07:09.179304 | finish at 2026-10-10 14:50:50 + [2026-10-06 08:45:07] iteration 2587/ 6855 | consumed samples: 331136 | elapsed time per iteration (ms): 86248.5 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.49% | FLOPs estimator: workload-aware | logical tokens/s: 194521.4 | valid tokens/s: 194521.4 | physical tokens/s: 195027.1 | loss tokens/s: 144171.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.256030E-01 | lm-global-supervised-tokens: 1.243455E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.263972E-02 | lm-token-buffer-mib: 5.819508E+01 | global_load_balancing_loss: 1.001047E+00 | loss scale: 1.0 | grad norm: 0.092 | params norm: 11473.068 | params rms: 0.062123 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 6:15:08.637132 | finish at 2026-10-10 15:00:16 + [2026-10-06 08:46:35] iteration 2588/ 6855 | consumed samples: 331264 | elapsed time per iteration (ms): 88144.0 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.59% | FLOPs estimator: workload-aware | logical tokens/s: 190338.4 | valid tokens/s: 190338.4 | physical tokens/s: 190814.6 | loss tokens/s: 143375.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.011055E-01 | lm-global-supervised-tokens: 1.263768E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.520767E-02 | lm-token-buffer-mib: 5.809872E+01 | global_load_balancing_loss: 1.000287E+00 | loss scale: 1.0 | grad norm: 0.053 | params norm: 11473.002 | params rms: 0.062123 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 8:28:30.639486 | finish at 2026-10-10 17:15:06 + [2026-10-06 08:47:59] iteration 2589/ 6855 | consumed samples: 331392 | elapsed time per iteration (ms): 83772.6 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.79% | FLOPs estimator: workload-aware | logical tokens/s: 200270.1 | valid tokens/s: 200270.1 | physical tokens/s: 200890.4 | loss tokens/s: 149476.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.344836E-01 | lm-global-supervised-tokens: 1.252206E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.833415E-02 | lm-token-buffer-mib: 5.807070E+01 | global_load_balancing_loss: 1.000146E+00 | loss scale: 1.0 | grad norm: 0.043 | params norm: 11472.936 | params rms: 0.062122 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 3:16:13.839111 | finish at 2026-10-10 12:04:13 + [2026-10-06 08:49:46] iteration 2590/ 6855 | consumed samples: 331520 | elapsed time per iteration (ms): 106915.2 | throughput per GPU (TFLOP/s/GPU): 33.3 | MFU 10.67% | FLOPs estimator: workload-aware | logical tokens/s: 156920.4 | valid tokens/s: 156920.3 | physical tokens/s: 157373.4 | loss tokens/s: 115261.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.509203E-01 | lm-global-supervised-tokens: 1.232321E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.134863E-02 | lm-token-buffer-mib: 5.817955E+01 | global_load_balancing_loss: 1.000273E+00 | loss scale: 1.0 | grad norm: 0.081 | params norm: 11472.871 | params rms: 0.062122 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 6:39:53.506945 | finish at 2026-10-11 15:29:39 + [2026-10-06 08:51:13] iteration 2591/ 6855 | consumed samples: 331648 | elapsed time per iteration (ms): 87484.8 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.57% | FLOPs estimator: workload-aware | logical tokens/s: 191772.6 | valid tokens/s: 191772.6 | physical tokens/s: 192114.4 | loss tokens/s: 149230.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.993818E-01 | lm-global-supervised-tokens: 1.305538E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.452379E-02 | lm-token-buffer-mib: 5.819189E+01 | global_load_balancing_loss: 9.984543E-01 | loss scale: 1.0 | grad norm: 0.043 | params norm: 11472.806 | params rms: 0.062122 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 7:37:15.066650 | finish at 2026-10-10 16:28:28 + [2026-10-06 08:52:40] iteration 2592/ 6855 | consumed samples: 331776 | elapsed time per iteration (ms): 86907.4 | throughput per GPU (TFLOP/s/GPU): 43.2 | MFU 13.86% | FLOPs estimator: workload-aware | logical tokens/s: 193046.0 | valid tokens/s: 193046.0 | physical tokens/s: 193463.3 | loss tokens/s: 139460.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.803116E-01 | lm-global-supervised-tokens: 1.212011E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.546107E-02 | lm-token-buffer-mib: 5.816319E+01 | global_load_balancing_loss: 9.994630E-01 | loss scale: 1.0 | grad norm: 0.129 | params norm: 11472.740 | params rms: 0.062121 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 6:54:46.270136 | finish at 2026-10-10 15:47:26 + [2026-10-06 08:54:08] iteration 2593/ 6855 | consumed samples: 331904 | elapsed time per iteration (ms): 88189.8 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.58% | FLOPs estimator: workload-aware | logical tokens/s: 190239.3 | valid tokens/s: 190239.3 | physical tokens/s: 190567.6 | loss tokens/s: 140798.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.449732E-01 | lm-global-supervised-tokens: 1.241702E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.384541E-02 | lm-token-buffer-mib: 5.810713E+01 | global_load_balancing_loss: 9.997696E-01 | loss scale: 1.0 | grad norm: 0.080 | params norm: 11472.670 | params rms: 0.062121 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 8:24:24.828121 | finish at 2026-10-10 17:18:33 + [2026-10-06 08:55:35] iteration 2594/ 6855 | consumed samples: 332032 | elapsed time per iteration (ms): 86265.7 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.67% | FLOPs estimator: workload-aware | logical tokens/s: 194483.0 | valid tokens/s: 194483.0 | physical tokens/s: 194905.2 | loss tokens/s: 144789.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.537256E-01 | lm-global-supervised-tokens: 1.249037E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.171704E-02 | lm-token-buffer-mib: 5.822448E+01 | global_load_balancing_loss: 1.000528E+00 | loss scale: 1.0 | grad norm: 0.062 | params norm: 11472.601 | params rms: 0.062120 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 6:06:18.070925 | finish at 2026-10-10 15:01:53 + [2026-10-06 08:57:04] iteration 2595/ 6855 | consumed samples: 332160 | elapsed time per iteration (ms): 89846.0 | throughput per GPU (TFLOP/s/GPU): 41.4 | MFU 13.25% | FLOPs estimator: workload-aware | logical tokens/s: 186732.9 | valid tokens/s: 186732.9 | physical tokens/s: 187138.7 | loss tokens/s: 139119.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.406311E-01 | lm-global-supervised-tokens: 1.249931E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.168702E-02 | lm-token-buffer-mib: 5.822895E+01 | global_load_balancing_loss: 1.000459E+00 | loss scale: 1.0 | grad norm: 0.074 | params norm: 11472.533 | params rms: 0.062120 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 10:19:03.891764 | finish at 2026-10-10 19:16:08 + [2026-10-06 08:58:30] iteration 2596/ 6855 | consumed samples: 332288 | elapsed time per iteration (ms): 85724.4 | throughput per GPU (TFLOP/s/GPU): 43.6 | MFU 13.97% | FLOPs estimator: workload-aware | logical tokens/s: 195711.1 | valid tokens/s: 195711.1 | physical tokens/s: 196112.8 | loss tokens/s: 143979.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.603746E-01 | lm-global-supervised-tokens: 1.234257E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.504210E-02 | lm-token-buffer-mib: 5.808248E+01 | global_load_balancing_loss: 1.000958E+00 | loss scale: 1.0 | grad norm: 0.048 | params norm: 11472.465 | params rms: 0.062120 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 5:25:00.118243 | finish at 2026-10-10 14:23:30 + [2026-10-06 09:00:00] iteration 2597/ 6855 | consumed samples: 332416 | elapsed time per iteration (ms): 89976.0 | throughput per GPU (TFLOP/s/GPU): 41.7 | MFU 13.36% | FLOPs estimator: workload-aware | logical tokens/s: 186463.1 | valid tokens/s: 186463.1 | physical tokens/s: 186940.7 | loss tokens/s: 134362.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.430611E-01 | lm-global-supervised-tokens: 1.208937E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.550122E-02 | lm-token-buffer-mib: 5.820570E+01 | global_load_balancing_loss: 1.001583E+00 | loss scale: 1.0 | grad norm: 0.082 | params norm: 11472.396 | params rms: 0.062119 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 10:25:17.708812 | finish at 2026-10-10 19:25:18 + [2026-10-06 09:01:26] iteration 2598/ 6855 | consumed samples: 332544 | elapsed time per iteration (ms): 85854.7 | throughput per GPU (TFLOP/s/GPU): 43.1 | MFU 13.80% | FLOPs estimator: workload-aware | logical tokens/s: 195413.6 | valid tokens/s: 195413.6 | physical tokens/s: 195754.9 | loss tokens/s: 141499.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.229165E-01 | lm-global-supervised-tokens: 1.214838E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.274393E-02 | lm-token-buffer-mib: 5.804200E+01 | global_load_balancing_loss: 1.001216E+00 | loss scale: 1.0 | grad norm: 0.114 | params norm: 11472.327 | params rms: 0.062119 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 5:31:23.482635 | finish at 2026-10-10 14:32:50 + [2026-10-06 09:02:55] iteration 2599/ 6855 | consumed samples: 332672 | elapsed time per iteration (ms): 89338.8 | throughput per GPU (TFLOP/s/GPU): 41.5 | MFU 13.30% | FLOPs estimator: workload-aware | logical tokens/s: 187792.4 | valid tokens/s: 187792.4 | physical tokens/s: 188276.7 | loss tokens/s: 137230.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.824981E-01 | lm-global-supervised-tokens: 1.225999E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.344772E-02 | lm-token-buffer-mib: 5.812630E+01 | global_load_balancing_loss: 9.999601E-01 | loss scale: 1.0 | grad norm: 0.053 | params norm: 11472.262 | params rms: 0.062119 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 9:37:05.878822 | finish at 2026-10-10 18:40:01 + [2026-10-06 09:04:46] iteration 2600/ 6855 | consumed samples: 332800 | elapsed time per iteration (ms): 110562.6 | throughput per GPU (TFLOP/s/GPU): 33.7 | MFU 10.82% | FLOPs estimator: workload-aware | logical tokens/s: 151743.7 | valid tokens/s: 151743.7 | physical tokens/s: 152066.9 | loss tokens/s: 110008.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.956031E-01 | lm-global-supervised-tokens: 1.216283E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.406074E-02 | lm-token-buffer-mib: 5.806844E+01 | global_load_balancing_loss: 1.000110E+00 | loss scale: 1.0 | grad norm: 0.078 | params norm: 11472.197 | params rms: 0.062118 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 10:40:43.799666 | finish at 2026-10-11 19:45:30 +times across ranks (ms): + save-checkpoint: + rank 0: 26642.55 + rank 1: 26642.53 + rank 2: 26642.60 + rank 3: 26642.56 + rank 4: 26642.53 + rank 5: 26642.53 + rank 6: 26642.52 + rank 7: 26642.55 + rank 8: 26642.13 + rank 9: 26642.09 + rank 10: 26642.09 + rank 11: 26642.11 + rank 12: 26642.12 + rank 13: 26642.08 + rank 14: 26642.11 + rank 15: 26642.07 + rank 16: 26642.30 + rank 17: 26642.32 + rank 18: 26642.27 + rank 19: 26642.27 + rank 20: 26642.26 + rank 21: 26642.29 + rank 22: 26642.26 + rank 23: 26642.25 + rank 24: 26642.25 + rank 25: 26642.29 + rank 26: 26642.39 + rank 27: 26642.37 + rank 28: 26642.28 + rank 29: 26642.33 + rank 30: 26642.28 + rank 31: 26642.31 + rank 32: 26642.48 + rank 33: 26642.46 + rank 34: 26642.48 + rank 35: 26642.45 + rank 36: 26642.36 + rank 37: 26642.37 + rank 38: 26642.29 + rank 39: 26642.38 + rank 40: 26642.11 + rank 41: 26642.07 + rank 42: 26642.07 + rank 43: 26642.11 + rank 44: 26642.07 + rank 45: 26642.10 + rank 46: 26642.08 + rank 47: 26642.06 + rank 48: 26642.23 + rank 49: 26642.11 + rank 50: 26642.15 + rank 51: 26642.01 + rank 52: 26642.22 + rank 53: 26642.15 + rank 54: 26642.23 + rank 55: 26642.18 + rank 56: 26641.75 + rank 57: 26641.75 + rank 58: 26641.72 + rank 59: 26641.73 + rank 60: 26641.72 + rank 61: 26641.73 + rank 62: 26641.73 + rank 63: 26641.58 + [2026-10-06 09:06:37] iteration 2601/ 6855 | consumed samples: 332928 | elapsed time per iteration (ms): 84179.7 | throughput per GPU (TFLOP/s/GPU): 44.1 | MFU 14.14% | FLOPs estimator: workload-aware | logical tokens/s: 199302.1 | valid tokens/s: 199302.1 | physical tokens/s: 199810.2 | loss tokens/s: 150979.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.065182E-01 | lm-global-supervised-tokens: 1.270939E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.133258E+00 | lm-token-buffer-mib: 5.831264E+01 | global_load_balancing_loss: 9.986618E-01 | loss scale: 1.0 | grad norm: 0.121 | params norm: 11472.131 | params rms: 0.062118 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 3:28:20.487991 | finish at 2026-10-10 12:34:57 + [2026-10-06 09:08:07] iteration 2602/ 6855 | consumed samples: 333056 | elapsed time per iteration (ms): 89988.1 | throughput per GPU (TFLOP/s/GPU): 41.0 | MFU 13.13% | FLOPs estimator: workload-aware | logical tokens/s: 186437.6 | valid tokens/s: 186437.6 | physical tokens/s: 186891.1 | loss tokens/s: 134445.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.292445E-01 | lm-global-supervised-tokens: 1.209850E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.294363E-02 | lm-token-buffer-mib: 5.812077E+01 | global_load_balancing_loss: 1.000369E+00 | loss scale: 1.0 | grad norm: 0.132 | params norm: 11472.064 | params rms: 0.062117 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 10:18:39.595361 | finish at 2026-10-10 19:26:47 + [2026-10-06 09:09:35] iteration 2603/ 6855 | consumed samples: 333184 | elapsed time per iteration (ms): 88325.9 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.57% | FLOPs estimator: workload-aware | logical tokens/s: 189946.4 | valid tokens/s: 189946.3 | physical tokens/s: 190387.9 | loss tokens/s: 135718.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.494811E-01 | lm-global-supervised-tokens: 1.198746E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.473573E-02 | lm-token-buffer-mib: 5.820670E+01 | global_load_balancing_loss: 1.000200E+00 | loss scale: 1.0 | grad norm: 0.064 | params norm: 11471.999 | params rms: 0.062117 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 8:19:21.656168 | finish at 2026-10-10 17:28:57 + [2026-10-06 09:11:00] iteration 2604/ 6855 | consumed samples: 333312 | elapsed time per iteration (ms): 84350.4 | throughput per GPU (TFLOP/s/GPU): 43.3 | MFU 13.89% | FLOPs estimator: workload-aware | logical tokens/s: 198898.7 | valid tokens/s: 198898.7 | physical tokens/s: 199485.7 | loss tokens/s: 148160.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.741562E-01 | lm-global-supervised-tokens: 1.249738E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.160624E-02 | lm-token-buffer-mib: 5.812252E+01 | global_load_balancing_loss: 9.994652E-01 | loss scale: 1.0 | grad norm: 0.058 | params norm: 11471.936 | params rms: 0.062117 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 3:36:13.614128 | finish at 2026-10-10 12:47:13 + [2026-10-06 09:12:26] iteration 2605/ 6855 | consumed samples: 333440 | elapsed time per iteration (ms): 86804.9 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.44% | FLOPs estimator: workload-aware | logical tokens/s: 193274.5 | valid tokens/s: 193274.5 | physical tokens/s: 193691.4 | loss tokens/s: 146211.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.475227E-01 | lm-global-supervised-tokens: 1.269192E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.107628E-02 | lm-token-buffer-mib: 5.816165E+01 | global_load_balancing_loss: 1.001273E+00 | loss scale: 1.0 | grad norm: 0.091 | params norm: 11471.872 | params rms: 0.062116 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 6:28:41.026349 | finish at 2026-10-10 15:41:07 + [2026-10-06 09:13:50] iteration 2606/ 6855 | consumed samples: 333568 | elapsed time per iteration (ms): 83373.6 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.78% | FLOPs estimator: workload-aware | logical tokens/s: 201229.1 | valid tokens/s: 201229.1 | physical tokens/s: 201738.4 | loss tokens/s: 148553.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.231933E-01 | lm-global-supervised-tokens: 1.238540E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.387588E-02 | lm-token-buffer-mib: 5.824593E+01 | global_load_balancing_loss: 1.000489E+00 | loss scale: 1.0 | grad norm: 0.040 | params norm: 11471.807 | params rms: 0.062116 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 2:24:14.328161 | finish at 2026-10-10 11:38:04 + [2026-10-06 09:15:15] iteration 2607/ 6855 | consumed samples: 333696 | elapsed time per iteration (ms): 84865.9 | throughput per GPU (TFLOP/s/GPU): 43.5 | MFU 13.94% | FLOPs estimator: workload-aware | logical tokens/s: 197690.4 | valid tokens/s: 197690.4 | physical tokens/s: 198010.8 | loss tokens/s: 152954.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.759672E-01 | lm-global-supervised-tokens: 1.298064E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.221738E-02 | lm-token-buffer-mib: 5.812351E+01 | global_load_balancing_loss: 1.000172E+00 | loss scale: 1.0 | grad norm: 0.051 | params norm: 11471.743 | params rms: 0.062116 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 4:08:30.456802 | finish at 2026-10-10 13:23:45 + [2026-10-06 09:16:42] iteration 2608/ 6855 | consumed samples: 333824 | elapsed time per iteration (ms): 86982.1 | throughput per GPU (TFLOP/s/GPU): 43.1 | MFU 13.81% | FLOPs estimator: workload-aware | logical tokens/s: 192880.5 | valid tokens/s: 192880.5 | physical tokens/s: 193258.7 | loss tokens/s: 138941.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.154481E-01 | lm-global-supervised-tokens: 1.208545E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.019991E-02 | lm-token-buffer-mib: 5.814156E+01 | global_load_balancing_loss: 9.999821E-01 | loss scale: 1.0 | grad norm: 0.055 | params norm: 11471.679 | params rms: 0.062115 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 6:36:52.908875 | finish at 2026-10-10 15:53:35 + [2026-10-06 09:18:10] iteration 2609/ 6855 | consumed samples: 333952 | elapsed time per iteration (ms): 88414.6 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.54% | FLOPs estimator: workload-aware | logical tokens/s: 189755.9 | valid tokens/s: 189755.9 | physical tokens/s: 190184.4 | loss tokens/s: 141379.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.873891E-01 | lm-global-supervised-tokens: 1.250000E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.221815E-02 | lm-token-buffer-mib: 5.821318E+01 | global_load_balancing_loss: 1.000767E+00 | loss scale: 1.0 | grad norm: 0.101 | params norm: 11471.615 | params rms: 0.062115 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 8:16:48.240320 | finish at 2026-10-10 17:34:58 + [2026-10-06 09:20:00] iteration 2610/ 6855 | consumed samples: 334080 | elapsed time per iteration (ms): 110198.8 | throughput per GPU (TFLOP/s/GPU): 33.7 | MFU 10.79% | FLOPs estimator: workload-aware | logical tokens/s: 152244.7 | valid tokens/s: 152244.7 | physical tokens/s: 152699.2 | loss tokens/s: 109748.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.733234E-01 | lm-global-supervised-tokens: 1.209413E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.196091E-02 | lm-token-buffer-mib: 5.806270E+01 | global_load_balancing_loss: 1.000047E+00 | loss scale: 1.0 | grad norm: 0.069 | params norm: 11471.547 | params rms: 0.062115 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 9:56:34.079436 | finish at 2026-10-11 19:16:34 + [2026-10-06 09:21:27] iteration 2611/ 6855 | consumed samples: 334208 | elapsed time per iteration (ms): 86997.5 | throughput per GPU (TFLOP/s/GPU): 43.7 | MFU 14.00% | FLOPs estimator: workload-aware | logical tokens/s: 192846.8 | valid tokens/s: 192846.8 | physical tokens/s: 193384.3 | loss tokens/s: 140527.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.805222E-01 | lm-global-supervised-tokens: 1.222552E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.485226E-02 | lm-token-buffer-mib: 5.816968E+01 | global_load_balancing_loss: 9.997775E-01 | loss scale: 1.0 | grad norm: 0.126 | params norm: 11471.479 | params rms: 0.062114 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 6:33:37.332082 | finish at 2026-10-10 15:55:05 + [2026-10-06 09:22:55] iteration 2612/ 6855 | consumed samples: 334336 | elapsed time per iteration (ms): 88140.3 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.40% | FLOPs estimator: workload-aware | logical tokens/s: 190346.1 | valid tokens/s: 190346.1 | physical tokens/s: 190872.9 | loss tokens/s: 140461.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.600792E-01 | lm-global-supervised-tokens: 1.238029E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.162248E-02 | lm-token-buffer-mib: 5.818888E+01 | global_load_balancing_loss: 9.987919E-01 | loss scale: 1.0 | grad norm: 0.351 | params norm: 11471.410 | params rms: 0.062114 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 7:52:59.482222 | finish at 2026-10-10 17:15:55 + [2026-10-06 09:24:22] iteration 2613/ 6855 | consumed samples: 334464 | elapsed time per iteration (ms): 86772.8 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.44% | FLOPs estimator: workload-aware | logical tokens/s: 193345.9 | valid tokens/s: 193345.9 | physical tokens/s: 193807.4 | loss tokens/s: 149018.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.757562E-01 | lm-global-supervised-tokens: 1.293078E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.366403E-02 | lm-token-buffer-mib: 5.812135E+01 | global_load_balancing_loss: 1.000727E+00 | loss scale: 1.0 | grad norm: 0.371 | params norm: 11471.340 | params rms: 0.062114 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 6:14:50.278150 | finish at 2026-10-10 15:39:12 + [2026-10-06 09:25:55] iteration 2614/ 6855 | consumed samples: 334592 | elapsed time per iteration (ms): 93164.0 | throughput per GPU (TFLOP/s/GPU): 40.4 | MFU 12.94% | FLOPs estimator: workload-aware | logical tokens/s: 180082.6 | valid tokens/s: 180082.6 | physical tokens/s: 180549.4 | loss tokens/s: 129466.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.395643E-01 | lm-global-supervised-tokens: 1.206159E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.418458E-02 | lm-token-buffer-mib: 5.807533E+01 | global_load_balancing_loss: 1.000420E+00 | loss scale: 1.0 | grad norm: 0.062 | params norm: 11471.270 | params rms: 0.062113 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 13:45:08.542346 | finish at 2026-10-10 23:11:04 + [2026-10-06 09:27:27] iteration 2615/ 6855 | consumed samples: 334720 | elapsed time per iteration (ms): 91611.8 | throughput per GPU (TFLOP/s/GPU): 40.5 | MFU 12.99% | FLOPs estimator: workload-aware | logical tokens/s: 183132.5 | valid tokens/s: 183132.5 | physical tokens/s: 183518.1 | loss tokens/s: 135123.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 5.987370E-01 | lm-global-supervised-tokens: 1.237889E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.297750E-02 | lm-token-buffer-mib: 5.810452E+01 | global_load_balancing_loss: 1.001681E+00 | loss scale: 1.0 | grad norm: 0.208 | params norm: 11471.201 | params rms: 0.062113 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 11:53:53.963070 | finish at 2026-10-10 21:21:21 + [2026-10-06 09:28:58] iteration 2616/ 6855 | consumed samples: 334848 | elapsed time per iteration (ms): 91343.3 | throughput per GPU (TFLOP/s/GPU): 40.6 | MFU 13.02% | FLOPs estimator: workload-aware | logical tokens/s: 183671.8 | valid tokens/s: 183671.8 | physical tokens/s: 184215.5 | loss tokens/s: 139734.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.286337E-01 | lm-global-supervised-tokens: 1.276379E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.211571E-02 | lm-token-buffer-mib: 5.804125E+01 | global_load_balancing_loss: 9.989634E-01 | loss scale: 1.0 | grad norm: 0.063 | params norm: 11471.133 | params rms: 0.062112 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 11:33:24.063181 | finish at 2026-10-10 21:02:22 + [2026-10-06 09:30:24] iteration 2617/ 6855 | consumed samples: 334976 | elapsed time per iteration (ms): 85679.3 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.41% | FLOPs estimator: workload-aware | logical tokens/s: 195812.8 | valid tokens/s: 195812.8 | physical tokens/s: 196487.9 | loss tokens/s: 152533.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.954208E-01 | lm-global-supervised-tokens: 1.306900E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.202920E-02 | lm-token-buffer-mib: 5.816370E+01 | global_load_balancing_loss: 1.000162E+00 | loss scale: 1.0 | grad norm: 0.196 | params norm: 11471.066 | params rms: 0.062112 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 4:51:48.836310 | finish at 2026-10-10 14:22:13 + [2026-10-06 09:31:51] iteration 2618/ 6855 | consumed samples: 335104 | elapsed time per iteration (ms): 87280.2 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.51% | FLOPs estimator: workload-aware | logical tokens/s: 192222.4 | valid tokens/s: 192222.4 | physical tokens/s: 192663.6 | loss tokens/s: 140891.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.653938E-01 | lm-global-supervised-tokens: 1.229701E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.302162E-02 | lm-token-buffer-mib: 5.814901E+01 | global_load_balancing_loss: 9.998900E-01 | loss scale: 1.0 | grad norm: 0.100 | params norm: 11470.997 | params rms: 0.062112 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 6:43:26.116503 | finish at 2026-10-10 16:15:17 + [2026-10-06 09:33:23] iteration 2619/ 6855 | consumed samples: 335232 | elapsed time per iteration (ms): 91502.0 | throughput per GPU (TFLOP/s/GPU): 41.5 | MFU 13.31% | FLOPs estimator: workload-aware | logical tokens/s: 183352.5 | valid tokens/s: 183352.5 | physical tokens/s: 183787.7 | loss tokens/s: 131732.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.819074E-01 | lm-global-supervised-tokens: 1.205378E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.262402E-02 | lm-token-buffer-mib: 5.808633E+01 | global_load_balancing_loss: 1.000040E+00 | loss scale: 1.0 | grad norm: 0.147 | params norm: 11470.927 | params rms: 0.062111 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 11:40:02.487545 | finish at 2026-10-10 21:13:25 + [2026-10-06 09:35:10] iteration 2620/ 6855 | consumed samples: 335360 | elapsed time per iteration (ms): 107189.9 | throughput per GPU (TFLOP/s/GPU): 33.9 | MFU 10.87% | FLOPs estimator: workload-aware | logical tokens/s: 156518.0 | valid tokens/s: 156518.0 | physical tokens/s: 156836.0 | loss tokens/s: 119245.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.769346E-01 | lm-global-supervised-tokens: 1.278190E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.363332E-02 | lm-token-buffer-mib: 5.807382E+01 | global_load_balancing_loss: 1.000515E+00 | loss scale: 1.0 | grad norm: 0.078 | params norm: 11470.856 | params rms: 0.062111 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 6:05:49.042401 | finish at 2026-10-11 15:40:59 + [2026-10-06 09:36:36] iteration 2621/ 6855 | consumed samples: 335488 | elapsed time per iteration (ms): 86073.6 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.73% | FLOPs estimator: workload-aware | logical tokens/s: 194916.7 | valid tokens/s: 194916.7 | physical tokens/s: 195428.6 | loss tokens/s: 147683.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.880881E-01 | lm-global-supervised-tokens: 1.271168E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.469762E-02 | lm-token-buffer-mib: 5.811533E+01 | global_load_balancing_loss: 1.000625E+00 | loss scale: 1.0 | grad norm: 0.102 | params norm: 11470.786 | params rms: 0.062111 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 5:13:55.454047 | finish at 2026-10-10 14:50:31 + [2026-10-06 09:38:03] iteration 2622/ 6855 | consumed samples: 335616 | elapsed time per iteration (ms): 86884.9 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.56% | FLOPs estimator: workload-aware | logical tokens/s: 193096.9 | valid tokens/s: 193096.8 | physical tokens/s: 193525.3 | loss tokens/s: 143268.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.650407E-01 | lm-global-supervised-tokens: 1.244789E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.082685E-02 | lm-token-buffer-mib: 5.806604E+01 | global_load_balancing_loss: 1.001092E+00 | loss scale: 1.0 | grad norm: 0.107 | params norm: 11470.717 | params rms: 0.062110 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 6:09:43.878980 | finish at 2026-10-10 15:47:47 + [2026-10-06 09:39:32] iteration 2623/ 6855 | consumed samples: 335744 | elapsed time per iteration (ms): 88812.7 | throughput per GPU (TFLOP/s/GPU): 41.6 | MFU 13.34% | FLOPs estimator: workload-aware | logical tokens/s: 188905.4 | valid tokens/s: 188905.4 | physical tokens/s: 189642.2 | loss tokens/s: 136197.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.479408E-01 | lm-global-supervised-tokens: 1.209609E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.151840E-02 | lm-token-buffer-mib: 5.810899E+01 | global_load_balancing_loss: 1.000636E+00 | loss scale: 1.0 | grad norm: 0.172 | params norm: 11470.647 | params rms: 0.062110 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 8:24:15.463583 | finish at 2026-10-10 18:03:47 + [2026-10-06 09:40:58] iteration 2624/ 6855 | consumed samples: 335872 | elapsed time per iteration (ms): 86181.2 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.75% | FLOPs estimator: workload-aware | logical tokens/s: 194673.5 | valid tokens/s: 194673.5 | physical tokens/s: 195123.5 | loss tokens/s: 151430.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.923991E-01 | lm-global-supervised-tokens: 1.305047E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.537501E-02 | lm-token-buffer-mib: 5.810308E+01 | global_load_balancing_loss: 9.991420E-01 | loss scale: 1.0 | grad norm: 0.075 | params norm: 11470.577 | params rms: 0.062109 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 5:17:12.818716 | finish at 2026-10-10 14:58:11 + [2026-10-06 09:42:27] iteration 2625/ 6855 | consumed samples: 336000 | elapsed time per iteration (ms): 89474.3 | throughput per GPU (TFLOP/s/GPU): 41.6 | MFU 13.32% | FLOPs estimator: workload-aware | logical tokens/s: 187508.0 | valid tokens/s: 187508.0 | physical tokens/s: 187814.2 | loss tokens/s: 136782.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.396794E-01 | lm-global-supervised-tokens: 1.223853E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.190072E-02 | lm-token-buffer-mib: 5.810155E+01 | global_load_balancing_loss: 9.997013E-01 | loss scale: 1.0 | grad norm: 0.061 | params norm: 11470.505 | params rms: 0.062109 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 9:07:56.360214 | finish at 2026-10-10 18:50:24 + [2026-10-06 09:43:54] iteration 2626/ 6855 | consumed samples: 336128 | elapsed time per iteration (ms): 86156.4 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.62% | FLOPs estimator: workload-aware | logical tokens/s: 194729.8 | valid tokens/s: 194729.8 | physical tokens/s: 195292.8 | loss tokens/s: 145813.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.849002E-01 | lm-global-supervised-tokens: 1.256275E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.145701E-02 | lm-token-buffer-mib: 5.814891E+01 | global_load_balancing_loss: 9.999342E-01 | loss scale: 1.0 | grad norm: 0.073 | params norm: 11470.435 | params rms: 0.062109 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 5:12:35.398313 | finish at 2026-10-10 14:56:29 + [2026-10-06 09:45:16] iteration 2627/ 6855 | consumed samples: 336256 | elapsed time per iteration (ms): 82921.6 | throughput per GPU (TFLOP/s/GPU): 44.0 | MFU 14.12% | FLOPs estimator: workload-aware | logical tokens/s: 202325.9 | valid tokens/s: 202325.9 | physical tokens/s: 202783.9 | loss tokens/s: 152360.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.335475E-01 | lm-global-supervised-tokens: 1.263399E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.682815E-02 | lm-token-buffer-mib: 5.811764E+01 | global_load_balancing_loss: 9.995899E-01 | loss scale: 1.0 | grad norm: 0.048 | params norm: 11470.368 | params rms: 0.062108 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 1:23:12.543382 | finish at 2026-10-10 11:08:29 + [2026-10-06 09:46:46] iteration 2628/ 6855 | consumed samples: 336384 | elapsed time per iteration (ms): 89535.5 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.48% | FLOPs estimator: workload-aware | logical tokens/s: 187379.9 | valid tokens/s: 187379.9 | physical tokens/s: 187910.8 | loss tokens/s: 133176.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.852966E-01 | lm-global-supervised-tokens: 1.192400E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.498651E-02 | lm-token-buffer-mib: 5.832095E+01 | global_load_balancing_loss: 9.996146E-01 | loss scale: 1.0 | grad norm: 0.056 | params norm: 11470.298 | params rms: 0.062108 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 9:07:46.471031 | finish at 2026-10-10 18:54:32 + [2026-10-06 09:48:11] iteration 2629/ 6855 | consumed samples: 336512 | elapsed time per iteration (ms): 85297.6 | throughput per GPU (TFLOP/s/GPU): 43.2 | MFU 13.83% | FLOPs estimator: workload-aware | logical tokens/s: 196688.9 | valid tokens/s: 196688.9 | physical tokens/s: 197152.4 | loss tokens/s: 143363.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.141927E-01 | lm-global-supervised-tokens: 1.222859E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.325564E-02 | lm-token-buffer-mib: 5.808717E+01 | global_load_balancing_loss: 1.000985E+00 | loss scale: 1.0 | grad norm: 0.047 | params norm: 11470.229 | params rms: 0.062108 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 4:07:47.512642 | finish at 2026-10-10 13:55:59 + [2026-10-06 09:50:01] iteration 2630/ 6855 | consumed samples: 336640 | elapsed time per iteration (ms): 109384.7 | throughput per GPU (TFLOP/s/GPU): 33.2 | MFU 10.64% | FLOPs estimator: workload-aware | logical tokens/s: 153378.1 | valid tokens/s: 153378.1 | physical tokens/s: 153735.3 | loss tokens/s: 113804.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.331423E-01 | lm-global-supervised-tokens: 1.244843E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.153658E-02 | lm-token-buffer-mib: 5.817510E+01 | global_load_balancing_loss: 1.000427E+00 | loss scale: 1.0 | grad norm: 0.046 | params norm: 11470.162 | params rms: 0.062107 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 8:22:30.289255 | finish at 2026-10-11 18:12:31 + [2026-10-06 09:51:25] iteration 2631/ 6855 | consumed samples: 336768 | elapsed time per iteration (ms): 84400.8 | throughput per GPU (TFLOP/s/GPU): 43.1 | MFU 13.81% | FLOPs estimator: workload-aware | logical tokens/s: 198779.5 | valid tokens/s: 198779.5 | physical tokens/s: 199196.0 | loss tokens/s: 147239.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.567803E-01 | lm-global-supervised-tokens: 1.242710E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.561306E-02 | lm-token-buffer-mib: 5.812004E+01 | global_load_balancing_loss: 1.000533E+00 | loss scale: 1.0 | grad norm: 0.044 | params norm: 11470.096 | params rms: 0.062107 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 3:01:48.996277 | finish at 2026-10-10 12:53:14 + [2026-10-06 09:52:52] iteration 2632/ 6855 | consumed samples: 336896 | elapsed time per iteration (ms): 87017.1 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.64% | FLOPs estimator: workload-aware | logical tokens/s: 192802.9 | valid tokens/s: 192802.9 | physical tokens/s: 193284.0 | loss tokens/s: 140430.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.533206E-01 | lm-global-supervised-tokens: 1.221981E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.150464E-02 | lm-token-buffer-mib: 5.824480E+01 | global_load_balancing_loss: 1.000205E+00 | loss scale: 1.0 | grad norm: 0.069 | params norm: 11470.028 | params rms: 0.062106 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 6:04:33.406011 | finish at 2026-10-10 15:57:26 + [2026-10-06 09:54:21] iteration 2633/ 6855 | consumed samples: 337024 | elapsed time per iteration (ms): 88972.5 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.57% | FLOPs estimator: workload-aware | logical tokens/s: 188565.7 | valid tokens/s: 188565.7 | physical tokens/s: 188934.8 | loss tokens/s: 140047.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.913882E-01 | lm-global-supervised-tokens: 1.246038E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.437801E-02 | lm-token-buffer-mib: 5.813747E+01 | global_load_balancing_loss: 1.000245E+00 | loss scale: 1.0 | grad norm: 0.052 | params norm: 11469.962 | params rms: 0.062106 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 8:20:41.907442 | finish at 2026-10-10 18:15:03 + [2026-10-06 09:55:52] iteration 2634/ 6855 | consumed samples: 337152 | elapsed time per iteration (ms): 91279.1 | throughput per GPU (TFLOP/s/GPU): 41.6 | MFU 13.34% | FLOPs estimator: workload-aware | logical tokens/s: 183800.4 | valid tokens/s: 183800.4 | physical tokens/s: 184206.4 | loss tokens/s: 127255.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.064581E-01 | lm-global-supervised-tokens: 1.161572E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.914175E-02 | lm-token-buffer-mib: 5.811487E+01 | global_load_balancing_loss: 1.000404E+00 | loss scale: 1.0 | grad norm: 0.043 | params norm: 11469.896 | params rms: 0.062106 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 11:01:29.103998 | finish at 2026-10-10 20:57:21 + [2026-10-06 09:57:19] iteration 2635/ 6855 | consumed samples: 337280 | elapsed time per iteration (ms): 86664.8 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.55% | FLOPs estimator: workload-aware | logical tokens/s: 193585.7 | valid tokens/s: 193585.7 | physical tokens/s: 194071.6 | loss tokens/s: 143127.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.209583E-01 | lm-global-supervised-tokens: 1.240411E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.383959E-02 | lm-token-buffer-mib: 5.808002E+01 | global_load_balancing_loss: 9.999371E-01 | loss scale: 1.0 | grad norm: 0.040 | params norm: 11469.829 | params rms: 0.062105 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 5:35:25.635796 | finish at 2026-10-10 15:32:45 + [2026-10-06 09:58:43] iteration 2636/ 6855 | consumed samples: 337408 | elapsed time per iteration (ms): 83768.6 | throughput per GPU (TFLOP/s/GPU): 43.6 | MFU 13.99% | FLOPs estimator: workload-aware | logical tokens/s: 200279.9 | valid tokens/s: 200279.9 | physical tokens/s: 200838.5 | loss tokens/s: 148403.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.068769E-01 | lm-global-supervised-tokens: 1.243152E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.049195E-02 | lm-token-buffer-mib: 5.834827E+01 | global_load_balancing_loss: 1.000652E+00 | loss scale: 1.0 | grad norm: 0.121 | params norm: 11469.761 | params rms: 0.062105 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 2:10:19.933576 | finish at 2026-10-10 12:09:03 + [2026-10-06 10:00:09] iteration 2637/ 6855 | consumed samples: 337536 | elapsed time per iteration (ms): 85807.4 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.63% | FLOPs estimator: workload-aware | logical tokens/s: 195521.0 | valid tokens/s: 195521.0 | physical tokens/s: 195934.4 | loss tokens/s: 146351.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.423662E-01 | lm-global-supervised-tokens: 1.255802E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.505051E-02 | lm-token-buffer-mib: 5.822920E+01 | global_load_balancing_loss: 9.997692E-01 | loss scale: 1.0 | grad norm: 0.165 | params norm: 11469.693 | params rms: 0.062105 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 4:32:15.743885 | finish at 2026-10-10 14:32:24 + [2026-10-06 10:01:36] iteration 2638/ 6855 | consumed samples: 337664 | elapsed time per iteration (ms): 87678.6 | throughput per GPU (TFLOP/s/GPU): 43.2 | MFU 13.85% | FLOPs estimator: workload-aware | logical tokens/s: 191348.2 | valid tokens/s: 191348.2 | physical tokens/s: 191728.2 | loss tokens/s: 139985.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.516926E-01 | lm-global-supervised-tokens: 1.227376E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.598235E-02 | lm-token-buffer-mib: 5.809550E+01 | global_load_balancing_loss: 9.994043E-01 | loss scale: 1.0 | grad norm: 0.066 | params norm: 11469.627 | params rms: 0.062104 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 6:42:20.772130 | finish at 2026-10-10 16:43:57 + [2026-10-06 10:03:02] iteration 2639/ 6855 | consumed samples: 337792 | elapsed time per iteration (ms): 85479.8 | throughput per GPU (TFLOP/s/GPU): 43.4 | MFU 13.91% | FLOPs estimator: workload-aware | logical tokens/s: 196270.9 | valid tokens/s: 196270.9 | physical tokens/s: 196703.4 | loss tokens/s: 145237.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.281955E-01 | lm-global-supervised-tokens: 1.241485E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.150056E-02 | lm-token-buffer-mib: 5.805917E+01 | global_load_balancing_loss: 9.993079E-01 | loss scale: 1.0 | grad norm: 0.156 | params norm: 11469.559 | params rms: 0.062104 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 4:06:22.943289 | finish at 2026-10-10 14:09:25 + [2026-10-06 10:04:50] iteration 2640/ 6855 | consumed samples: 337920 | elapsed time per iteration (ms): 108345.0 | throughput per GPU (TFLOP/s/GPU): 33.8 | MFU 10.84% | FLOPs estimator: workload-aware | logical tokens/s: 154849.8 | valid tokens/s: 154849.8 | physical tokens/s: 155183.5 | loss tokens/s: 116234.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.838179E-01 | lm-global-supervised-tokens: 1.259345E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.398543E-02 | lm-token-buffer-mib: 5.809641E+01 | global_load_balancing_loss: 1.000832E+00 | loss scale: 1.0 | grad norm: 0.120 | params norm: 11469.493 | params rms: 0.062104 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 6:51:14.331890 | finish at 2026-10-11 16:56:04 +times across ranks (ms): + save-checkpoint: + rank 0: 24180.04 + rank 1: 24180.06 + rank 2: 24180.07 + rank 3: 24180.06 + rank 4: 24180.07 + rank 5: 24180.03 + rank 6: 24180.06 + rank 7: 24180.05 + rank 8: 24180.16 + rank 9: 24180.12 + rank 10: 24180.12 + rank 11: 24180.14 + rank 12: 24180.12 + rank 13: 24180.14 + rank 14: 24180.14 + rank 15: 24180.12 + rank 16: 24179.61 + rank 17: 24179.60 + rank 18: 24179.55 + rank 19: 24179.57 + rank 20: 24179.55 + rank 21: 24179.60 + rank 22: 24179.55 + rank 23: 24179.57 + rank 24: 24180.07 + rank 25: 24180.11 + rank 26: 24180.10 + rank 27: 24180.08 + rank 28: 24180.11 + rank 29: 24180.05 + rank 30: 24180.09 + rank 31: 24180.08 + rank 32: 24179.82 + rank 33: 24179.81 + rank 34: 24179.80 + rank 35: 24179.79 + rank 36: 24179.77 + rank 37: 24179.78 + rank 38: 24179.80 + rank 39: 24179.79 + rank 40: 24179.39 + rank 41: 24179.38 + rank 42: 24179.37 + rank 43: 24179.38 + rank 44: 24179.37 + rank 45: 24179.38 + rank 46: 24179.37 + rank 47: 24179.36 + rank 48: 24179.69 + rank 49: 24179.68 + rank 50: 24179.64 + rank 51: 24179.62 + rank 52: 24179.76 + rank 53: 24179.64 + rank 54: 24179.62 + rank 55: 24179.70 + rank 56: 24179.76 + rank 57: 24179.72 + rank 58: 24179.73 + rank 59: 24179.73 + rank 60: 24179.74 + rank 61: 24179.73 + rank 62: 24179.73 + rank 63: 24179.76 + [2026-10-06 10:07:10] iteration 2641/ 6855 | consumed samples: 338048 | elapsed time per iteration (ms): 115436.2 | throughput per GPU (TFLOP/s/GPU): 31.1 | MFU 9.97% | FLOPs estimator: workload-aware | logical tokens/s: 145337.2 | valid tokens/s: 145337.2 | physical tokens/s: 145723.4 | loss tokens/s: 111568.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.659430E-01 | lm-global-supervised-tokens: 1.287900E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 4.694672E-01 | lm-token-buffer-mib: 5.803674E+01 | global_load_balancing_loss: 1.000346E+00 | loss scale: 1.0 | grad norm: 0.132 | params norm: 11469.427 | params rms: 0.062103 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 15:07:27.960525 | finish at 2026-10-12 01:14:38 + [2026-10-06 10:08:42] iteration 2642/ 6855 | consumed samples: 338176 | elapsed time per iteration (ms): 92325.7 | throughput per GPU (TFLOP/s/GPU): 39.9 | MFU 12.78% | FLOPs estimator: workload-aware | logical tokens/s: 181717.1 | valid tokens/s: 181717.1 | physical tokens/s: 182105.2 | loss tokens/s: 135834.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.644173E-01 | lm-global-supervised-tokens: 1.254105E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.448648E-02 | lm-token-buffer-mib: 5.817388E+01 | global_load_balancing_loss: 1.000671E+00 | loss scale: 1.0 | grad norm: 0.079 | params norm: 11469.361 | params rms: 0.062103 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 12:02:48.272725 | finish at 2026-10-10 22:11:31 + [2026-10-06 10:10:14] iteration 2643/ 6855 | consumed samples: 338304 | elapsed time per iteration (ms): 91858.3 | throughput per GPU (TFLOP/s/GPU): 41.7 | MFU 13.36% | FLOPs estimator: workload-aware | logical tokens/s: 182641.8 | valid tokens/s: 182641.8 | physical tokens/s: 182949.7 | loss tokens/s: 134232.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.622587E-01 | lm-global-supervised-tokens: 1.233037E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.270139E-02 | lm-token-buffer-mib: 5.802887E+01 | global_load_balancing_loss: 1.001464E+00 | loss scale: 1.0 | grad norm: 0.085 | params norm: 11469.295 | params rms: 0.062102 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 11:28:27.252869 | finish at 2026-10-10 21:38:41 + [2026-10-06 10:11:40] iteration 2644/ 6855 | consumed samples: 338432 | elapsed time per iteration (ms): 85385.2 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.52% | FLOPs estimator: workload-aware | logical tokens/s: 196488.3 | valid tokens/s: 196488.3 | physical tokens/s: 196993.7 | loss tokens/s: 151503.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.427241E-01 | lm-global-supervised-tokens: 1.293613E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.641746E-02 | lm-token-buffer-mib: 5.810496E+01 | global_load_balancing_loss: 1.000129E+00 | loss scale: 1.0 | grad norm: 0.070 | params norm: 11469.229 | params rms: 0.062102 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 3:52:36.918671 | finish at 2026-10-10 14:04:16 + [2026-10-06 10:13:05] iteration 2645/ 6855 | consumed samples: 338560 | elapsed time per iteration (ms): 85813.7 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.55% | FLOPs estimator: workload-aware | logical tokens/s: 195507.1 | valid tokens/s: 195507.1 | physical tokens/s: 195902.2 | loss tokens/s: 149303.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.652876E-01 | lm-global-supervised-tokens: 1.281229E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.145792E-02 | lm-token-buffer-mib: 5.805180E+01 | global_load_balancing_loss: 1.000178E+00 | loss scale: 1.0 | grad norm: 0.133 | params norm: 11469.165 | params rms: 0.062102 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 4:21:15.518243 | finish at 2026-10-10 14:34:21 + [2026-10-06 10:14:33] iteration 2646/ 6855 | consumed samples: 338688 | elapsed time per iteration (ms): 87783.8 | throughput per GPU (TFLOP/s/GPU): 42.8 | MFU 13.72% | FLOPs estimator: workload-aware | logical tokens/s: 191118.5 | valid tokens/s: 191118.4 | physical tokens/s: 191550.3 | loss tokens/s: 142159.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.589521E-01 | lm-global-supervised-tokens: 1.247932E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.173054E-02 | lm-token-buffer-mib: 5.816612E+01 | global_load_balancing_loss: 9.995903E-01 | loss scale: 1.0 | grad norm: 0.114 | params norm: 11469.100 | params rms: 0.062101 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 6:38:02.195357 | finish at 2026-10-10 16:52:35 + [2026-10-06 10:16:00] iteration 2647/ 6855 | consumed samples: 338816 | elapsed time per iteration (ms): 87035.9 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.58% | FLOPs estimator: workload-aware | logical tokens/s: 192761.3 | valid tokens/s: 192761.3 | physical tokens/s: 193068.1 | loss tokens/s: 147653.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.106867E-01 | lm-global-supervised-tokens: 1.285114E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.744605E-02 | lm-token-buffer-mib: 5.805016E+01 | global_load_balancing_loss: 1.000077E+00 | loss scale: 1.0 | grad norm: 0.039 | params norm: 11469.035 | params rms: 0.062101 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 5:44:07.056652 | finish at 2026-10-10 16:00:07 + [2026-10-06 10:17:27] iteration 2648/ 6855 | consumed samples: 338944 | elapsed time per iteration (ms): 87200.8 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.42% | FLOPs estimator: workload-aware | logical tokens/s: 192397.4 | valid tokens/s: 192397.4 | physical tokens/s: 192868.0 | loss tokens/s: 144692.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.607668E-01 | lm-global-supervised-tokens: 1.261731E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.296790E-02 | lm-token-buffer-mib: 5.820976E+01 | global_load_balancing_loss: 9.999442E-01 | loss scale: 1.0 | grad norm: 0.077 | params norm: 11468.970 | params rms: 0.062101 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 5:54:13.772377 | finish at 2026-10-10 16:11:41 + [2026-10-06 10:18:54] iteration 2649/ 6855 | consumed samples: 339072 | elapsed time per iteration (ms): 86909.2 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.50% | FLOPs estimator: workload-aware | logical tokens/s: 193042.9 | valid tokens/s: 193042.9 | physical tokens/s: 193521.2 | loss tokens/s: 143047.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.457444E-01 | lm-global-supervised-tokens: 1.243217E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.292879E-02 | lm-token-buffer-mib: 5.804869E+01 | global_load_balancing_loss: 1.000106E+00 | loss scale: 1.0 | grad norm: 0.086 | params norm: 11468.907 | params rms: 0.062100 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 5:32:20.290546 | finish at 2026-10-10 15:51:15 + [2026-10-06 10:20:42] iteration 2650/ 6855 | consumed samples: 339200 | elapsed time per iteration (ms): 107451.8 | throughput per GPU (TFLOP/s/GPU): 33.8 | MFU 10.84% | FLOPs estimator: workload-aware | logical tokens/s: 156136.9 | valid tokens/s: 156136.9 | physical tokens/s: 156492.6 | loss tokens/s: 119525.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.305454E-01 | lm-global-supervised-tokens: 1.284318E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.141648E-02 | lm-token-buffer-mib: 5.805466E+01 | global_load_balancing_loss: 9.994556E-01 | loss scale: 1.0 | grad norm: 0.084 | params norm: 11468.843 | params rms: 0.062100 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 5:30:34.627967 | finish at 2026-10-11 15:51:16 + [2026-10-06 10:22:10] iteration 2651/ 6855 | consumed samples: 339328 | elapsed time per iteration (ms): 88676.2 | throughput per GPU (TFLOP/s/GPU): 40.9 | MFU 13.11% | FLOPs estimator: workload-aware | logical tokens/s: 189195.5 | valid tokens/s: 189195.5 | physical tokens/s: 189567.7 | loss tokens/s: 143260.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.252101E-01 | lm-global-supervised-tokens: 1.270378E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.503694E-02 | lm-token-buffer-mib: 5.814705E+01 | global_load_balancing_loss: 1.000326E+00 | loss scale: 1.0 | grad norm: 0.113 | params norm: 11468.779 | params rms: 0.062100 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 7:33:14.682291 | finish at 2026-10-10 17:55:25 + [2026-10-06 10:23:37] iteration 2652/ 6855 | consumed samples: 339456 | elapsed time per iteration (ms): 86165.6 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.59% | FLOPs estimator: workload-aware | logical tokens/s: 194708.8 | valid tokens/s: 194708.8 | physical tokens/s: 195413.6 | loss tokens/s: 151270.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.846163E-01 | lm-global-supervised-tokens: 1.303430E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.275932E-02 | lm-token-buffer-mib: 5.817564E+01 | global_load_balancing_loss: 1.000309E+00 | loss scale: 1.0 | grad norm: 0.139 | params norm: 11468.716 | params rms: 0.062099 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 4:35:53.862739 | finish at 2026-10-10 14:59:30 + [2026-10-06 10:25:06] iteration 2653/ 6855 | consumed samples: 339584 | elapsed time per iteration (ms): 89424.9 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.44% | FLOPs estimator: workload-aware | logical tokens/s: 187611.1 | valid tokens/s: 187611.1 | physical tokens/s: 188237.1 | loss tokens/s: 136361.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.621795E-01 | lm-global-supervised-tokens: 1.219409E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.160178E-02 | lm-token-buffer-mib: 5.802399E+01 | global_load_balancing_loss: 1.000690E+00 | loss scale: 1.0 | grad norm: 0.043 | params norm: 11468.649 | params rms: 0.062099 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 8:22:43.247697 | finish at 2026-10-10 18:47:49 + [2026-10-06 10:26:33] iteration 2654/ 6855 | consumed samples: 339712 | elapsed time per iteration (ms): 86882.8 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.41% | FLOPs estimator: workload-aware | logical tokens/s: 193101.7 | valid tokens/s: 193101.7 | physical tokens/s: 193576.9 | loss tokens/s: 141332.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.446972E-01 | lm-global-supervised-tokens: 1.227933E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.046731E-02 | lm-token-buffer-mib: 5.824988E+01 | global_load_balancing_loss: 1.000907E+00 | loss scale: 1.0 | grad norm: 0.126 | params norm: 11468.583 | params rms: 0.062099 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 5:23:14.497998 | finish at 2026-10-10 15:49:47 + [2026-10-06 10:28:00] iteration 2655/ 6855 | consumed samples: 339840 | elapsed time per iteration (ms): 87024.1 | throughput per GPU (TFLOP/s/GPU): 41.6 | MFU 13.33% | FLOPs estimator: workload-aware | logical tokens/s: 192788.2 | valid tokens/s: 192788.2 | physical tokens/s: 193277.5 | loss tokens/s: 145765.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.748611E-01 | lm-global-supervised-tokens: 1.268509E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.715498E-02 | lm-token-buffer-mib: 5.808836E+01 | global_load_balancing_loss: 1.001387E+00 | loss scale: 1.0 | grad norm: 0.120 | params norm: 11468.518 | params rms: 0.062098 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 5:31:41.102114 | finish at 2026-10-10 15:59:41 + [2026-10-06 10:29:25] iteration 2656/ 6855 | consumed samples: 339968 | elapsed time per iteration (ms): 85456.0 | throughput per GPU (TFLOP/s/GPU): 43.4 | MFU 13.91% | FLOPs estimator: workload-aware | logical tokens/s: 196325.4 | valid tokens/s: 196325.4 | physical tokens/s: 196848.4 | loss tokens/s: 146059.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.335792E-01 | lm-global-supervised-tokens: 1.248165E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.206759E-02 | lm-token-buffer-mib: 5.830247E+01 | global_load_balancing_loss: 1.000529E+00 | loss scale: 1.0 | grad norm: 0.086 | params norm: 11468.453 | params rms: 0.062098 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 3:40:29.708336 | finish at 2026-10-10 14:09:55 + [2026-10-06 10:30:58] iteration 2657/ 6855 | consumed samples: 340096 | elapsed time per iteration (ms): 92579.1 | throughput per GPU (TFLOP/s/GPU): 40.5 | MFU 12.98% | FLOPs estimator: workload-aware | logical tokens/s: 181219.7 | valid tokens/s: 181219.7 | physical tokens/s: 181718.1 | loss tokens/s: 128533.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.848592E-01 | lm-global-supervised-tokens: 1.189955E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.211166E-02 | lm-token-buffer-mib: 5.814922E+01 | global_load_balancing_loss: 9.997970E-01 | loss scale: 1.0 | grad norm: 0.071 | params norm: 11468.389 | params rms: 0.062098 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 11:57:27.002301 | finish at 2026-10-10 22:28:25 + [2026-10-06 10:32:22] iteration 2658/ 6855 | consumed samples: 340224 | elapsed time per iteration (ms): 84378.9 | throughput per GPU (TFLOP/s/GPU): 43.7 | MFU 14.02% | FLOPs estimator: workload-aware | logical tokens/s: 198831.7 | valid tokens/s: 198831.7 | physical tokens/s: 199155.8 | loss tokens/s: 157888.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.940331E-01 | lm-global-supervised-tokens: 1.332242E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.141742E-02 | lm-token-buffer-mib: 5.805250E+01 | global_load_balancing_loss: 9.979184E-01 | loss scale: 1.0 | grad norm: 0.044 | params norm: 11468.325 | params rms: 0.062097 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 2:22:18.102424 | finish at 2026-10-10 12:54:40 + [2026-10-06 10:33:51] iteration 2659/ 6855 | consumed samples: 340352 | elapsed time per iteration (ms): 89067.9 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.47% | FLOPs estimator: workload-aware | logical tokens/s: 188363.1 | valid tokens/s: 188363.1 | physical tokens/s: 188843.5 | loss tokens/s: 137442.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.644891E-01 | lm-global-supervised-tokens: 1.224174E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.300794E-02 | lm-token-buffer-mib: 5.806336E+01 | global_load_balancing_loss: 9.999021E-01 | loss scale: 1.0 | grad norm: 0.048 | params norm: 11468.261 | params rms: 0.062097 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 7:48:48.702075 | finish at 2026-10-10 18:22:40 + [2026-10-06 10:35:40] iteration 2660/ 6855 | consumed samples: 340480 | elapsed time per iteration (ms): 109056.6 | throughput per GPU (TFLOP/s/GPU): 34.3 | MFU 10.98% | FLOPs estimator: workload-aware | logical tokens/s: 153838.5 | valid tokens/s: 153838.5 | physical tokens/s: 154200.1 | loss tokens/s: 112243.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.773018E-01 | lm-global-supervised-tokens: 1.224088E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.149092E-02 | lm-token-buffer-mib: 5.822151E+01 | global_load_balancing_loss: 1.000156E+00 | loss scale: 1.0 | grad norm: 0.036 | params norm: 11468.193 | params rms: 0.062097 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 7:04:52.587419 | finish at 2026-10-11 17:40:33 + [2026-10-06 10:37:08] iteration 2661/ 6855 | consumed samples: 340608 | elapsed time per iteration (ms): 87320.5 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.77% | FLOPs estimator: workload-aware | logical tokens/s: 192132.8 | valid tokens/s: 192132.8 | physical tokens/s: 192473.7 | loss tokens/s: 141307.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.710302E-01 | lm-global-supervised-tokens: 1.233904E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.410690E-02 | lm-token-buffer-mib: 5.803450E+01 | global_load_balancing_loss: 1.000989E+00 | loss scale: 1.0 | grad norm: 0.047 | params norm: 11468.128 | params rms: 0.062096 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 5:43:42.363554 | finish at 2026-10-10 16:20:50 + [2026-10-06 10:38:35] iteration 2662/ 6855 | consumed samples: 340736 | elapsed time per iteration (ms): 86914.8 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.39% | FLOPs estimator: workload-aware | logical tokens/s: 193030.0 | valid tokens/s: 193030.0 | physical tokens/s: 193526.8 | loss tokens/s: 146611.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.645978E-01 | lm-global-supervised-tokens: 1.274270E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.357203E-02 | lm-token-buffer-mib: 5.810652E+01 | global_load_balancing_loss: 1.001366E+00 | loss scale: 1.0 | grad norm: 0.048 | params norm: 11468.064 | params rms: 0.062096 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 5:13:53.868066 | finish at 2026-10-10 15:52:29 + [2026-10-06 10:40:05] iteration 2663/ 6855 | consumed samples: 340864 | elapsed time per iteration (ms): 89960.2 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.42% | FLOPs estimator: workload-aware | logical tokens/s: 186495.3 | valid tokens/s: 186495.3 | physical tokens/s: 187060.7 | loss tokens/s: 137892.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.037497E-01 | lm-global-supervised-tokens: 1.240483E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.286561E-02 | lm-token-buffer-mib: 5.808833E+01 | global_load_balancing_loss: 1.000043E+00 | loss scale: 1.0 | grad norm: 0.045 | params norm: 11468.000 | params rms: 0.062095 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 8:45:13.055756 | finish at 2026-10-10 19:25:18 + [2026-10-06 10:41:34] iteration 2664/ 6855 | consumed samples: 340992 | elapsed time per iteration (ms): 88911.4 | throughput per GPU (TFLOP/s/GPU): 41.2 | MFU 13.20% | FLOPs estimator: workload-aware | logical tokens/s: 188695.1 | valid tokens/s: 188695.1 | physical tokens/s: 189157.7 | loss tokens/s: 137480.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.693359E-01 | lm-global-supervised-tokens: 1.222359E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.317382E-02 | lm-token-buffer-mib: 5.817304E+01 | global_load_balancing_loss: 1.000320E+00 | loss scale: 1.0 | grad norm: 0.056 | params norm: 11467.934 | params rms: 0.062095 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 7:30:27.686727 | finish at 2026-10-10 18:12:01 + [2026-10-06 10:43:01] iteration 2665/ 6855 | consumed samples: 341120 | elapsed time per iteration (ms): 87029.5 | throughput per GPU (TFLOP/s/GPU): 41.7 | MFU 13.37% | FLOPs estimator: workload-aware | logical tokens/s: 192775.9 | valid tokens/s: 192775.9 | physical tokens/s: 193234.0 | loss tokens/s: 143678.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.439942E-01 | lm-global-supervised-tokens: 1.250427E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.180066E-02 | lm-token-buffer-mib: 5.820354E+01 | global_load_balancing_loss: 9.998816E-01 | loss scale: 1.0 | grad norm: 0.045 | params norm: 11467.871 | params rms: 0.062095 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 5:17:33.606031 | finish at 2026-10-10 16:00:34 + [2026-10-06 10:44:29] iteration 2666/ 6855 | consumed samples: 341248 | elapsed time per iteration (ms): 88155.4 | throughput per GPU (TFLOP/s/GPU): 41.5 | MFU 13.29% | FLOPs estimator: workload-aware | logical tokens/s: 190313.4 | valid tokens/s: 190313.4 | physical tokens/s: 190678.1 | loss tokens/s: 145884.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.992760E-01 | lm-global-supervised-tokens: 1.286047E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.542804E-02 | lm-token-buffer-mib: 5.820737E+01 | global_load_balancing_loss: 9.987816E-01 | loss scale: 1.0 | grad norm: 0.052 | params norm: 11467.809 | params rms: 0.062094 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 6:34:42.775006 | finish at 2026-10-10 17:19:12 + [2026-10-06 10:45:58] iteration 2667/ 6855 | consumed samples: 341376 | elapsed time per iteration (ms): 88950.9 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.60% | FLOPs estimator: workload-aware | logical tokens/s: 188611.7 | valid tokens/s: 188611.7 | physical tokens/s: 189032.8 | loss tokens/s: 140825.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.591599E-01 | lm-global-supervised-tokens: 1.252655E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.372981E-02 | lm-token-buffer-mib: 5.826072E+01 | global_load_balancing_loss: 9.998370E-01 | loss scale: 1.0 | grad norm: 0.050 | params norm: 11467.745 | params rms: 0.062094 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 7:28:46.510314 | finish at 2026-10-10 18:14:44 + [2026-10-06 10:47:28] iteration 2668/ 6855 | consumed samples: 341504 | elapsed time per iteration (ms): 90718.9 | throughput per GPU (TFLOP/s/GPU): 40.9 | MFU 13.12% | FLOPs estimator: workload-aware | logical tokens/s: 184935.4 | valid tokens/s: 184935.4 | physical tokens/s: 185444.0 | loss tokens/s: 130117.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.124396E-01 | lm-global-supervised-tokens: 1.180407E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.317374E-02 | lm-token-buffer-mib: 5.824989E+01 | global_load_balancing_loss: 1.001089E+00 | loss scale: 1.0 | grad norm: 0.101 | params norm: 11467.680 | params rms: 0.062094 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 9:30:40.157939 | finish at 2026-10-10 20:18:09 + [2026-10-06 10:48:54] iteration 2669/ 6855 | consumed samples: 341632 | elapsed time per iteration (ms): 85681.8 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.77% | FLOPs estimator: workload-aware | logical tokens/s: 195807.9 | valid tokens/s: 195807.9 | physical tokens/s: 196438.4 | loss tokens/s: 144954.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.016894E-01 | lm-global-supervised-tokens: 1.241996E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.365970E-02 | lm-token-buffer-mib: 5.817181E+01 | global_load_balancing_loss: 1.001151E+00 | loss scale: 1.0 | grad norm: 0.179 | params norm: 11467.616 | params rms: 0.062093 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 3:37:43.824710 | finish at 2026-10-10 14:26:38 + [2026-10-06 10:50:43] iteration 2670/ 6855 | consumed samples: 341760 | elapsed time per iteration (ms): 109286.8 | throughput per GPU (TFLOP/s/GPU): 33.9 | MFU 10.86% | FLOPs estimator: workload-aware | logical tokens/s: 153515.1 | valid tokens/s: 153515.1 | physical tokens/s: 153887.1 | loss tokens/s: 116075.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.506398E-01 | lm-global-supervised-tokens: 1.268547E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.152409E-02 | lm-token-buffer-mib: 5.815955E+01 | global_load_balancing_loss: 1.000924E+00 | loss scale: 1.0 | grad norm: 0.308 | params norm: 11467.554 | params rms: 0.062093 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 7:02:45.448190 | finish at 2026-10-11 17:53:29 + [2026-10-06 10:52:08] iteration 2671/ 6855 | consumed samples: 341888 | elapsed time per iteration (ms): 84166.9 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.73% | FLOPs estimator: workload-aware | logical tokens/s: 199332.4 | valid tokens/s: 199332.4 | physical tokens/s: 199800.4 | loss tokens/s: 150788.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.416495E-01 | lm-global-supervised-tokens: 1.269142E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.741983E-02 | lm-token-buffer-mib: 5.812433E+01 | global_load_balancing_loss: 1.000705E+00 | loss scale: 1.0 | grad norm: 0.165 | params norm: 11467.492 | params rms: 0.062093 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 1:49:14.203524 | finish at 2026-10-10 12:41:22 + [2026-10-06 10:53:36] iteration 2672/ 6855 | consumed samples: 342016 | elapsed time per iteration (ms): 88384.7 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.47% | FLOPs estimator: workload-aware | logical tokens/s: 189819.9 | valid tokens/s: 189819.9 | physical tokens/s: 190367.3 | loss tokens/s: 146582.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.554541E-01 | lm-global-supervised-tokens: 1.295564E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.131663E-02 | lm-token-buffer-mib: 5.810499E+01 | global_load_balancing_loss: 1.000276E+00 | loss scale: 1.0 | grad norm: 0.198 | params norm: 11467.430 | params rms: 0.062092 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 6:41:53.084663 | finish at 2026-10-10 17:35:29 + [2026-10-06 10:55:01] iteration 2673/ 6855 | consumed samples: 342144 | elapsed time per iteration (ms): 85185.2 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.59% | FLOPs estimator: workload-aware | logical tokens/s: 196949.0 | valid tokens/s: 196949.0 | physical tokens/s: 197363.3 | loss tokens/s: 148429.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.589693E-01 | lm-global-supervised-tokens: 1.264401E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.540112E-02 | lm-token-buffer-mib: 5.810725E+01 | global_load_balancing_loss: 1.000940E+00 | loss scale: 1.0 | grad norm: 0.157 | params norm: 11467.366 | params rms: 0.062092 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 2:57:24.536211 | finish at 2026-10-10 13:52:26 + [2026-10-06 10:56:25] iteration 2674/ 6855 | consumed samples: 342272 | elapsed time per iteration (ms): 83435.3 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.78% | FLOPs estimator: workload-aware | logical tokens/s: 201080.3 | valid tokens/s: 201080.3 | physical tokens/s: 201465.3 | loss tokens/s: 156076.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.395624E-01 | lm-global-supervised-tokens: 1.302231E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.357349E-02 | lm-token-buffer-mib: 5.816641E+01 | global_load_balancing_loss: 9.985808E-01 | loss scale: 1.0 | grad norm: 0.177 | params norm: 11467.304 | params rms: 0.062092 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 0:54:02.876129 | finish at 2026-10-10 11:50:27 + [2026-10-06 10:57:56] iteration 2675/ 6855 | consumed samples: 342400 | elapsed time per iteration (ms): 90926.2 | throughput per GPU (TFLOP/s/GPU): 40.8 | MFU 13.08% | FLOPs estimator: workload-aware | logical tokens/s: 184514.6 | valid tokens/s: 184514.6 | physical tokens/s: 184811.7 | loss tokens/s: 129502.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.632735E-01 | lm-global-supervised-tokens: 1.177514E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.162295E-02 | lm-token-buffer-mib: 5.806029E+01 | global_load_balancing_loss: 9.993975E-01 | loss scale: 1.0 | grad norm: 0.047 | params norm: 11467.240 | params rms: 0.062091 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 9:34:31.518626 | finish at 2026-10-10 20:32:27 + [2026-10-06 10:59:22] iteration 2676/ 6855 | consumed samples: 342528 | elapsed time per iteration (ms): 86803.9 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.66% | FLOPs estimator: workload-aware | logical tokens/s: 193275.7 | valid tokens/s: 193275.7 | physical tokens/s: 193752.9 | loss tokens/s: 143387.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.525771E-01 | lm-global-supervised-tokens: 1.244655E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.190018E-02 | lm-token-buffer-mib: 5.815453E+01 | global_load_balancing_loss: 9.997054E-01 | loss scale: 1.0 | grad norm: 0.231 | params norm: 11467.175 | params rms: 0.062091 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 4:45:53.531990 | finish at 2026-10-10 15:45:16 + [2026-10-06 11:00:47] iteration 2677/ 6855 | consumed samples: 342656 | elapsed time per iteration (ms): 84559.9 | throughput per GPU (TFLOP/s/GPU): 42.8 | MFU 13.71% | FLOPs estimator: workload-aware | logical tokens/s: 198406.1 | valid tokens/s: 198406.1 | physical tokens/s: 199028.3 | loss tokens/s: 145403.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.717340E-01 | lm-global-supervised-tokens: 1.229528E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.172118E-02 | lm-token-buffer-mib: 5.811044E+01 | global_load_balancing_loss: 1.000307E+00 | loss scale: 1.0 | grad norm: 0.157 | params norm: 11467.109 | params rms: 0.062091 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 2:08:11.435713 | finish at 2026-10-10 13:08:58 + [2026-10-06 11:02:09] iteration 2678/ 6855 | consumed samples: 342784 | elapsed time per iteration (ms): 81851.1 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.68% | FLOPs estimator: workload-aware | logical tokens/s: 204972.0 | valid tokens/s: 204972.0 | physical tokens/s: 205525.2 | loss tokens/s: 166367.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.442959E-01 | lm-global-supervised-tokens: 1.361735E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.345203E-02 | lm-token-buffer-mib: 5.823836E+01 | global_load_balancing_loss: 9.998181E-01 | loss scale: 1.0 | grad norm: 0.167 | params norm: 11467.043 | params rms: 0.062090 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 22:58:12.097347 | finish at 2026-10-10 10:00:21 + [2026-10-06 11:03:39] iteration 2679/ 6855 | consumed samples: 342912 | elapsed time per iteration (ms): 90692.6 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.39% | FLOPs estimator: workload-aware | logical tokens/s: 184989.6 | valid tokens/s: 184989.6 | physical tokens/s: 185313.4 | loss tokens/s: 143548.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.481789E-01 | lm-global-supervised-tokens: 1.301875E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.299141E-02 | lm-token-buffer-mib: 5.808334E+01 | global_load_balancing_loss: 1.000639E+00 | loss scale: 1.0 | grad norm: 0.098 | params norm: 11466.976 | params rms: 0.062090 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 9:12:12.434052 | finish at 2026-10-10 20:15:52 + [2026-10-06 11:05:26] iteration 2680/ 6855 | consumed samples: 343040 | elapsed time per iteration (ms): 106566.8 | throughput per GPU (TFLOP/s/GPU): 34.3 | MFU 10.98% | FLOPs estimator: workload-aware | logical tokens/s: 157433.6 | valid tokens/s: 157433.6 | physical tokens/s: 157680.4 | loss tokens/s: 112013.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.685914E-01 | lm-global-supervised-tokens: 1.193692E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.284053E-02 | lm-token-buffer-mib: 5.813523E+01 | global_load_balancing_loss: 1.002255E+00 | loss scale: 1.0 | grad norm: 0.083 | params norm: 11466.906 | params rms: 0.062090 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 3:35:16.323799 | finish at 2026-10-11 14:40:42 +times across ranks (ms): + save-checkpoint: + rank 0: 24431.65 + rank 1: 24431.63 + rank 2: 24431.66 + rank 3: 24431.67 + rank 4: 24431.63 + rank 5: 24431.62 + rank 6: 24431.60 + rank 7: 24431.64 + rank 8: 24432.56 + rank 9: 24432.53 + rank 10: 24432.51 + rank 11: 24432.51 + rank 12: 24432.57 + rank 13: 24432.52 + rank 14: 24432.52 + rank 15: 24432.51 + rank 16: 24432.08 + rank 17: 24432.11 + rank 18: 24432.10 + rank 19: 24432.08 + rank 20: 24432.12 + rank 21: 24432.12 + rank 22: 24432.09 + rank 23: 24432.11 + rank 24: 24432.66 + rank 25: 24432.59 + rank 26: 24432.66 + rank 27: 24432.66 + rank 28: 24432.53 + rank 29: 24432.68 + rank 30: 24432.65 + rank 31: 24432.63 + rank 32: 24432.67 + rank 33: 24432.79 + rank 34: 24432.57 + rank 35: 24432.60 + rank 36: 24432.59 + rank 37: 24432.64 + rank 38: 24432.54 + rank 39: 24432.61 + rank 40: 24432.11 + rank 41: 24432.12 + rank 42: 24432.08 + rank 43: 24432.12 + rank 44: 24432.08 + rank 45: 24432.10 + rank 46: 24432.10 + rank 47: 24432.18 + rank 48: 24432.96 + rank 49: 24432.92 + rank 50: 24432.88 + rank 51: 24432.76 + rank 52: 24432.93 + rank 53: 24432.88 + rank 54: 24432.81 + rank 55: 24432.98 + rank 56: 24432.59 + rank 57: 24432.58 + rank 58: 24432.55 + rank 59: 24432.54 + rank 60: 24432.54 + rank 61: 24432.57 + rank 62: 24432.42 + rank 63: 24432.64 + [2026-10-06 11:07:20] iteration 2681/ 6855 | consumed samples: 343168 | elapsed time per iteration (ms): 89824.6 | throughput per GPU (TFLOP/s/GPU): 40.1 | MFU 12.84% | FLOPs estimator: workload-aware | logical tokens/s: 186776.4 | valid tokens/s: 186776.4 | physical tokens/s: 187225.8 | loss tokens/s: 139925.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 5.968603E-01 | lm-global-supervised-tokens: 1.256880E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 3.316227E-02 | lm-token-buffer-mib: 5.803266E+01 | global_load_balancing_loss: 1.000928E+00 | loss scale: 1.0 | grad norm: 0.080 | params norm: 11466.836 | params rms: 0.062089 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 8:08:48.026610 | finish at 2026-10-10 19:16:08 + [2026-10-06 11:08:55] iteration 2682/ 6855 | consumed samples: 343296 | elapsed time per iteration (ms): 94845.3 | throughput per GPU (TFLOP/s/GPU): 40.0 | MFU 12.81% | FLOPs estimator: workload-aware | logical tokens/s: 176889.6 | valid tokens/s: 176889.6 | physical tokens/s: 177233.8 | loss tokens/s: 129590.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.227718E-01 | lm-global-supervised-tokens: 1.229102E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.522689E-02 | lm-token-buffer-mib: 5.811536E+01 | global_load_balancing_loss: 1.000023E+00 | loss scale: 1.0 | grad norm: 0.091 | params norm: 11466.767 | params rms: 0.062089 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 13:56:29.519308 | finish at 2026-10-11 01:05:25 + [2026-10-06 11:10:23] iteration 2683/ 6855 | consumed samples: 343424 | elapsed time per iteration (ms): 87306.9 | throughput per GPU (TFLOP/s/GPU): 41.2 | MFU 13.21% | FLOPs estimator: workload-aware | logical tokens/s: 192163.5 | valid tokens/s: 192163.5 | physical tokens/s: 192586.0 | loss tokens/s: 144625.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.571124E-01 | lm-global-supervised-tokens: 1.262681E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.856415E-02 | lm-token-buffer-mib: 5.816168E+01 | global_load_balancing_loss: 9.987206E-01 | loss scale: 1.0 | grad norm: 0.111 | params norm: 11466.696 | params rms: 0.062088 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 5:10:44.261572 | finish at 2026-10-10 16:21:07 + [2026-10-06 11:11:49] iteration 2684/ 6855 | consumed samples: 343552 | elapsed time per iteration (ms): 86834.1 | throughput per GPU (TFLOP/s/GPU): 43.1 | MFU 13.82% | FLOPs estimator: workload-aware | logical tokens/s: 193210.0 | valid tokens/s: 193210.0 | physical tokens/s: 193552.8 | loss tokens/s: 143336.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.030741E-01 | lm-global-supervised-tokens: 1.244650E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.192585E-02 | lm-token-buffer-mib: 5.808297E+01 | global_load_balancing_loss: 9.983432E-01 | loss scale: 1.0 | grad norm: 0.051 | params norm: 11466.626 | params rms: 0.062088 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 4:36:24.890917 | finish at 2026-10-10 15:48:14 + [2026-10-06 11:13:14] iteration 2685/ 6855 | consumed samples: 343680 | elapsed time per iteration (ms): 84658.4 | throughput per GPU (TFLOP/s/GPU): 43.5 | MFU 13.93% | FLOPs estimator: workload-aware | logical tokens/s: 198175.4 | valid tokens/s: 198175.4 | physical tokens/s: 198694.9 | loss tokens/s: 149644.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.475175E-01 | lm-global-supervised-tokens: 1.266866E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.029434E-02 | lm-token-buffer-mib: 5.817532E+01 | global_load_balancing_loss: 9.997107E-01 | loss scale: 1.0 | grad norm: 0.120 | params norm: 11466.556 | params rms: 0.062088 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 2:03:45.426836 | finish at 2026-10-10 13:17:00 + [2026-10-06 11:14:40] iteration 2686/ 6855 | consumed samples: 343808 | elapsed time per iteration (ms): 85892.1 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.49% | FLOPs estimator: workload-aware | logical tokens/s: 195328.9 | valid tokens/s: 195328.9 | physical tokens/s: 195798.2 | loss tokens/s: 141585.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.088786E-01 | lm-global-supervised-tokens: 1.216111E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.362674E-02 | lm-token-buffer-mib: 5.809850E+01 | global_load_balancing_loss: 1.000929E+00 | loss scale: 1.0 | grad norm: 0.085 | params norm: 11466.486 | params rms: 0.062087 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 3:28:04.218973 | finish at 2026-10-10 14:42:44 + [2026-10-06 11:16:06] iteration 2687/ 6855 | consumed samples: 343936 | elapsed time per iteration (ms): 85774.8 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.48% | FLOPs estimator: workload-aware | logical tokens/s: 195595.6 | valid tokens/s: 195595.5 | physical tokens/s: 196107.3 | loss tokens/s: 147086.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.441898E-01 | lm-global-supervised-tokens: 1.261631E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.226613E-02 | lm-token-buffer-mib: 5.805363E+01 | global_load_balancing_loss: 1.000973E+00 | loss scale: 1.0 | grad norm: 0.097 | params norm: 11466.416 | params rms: 0.062087 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 3:18:29.221575 | finish at 2026-10-10 14:34:35 + [2026-10-06 11:17:35] iteration 2688/ 6855 | consumed samples: 344064 | elapsed time per iteration (ms): 88821.1 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.52% | FLOPs estimator: workload-aware | logical tokens/s: 188887.4 | valid tokens/s: 188887.4 | physical tokens/s: 189339.0 | loss tokens/s: 139546.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.846818E-01 | lm-global-supervised-tokens: 1.239463E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.398398E-02 | lm-token-buffer-mib: 5.819495E+01 | global_load_balancing_loss: 9.998742E-01 | loss scale: 1.0 | grad norm: 0.098 | params norm: 11466.349 | params rms: 0.062087 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 6:48:37.554484 | finish at 2026-10-10 18:06:12 + [2026-10-06 11:19:04] iteration 2689/ 6855 | consumed samples: 344192 | elapsed time per iteration (ms): 89185.8 | throughput per GPU (TFLOP/s/GPU): 41.5 | MFU 13.30% | FLOPs estimator: workload-aware | logical tokens/s: 188114.9 | valid tokens/s: 188114.9 | physical tokens/s: 188553.8 | loss tokens/s: 135614.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.155245E-01 | lm-global-supervised-tokens: 1.209485E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.251488E-02 | lm-token-buffer-mib: 5.808530E+01 | global_load_balancing_loss: 1.001518E+00 | loss scale: 1.0 | grad norm: 0.054 | params norm: 11466.282 | params rms: 0.062086 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 7:12:27.981533 | finish at 2026-10-10 18:31:32 + [2026-10-06 11:20:58] iteration 2690/ 6855 | consumed samples: 344320 | elapsed time per iteration (ms): 113855.1 | throughput per GPU (TFLOP/s/GPU): 34.7 | MFU 11.12% | FLOPs estimator: workload-aware | logical tokens/s: 147355.6 | valid tokens/s: 147355.5 | physical tokens/s: 147635.0 | loss tokens/s: 99470.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.917560E-01 | lm-global-supervised-tokens: 1.132523E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.312340E-02 | lm-token-buffer-mib: 5.819002E+01 | global_load_balancing_loss: 1.000841E+00 | loss scale: 1.0 | grad norm: 0.082 | params norm: 11466.213 | params rms: 0.062086 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 11:43:26.367025 | finish at 2026-10-11 23:04:24 + [2026-10-06 11:22:27] iteration 2691/ 6855 | consumed samples: 344448 | elapsed time per iteration (ms): 89052.3 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.53% | FLOPs estimator: workload-aware | logical tokens/s: 188397.2 | valid tokens/s: 188397.2 | physical tokens/s: 188894.7 | loss tokens/s: 136741.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.750484E-01 | lm-global-supervised-tokens: 1.217711E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.277550E-02 | lm-token-buffer-mib: 5.815623E+01 | global_load_balancing_loss: 9.993090E-01 | loss scale: 1.0 | grad norm: 0.050 | params norm: 11466.145 | params rms: 0.062085 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 7:00:13.663925 | finish at 2026-10-10 18:22:40 + [2026-10-06 11:23:54] iteration 2692/ 6855 | consumed samples: 344576 | elapsed time per iteration (ms): 87540.3 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.78% | FLOPs estimator: workload-aware | logical tokens/s: 191651.1 | valid tokens/s: 191651.1 | physical tokens/s: 192184.8 | loss tokens/s: 140413.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.545925E-01 | lm-global-supervised-tokens: 1.229181E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.163344E-02 | lm-token-buffer-mib: 5.807643E+01 | global_load_balancing_loss: 9.996132E-01 | loss scale: 1.0 | grad norm: 0.077 | params norm: 11466.077 | params rms: 0.062085 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 5:13:50.458027 | finish at 2026-10-10 16:37:45 + [2026-10-06 11:25:21] iteration 2693/ 6855 | consumed samples: 344704 | elapsed time per iteration (ms): 86650.4 | throughput per GPU (TFLOP/s/GPU): 43.2 | MFU 13.84% | FLOPs estimator: workload-aware | logical tokens/s: 193618.7 | valid tokens/s: 193618.7 | physical tokens/s: 194185.3 | loss tokens/s: 143133.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.897406E-01 | lm-global-supervised-tokens: 1.240253E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.393736E-02 | lm-token-buffer-mib: 5.825038E+01 | global_load_balancing_loss: 1.000613E+00 | loss scale: 1.0 | grad norm: 0.053 | params norm: 11466.009 | params rms: 0.062085 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 4:10:39.016080 | finish at 2026-10-10 15:36:00 + [2026-10-06 11:26:48] iteration 2694/ 6855 | consumed samples: 344832 | elapsed time per iteration (ms): 86980.8 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.50% | FLOPs estimator: workload-aware | logical tokens/s: 192883.4 | valid tokens/s: 192883.4 | physical tokens/s: 193499.3 | loss tokens/s: 147368.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.624067E-01 | lm-global-supervised-tokens: 1.281820E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.134331E-02 | lm-token-buffer-mib: 5.814388E+01 | global_load_balancing_loss: 1.000670E+00 | loss scale: 1.0 | grad norm: 0.069 | params norm: 11465.942 | params rms: 0.062084 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 4:32:07.015186 | finish at 2026-10-10 15:58:55 + [2026-10-06 11:28:16] iteration 2695/ 6855 | consumed samples: 344960 | elapsed time per iteration (ms): 87760.6 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.56% | FLOPs estimator: workload-aware | logical tokens/s: 191170.0 | valid tokens/s: 191170.0 | physical tokens/s: 191594.4 | loss tokens/s: 143006.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.593323E-01 | lm-global-supervised-tokens: 1.255030E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.418188E-02 | lm-token-buffer-mib: 5.811822E+01 | global_load_balancing_loss: 9.998518E-01 | loss scale: 1.0 | grad norm: 0.044 | params norm: 11465.878 | params rms: 0.062084 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 5:24:43.950577 | finish at 2026-10-10 16:53:00 + [2026-10-06 11:29:44] iteration 2696/ 6855 | consumed samples: 345088 | elapsed time per iteration (ms): 88753.1 | throughput per GPU (TFLOP/s/GPU): 41.5 | MFU 13.31% | FLOPs estimator: workload-aware | logical tokens/s: 189031.6 | valid tokens/s: 189031.6 | physical tokens/s: 189619.9 | loss tokens/s: 143957.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.361911E-01 | lm-global-supervised-tokens: 1.277666E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.230564E-02 | lm-token-buffer-mib: 5.816327E+01 | global_load_balancing_loss: 9.993355E-01 | loss scale: 1.0 | grad norm: 0.049 | params norm: 11465.814 | params rms: 0.062084 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 6:32:04.334928 | finish at 2026-10-10 18:01:49 + [2026-10-06 11:31:10] iteration 2697/ 6855 | consumed samples: 345216 | elapsed time per iteration (ms): 85615.6 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.66% | FLOPs estimator: workload-aware | logical tokens/s: 195959.5 | valid tokens/s: 195959.5 | physical tokens/s: 196692.0 | loss tokens/s: 147440.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.422836E-01 | lm-global-supervised-tokens: 1.262317E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.185330E-02 | lm-token-buffer-mib: 5.819994E+01 | global_load_balancing_loss: 1.000138E+00 | loss scale: 1.0 | grad norm: 0.042 | params norm: 11465.749 | params rms: 0.062083 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 2:53:09.588920 | finish at 2026-10-10 14:24:20 + [2026-10-06 11:32:37] iteration 2698/ 6855 | consumed samples: 345344 | elapsed time per iteration (ms): 86890.7 | throughput per GPU (TFLOP/s/GPU): 42.8 | MFU 13.70% | FLOPs estimator: workload-aware | logical tokens/s: 193084.1 | valid tokens/s: 193084.1 | physical tokens/s: 193379.4 | loss tokens/s: 137862.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.325431E-01 | lm-global-supervised-tokens: 1.197900E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.345533E-02 | lm-token-buffer-mib: 5.811877E+01 | global_load_balancing_loss: 1.000890E+00 | loss scale: 1.0 | grad norm: 0.035 | params norm: 11465.686 | params rms: 0.062083 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 4:20:04.536257 | finish at 2026-10-10 15:52:41 + [2026-10-06 11:34:03] iteration 2699/ 6855 | consumed samples: 345472 | elapsed time per iteration (ms): 86483.6 | throughput per GPU (TFLOP/s/GPU): 41.7 | MFU 13.38% | FLOPs estimator: workload-aware | logical tokens/s: 193992.1 | valid tokens/s: 193992.0 | physical tokens/s: 194803.2 | loss tokens/s: 141889.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.905338E-01 | lm-global-supervised-tokens: 1.227114E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.321831E-02 | lm-token-buffer-mib: 5.816289E+01 | global_load_balancing_loss: 1.000230E+00 | loss scale: 1.0 | grad norm: 0.063 | params norm: 11465.623 | params rms: 0.062083 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 3:50:25.885778 | finish at 2026-10-10 15:24:29 + [2026-10-06 11:35:51] iteration 2700/ 6855 | consumed samples: 345600 | elapsed time per iteration (ms): 107842.7 | throughput per GPU (TFLOP/s/GPU): 34.4 | MFU 11.04% | FLOPs estimator: workload-aware | logical tokens/s: 155570.8 | valid tokens/s: 155570.8 | physical tokens/s: 155940.6 | loss tokens/s: 116180.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.650993E-01 | lm-global-supervised-tokens: 1.252927E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.526815E-02 | lm-token-buffer-mib: 5.806808E+01 | global_load_balancing_loss: 9.996588E-01 | loss scale: 1.0 | grad norm: 0.091 | params norm: 11465.560 | params rms: 0.062082 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 4:28:06.570085 | finish at 2026-10-11 16:03:58 + [2026-10-06 11:37:16] iteration 2701/ 6855 | consumed samples: 345728 | elapsed time per iteration (ms): 85122.1 | throughput per GPU (TFLOP/s/GPU): 43.6 | MFU 13.99% | FLOPs estimator: workload-aware | logical tokens/s: 197095.5 | valid tokens/s: 197095.4 | physical tokens/s: 197426.8 | loss tokens/s: 143921.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.704280E-01 | lm-global-supervised-tokens: 1.225091E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.176168E-02 | lm-token-buffer-mib: 5.808429E+01 | global_load_balancing_loss: 9.998260E-01 | loss scale: 1.0 | grad norm: 0.130 | params norm: 11465.496 | params rms: 0.062082 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 2:13:17.318762 | finish at 2026-10-10 13:50:34 + [2026-10-06 11:38:41] iteration 2702/ 6855 | consumed samples: 345856 | elapsed time per iteration (ms): 84688.1 | throughput per GPU (TFLOP/s/GPU): 43.4 | MFU 13.92% | FLOPs estimator: workload-aware | logical tokens/s: 198105.9 | valid tokens/s: 198105.9 | physical tokens/s: 198453.7 | loss tokens/s: 145377.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.615937E-01 | lm-global-supervised-tokens: 1.231174E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.443433E-02 | lm-token-buffer-mib: 5.808443E+01 | global_load_balancing_loss: 1.000817E+00 | loss scale: 1.0 | grad norm: 0.071 | params norm: 11465.433 | params rms: 0.062082 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 1:41:49.840118 | finish at 2026-10-10 13:20:31 + [2026-10-06 11:40:07] iteration 2703/ 6855 | consumed samples: 345984 | elapsed time per iteration (ms): 86139.7 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.51% | FLOPs estimator: workload-aware | logical tokens/s: 194767.3 | valid tokens/s: 194767.3 | physical tokens/s: 195377.1 | loss tokens/s: 143428.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.993176E-01 | lm-global-supervised-tokens: 1.235493E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.160649E-02 | lm-token-buffer-mib: 5.821484E+01 | global_load_balancing_loss: 1.001557E+00 | loss scale: 1.0 | grad norm: 0.070 | params norm: 11465.369 | params rms: 0.062081 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 3:20:52.106398 | finish at 2026-10-10 15:00:59 + [2026-10-06 11:41:38] iteration 2704/ 6855 | consumed samples: 346112 | elapsed time per iteration (ms): 90492.6 | throughput per GPU (TFLOP/s/GPU): 41.7 | MFU 13.36% | FLOPs estimator: workload-aware | logical tokens/s: 185397.6 | valid tokens/s: 185397.6 | physical tokens/s: 185802.5 | loss tokens/s: 143750.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.008550E-01 | lm-global-supervised-tokens: 1.300836E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.186443E-02 | lm-token-buffer-mib: 5.802028E+01 | global_load_balancing_loss: 1.000960E+00 | loss scale: 1.0 | grad norm: 0.065 | params norm: 11465.307 | params rms: 0.062081 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 8:20:34.669628 | finish at 2026-10-10 20:02:12 + [2026-10-06 11:43:06] iteration 2705/ 6855 | consumed samples: 346240 | elapsed time per iteration (ms): 88239.6 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.74% | FLOPs estimator: workload-aware | logical tokens/s: 190132.3 | valid tokens/s: 190132.3 | physical tokens/s: 190563.4 | loss tokens/s: 133099.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.642218E-01 | lm-global-supervised-tokens: 1.174463E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.183397E-02 | lm-token-buffer-mib: 5.807088E+01 | global_load_balancing_loss: 1.000350E+00 | loss scale: 1.0 | grad norm: 0.076 | params norm: 11465.245 | params rms: 0.062081 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 5:43:14.539630 | finish at 2026-10-10 17:26:20 + [2026-10-06 11:44:37] iteration 2706/ 6855 | consumed samples: 346368 | elapsed time per iteration (ms): 90978.3 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.43% | FLOPs estimator: workload-aware | logical tokens/s: 184409.0 | valid tokens/s: 184409.0 | physical tokens/s: 184756.7 | loss tokens/s: 130010.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.899720E-01 | lm-global-supervised-tokens: 1.182813E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.649717E-02 | lm-token-buffer-mib: 5.809160E+01 | global_load_balancing_loss: 9.999844E-01 | loss scale: 1.0 | grad norm: 0.060 | params norm: 11465.181 | params rms: 0.062080 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 8:51:08.920600 | finish at 2026-10-10 20:35:46 + [2026-10-06 11:46:05] iteration 2707/ 6855 | consumed samples: 346496 | elapsed time per iteration (ms): 88435.2 | throughput per GPU (TFLOP/s/GPU): 40.9 | MFU 13.12% | FLOPs estimator: workload-aware | logical tokens/s: 189711.3 | valid tokens/s: 189711.3 | physical tokens/s: 190255.7 | loss tokens/s: 139190.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.340161E-01 | lm-global-supervised-tokens: 1.230930E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.233551E-02 | lm-token-buffer-mib: 5.812286E+01 | global_load_balancing_loss: 9.993050E-01 | loss scale: 1.0 | grad norm: 0.093 | params norm: 11465.116 | params rms: 0.062080 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 5:53:49.144229 | finish at 2026-10-10 17:39:54 + [2026-10-06 11:47:32] iteration 2708/ 6855 | consumed samples: 346624 | elapsed time per iteration (ms): 86880.6 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.50% | FLOPs estimator: workload-aware | logical tokens/s: 193105.8 | valid tokens/s: 193105.7 | physical tokens/s: 193557.4 | loss tokens/s: 145987.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.332642E-01 | lm-global-supervised-tokens: 1.268346E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.401246E-02 | lm-token-buffer-mib: 5.813443E+01 | global_load_balancing_loss: 9.989707E-01 | loss scale: 1.0 | grad norm: 0.067 | params norm: 11465.054 | params rms: 0.062080 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 4:04:53.794708 | finish at 2026-10-10 15:52:26 + [2026-10-06 11:48:57] iteration 2709/ 6855 | consumed samples: 346752 | elapsed time per iteration (ms): 84472.3 | throughput per GPU (TFLOP/s/GPU): 43.1 | MFU 13.82% | FLOPs estimator: workload-aware | logical tokens/s: 198612.0 | valid tokens/s: 198612.0 | physical tokens/s: 198964.8 | loss tokens/s: 148666.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.276242E-01 | lm-global-supervised-tokens: 1.255824E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.321192E-02 | lm-token-buffer-mib: 5.812165E+01 | global_load_balancing_loss: 9.994240E-01 | loss scale: 1.0 | grad norm: 0.093 | params norm: 11464.993 | params rms: 0.062079 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 1:17:02.080894 | finish at 2026-10-10 13:05:59 + [2026-10-06 11:50:45] iteration 2710/ 6855 | consumed samples: 346880 | elapsed time per iteration (ms): 108771.9 | throughput per GPU (TFLOP/s/GPU): 34.1 | MFU 10.92% | FLOPs estimator: workload-aware | logical tokens/s: 154241.9 | valid tokens/s: 154241.9 | physical tokens/s: 154660.2 | loss tokens/s: 114482.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.752001E-01 | lm-global-supervised-tokens: 1.245250E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.339268E-02 | lm-token-buffer-mib: 5.824129E+01 | global_load_balancing_loss: 1.000438E+00 | loss scale: 1.0 | grad norm: 0.036 | params norm: 11464.931 | params rms: 0.062079 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 5:14:19.640870 | finish at 2026-10-11 17:05:05 + [2026-10-06 11:52:13] iteration 2711/ 6855 | consumed samples: 347008 | elapsed time per iteration (ms): 87928.8 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.55% | FLOPs estimator: workload-aware | logical tokens/s: 190804.3 | valid tokens/s: 190804.3 | physical tokens/s: 191232.5 | loss tokens/s: 138577.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.817302E-01 | lm-global-supervised-tokens: 1.218496E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.172196E-02 | lm-token-buffer-mib: 5.807606E+01 | global_load_balancing_loss: 1.001706E+00 | loss scale: 1.0 | grad norm: 0.054 | params norm: 11464.867 | params rms: 0.062079 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 5:12:56.946651 | finish at 2026-10-10 17:05:10 + [2026-10-06 11:53:44] iteration 2712/ 6855 | consumed samples: 347136 | elapsed time per iteration (ms): 90677.2 | throughput per GPU (TFLOP/s/GPU): 41.1 | MFU 13.16% | FLOPs estimator: workload-aware | logical tokens/s: 185021.4 | valid tokens/s: 185021.4 | physical tokens/s: 185393.7 | loss tokens/s: 136342.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.270564E-01 | lm-global-supervised-tokens: 1.236313E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.106562E-02 | lm-token-buffer-mib: 5.813452E+01 | global_load_balancing_loss: 1.000615E+00 | loss scale: 1.0 | grad norm: 0.049 | params norm: 11464.804 | params rms: 0.062078 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 8:21:15.536211 | finish at 2026-10-10 20:15:00 + [2026-10-06 11:55:11] iteration 2713/ 6855 | consumed samples: 347264 | elapsed time per iteration (ms): 87084.4 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.52% | FLOPs estimator: workload-aware | logical tokens/s: 192654.1 | valid tokens/s: 192654.1 | physical tokens/s: 193220.1 | loss tokens/s: 139469.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.523246E-01 | lm-global-supervised-tokens: 1.214562E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.278024E-02 | lm-token-buffer-mib: 5.807207E+01 | global_load_balancing_loss: 1.000741E+00 | loss scale: 1.0 | grad norm: 0.055 | params norm: 11464.741 | params rms: 0.062078 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 4:11:43.772176 | finish at 2026-10-10 16:06:55 + [2026-10-06 11:56:40] iteration 2714/ 6855 | consumed samples: 347392 | elapsed time per iteration (ms): 88523.8 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.70% | FLOPs estimator: workload-aware | logical tokens/s: 189522.1 | valid tokens/s: 189522.1 | physical tokens/s: 189894.3 | loss tokens/s: 138989.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.749527E-01 | lm-global-supervised-tokens: 1.230388E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.375871E-02 | lm-token-buffer-mib: 5.803177E+01 | global_load_balancing_loss: 9.996548E-01 | loss scale: 1.0 | grad norm: 0.079 | params norm: 11464.678 | params rms: 0.062077 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 5:49:36.963552 | finish at 2026-10-10 17:46:17 + [2026-10-06 11:58:05] iteration 2715/ 6855 | consumed samples: 347520 | elapsed time per iteration (ms): 85478.6 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.58% | FLOPs estimator: workload-aware | logical tokens/s: 196273.1 | valid tokens/s: 196273.1 | physical tokens/s: 196881.8 | loss tokens/s: 146933.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.875667E-01 | lm-global-supervised-tokens: 1.255963E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.300194E-02 | lm-token-buffer-mib: 5.825711E+01 | global_load_balancing_loss: 1.000548E+00 | loss scale: 1.0 | grad norm: 0.067 | params norm: 11464.615 | params rms: 0.062077 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 2:18:01.463327 | finish at 2026-10-10 14:16:07 + [2026-10-06 11:59:32] iteration 2716/ 6855 | consumed samples: 347648 | elapsed time per iteration (ms): 87001.2 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.53% | FLOPs estimator: workload-aware | logical tokens/s: 192838.7 | valid tokens/s: 192838.7 | physical tokens/s: 193406.3 | loss tokens/s: 143247.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.737794E-01 | lm-global-supervised-tokens: 1.246273E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.135112E-02 | lm-token-buffer-mib: 5.821004E+01 | global_load_balancing_loss: 9.996170E-01 | loss scale: 1.0 | grad norm: 0.091 | params norm: 11464.552 | params rms: 0.062077 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 4:01:37.890653 | finish at 2026-10-10 16:01:10 + [2026-10-06 12:01:00] iteration 2717/ 6855 | consumed samples: 347776 | elapsed time per iteration (ms): 88039.7 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.55% | FLOPs estimator: workload-aware | logical tokens/s: 190564.1 | valid tokens/s: 190564.1 | physical tokens/s: 191045.9 | loss tokens/s: 144415.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.468596E-01 | lm-global-supervised-tokens: 1.271430E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.026662E-02 | lm-token-buffer-mib: 5.813763E+01 | global_load_balancing_loss: 1.000320E+00 | loss scale: 1.0 | grad norm: 0.223 | params norm: 11464.488 | params rms: 0.062076 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 5:11:48.380347 | finish at 2026-10-10 17:12:49 + [2026-10-06 12:02:29] iteration 2718/ 6855 | consumed samples: 347904 | elapsed time per iteration (ms): 88614.0 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.75% | FLOPs estimator: workload-aware | logical tokens/s: 189328.9 | valid tokens/s: 189328.9 | physical tokens/s: 189787.7 | loss tokens/s: 141823.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.635298E-01 | lm-global-supervised-tokens: 1.256754E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.122958E-02 | lm-token-buffer-mib: 5.809550E+01 | global_load_balancing_loss: 9.996268E-01 | loss scale: 1.0 | grad norm: 0.245 | params norm: 11464.423 | params rms: 0.062076 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 5:49:55.983211 | finish at 2026-10-10 17:52:25 + [2026-10-06 12:03:55] iteration 2719/ 6855 | consumed samples: 348032 | elapsed time per iteration (ms): 86198.8 | throughput per GPU (TFLOP/s/GPU): 43.4 | MFU 13.90% | FLOPs estimator: workload-aware | logical tokens/s: 194633.8 | valid tokens/s: 194633.8 | physical tokens/s: 195051.8 | loss tokens/s: 140390.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.676507E-01 | lm-global-supervised-tokens: 1.210153E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 4.228682E-02 | lm-token-buffer-mib: 5.808973E+01 | global_load_balancing_loss: 1.000776E+00 | loss scale: 1.0 | grad norm: 0.118 | params norm: 11464.357 | params rms: 0.062076 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 3:01:58.345631 | finish at 2026-10-10 15:05:53 + [2026-10-06 12:05:44] iteration 2720/ 6855 | consumed samples: 348160 | elapsed time per iteration (ms): 109316.7 | throughput per GPU (TFLOP/s/GPU): 33.9 | MFU 10.86% | FLOPs estimator: workload-aware | logical tokens/s: 153473.4 | valid tokens/s: 153473.4 | physical tokens/s: 153883.8 | loss tokens/s: 117540.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.539835E-01 | lm-global-supervised-tokens: 1.284915E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.141574E-02 | lm-token-buffer-mib: 5.812533E+01 | global_load_balancing_loss: 1.000780E+00 | loss scale: 1.0 | grad norm: 0.138 | params norm: 11464.288 | params rms: 0.062075 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 5:33:44.706247 | finish at 2026-10-11 17:39:29 +times across ranks (ms): + save-checkpoint: + rank 0: 26011.47 + rank 1: 26011.50 + rank 2: 26011.49 + rank 3: 26011.50 + rank 4: 26011.46 + rank 5: 26011.45 + rank 6: 26011.44 + rank 7: 26011.48 + rank 8: 26012.19 + rank 9: 26012.11 + rank 10: 26012.11 + rank 11: 26012.13 + rank 12: 26012.11 + rank 13: 26012.20 + rank 14: 26012.21 + rank 15: 26012.12 + rank 16: 26011.32 + rank 17: 26011.28 + rank 18: 26011.27 + rank 19: 26011.31 + rank 20: 26011.27 + rank 21: 26011.37 + rank 22: 26011.28 + rank 23: 26011.26 + rank 24: 26010.95 + rank 25: 26010.88 + rank 26: 26010.66 + rank 27: 26010.99 + rank 28: 26010.96 + rank 29: 26010.96 + rank 30: 26010.94 + rank 31: 26011.07 + rank 32: 26012.20 + rank 33: 26012.21 + rank 34: 26012.22 + rank 35: 26012.21 + rank 36: 26012.12 + rank 37: 26012.10 + rank 38: 26012.13 + rank 39: 26012.14 + rank 40: 26013.05 + rank 41: 26013.04 + rank 42: 26013.11 + rank 43: 26013.05 + rank 44: 26013.03 + rank 45: 26013.36 + rank 46: 26013.04 + rank 47: 26013.03 + rank 48: 26012.43 + rank 49: 26012.51 + rank 50: 26012.32 + rank 51: 26012.27 + rank 52: 26012.37 + rank 53: 26012.35 + rank 54: 26012.37 + rank 55: 26012.41 + rank 56: 26012.04 + rank 57: 26012.03 + rank 58: 26012.00 + rank 59: 26012.03 + rank 60: 26011.99 + rank 61: 26012.05 + rank 62: 26012.02 + rank 63: 26012.03 + [2026-10-06 12:07:50] iteration 2721/ 6855 | consumed samples: 348288 | elapsed time per iteration (ms): 99430.7 | throughput per GPU (TFLOP/s/GPU): 38.2 | MFU 12.23% | FLOPs estimator: workload-aware | logical tokens/s: 168732.0 | valid tokens/s: 168732.0 | physical tokens/s: 169194.4 | loss tokens/s: 119856.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.386036E-01 | lm-global-supervised-tokens: 1.191738E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 3.440136E-01 | lm-token-buffer-mib: 5.811583E+01 | global_load_balancing_loss: 1.001170E+00 | loss scale: 1.0 | grad norm: 0.194 | params norm: 11464.220 | params rms: 0.062075 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 18:10:46.457883 | finish at 2026-10-11 06:18:36 + [2026-10-06 12:09:22] iteration 2722/ 6855 | consumed samples: 348416 | elapsed time per iteration (ms): 91760.0 | throughput per GPU (TFLOP/s/GPU): 40.6 | MFU 13.01% | FLOPs estimator: workload-aware | logical tokens/s: 182837.2 | valid tokens/s: 182837.2 | physical tokens/s: 183412.2 | loss tokens/s: 136175.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.607006E-01 | lm-global-supervised-tokens: 1.249545E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.187622E-02 | lm-token-buffer-mib: 5.815808E+01 | global_load_balancing_loss: 1.000012E+00 | loss scale: 1.0 | grad norm: 0.077 | params norm: 11464.153 | params rms: 0.062075 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 9:20:44.126274 | finish at 2026-10-10 21:30:06 + [2026-10-06 12:10:46] iteration 2723/ 6855 | consumed samples: 348544 | elapsed time per iteration (ms): 84648.2 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.79% | FLOPs estimator: workload-aware | logical tokens/s: 198198.9 | valid tokens/s: 198198.9 | physical tokens/s: 198595.0 | loss tokens/s: 149297.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.684560E-01 | lm-global-supervised-tokens: 1.263774E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.602473E-02 | lm-token-buffer-mib: 5.809047E+01 | global_load_balancing_loss: 1.000240E+00 | loss scale: 1.0 | grad norm: 0.083 | params norm: 11464.086 | params rms: 0.062074 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 1:09:26.220684 | finish at 2026-10-10 13:20:13 + [2026-10-06 12:12:14] iteration 2724/ 6855 | consumed samples: 348672 | elapsed time per iteration (ms): 87320.6 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.74% | FLOPs estimator: workload-aware | logical tokens/s: 192133.2 | valid tokens/s: 192133.2 | physical tokens/s: 192496.5 | loss tokens/s: 140271.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.865113E-01 | lm-global-supervised-tokens: 1.224861E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.163378E-02 | lm-token-buffer-mib: 5.815816E+01 | global_load_balancing_loss: 9.995664E-01 | loss scale: 1.0 | grad norm: 0.129 | params norm: 11464.021 | params rms: 0.062074 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 4:12:01.498211 | finish at 2026-10-10 16:24:15 + [2026-10-06 12:13:40] iteration 2725/ 6855 | consumed samples: 348800 | elapsed time per iteration (ms): 86657.5 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.79% | FLOPs estimator: workload-aware | logical tokens/s: 193603.2 | valid tokens/s: 193603.2 | physical tokens/s: 193945.3 | loss tokens/s: 143615.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.785939E-01 | lm-global-supervised-tokens: 1.244536E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.156959E-02 | lm-token-buffer-mib: 5.813834E+01 | global_load_balancing_loss: 9.987766E-01 | loss scale: 1.0 | grad norm: 0.049 | params norm: 11463.954 | params rms: 0.062074 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 3:24:55.496781 | finish at 2026-10-10 15:38:36 + [2026-10-06 12:15:05] iteration 2726/ 6855 | consumed samples: 348928 | elapsed time per iteration (ms): 84990.0 | throughput per GPU (TFLOP/s/GPU): 43.1 | MFU 13.82% | FLOPs estimator: workload-aware | logical tokens/s: 197401.4 | valid tokens/s: 197401.4 | physical tokens/s: 198031.3 | loss tokens/s: 146462.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.590715E-01 | lm-global-supervised-tokens: 1.244789E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.616442E-02 | lm-token-buffer-mib: 5.832779E+01 | global_load_balancing_loss: 9.995444E-01 | loss scale: 1.0 | grad norm: 0.195 | params norm: 11463.887 | params rms: 0.062073 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 1:28:43.895112 | finish at 2026-10-10 13:43:49 + [2026-10-06 12:16:28] iteration 2727/ 6855 | consumed samples: 349056 | elapsed time per iteration (ms): 82409.8 | throughput per GPU (TFLOP/s/GPU): 44.2 | MFU 14.17% | FLOPs estimator: workload-aware | logical tokens/s: 203582.4 | valid tokens/s: 203582.4 | physical tokens/s: 203916.5 | loss tokens/s: 158531.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.997858E-01 | lm-global-supervised-tokens: 1.306453E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.209464E-02 | lm-token-buffer-mib: 5.815870E+01 | global_load_balancing_loss: 9.997780E-01 | loss scale: 1.0 | grad norm: 0.084 | params norm: 11463.820 | params rms: 0.062073 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 22:29:47.705795 | finish at 2026-10-10 10:46:15 + [2026-10-06 12:17:52] iteration 2728/ 6855 | consumed samples: 349184 | elapsed time per iteration (ms): 84676.8 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.76% | FLOPs estimator: workload-aware | logical tokens/s: 198131.5 | valid tokens/s: 198131.5 | physical tokens/s: 198637.8 | loss tokens/s: 149312.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.577920E-01 | lm-global-supervised-tokens: 1.264327E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.492943E-02 | lm-token-buffer-mib: 5.807813E+01 | global_load_balancing_loss: 1.000808E+00 | loss scale: 1.0 | grad norm: 0.165 | params norm: 11463.753 | params rms: 0.062072 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 1:04:21.298293 | finish at 2026-10-10 13:22:14 + [2026-10-06 12:19:21] iteration 2729/ 6855 | consumed samples: 349312 | elapsed time per iteration (ms): 88517.0 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.39% | FLOPs estimator: workload-aware | logical tokens/s: 189536.3 | valid tokens/s: 189536.3 | physical tokens/s: 189910.4 | loss tokens/s: 135130.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.536247E-01 | lm-global-supervised-tokens: 1.196132E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.157536E-02 | lm-token-buffer-mib: 5.826640E+01 | global_load_balancing_loss: 1.000609E+00 | loss scale: 1.0 | grad norm: 0.101 | params norm: 11463.684 | params rms: 0.062072 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 5:27:00.944108 | finish at 2026-10-10 17:46:22 + [2026-10-06 12:21:10] iteration 2730/ 6855 | consumed samples: 349440 | elapsed time per iteration (ms): 108623.1 | throughput per GPU (TFLOP/s/GPU): 33.4 | MFU 10.71% | FLOPs estimator: workload-aware | logical tokens/s: 154453.2 | valid tokens/s: 154453.2 | physical tokens/s: 154756.9 | loss tokens/s: 115691.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.385480E-01 | lm-global-supervised-tokens: 1.256674E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.152985E-02 | lm-token-buffer-mib: 5.808052E+01 | global_load_balancing_loss: 1.000997E+00 | loss scale: 1.0 | grad norm: 0.073 | params norm: 11463.613 | params rms: 0.062072 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 4:27:50.312262 | finish at 2026-10-11 16:49:00 + [2026-10-06 12:22:33] iteration 2731/ 6855 | consumed samples: 349568 | elapsed time per iteration (ms): 83156.5 | throughput per GPU (TFLOP/s/GPU): 43.7 | MFU 14.00% | FLOPs estimator: workload-aware | logical tokens/s: 201754.2 | valid tokens/s: 201754.2 | physical tokens/s: 202227.0 | loss tokens/s: 155238.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.838655E-01 | lm-global-supervised-tokens: 1.290907E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.637069E-02 | lm-token-buffer-mib: 5.804266E+01 | global_load_balancing_loss: 1.000131E+00 | loss scale: 1.0 | grad norm: 0.086 | params norm: 11463.544 | params rms: 0.062071 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 23:15:37.537188 | finish at 2026-10-10 11:38:10 + [2026-10-06 12:23:57] iteration 2732/ 6855 | consumed samples: 349696 | elapsed time per iteration (ms): 84570.1 | throughput per GPU (TFLOP/s/GPU): 43.2 | MFU 13.85% | FLOPs estimator: workload-aware | logical tokens/s: 198380.9 | valid tokens/s: 198380.9 | physical tokens/s: 198951.2 | loss tokens/s: 141961.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.466378E-01 | lm-global-supervised-tokens: 1.200570E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.401003E-02 | lm-token-buffer-mib: 5.810075E+01 | global_load_balancing_loss: 1.000819E+00 | loss scale: 1.0 | grad norm: 0.088 | params norm: 11463.475 | params rms: 0.062071 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 0:51:22.715253 | finish at 2026-10-10 13:15:20 + [2026-10-06 12:25:22] iteration 2733/ 6855 | consumed samples: 349824 | elapsed time per iteration (ms): 84854.5 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.65% | FLOPs estimator: workload-aware | logical tokens/s: 197716.4 | valid tokens/s: 197716.4 | physical tokens/s: 198304.4 | loss tokens/s: 140415.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 5.728860E-01 | lm-global-supervised-tokens: 1.191492E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.449868E-02 | lm-token-buffer-mib: 5.814114E+01 | global_load_balancing_loss: 1.000501E+00 | loss scale: 1.0 | grad norm: 0.055 | params norm: 11463.407 | params rms: 0.062071 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 1:09:30.400573 | finish at 2026-10-10 13:34:53 + [2026-10-06 12:26:51] iteration 2734/ 6855 | consumed samples: 349952 | elapsed time per iteration (ms): 88575.8 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.40% | FLOPs estimator: workload-aware | logical tokens/s: 189410.3 | valid tokens/s: 189410.3 | physical tokens/s: 189809.7 | loss tokens/s: 140673.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.787985E-01 | lm-global-supervised-tokens: 1.246023E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.170496E-02 | lm-token-buffer-mib: 5.812914E+01 | global_load_balancing_loss: 1.000775E+00 | loss scale: 1.0 | grad norm: 0.162 | params norm: 11463.337 | params rms: 0.062070 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 5:23:40.758571 | finish at 2026-10-10 17:50:31 + [2026-10-06 12:28:17] iteration 2735/ 6855 | consumed samples: 350080 | elapsed time per iteration (ms): 86555.8 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.57% | FLOPs estimator: workload-aware | logical tokens/s: 193830.2 | valid tokens/s: 193830.1 | physical tokens/s: 194307.6 | loss tokens/s: 136233.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 5.910894E-01 | lm-global-supervised-tokens: 1.179182E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.330446E-02 | lm-token-buffer-mib: 5.820641E+01 | global_load_balancing_loss: 1.001336E+00 | loss scale: 1.0 | grad norm: 0.114 | params norm: 11463.268 | params rms: 0.062070 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 3:03:29.820204 | finish at 2026-10-10 15:31:47 + [2026-10-06 12:29:43] iteration 2736/ 6855 | consumed samples: 350208 | elapsed time per iteration (ms): 86142.7 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.68% | FLOPs estimator: workload-aware | logical tokens/s: 194760.8 | valid tokens/s: 194760.8 | physical tokens/s: 195182.8 | loss tokens/s: 147396.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.842903E-01 | lm-global-supervised-tokens: 1.269710E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.136876E-02 | lm-token-buffer-mib: 5.817584E+01 | global_load_balancing_loss: 9.990906E-01 | loss scale: 1.0 | grad norm: 0.031 | params norm: 11463.201 | params rms: 0.062070 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 2:33:41.613193 | finish at 2026-10-10 15:03:25 + [2026-10-06 12:31:12] iteration 2737/ 6855 | consumed samples: 350336 | elapsed time per iteration (ms): 88424.5 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.40% | FLOPs estimator: workload-aware | logical tokens/s: 189734.3 | valid tokens/s: 189734.3 | physical tokens/s: 190073.4 | loss tokens/s: 140878.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.175608E-01 | lm-global-supervised-tokens: 1.245708E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.256813E-02 | lm-token-buffer-mib: 5.807949E+01 | global_load_balancing_loss: 9.993825E-01 | loss scale: 1.0 | grad norm: 0.054 | params norm: 11463.135 | params rms: 0.062069 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 5:08:52.177352 | finish at 2026-10-10 17:40:04 + [2026-10-06 12:32:41] iteration 2738/ 6855 | consumed samples: 350464 | elapsed time per iteration (ms): 88855.3 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.45% | FLOPs estimator: workload-aware | logical tokens/s: 188814.9 | valid tokens/s: 188814.9 | physical tokens/s: 189442.8 | loss tokens/s: 143135.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.883283E-01 | lm-global-supervised-tokens: 1.271832E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.774793E-02 | lm-token-buffer-mib: 5.808620E+01 | global_load_balancing_loss: 9.983675E-01 | loss scale: 1.0 | grad norm: 0.086 | params norm: 11463.065 | params rms: 0.062069 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 5:36:57.423999 | finish at 2026-10-10 18:09:38 + [2026-10-06 12:34:08] iteration 2739/ 6855 | consumed samples: 350592 | elapsed time per iteration (ms): 87057.1 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.49% | FLOPs estimator: workload-aware | logical tokens/s: 192714.2 | valid tokens/s: 192714.2 | physical tokens/s: 193282.5 | loss tokens/s: 139304.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 5.914221E-01 | lm-global-supervised-tokens: 1.212743E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.709836E-02 | lm-token-buffer-mib: 5.817207E+01 | global_load_balancing_loss: 1.000206E+00 | loss scale: 1.0 | grad norm: 0.210 | params norm: 11462.994 | params rms: 0.062068 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 3:32:06.961032 | finish at 2026-10-10 16:06:15 + [2026-10-06 12:35:55] iteration 2740/ 6855 | consumed samples: 350720 | elapsed time per iteration (ms): 107263.2 | throughput per GPU (TFLOP/s/GPU): 34.9 | MFU 11.18% | FLOPs estimator: workload-aware | logical tokens/s: 156411.7 | valid tokens/s: 156411.7 | physical tokens/s: 156787.3 | loss tokens/s: 110078.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.852151E-01 | lm-global-supervised-tokens: 1.180742E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.467773E-02 | lm-token-buffer-mib: 5.823295E+01 | global_load_balancing_loss: 9.998184E-01 | loss scale: 1.0 | grad norm: 0.507 | params norm: 11462.926 | params rms: 0.062068 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 2:36:27.980866 | finish at 2026-10-11 15:12:23 + [2026-10-06 12:37:21] iteration 2741/ 6855 | consumed samples: 350848 | elapsed time per iteration (ms): 86139.3 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.56% | FLOPs estimator: workload-aware | logical tokens/s: 194768.0 | valid tokens/s: 194768.0 | physical tokens/s: 195169.3 | loss tokens/s: 146889.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.578902E-01 | lm-global-supervised-tokens: 1.265298E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.424192E-02 | lm-token-buffer-mib: 5.814084E+01 | global_load_balancing_loss: 1.000581E+00 | loss scale: 1.0 | grad norm: 0.310 | params norm: 11462.858 | params rms: 0.062068 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 2:26:17.089472 | finish at 2026-10-10 15:03:38 + [2026-10-06 12:38:46] iteration 2742/ 6855 | consumed samples: 350976 | elapsed time per iteration (ms): 85016.2 | throughput per GPU (TFLOP/s/GPU): 43.5 | MFU 13.94% | FLOPs estimator: workload-aware | logical tokens/s: 197340.6 | valid tokens/s: 197340.6 | physical tokens/s: 197687.4 | loss tokens/s: 147300.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.589355E-01 | lm-global-supervised-tokens: 1.252292E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.459235E-02 | lm-token-buffer-mib: 5.818666E+01 | global_load_balancing_loss: 1.001673E+00 | loss scale: 1.0 | grad norm: 0.294 | params norm: 11462.790 | params rms: 0.062067 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 1:07:51.472991 | finish at 2026-10-10 13:46:38 + [2026-10-06 12:40:15] iteration 2743/ 6855 | consumed samples: 351104 | elapsed time per iteration (ms): 88494.2 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.45% | FLOPs estimator: workload-aware | logical tokens/s: 189584.2 | valid tokens/s: 189584.2 | physical tokens/s: 190103.7 | loss tokens/s: 139555.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.581040E-01 | lm-global-supervised-tokens: 1.234982E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.319368E-02 | lm-token-buffer-mib: 5.806578E+01 | global_load_balancing_loss: 1.001528E+00 | loss scale: 1.0 | grad norm: 0.288 | params norm: 11462.724 | params rms: 0.062067 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 5:04:48.058208 | finish at 2026-10-10 17:45:03 + [2026-10-06 12:41:43] iteration 2744/ 6855 | consumed samples: 351232 | elapsed time per iteration (ms): 88674.0 | throughput per GPU (TFLOP/s/GPU): 41.3 | MFU 13.25% | FLOPs estimator: workload-aware | logical tokens/s: 189201.1 | valid tokens/s: 189201.1 | physical tokens/s: 189751.8 | loss tokens/s: 143184.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.193250E-01 | lm-global-supervised-tokens: 1.269673E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.590350E-02 | lm-token-buffer-mib: 5.808418E+01 | global_load_balancing_loss: 1.000677E+00 | loss scale: 1.0 | grad norm: 0.231 | params norm: 11462.655 | params rms: 0.062067 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 5:15:38.735691 | finish at 2026-10-10 17:57:22 + [2026-10-06 12:43:12] iteration 2745/ 6855 | consumed samples: 351360 | elapsed time per iteration (ms): 88942.6 | throughput per GPU (TFLOP/s/GPU): 41.4 | MFU 13.25% | FLOPs estimator: workload-aware | logical tokens/s: 188629.6 | valid tokens/s: 188629.6 | physical tokens/s: 189243.1 | loss tokens/s: 134213.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.505381E-01 | lm-global-supervised-tokens: 1.193731E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.440575E-02 | lm-token-buffer-mib: 5.805256E+01 | global_load_balancing_loss: 1.000351E+00 | loss scale: 1.0 | grad norm: 0.109 | params norm: 11462.587 | params rms: 0.062066 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 5:32:34.234014 | finish at 2026-10-10 18:15:46 + [2026-10-06 12:44:37] iteration 2746/ 6855 | consumed samples: 351488 | elapsed time per iteration (ms): 84743.1 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.74% | FLOPs estimator: workload-aware | logical tokens/s: 197977.5 | valid tokens/s: 197977.5 | physical tokens/s: 198411.4 | loss tokens/s: 146823.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.649021E-01 | lm-global-supervised-tokens: 1.244230E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.419143E-02 | lm-token-buffer-mib: 5.819979E+01 | global_load_balancing_loss: 1.000239E+00 | loss scale: 1.0 | grad norm: 0.284 | params norm: 11462.518 | params rms: 0.062066 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 0:43:29.203631 | finish at 2026-10-10 13:28:06 + [2026-10-06 12:46:05] iteration 2747/ 6855 | consumed samples: 351616 | elapsed time per iteration (ms): 88207.8 | throughput per GPU (TFLOP/s/GPU): 41.7 | MFU 13.35% | FLOPs estimator: workload-aware | logical tokens/s: 190201.1 | valid tokens/s: 190201.1 | physical tokens/s: 190731.7 | loss tokens/s: 137703.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.364622E-01 | lm-global-supervised-tokens: 1.214647E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.233562E-02 | lm-token-buffer-mib: 5.810908E+01 | global_load_balancing_loss: 1.000264E+00 | loss scale: 1.0 | grad norm: 0.172 | params norm: 11462.452 | params rms: 0.062065 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 4:39:17.439296 | finish at 2026-10-10 17:25:23 + [2026-10-06 12:47:35] iteration 2748/ 6855 | consumed samples: 351744 | elapsed time per iteration (ms): 89401.6 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.57% | FLOPs estimator: workload-aware | logical tokens/s: 187660.9 | valid tokens/s: 187660.9 | physical tokens/s: 188017.2 | loss tokens/s: 130843.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.567122E-01 | lm-global-supervised-tokens: 1.169765E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.155019E-02 | lm-token-buffer-mib: 5.807564E+01 | global_load_balancing_loss: 9.992942E-01 | loss scale: 1.0 | grad norm: 0.147 | params norm: 11462.388 | params rms: 0.062065 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 5:59:32.349182 | finish at 2026-10-10 18:47:07 + [2026-10-06 12:49:01] iteration 2749/ 6855 | consumed samples: 351872 | elapsed time per iteration (ms): 86791.7 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.61% | FLOPs estimator: workload-aware | logical tokens/s: 193303.8 | valid tokens/s: 193303.8 | physical tokens/s: 193663.4 | loss tokens/s: 147469.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.915382E-01 | lm-global-supervised-tokens: 1.279914E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.137591E-02 | lm-token-buffer-mib: 5.804025E+01 | global_load_balancing_loss: 9.985737E-01 | loss scale: 1.0 | grad norm: 0.177 | params norm: 11462.325 | params rms: 0.062065 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 2:59:26.569954 | finish at 2026-10-10 15:48:28 + [2026-10-06 12:50:49] iteration 2750/ 6855 | consumed samples: 352000 | elapsed time per iteration (ms): 107777.1 | throughput per GPU (TFLOP/s/GPU): 34.4 | MFU 11.02% | FLOPs estimator: workload-aware | logical tokens/s: 155664.6 | valid tokens/s: 155664.6 | physical tokens/s: 156003.2 | loss tokens/s: 109242.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.323543E-01 | lm-global-supervised-tokens: 1.177382E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.334979E-02 | lm-token-buffer-mib: 5.803863E+01 | global_load_balancing_loss: 9.994128E-01 | loss scale: 1.0 | grad norm: 0.211 | params norm: 11462.263 | params rms: 0.062064 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 2:53:44.993896 | finish at 2026-10-11 15:44:34 + [2026-10-06 12:52:15] iteration 2751/ 6855 | consumed samples: 352128 | elapsed time per iteration (ms): 85949.7 | throughput per GPU (TFLOP/s/GPU): 43.2 | MFU 13.83% | FLOPs estimator: workload-aware | logical tokens/s: 195197.2 | valid tokens/s: 195197.2 | physical tokens/s: 195712.0 | loss tokens/s: 149021.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.601635E-01 | lm-global-supervised-tokens: 1.280836E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.412295E-02 | lm-token-buffer-mib: 5.813259E+01 | global_load_balancing_loss: 9.985740E-01 | loss scale: 1.0 | grad norm: 0.251 | params norm: 11462.201 | params rms: 0.062064 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 1:58:57.586893 | finish at 2026-10-10 14:51:13 + [2026-10-06 12:53:41] iteration 2752/ 6855 | consumed samples: 352256 | elapsed time per iteration (ms): 85559.5 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.55% | FLOPs estimator: workload-aware | logical tokens/s: 196088.1 | valid tokens/s: 196088.1 | physical tokens/s: 196585.5 | loss tokens/s: 144392.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.658616E-01 | lm-global-supervised-tokens: 1.235414E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.579084E-02 | lm-token-buffer-mib: 5.811629E+01 | global_load_balancing_loss: 1.001436E+00 | loss scale: 1.0 | grad norm: 0.189 | params norm: 11462.140 | params rms: 0.062064 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 1:30:50.708629 | finish at 2026-10-10 14:24:31 + [2026-10-06 12:55:06] iteration 2753/ 6855 | consumed samples: 352384 | elapsed time per iteration (ms): 85412.0 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.66% | FLOPs estimator: workload-aware | logical tokens/s: 196426.7 | valid tokens/s: 196426.7 | physical tokens/s: 196867.2 | loss tokens/s: 145680.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.105447E-01 | lm-global-supervised-tokens: 1.244281E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.136295E-02 | lm-token-buffer-mib: 5.812348E+01 | global_load_balancing_loss: 1.001069E+00 | loss scale: 1.0 | grad norm: 0.146 | params norm: 11462.079 | params rms: 0.062063 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 1:19:19.851631 | finish at 2026-10-10 14:14:26 + [2026-10-06 12:56:31] iteration 2754/ 6855 | consumed samples: 352512 | elapsed time per iteration (ms): 84854.7 | throughput per GPU (TFLOP/s/GPU): 43.2 | MFU 13.85% | FLOPs estimator: workload-aware | logical tokens/s: 197716.1 | valid tokens/s: 197716.1 | physical tokens/s: 198253.3 | loss tokens/s: 152903.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.908548E-01 | lm-global-supervised-tokens: 1.297455E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.274162E-02 | lm-token-buffer-mib: 5.821455E+01 | global_load_balancing_loss: 1.000724E+00 | loss scale: 1.0 | grad norm: 0.187 | params norm: 11462.017 | params rms: 0.062063 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 0:39:49.020443 | finish at 2026-10-10 13:36:20 + [2026-10-06 12:57:56] iteration 2755/ 6855 | consumed samples: 352640 | elapsed time per iteration (ms): 85528.1 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.68% | FLOPs estimator: workload-aware | logical tokens/s: 196159.9 | valid tokens/s: 196159.9 | physical tokens/s: 196757.5 | loss tokens/s: 144880.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.128241E-01 | lm-global-supervised-tokens: 1.239134E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.145245E-02 | lm-token-buffer-mib: 5.837414E+01 | global_load_balancing_loss: 1.001204E+00 | loss scale: 1.0 | grad norm: 0.156 | params norm: 11461.955 | params rms: 0.062063 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 1:24:25.131855 | finish at 2026-10-10 14:22:22 + [2026-10-06 12:59:27] iteration 2756/ 6855 | consumed samples: 352768 | elapsed time per iteration (ms): 90031.6 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.64% | FLOPs estimator: workload-aware | logical tokens/s: 186347.2 | valid tokens/s: 186347.2 | physical tokens/s: 186695.2 | loss tokens/s: 128053.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.650079E-01 | lm-global-supervised-tokens: 1.152890E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.149720E-02 | lm-token-buffer-mib: 5.810255E+01 | global_load_balancing_loss: 1.001305E+00 | loss scale: 1.0 | grad norm: 0.123 | params norm: 11461.893 | params rms: 0.062062 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 6:30:39.717986 | finish at 2026-10-10 19:30:06 + [2026-10-06 13:00:52] iteration 2757/ 6855 | consumed samples: 352896 | elapsed time per iteration (ms): 85505.3 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.78% | FLOPs estimator: workload-aware | logical tokens/s: 196211.0 | valid tokens/s: 196211.0 | physical tokens/s: 196938.4 | loss tokens/s: 150361.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.733291E-01 | lm-global-supervised-tokens: 1.285673E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.170058E-02 | lm-token-buffer-mib: 5.818015E+01 | global_load_balancing_loss: 9.997064E-01 | loss scale: 1.0 | grad norm: 0.240 | params norm: 11461.830 | params rms: 0.062062 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 1:20:00.804587 | finish at 2026-10-10 14:20:53 + [2026-10-06 13:02:20] iteration 2758/ 6855 | consumed samples: 353024 | elapsed time per iteration (ms): 88074.1 | throughput per GPU (TFLOP/s/GPU): 41.6 | MFU 13.33% | FLOPs estimator: workload-aware | logical tokens/s: 190489.4 | valid tokens/s: 190489.4 | physical tokens/s: 190996.8 | loss tokens/s: 143265.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.677883E-01 | lm-global-supervised-tokens: 1.261794E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.671452E-02 | lm-token-buffer-mib: 5.834271E+01 | global_load_balancing_loss: 1.000505E+00 | loss scale: 1.0 | grad norm: 0.120 | params norm: 11461.767 | params rms: 0.062062 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 4:13:59.575073 | finish at 2026-10-10 17:16:20 + [2026-10-06 13:03:49] iteration 2759/ 6855 | consumed samples: 353152 | elapsed time per iteration (ms): 89157.8 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.45% | FLOPs estimator: workload-aware | logical tokens/s: 188173.9 | valid tokens/s: 188173.9 | physical tokens/s: 188643.8 | loss tokens/s: 137918.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.197977E-01 | lm-global-supervised-tokens: 1.229650E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.293077E-02 | lm-token-buffer-mib: 5.804019E+01 | global_load_balancing_loss: 9.994407E-01 | loss scale: 1.0 | grad norm: 0.106 | params norm: 11461.703 | params rms: 0.062061 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 5:26:30.470703 | finish at 2026-10-10 18:30:20 + [2026-10-06 13:05:37] iteration 2760/ 6855 | consumed samples: 353280 | elapsed time per iteration (ms): 107305.8 | throughput per GPU (TFLOP/s/GPU): 34.5 | MFU 11.04% | FLOPs estimator: workload-aware | logical tokens/s: 156349.0 | valid tokens/s: 156349.0 | physical tokens/s: 156752.2 | loss tokens/s: 113065.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.382810E-01 | lm-global-supervised-tokens: 1.213252E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.676325E-02 | lm-token-buffer-mib: 5.808817E+01 | global_load_balancing_loss: 1.000128E+00 | loss scale: 1.0 | grad norm: 0.311 | params norm: 11461.638 | params rms: 0.062061 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 2:03:37.108297 | finish at 2026-10-11 15:09:14 +times across ranks (ms): + save-checkpoint: + rank 0: 23806.86 + rank 1: 23806.86 + rank 2: 23806.88 + rank 3: 23806.89 + rank 4: 23806.91 + rank 5: 23806.85 + rank 6: 23806.87 + rank 7: 23806.87 + rank 8: 23806.33 + rank 9: 23806.29 + rank 10: 23806.30 + rank 11: 23806.31 + rank 12: 23806.31 + rank 13: 23806.33 + rank 14: 23806.31 + rank 15: 23806.30 + rank 16: 23808.21 + rank 17: 23808.19 + rank 18: 23808.18 + rank 19: 23808.20 + rank 20: 23808.19 + rank 21: 23808.22 + rank 22: 23808.20 + rank 23: 23808.19 + rank 24: 23807.63 + rank 25: 23807.61 + rank 26: 23807.62 + rank 27: 23807.64 + rank 28: 23807.59 + rank 29: 23807.72 + rank 30: 23807.60 + rank 31: 23807.70 + rank 32: 23807.00 + rank 33: 23807.01 + rank 34: 23806.99 + rank 35: 23806.96 + rank 36: 23806.95 + rank 37: 23806.96 + rank 38: 23806.98 + rank 39: 23806.99 + rank 40: 23807.19 + rank 41: 23807.20 + rank 42: 23807.18 + rank 43: 23807.21 + rank 44: 23807.20 + rank 45: 23807.18 + rank 46: 23807.17 + rank 47: 23807.17 + rank 48: 23806.88 + rank 49: 23806.86 + rank 50: 23806.86 + rank 51: 23806.86 + rank 52: 23806.85 + rank 53: 23806.85 + rank 54: 23806.86 + rank 55: 23806.86 + rank 56: 23806.71 + rank 57: 23806.69 + rank 58: 23806.70 + rank 59: 23806.69 + rank 60: 23806.69 + rank 61: 23806.69 + rank 62: 23806.69 + rank 63: 23806.70 + [2026-10-06 13:07:24] iteration 2761/ 6855 | consumed samples: 353408 | elapsed time per iteration (ms): 82639.6 | throughput per GPU (TFLOP/s/GPU): 44.3 | MFU 14.21% | FLOPs estimator: workload-aware | logical tokens/s: 203016.3 | valid tokens/s: 203016.2 | physical tokens/s: 203741.2 | loss tokens/s: 148753.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.671453E-01 | lm-global-supervised-tokens: 1.229289E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 5.642865E-01 | lm-token-buffer-mib: 5.803277E+01 | global_load_balancing_loss: 9.991871E-01 | loss scale: 1.0 | grad norm: 0.064 | params norm: 11461.574 | params rms: 0.062061 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 21:58:46.502060 | finish at 2026-10-10 11:06:10 + [2026-10-06 13:08:55] iteration 2762/ 6855 | consumed samples: 353536 | elapsed time per iteration (ms): 91501.6 | throughput per GPU (TFLOP/s/GPU): 40.2 | MFU 12.88% | FLOPs estimator: workload-aware | logical tokens/s: 183354.2 | valid tokens/s: 183354.1 | physical tokens/s: 183882.6 | loss tokens/s: 127676.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.126703E-01 | lm-global-supervised-tokens: 1.168260E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.343090E-02 | lm-token-buffer-mib: 5.836998E+01 | global_load_balancing_loss: 9.993010E-01 | loss scale: 1.0 | grad norm: 0.141 | params norm: 11461.510 | params rms: 0.062060 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 8:01:56.102582 | finish at 2026-10-10 21:10:51 + [2026-10-06 13:10:21] iteration 2763/ 6855 | consumed samples: 353664 | elapsed time per iteration (ms): 85865.7 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.77% | FLOPs estimator: workload-aware | logical tokens/s: 195388.5 | valid tokens/s: 195388.5 | physical tokens/s: 195857.4 | loss tokens/s: 144858.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.759003E-01 | lm-global-supervised-tokens: 1.243839E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.155395E-02 | lm-token-buffer-mib: 5.810152E+01 | global_load_balancing_loss: 1.000143E+00 | loss scale: 1.0 | grad norm: 0.230 | params norm: 11461.445 | params rms: 0.062060 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 1:36:02.621012 | finish at 2026-10-10 14:46:24 + [2026-10-06 13:11:45] iteration 2764/ 6855 | consumed samples: 353792 | elapsed time per iteration (ms): 83729.3 | throughput per GPU (TFLOP/s/GPU): 43.4 | MFU 13.90% | FLOPs estimator: workload-aware | logical tokens/s: 200373.6 | valid tokens/s: 200373.6 | physical tokens/s: 200980.7 | loss tokens/s: 145117.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.710317E-01 | lm-global-supervised-tokens: 1.215063E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.157558E-02 | lm-token-buffer-mib: 5.814104E+01 | global_load_balancing_loss: 1.001182E+00 | loss scale: 1.0 | grad norm: 0.121 | params norm: 11461.380 | params rms: 0.062060 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 23:08:56.752686 | finish at 2026-10-10 12:20:42 + [2026-10-06 13:13:09] iteration 2765/ 6855 | consumed samples: 353920 | elapsed time per iteration (ms): 83914.4 | throughput per GPU (TFLOP/s/GPU): 43.1 | MFU 13.81% | FLOPs estimator: workload-aware | logical tokens/s: 199931.7 | valid tokens/s: 199931.7 | physical tokens/s: 200319.1 | loss tokens/s: 153429.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.792017E-01 | lm-global-supervised-tokens: 1.287498E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.617210E-02 | lm-token-buffer-mib: 5.811682E+01 | global_load_balancing_loss: 1.000705E+00 | loss scale: 1.0 | grad norm: 0.137 | params norm: 11461.315 | params rms: 0.062059 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 23:20:10.030980 | finish at 2026-10-10 12:33:19 + [2026-10-06 13:14:37] iteration 2766/ 6855 | consumed samples: 354048 | elapsed time per iteration (ms): 87972.2 | throughput per GPU (TFLOP/s/GPU): 41.7 | MFU 13.36% | FLOPs estimator: workload-aware | logical tokens/s: 190710.4 | valid tokens/s: 190710.4 | physical tokens/s: 191133.6 | loss tokens/s: 138171.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.413150E-01 | lm-global-supervised-tokens: 1.215523E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.213117E-02 | lm-token-buffer-mib: 5.807719E+01 | global_load_balancing_loss: 1.001152E+00 | loss scale: 1.0 | grad norm: 0.063 | params norm: 11461.249 | params rms: 0.062059 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 3:55:18.369330 | finish at 2026-10-10 17:09:55 + [2026-10-06 13:16:05] iteration 2767/ 6855 | consumed samples: 354176 | elapsed time per iteration (ms): 87800.0 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.43% | FLOPs estimator: workload-aware | logical tokens/s: 191084.5 | valid tokens/s: 191084.5 | physical tokens/s: 191649.8 | loss tokens/s: 137475.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.428433E-01 | lm-global-supervised-tokens: 1.207031E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.496139E-02 | lm-token-buffer-mib: 5.816804E+01 | global_load_balancing_loss: 1.000915E+00 | loss scale: 1.0 | grad norm: 0.157 | params norm: 11461.182 | params rms: 0.062059 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 3:42:06.312086 | finish at 2026-10-10 16:58:11 + [2026-10-06 13:17:30] iteration 2768/ 6855 | consumed samples: 354304 | elapsed time per iteration (ms): 85036.2 | throughput per GPU (TFLOP/s/GPU): 43.6 | MFU 13.98% | FLOPs estimator: workload-aware | logical tokens/s: 197294.5 | valid tokens/s: 197294.5 | physical tokens/s: 197676.8 | loss tokens/s: 136498.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.530079E-01 | lm-global-supervised-tokens: 1.160728E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.472227E-02 | lm-token-buffer-mib: 5.815002E+01 | global_load_balancing_loss: 9.999508E-01 | loss scale: 1.0 | grad norm: 0.037 | params norm: 11461.114 | params rms: 0.062058 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 0:32:22.829737 | finish at 2026-10-10 13:49:52 + [2026-10-06 13:18:57] iteration 2769/ 6855 | consumed samples: 354432 | elapsed time per iteration (ms): 87710.7 | throughput per GPU (TFLOP/s/GPU): 42.8 | MFU 13.72% | FLOPs estimator: workload-aware | logical tokens/s: 191279.0 | valid tokens/s: 191279.0 | physical tokens/s: 191656.9 | loss tokens/s: 142209.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.205360E-01 | lm-global-supervised-tokens: 1.247328E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.198461E-02 | lm-token-buffer-mib: 5.819039E+01 | global_load_balancing_loss: 9.986938E-01 | loss scale: 1.0 | grad norm: 0.103 | params norm: 11461.047 | params rms: 0.062058 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 3:33:05.726482 | finish at 2026-10-10 16:52:03 + [2026-10-06 13:20:46] iteration 2770/ 6855 | consumed samples: 354560 | elapsed time per iteration (ms): 108599.6 | throughput per GPU (TFLOP/s/GPU): 33.9 | MFU 10.86% | FLOPs estimator: workload-aware | logical tokens/s: 154486.6 | valid tokens/s: 154486.6 | physical tokens/s: 154887.7 | loss tokens/s: 110335.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.314927E-01 | lm-global-supervised-tokens: 1.198233E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.105260E-02 | lm-token-buffer-mib: 5.810597E+01 | global_load_balancing_loss: 9.997292E-01 | loss scale: 1.0 | grad norm: 0.067 | params norm: 11460.980 | params rms: 0.062057 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 3:13:49.193926 | finish at 2026-10-11 16:34:35 + [2026-10-06 13:22:14] iteration 2771/ 6855 | consumed samples: 354688 | elapsed time per iteration (ms): 87622.8 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.48% | FLOPs estimator: workload-aware | logical tokens/s: 191469.9 | valid tokens/s: 191469.9 | physical tokens/s: 191987.0 | loss tokens/s: 141250.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.955723E-01 | lm-global-supervised-tokens: 1.237679E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.405654E-02 | lm-token-buffer-mib: 5.817594E+01 | global_load_balancing_loss: 9.983582E-01 | loss scale: 1.0 | grad norm: 0.104 | params norm: 11460.912 | params rms: 0.062057 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 3:24:11.410497 | finish at 2026-10-10 16:46:25 + [2026-10-06 13:23:36] iteration 2772/ 6855 | consumed samples: 354816 | elapsed time per iteration (ms): 82784.3 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.74% | FLOPs estimator: workload-aware | logical tokens/s: 202660.9 | valid tokens/s: 202660.9 | physical tokens/s: 203070.9 | loss tokens/s: 150007.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 5.769316E-01 | lm-global-supervised-tokens: 1.241830E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.402126E-02 | lm-token-buffer-mib: 5.809903E+01 | global_load_balancing_loss: 9.993545E-01 | loss scale: 1.0 | grad norm: 0.123 | params norm: 11460.844 | params rms: 0.062057 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 21:53:28.399477 | finish at 2026-10-10 11:17:05 + [2026-10-06 13:25:05] iteration 2773/ 6855 | consumed samples: 354944 | elapsed time per iteration (ms): 89124.8 | throughput per GPU (TFLOP/s/GPU): 40.9 | MFU 13.11% | FLOPs estimator: workload-aware | logical tokens/s: 188243.6 | valid tokens/s: 188243.5 | physical tokens/s: 188738.3 | loss tokens/s: 135961.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.539928E-01 | lm-global-supervised-tokens: 1.211757E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.152646E-02 | lm-token-buffer-mib: 5.805637E+01 | global_load_balancing_loss: 1.000792E+00 | loss scale: 1.0 | grad norm: 0.065 | params norm: 11460.776 | params rms: 0.062056 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 5:03:27.361446 | finish at 2026-10-10 18:28:33 + [2026-10-06 13:26:37] iteration 2774/ 6855 | consumed samples: 355072 | elapsed time per iteration (ms): 91532.6 | throughput per GPU (TFLOP/s/GPU): 41.1 | MFU 13.16% | FLOPs estimator: workload-aware | logical tokens/s: 183292.3 | valid tokens/s: 183292.0 | physical tokens/s: 183693.1 | loss tokens/s: 129553.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.544285E-01 | lm-global-supervised-tokens: 1.185834E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.362531E-02 | lm-token-buffer-mib: 5.820729E+01 | global_load_balancing_loss: 1.002633E+00 | loss scale: 1.0 | grad norm: 0.108 | params norm: 11460.707 | params rms: 0.062056 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 7:45:44.512081 | finish at 2026-10-10 21:12:21 + [2026-10-06 13:28:03] iteration 2775/ 6855 | consumed samples: 355200 | elapsed time per iteration (ms): 85731.6 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.69% | FLOPs estimator: workload-aware | logical tokens/s: 195694.2 | valid tokens/s: 195694.2 | physical tokens/s: 196070.4 | loss tokens/s: 142844.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.510167E-01 | lm-global-supervised-tokens: 1.224627E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.315233E-02 | lm-token-buffer-mib: 5.811987E+01 | global_load_balancing_loss: 1.001387E+00 | loss scale: 1.0 | grad norm: 0.058 | params norm: 11460.638 | params rms: 0.062056 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 1:09:45.094528 | finish at 2026-10-10 14:37:48 + [2026-10-06 13:29:29] iteration 2776/ 6855 | consumed samples: 355328 | elapsed time per iteration (ms): 86442.4 | throughput per GPU (TFLOP/s/GPU): 41.6 | MFU 13.34% | FLOPs estimator: workload-aware | logical tokens/s: 194085.5 | valid tokens/s: 194085.5 | physical tokens/s: 194477.6 | loss tokens/s: 142778.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.158873E-01 | lm-global-supervised-tokens: 1.234208E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.146443E-02 | lm-token-buffer-mib: 5.803121E+01 | global_load_balancing_loss: 1.001735E+00 | loss scale: 1.0 | grad norm: 0.052 | params norm: 11460.569 | params rms: 0.062055 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 1:56:38.406554 | finish at 2026-10-10 15:26:08 + [2026-10-06 13:30:56] iteration 2777/ 6855 | consumed samples: 355456 | elapsed time per iteration (ms): 87306.6 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.43% | FLOPs estimator: workload-aware | logical tokens/s: 192164.2 | valid tokens/s: 192164.2 | physical tokens/s: 192639.3 | loss tokens/s: 142376.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.130536E-01 | lm-global-supervised-tokens: 1.243042E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.745727E-02 | lm-token-buffer-mib: 5.825298E+01 | global_load_balancing_loss: 1.000499E+00 | loss scale: 1.0 | grad norm: 0.132 | params norm: 11460.497 | params rms: 0.062055 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 2:53:56.474635 | finish at 2026-10-10 16:24:53 + [2026-10-06 13:32:17] iteration 2778/ 6855 | consumed samples: 355584 | elapsed time per iteration (ms): 80711.7 | throughput per GPU (TFLOP/s/GPU): 43.7 | MFU 14.01% | FLOPs estimator: workload-aware | logical tokens/s: 207865.7 | valid tokens/s: 207865.7 | physical tokens/s: 208555.7 | loss tokens/s: 161391.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.354876E-01 | lm-global-supervised-tokens: 1.302621E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.144247E-02 | lm-token-buffer-mib: 5.807942E+01 | global_load_balancing_loss: 9.982044E-01 | loss scale: 1.0 | grad norm: 0.089 | params norm: 11460.422 | params rms: 0.062054 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 19:24:21.447166 | finish at 2026-10-10 08:56:39 + [2026-10-06 13:33:44] iteration 2779/ 6855 | consumed samples: 355712 | elapsed time per iteration (ms): 86759.7 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.39% | FLOPs estimator: workload-aware | logical tokens/s: 193375.7 | valid tokens/s: 193375.7 | physical tokens/s: 193916.8 | loss tokens/s: 145822.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.818441E-01 | lm-global-supervised-tokens: 1.265151E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.156806E-02 | lm-token-buffer-mib: 5.811969E+01 | global_load_balancing_loss: 9.985992E-01 | loss scale: 1.0 | grad norm: 0.054 | params norm: 11460.352 | params rms: 0.062054 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 2:13:52.449011 | finish at 2026-10-10 15:47:36 + [2026-10-06 13:35:33] iteration 2780/ 6855 | consumed samples: 355840 | elapsed time per iteration (ms): 108742.9 | throughput per GPU (TFLOP/s/GPU): 34.9 | MFU 11.20% | FLOPs estimator: workload-aware | logical tokens/s: 154283.1 | valid tokens/s: 154283.1 | physical tokens/s: 154565.6 | loss tokens/s: 116273.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.334941E-01 | lm-global-supervised-tokens: 1.264392E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.348298E-02 | lm-token-buffer-mib: 5.804490E+01 | global_load_balancing_loss: 9.995007E-01 | loss scale: 1.0 | grad norm: 0.045 | params norm: 11460.281 | params rms: 0.062054 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 3:05:27.169394 | finish at 2026-10-11 16:41:00 + [2026-10-06 13:36:58] iteration 2781/ 6855 | consumed samples: 355968 | elapsed time per iteration (ms): 84983.7 | throughput per GPU (TFLOP/s/GPU): 41.6 | MFU 13.34% | FLOPs estimator: workload-aware | logical tokens/s: 197416.0 | valid tokens/s: 197416.0 | physical tokens/s: 198069.3 | loss tokens/s: 148986.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.717516E-01 | lm-global-supervised-tokens: 1.266142E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.259753E-02 | lm-token-buffer-mib: 5.822853E+01 | global_load_balancing_loss: 1.000107E+00 | loss scale: 1.0 | grad norm: 0.073 | params norm: 11460.210 | params rms: 0.062053 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 0:10:23.422999 | finish at 2026-10-10 13:47:21 + [2026-10-06 13:38:27] iteration 2782/ 6855 | consumed samples: 356096 | elapsed time per iteration (ms): 89508.2 | throughput per GPU (TFLOP/s/GPU): 43.1 | MFU 13.80% | FLOPs estimator: workload-aware | logical tokens/s: 187437.7 | valid tokens/s: 187437.7 | physical tokens/s: 187796.2 | loss tokens/s: 132949.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.022473E-01 | lm-global-supervised-tokens: 1.190002E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.138804E-02 | lm-token-buffer-mib: 5.817001E+01 | global_load_balancing_loss: 1.000466E+00 | loss scale: 1.0 | grad norm: 0.084 | params norm: 11460.141 | params rms: 0.062053 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 5:16:06.722550 | finish at 2026-10-10 18:54:34 + [2026-10-06 13:39:56] iteration 2783/ 6855 | consumed samples: 356224 | elapsed time per iteration (ms): 88991.5 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.57% | FLOPs estimator: workload-aware | logical tokens/s: 188525.4 | valid tokens/s: 188525.4 | physical tokens/s: 188876.0 | loss tokens/s: 130912.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.394724E-01 | lm-global-supervised-tokens: 1.165005E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.647464E-02 | lm-token-buffer-mib: 5.806981E+01 | global_load_balancing_loss: 1.000515E+00 | loss scale: 1.0 | grad norm: 0.050 | params norm: 11460.074 | params rms: 0.062053 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 4:39:33.218670 | finish at 2026-10-10 18:19:29 + [2026-10-06 13:41:26] iteration 2784/ 6855 | consumed samples: 356352 | elapsed time per iteration (ms): 89785.7 | throughput per GPU (TFLOP/s/GPU): 42.8 | MFU 13.72% | FLOPs estimator: workload-aware | logical tokens/s: 186857.7 | valid tokens/s: 186857.6 | physical tokens/s: 187230.4 | loss tokens/s: 134764.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.430768E-01 | lm-global-supervised-tokens: 1.209989E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.562725E-02 | lm-token-buffer-mib: 5.812875E+01 | global_load_balancing_loss: 1.001034E+00 | loss scale: 1.0 | grad norm: 0.081 | params norm: 11460.007 | params rms: 0.062052 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 5:31:57.454006 | finish at 2026-10-10 19:13:23 + [2026-10-06 13:42:51] iteration 2785/ 6855 | consumed samples: 356480 | elapsed time per iteration (ms): 85355.1 | throughput per GPU (TFLOP/s/GPU): 43.9 | MFU 14.06% | FLOPs estimator: workload-aware | logical tokens/s: 196558.0 | valid tokens/s: 196557.9 | physical tokens/s: 197029.4 | loss tokens/s: 144440.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.659932E-01 | lm-global-supervised-tokens: 1.232869E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.262014E-02 | lm-token-buffer-mib: 5.809958E+01 | global_load_balancing_loss: 1.000632E+00 | loss scale: 1.0 | grad norm: 0.034 | params norm: 11459.939 | params rms: 0.062052 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 0:29:55.063558 | finish at 2026-10-10 14:12:46 + [2026-10-06 13:44:17] iteration 2786/ 6855 | consumed samples: 356608 | elapsed time per iteration (ms): 86197.7 | throughput per GPU (TFLOP/s/GPU): 43.2 | MFU 13.83% | FLOPs estimator: workload-aware | logical tokens/s: 194636.3 | valid tokens/s: 194636.2 | physical tokens/s: 195055.9 | loss tokens/s: 144842.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.743873E-01 | lm-global-supervised-tokens: 1.248513E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.539004E-02 | lm-token-buffer-mib: 5.809373E+01 | global_load_balancing_loss: 9.998829E-01 | loss scale: 1.0 | grad norm: 0.110 | params norm: 11459.873 | params rms: 0.062051 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 1:25:38.508335 | finish at 2026-10-10 15:09:56 + [2026-10-06 13:45:46] iteration 2787/ 6855 | consumed samples: 356736 | elapsed time per iteration (ms): 88374.0 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.67% | FLOPs estimator: workload-aware | logical tokens/s: 189842.6 | valid tokens/s: 189842.6 | physical tokens/s: 190188.2 | loss tokens/s: 137713.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.032757E-01 | lm-global-supervised-tokens: 1.217031E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.153717E-02 | lm-token-buffer-mib: 5.817995E+01 | global_load_balancing_loss: 9.999392E-01 | loss scale: 1.0 | grad norm: 0.071 | params norm: 11459.809 | params rms: 0.062051 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 3:51:45.481759 | finish at 2026-10-10 17:37:31 + [2026-10-06 13:47:12] iteration 2788/ 6855 | consumed samples: 356864 | elapsed time per iteration (ms): 85785.9 | throughput per GPU (TFLOP/s/GPU): 43.3 | MFU 13.88% | FLOPs estimator: workload-aware | logical tokens/s: 195570.6 | valid tokens/s: 195570.6 | physical tokens/s: 195916.3 | loss tokens/s: 143205.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.606758E-01 | lm-global-supervised-tokens: 1.228504E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.316116E-02 | lm-token-buffer-mib: 5.804238E+01 | global_load_balancing_loss: 9.992839E-01 | loss scale: 1.0 | grad norm: 0.056 | params norm: 11459.744 | params rms: 0.062051 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 0:54:51.405097 | finish at 2026-10-10 14:42:03 + [2026-10-06 13:48:41] iteration 2789/ 6855 | consumed samples: 356992 | elapsed time per iteration (ms): 89149.1 | throughput per GPU (TFLOP/s/GPU): 41.3 | MFU 13.22% | FLOPs estimator: workload-aware | logical tokens/s: 188192.3 | valid tokens/s: 188192.3 | physical tokens/s: 188683.6 | loss tokens/s: 139563.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.380001E-01 | lm-global-supervised-tokens: 1.244199E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.295778E-02 | lm-token-buffer-mib: 5.812424E+01 | global_load_balancing_loss: 1.000564E+00 | loss scale: 1.0 | grad norm: 0.100 | params norm: 11459.680 | params rms: 0.062050 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 4:41:20.314540 | finish at 2026-10-10 18:30:01 + [2026-10-06 13:50:25] iteration 2790/ 6855 | consumed samples: 357120 | elapsed time per iteration (ms): 104601.3 | throughput per GPU (TFLOP/s/GPU): 34.4 | MFU 11.04% | FLOPs estimator: workload-aware | logical tokens/s: 160391.4 | valid tokens/s: 160391.4 | physical tokens/s: 160752.6 | loss tokens/s: 122848.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.432101E-01 | lm-global-supervised-tokens: 1.285010E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.200889E-02 | lm-token-buffer-mib: 5.812111E+01 | global_load_balancing_loss: 9.999226E-01 | loss scale: 1.0 | grad norm: 0.097 | params norm: 11459.617 | params rms: 0.062050 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 22:06:44.195341 | finish at 2026-10-11 11:57:10 + [2026-10-06 13:51:57] iteration 2791/ 6855 | consumed samples: 357248 | elapsed time per iteration (ms): 91764.2 | throughput per GPU (TFLOP/s/GPU): 40.4 | MFU 12.95% | FLOPs estimator: workload-aware | logical tokens/s: 182829.6 | valid tokens/s: 182829.6 | physical tokens/s: 183239.9 | loss tokens/s: 136282.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.686305E-01 | lm-global-supervised-tokens: 1.250583E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.686285E-02 | lm-token-buffer-mib: 5.813622E+01 | global_load_balancing_loss: 1.000754E+00 | loss scale: 1.0 | grad norm: 0.041 | params norm: 11459.555 | params rms: 0.062050 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 7:35:29.642799 | finish at 2026-10-10 21:27:27 + [2026-10-06 13:53:23] iteration 2792/ 6855 | consumed samples: 357376 | elapsed time per iteration (ms): 85945.8 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.62% | FLOPs estimator: workload-aware | logical tokens/s: 195205.8 | valid tokens/s: 195205.8 | physical tokens/s: 195865.8 | loss tokens/s: 146785.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.792047E-01 | lm-global-supervised-tokens: 1.261556E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.477961E-02 | lm-token-buffer-mib: 5.823756E+01 | global_load_balancing_loss: 1.000409E+00 | loss scale: 1.0 | grad norm: 0.070 | params norm: 11459.494 | params rms: 0.062049 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 0:59:57.750795 | finish at 2026-10-10 14:53:21 + [2026-10-06 13:54:47] iteration 2793/ 6855 | consumed samples: 357504 | elapsed time per iteration (ms): 83762.2 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.78% | FLOPs estimator: workload-aware | logical tokens/s: 200295.0 | valid tokens/s: 200295.0 | physical tokens/s: 201163.4 | loss tokens/s: 144499.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.292122E-01 | lm-global-supervised-tokens: 1.210362E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.544987E-02 | lm-token-buffer-mib: 5.804349E+01 | global_load_balancing_loss: 1.000605E+00 | loss scale: 1.0 | grad norm: 0.064 | params norm: 11459.432 | params rms: 0.062049 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 22:30:41.981336 | finish at 2026-10-10 12:25:29 + [2026-10-06 13:56:10] iteration 2794/ 6855 | consumed samples: 357632 | elapsed time per iteration (ms): 83349.2 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.63% | FLOPs estimator: workload-aware | logical tokens/s: 201287.5 | valid tokens/s: 201287.5 | physical tokens/s: 201960.4 | loss tokens/s: 146273.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 5.821138E-01 | lm-global-supervised-tokens: 1.219178E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.160539E-02 | lm-token-buffer-mib: 5.803300E+01 | global_load_balancing_loss: 1.000665E+00 | loss scale: 1.0 | grad norm: 0.230 | params norm: 11459.370 | params rms: 0.062049 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 22:01:21.241633 | finish at 2026-10-10 11:57:31 + [2026-10-06 13:57:38] iteration 2795/ 6855 | consumed samples: 357760 | elapsed time per iteration (ms): 88194.2 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.48% | FLOPs estimator: workload-aware | logical tokens/s: 190230.0 | valid tokens/s: 190230.0 | physical tokens/s: 190737.4 | loss tokens/s: 135688.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.760051E-01 | lm-global-supervised-tokens: 1.196692E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.380774E-02 | lm-token-buffer-mib: 5.820135E+01 | global_load_balancing_loss: 1.000176E+00 | loss scale: 1.0 | grad norm: 0.244 | params norm: 11459.307 | params rms: 0.062048 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 3:27:48.493781 | finish at 2026-10-10 17:25:27 + [2026-10-06 13:59:04] iteration 2796/ 6855 | consumed samples: 357888 | elapsed time per iteration (ms): 85145.2 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.69% | FLOPs estimator: workload-aware | logical tokens/s: 197041.9 | valid tokens/s: 197041.9 | physical tokens/s: 197676.5 | loss tokens/s: 146728.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.401246E-01 | lm-global-supervised-tokens: 1.249324E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.156500E-02 | lm-token-buffer-mib: 5.837468E+01 | global_load_balancing_loss: 9.997851E-01 | loss scale: 1.0 | grad norm: 0.040 | params norm: 11459.243 | params rms: 0.062048 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 0:00:04.341696 | finish at 2026-10-10 13:59:08 + [2026-10-06 14:00:31] iteration 2797/ 6855 | consumed samples: 358016 | elapsed time per iteration (ms): 87325.7 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.48% | FLOPs estimator: workload-aware | logical tokens/s: 192121.5 | valid tokens/s: 192121.5 | physical tokens/s: 192617.6 | loss tokens/s: 140592.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.601261E-01 | lm-global-supervised-tokens: 1.227736E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.519708E-02 | lm-token-buffer-mib: 5.806770E+01 | global_load_balancing_loss: 9.992962E-01 | loss scale: 1.0 | grad norm: 0.198 | params norm: 11459.178 | params rms: 0.062048 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 2:26:07.712066 | finish at 2026-10-10 16:26:39 + [2026-10-06 14:01:59] iteration 2798/ 6855 | consumed samples: 358144 | elapsed time per iteration (ms): 87984.8 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.62% | FLOPs estimator: workload-aware | logical tokens/s: 190683.2 | valid tokens/s: 190683.2 | physical tokens/s: 191069.5 | loss tokens/s: 132512.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.688536E-01 | lm-global-supervised-tokens: 1.165905E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.484283E-02 | lm-token-buffer-mib: 5.827803E+01 | global_load_balancing_loss: 9.990152E-01 | loss scale: 1.0 | grad norm: 0.041 | params norm: 11459.113 | params rms: 0.062047 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 3:09:14.140554 | finish at 2026-10-10 17:11:13 + [2026-10-06 14:03:27] iteration 2799/ 6855 | consumed samples: 358272 | elapsed time per iteration (ms): 88242.1 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.47% | FLOPs estimator: workload-aware | logical tokens/s: 190126.7 | valid tokens/s: 190126.6 | physical tokens/s: 190626.1 | loss tokens/s: 137906.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.965362E-01 | lm-global-supervised-tokens: 1.216917E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.249609E-02 | lm-token-buffer-mib: 5.801552E+01 | global_load_balancing_loss: 1.000692E+00 | loss scale: 1.0 | grad norm: 0.260 | params norm: 11459.047 | params rms: 0.062047 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 3:25:09.923756 | finish at 2026-10-10 17:28:37 + [2026-10-06 14:05:20] iteration 2800/ 6855 | consumed samples: 358400 | elapsed time per iteration (ms): 112643.8 | throughput per GPU (TFLOP/s/GPU): 32.9 | MFU 10.55% | FLOPs estimator: workload-aware | logical tokens/s: 148939.9 | valid tokens/s: 148939.9 | physical tokens/s: 149374.9 | loss tokens/s: 103116.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.297990E-01 | lm-global-supervised-tokens: 1.161539E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.013858E-02 | lm-token-buffer-mib: 5.816681E+01 | global_load_balancing_loss: 1.000647E+00 | loss scale: 1.0 | grad norm: 0.047 | params norm: 11458.979 | params rms: 0.062047 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 6:52:50.662255 | finish at 2026-10-11 20:58:10 +times across ranks (ms): + save-checkpoint: + rank 0: 24361.86 + rank 1: 24361.87 + rank 2: 24361.89 + rank 3: 24361.89 + rank 4: 24361.88 + rank 5: 24361.88 + rank 6: 24361.84 + rank 7: 24361.89 + rank 8: 24361.55 + rank 9: 24361.53 + rank 10: 24361.53 + rank 11: 24361.53 + rank 12: 24361.53 + rank 13: 24361.53 + rank 14: 24361.53 + rank 15: 24361.54 + rank 16: 24361.70 + rank 17: 24361.71 + rank 18: 24361.69 + rank 19: 24361.71 + rank 20: 24361.68 + rank 21: 24361.71 + rank 22: 24361.68 + rank 23: 24361.70 + rank 24: 24362.12 + rank 25: 24362.10 + rank 26: 24362.15 + rank 27: 24362.16 + rank 28: 24362.14 + rank 29: 24362.10 + rank 30: 24362.08 + rank 31: 24362.11 + rank 32: 24361.79 + rank 33: 24361.81 + rank 34: 24361.79 + rank 35: 24361.76 + rank 36: 24361.74 + rank 37: 24361.73 + rank 38: 24361.78 + rank 39: 24361.76 + rank 40: 24361.82 + rank 41: 24361.80 + rank 42: 24361.82 + rank 43: 24361.82 + rank 44: 24361.82 + rank 45: 24361.82 + rank 46: 24361.88 + rank 47: 24361.83 + rank 48: 24361.98 + rank 49: 24361.98 + rank 50: 24361.97 + rank 51: 24361.97 + rank 52: 24361.97 + rank 53: 24361.96 + rank 54: 24361.96 + rank 55: 24361.96 + rank 56: 24361.87 + rank 57: 24361.84 + rank 58: 24361.84 + rank 59: 24361.84 + rank 60: 24361.83 + rank 61: 24361.85 + rank 62: 24361.83 + rank 63: 24361.83 + [2026-10-06 14:07:16] iteration 2801/ 6855 | consumed samples: 358528 | elapsed time per iteration (ms): 91900.4 | throughput per GPU (TFLOP/s/GPU): 40.4 | MFU 12.96% | FLOPs estimator: workload-aware | logical tokens/s: 182558.7 | valid tokens/s: 182558.7 | physical tokens/s: 182818.1 | loss tokens/s: 133103.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.992216E-01 | lm-global-supervised-tokens: 1.223223E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.184181E-02 | lm-token-buffer-mib: 5.823520E+01 | global_load_balancing_loss: 1.000148E+00 | loss scale: 1.0 | grad norm: 0.169 | params norm: 11458.912 | params rms: 0.062046 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 7:29:24.146865 | finish at 2026-10-10 21:36:40 + [2026-10-06 14:08:49] iteration 2802/ 6855 | consumed samples: 358656 | elapsed time per iteration (ms): 92207.0 | throughput per GPU (TFLOP/s/GPU): 40.5 | MFU 12.98% | FLOPs estimator: workload-aware | logical tokens/s: 181951.4 | valid tokens/s: 181951.4 | physical tokens/s: 182419.5 | loss tokens/s: 135956.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.226515E-01 | lm-global-supervised-tokens: 1.253609E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.859066E-02 | lm-token-buffer-mib: 5.824702E+01 | global_load_balancing_loss: 1.000252E+00 | loss scale: 1.0 | grad norm: 0.126 | params norm: 11458.843 | params rms: 0.062046 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 7:48:34.927586 | finish at 2026-10-10 21:57:23 + [2026-10-06 14:10:15] iteration 2803/ 6855 | consumed samples: 358784 | elapsed time per iteration (ms): 86760.7 | throughput per GPU (TFLOP/s/GPU): 43.1 | MFU 13.83% | FLOPs estimator: workload-aware | logical tokens/s: 193372.5 | valid tokens/s: 193372.5 | physical tokens/s: 193850.9 | loss tokens/s: 141830.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.087263E-01 | lm-global-supervised-tokens: 1.230530E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.185957E-02 | lm-token-buffer-mib: 5.813210E+01 | global_load_balancing_loss: 1.000193E+00 | loss scale: 1.0 | grad norm: 0.047 | params norm: 11458.774 | params rms: 0.062046 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 1:39:14.309011 | finish at 2026-10-10 15:49:30 + [2026-10-06 14:11:43] iteration 2804/ 6855 | consumed samples: 358912 | elapsed time per iteration (ms): 88025.6 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.50% | FLOPs estimator: workload-aware | logical tokens/s: 190594.1 | valid tokens/s: 190594.1 | physical tokens/s: 191125.1 | loss tokens/s: 141221.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.890308E-01 | lm-global-supervised-tokens: 1.243111E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.223021E-02 | lm-token-buffer-mib: 5.804686E+01 | global_load_balancing_loss: 9.999203E-01 | loss scale: 1.0 | grad norm: 0.151 | params norm: 11458.704 | params rms: 0.062045 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 3:03:11.809734 | finish at 2026-10-10 17:14:55 + [2026-10-06 14:13:10] iteration 2805/ 6855 | consumed samples: 359040 | elapsed time per iteration (ms): 87157.1 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.67% | FLOPs estimator: workload-aware | logical tokens/s: 192493.1 | valid tokens/s: 192493.1 | physical tokens/s: 192982.1 | loss tokens/s: 141336.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.595885E-01 | lm-global-supervised-tokens: 1.231845E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.356441E-02 | lm-token-buffer-mib: 5.817184E+01 | global_load_balancing_loss: 1.001575E+00 | loss scale: 1.0 | grad norm: 0.114 | params norm: 11458.635 | params rms: 0.062045 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 2:03:06.225879 | finish at 2026-10-10 16:16:17 + [2026-10-06 14:14:39] iteration 2806/ 6855 | consumed samples: 359168 | elapsed time per iteration (ms): 88733.2 | throughput per GPU (TFLOP/s/GPU): 41.3 | MFU 13.24% | FLOPs estimator: workload-aware | logical tokens/s: 189074.3 | valid tokens/s: 189074.3 | physical tokens/s: 189541.0 | loss tokens/s: 138502.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.460629E-01 | lm-global-supervised-tokens: 1.228978E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.197657E-02 | lm-token-buffer-mib: 5.808100E+01 | global_load_balancing_loss: 1.000719E+00 | loss scale: 1.0 | grad norm: 0.039 | params norm: 11458.569 | params rms: 0.062044 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 3:48:00.607392 | finish at 2026-10-10 18:02:40 + [2026-10-06 14:16:05] iteration 2807/ 6855 | consumed samples: 359296 | elapsed time per iteration (ms): 86053.1 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.51% | FLOPs estimator: workload-aware | logical tokens/s: 194963.1 | valid tokens/s: 194963.1 | physical tokens/s: 195424.7 | loss tokens/s: 154071.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.654502E-01 | lm-global-supervised-tokens: 1.325832E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.603669E-02 | lm-token-buffer-mib: 5.806603E+01 | global_load_balancing_loss: 1.000808E+00 | loss scale: 1.0 | grad norm: 0.089 | params norm: 11458.504 | params rms: 0.062044 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 0:45:42.836323 | finish at 2026-10-10 15:01:48 + [2026-10-06 14:17:35] iteration 2808/ 6855 | consumed samples: 359424 | elapsed time per iteration (ms): 89962.7 | throughput per GPU (TFLOP/s/GPU): 40.9 | MFU 13.10% | FLOPs estimator: workload-aware | logical tokens/s: 186490.0 | valid tokens/s: 186490.0 | physical tokens/s: 186918.8 | loss tokens/s: 142110.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.985039E-01 | lm-global-supervised-tokens: 1.278462E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.140792E-02 | lm-token-buffer-mib: 5.828239E+01 | global_load_balancing_loss: 1.000063E+00 | loss scale: 1.0 | grad norm: 0.135 | params norm: 11458.437 | params rms: 0.062044 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 5:07:59.205661 | finish at 2026-10-10 19:25:34 + [2026-10-06 14:19:01] iteration 2809/ 6855 | consumed samples: 359552 | elapsed time per iteration (ms): 85867.7 | throughput per GPU (TFLOP/s/GPU): 42.8 | MFU 13.73% | FLOPs estimator: workload-aware | logical tokens/s: 195384.0 | valid tokens/s: 195384.0 | physical tokens/s: 195739.2 | loss tokens/s: 150265.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.882305E-01 | lm-global-supervised-tokens: 1.290293E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.582562E-02 | lm-token-buffer-mib: 5.807591E+01 | global_load_balancing_loss: 9.992383E-01 | loss scale: 1.0 | grad norm: 0.053 | params norm: 11458.370 | params rms: 0.062043 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 0:30:20.809139 | finish at 2026-10-10 14:49:22 + [2026-10-06 14:20:47] iteration 2810/ 6855 | consumed samples: 359680 | elapsed time per iteration (ms): 106344.2 | throughput per GPU (TFLOP/s/GPU): 34.7 | MFU 11.11% | FLOPs estimator: workload-aware | logical tokens/s: 157763.0 | valid tokens/s: 157763.0 | physical tokens/s: 158099.5 | loss tokens/s: 119789.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.625037E-01 | lm-global-supervised-tokens: 1.273894E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.155086E-02 | lm-token-buffer-mib: 5.807689E+01 | global_load_balancing_loss: 9.990134E-01 | loss scale: 1.0 | grad norm: 0.127 | params norm: 11458.303 | params rms: 0.062043 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 23:29:22.089912 | finish at 2026-10-11 13:50:10 + [2026-10-06 14:22:16] iteration 2811/ 6855 | consumed samples: 359808 | elapsed time per iteration (ms): 88152.6 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.42% | FLOPs estimator: workload-aware | logical tokens/s: 190319.9 | valid tokens/s: 190319.9 | physical tokens/s: 190861.0 | loss tokens/s: 143815.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.411831E-01 | lm-global-supervised-tokens: 1.267769E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.154083E-02 | lm-token-buffer-mib: 5.839357E+01 | global_load_balancing_loss: 1.001655E+00 | loss scale: 1.0 | grad norm: 0.100 | params norm: 11458.235 | params rms: 0.062043 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 3:01:29.095319 | finish at 2026-10-10 17:23:45 + [2026-10-06 14:23:44] iteration 2812/ 6855 | consumed samples: 359936 | elapsed time per iteration (ms): 88506.7 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.51% | FLOPs estimator: workload-aware | logical tokens/s: 189557.6 | valid tokens/s: 189557.6 | physical tokens/s: 190003.4 | loss tokens/s: 146643.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.197961E-01 | lm-global-supervised-tokens: 1.297894E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.250166E-02 | lm-token-buffer-mib: 5.809241E+01 | global_load_balancing_loss: 9.995574E-01 | loss scale: 1.0 | grad norm: 0.056 | params norm: 11458.166 | params rms: 0.062042 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 3:23:52.697181 | finish at 2026-10-10 17:47:37 + [2026-10-06 14:25:15] iteration 2813/ 6855 | consumed samples: 360064 | elapsed time per iteration (ms): 90440.0 | throughput per GPU (TFLOP/s/GPU): 41.5 | MFU 13.30% | FLOPs estimator: workload-aware | logical tokens/s: 185505.7 | valid tokens/s: 185505.7 | physical tokens/s: 185883.4 | loss tokens/s: 137396.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.633883E-01 | lm-global-supervised-tokens: 1.242613E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.642179E-02 | lm-token-buffer-mib: 5.810543E+01 | global_load_balancing_loss: 1.000362E+00 | loss scale: 1.0 | grad norm: 0.069 | params norm: 11458.097 | params rms: 0.062042 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 5:32:38.537089 | finish at 2026-10-10 19:57:53 + [2026-10-06 14:26:39] iteration 2814/ 6855 | consumed samples: 360192 | elapsed time per iteration (ms): 84064.1 | throughput per GPU (TFLOP/s/GPU): 43.7 | MFU 14.02% | FLOPs estimator: workload-aware | logical tokens/s: 199575.6 | valid tokens/s: 199575.6 | physical tokens/s: 199979.1 | loss tokens/s: 147226.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.945621E-01 | lm-global-supervised-tokens: 1.237644E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.165383E-02 | lm-token-buffer-mib: 5.811662E+01 | global_load_balancing_loss: 9.999629E-01 | loss scale: 1.0 | grad norm: 0.039 | params norm: 11458.028 | params rms: 0.062041 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 22:21:42.955951 | finish at 2026-10-10 12:48:22 + [2026-10-06 14:28:01] iteration 2815/ 6855 | consumed samples: 360320 | elapsed time per iteration (ms): 82337.4 | throughput per GPU (TFLOP/s/GPU): 43.3 | MFU 13.89% | FLOPs estimator: workload-aware | logical tokens/s: 203761.8 | valid tokens/s: 203761.8 | physical tokens/s: 204193.4 | loss tokens/s: 153755.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.591889E-01 | lm-global-supervised-tokens: 1.265980E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.143946E-02 | lm-token-buffer-mib: 5.808000E+01 | global_load_balancing_loss: 1.000041E+00 | loss scale: 1.0 | grad norm: 0.044 | params norm: 11457.959 | params rms: 0.062041 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 20:24:03.128586 | finish at 2026-10-10 10:52:04 + [2026-10-06 14:29:31] iteration 2816/ 6855 | consumed samples: 360448 | elapsed time per iteration (ms): 89626.9 | throughput per GPU (TFLOP/s/GPU): 41.6 | MFU 13.33% | FLOPs estimator: workload-aware | logical tokens/s: 187189.4 | valid tokens/s: 187189.4 | physical tokens/s: 187603.6 | loss tokens/s: 130794.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.381499E-01 | lm-global-supervised-tokens: 1.172271E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.256461E-02 | lm-token-buffer-mib: 5.818395E+01 | global_load_balancing_loss: 1.000794E+00 | loss scale: 1.0 | grad norm: 0.057 | params norm: 11457.891 | params rms: 0.062041 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 4:33:22.885224 | finish at 2026-10-10 19:02:53 + [2026-10-06 14:30:58] iteration 2817/ 6855 | consumed samples: 360576 | elapsed time per iteration (ms): 87041.1 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.52% | FLOPs estimator: workload-aware | logical tokens/s: 192750.1 | valid tokens/s: 192750.1 | physical tokens/s: 193111.8 | loss tokens/s: 138117.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.797015E-01 | lm-global-supervised-tokens: 1.202188E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.157044E-02 | lm-token-buffer-mib: 5.822868E+01 | global_load_balancing_loss: 9.999053E-01 | loss scale: 1.0 | grad norm: 0.041 | params norm: 11457.822 | params rms: 0.062040 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 1:37:52.079355 | finish at 2026-10-10 16:08:50 + [2026-10-06 14:32:26] iteration 2818/ 6855 | consumed samples: 360704 | elapsed time per iteration (ms): 88243.8 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.41% | FLOPs estimator: workload-aware | logical tokens/s: 190123.1 | valid tokens/s: 190123.1 | physical tokens/s: 190579.4 | loss tokens/s: 136525.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.862352E-01 | lm-global-supervised-tokens: 1.204750E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.146487E-02 | lm-token-buffer-mib: 5.802703E+01 | global_load_balancing_loss: 1.000105E+00 | loss scale: 1.0 | grad norm: 0.038 | params norm: 11457.752 | params rms: 0.062040 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 2:57:20.418571 | finish at 2026-10-10 17:29:46 + [2026-10-06 14:33:48] iteration 2819/ 6855 | consumed samples: 360832 | elapsed time per iteration (ms): 82464.1 | throughput per GPU (TFLOP/s/GPU): 43.7 | MFU 14.01% | FLOPs estimator: workload-aware | logical tokens/s: 203447.4 | valid tokens/s: 203447.4 | physical tokens/s: 203898.1 | loss tokens/s: 154747.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.791465E-01 | lm-global-supervised-tokens: 1.276110E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.488569E-02 | lm-token-buffer-mib: 5.808930E+01 | global_load_balancing_loss: 9.999702E-01 | loss scale: 1.0 | grad norm: 0.042 | params norm: 11457.683 | params rms: 0.062040 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 20:27:05.051321 | finish at 2026-10-10 11:00:53 + [2026-10-06 14:35:39] iteration 2820/ 6855 | consumed samples: 360960 | elapsed time per iteration (ms): 110685.2 | throughput per GPU (TFLOP/s/GPU): 34.0 | MFU 10.88% | FLOPs estimator: workload-aware | logical tokens/s: 151575.6 | valid tokens/s: 151575.6 | physical tokens/s: 151838.8 | loss tokens/s: 108538.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.630303E-01 | lm-global-supervised-tokens: 1.201359E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.271769E-02 | lm-token-buffer-mib: 5.806301E+01 | global_load_balancing_loss: 9.992694E-01 | loss scale: 1.0 | grad norm: 0.044 | params norm: 11457.614 | params rms: 0.062039 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 4:03:34.754004 | finish at 2026-10-11 18:39:14 + [2026-10-06 14:37:09] iteration 2821/ 6855 | consumed samples: 361088 | elapsed time per iteration (ms): 89593.1 | throughput per GPU (TFLOP/s/GPU): 41.6 | MFU 13.34% | FLOPs estimator: workload-aware | logical tokens/s: 187259.6 | valid tokens/s: 187259.6 | physical tokens/s: 187859.3 | loss tokens/s: 138150.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.920152E-01 | lm-global-supervised-tokens: 1.237734E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.496645E-02 | lm-token-buffer-mib: 5.816999E+01 | global_load_balancing_loss: 1.000327E+00 | loss scale: 1.0 | grad norm: 0.090 | params norm: 11457.547 | params rms: 0.062039 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 4:23:38.625317 | finish at 2026-10-10 19:00:47 + [2026-10-06 14:38:36] iteration 2822/ 6855 | consumed samples: 361216 | elapsed time per iteration (ms): 87197.1 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.68% | FLOPs estimator: workload-aware | logical tokens/s: 192405.2 | valid tokens/s: 192405.2 | physical tokens/s: 192923.9 | loss tokens/s: 140338.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.424596E-01 | lm-global-supervised-tokens: 1.223711E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.462575E-02 | lm-token-buffer-mib: 5.817713E+01 | global_load_balancing_loss: 1.000761E+00 | loss scale: 1.0 | grad norm: 0.167 | params norm: 11457.479 | params rms: 0.062039 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 1:41:05.881878 | finish at 2026-10-10 16:19:42 + [2026-10-06 14:40:04] iteration 2823/ 6855 | consumed samples: 361344 | elapsed time per iteration (ms): 88699.3 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.55% | FLOPs estimator: workload-aware | logical tokens/s: 189146.5 | valid tokens/s: 189146.5 | physical tokens/s: 189571.1 | loss tokens/s: 138589.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.273812E-01 | lm-global-supervised-tokens: 1.229282E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.535233E-02 | lm-token-buffer-mib: 5.809132E+01 | global_load_balancing_loss: 1.000615E+00 | loss scale: 1.0 | grad norm: 0.094 | params norm: 11457.410 | params rms: 0.062038 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 3:20:35.491287 | finish at 2026-10-10 18:00:40 + [2026-10-06 14:41:32] iteration 2824/ 6855 | consumed samples: 361472 | elapsed time per iteration (ms): 87928.0 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.55% | FLOPs estimator: workload-aware | logical tokens/s: 190806.0 | valid tokens/s: 190806.0 | physical tokens/s: 191375.3 | loss tokens/s: 137568.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.318681E-01 | lm-global-supervised-tokens: 1.209608E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.163919E-02 | lm-token-buffer-mib: 5.817970E+01 | global_load_balancing_loss: 1.001079E+00 | loss scale: 1.0 | grad norm: 0.074 | params norm: 11457.344 | params rms: 0.062038 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 2:27:17.793841 | finish at 2026-10-10 17:08:50 + [2026-10-06 14:43:00] iteration 2825/ 6855 | consumed samples: 361600 | elapsed time per iteration (ms): 87996.7 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.58% | FLOPs estimator: workload-aware | logical tokens/s: 190656.6 | valid tokens/s: 190656.6 | physical tokens/s: 191047.3 | loss tokens/s: 131255.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.746680E-01 | lm-global-supervised-tokens: 1.155000E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.656227E-02 | lm-token-buffer-mib: 5.815384E+01 | global_load_balancing_loss: 1.000984E+00 | loss scale: 1.0 | grad norm: 0.112 | params norm: 11457.278 | params rms: 0.062037 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 2:30:26.762688 | finish at 2026-10-10 17:13:27 + [2026-10-06 14:44:33] iteration 2826/ 6855 | consumed samples: 361728 | elapsed time per iteration (ms): 93011.9 | throughput per GPU (TFLOP/s/GPU): 41.4 | MFU 13.28% | FLOPs estimator: workload-aware | logical tokens/s: 180376.8 | valid tokens/s: 180376.8 | physical tokens/s: 180790.1 | loss tokens/s: 128506.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.516291E-01 | lm-global-supervised-tokens: 1.195260E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.538270E-02 | lm-token-buffer-mib: 5.816015E+01 | global_load_balancing_loss: 9.994612E-01 | loss scale: 1.0 | grad norm: 0.053 | params norm: 11457.210 | params rms: 0.062037 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 8:05:44.899743 | finish at 2026-10-10 22:50:18 + [2026-10-06 14:46:05] iteration 2827/ 6855 | consumed samples: 361856 | elapsed time per iteration (ms): 91685.8 | throughput per GPU (TFLOP/s/GPU): 41.6 | MFU 13.33% | FLOPs estimator: workload-aware | logical tokens/s: 182985.1 | valid tokens/s: 182985.1 | physical tokens/s: 183346.6 | loss tokens/s: 130481.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.173846E-01 | lm-global-supervised-tokens: 1.196330E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.166254E-02 | lm-token-buffer-mib: 5.817477E+01 | global_load_balancing_loss: 9.990192E-01 | loss scale: 1.0 | grad norm: 0.079 | params norm: 11457.142 | params rms: 0.062037 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 6:35:10.297066 | finish at 2026-10-10 21:21:15 + [2026-10-06 14:47:34] iteration 2828/ 6855 | consumed samples: 361984 | elapsed time per iteration (ms): 89366.1 | throughput per GPU (TFLOP/s/GPU): 41.4 | MFU 13.28% | FLOPs estimator: workload-aware | logical tokens/s: 187735.1 | valid tokens/s: 187735.1 | physical tokens/s: 188168.7 | loss tokens/s: 137796.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.580295E-01 | lm-global-supervised-tokens: 1.231437E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.292986E-02 | lm-token-buffer-mib: 5.802366E+01 | global_load_balancing_loss: 9.997441E-01 | loss scale: 1.0 | grad norm: 0.072 | params norm: 11457.077 | params rms: 0.062036 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 3:57:57.321478 | finish at 2026-10-10 18:45:32 + [2026-10-06 14:48:59] iteration 2829/ 6855 | consumed samples: 362112 | elapsed time per iteration (ms): 84277.0 | throughput per GPU (TFLOP/s/GPU): 43.4 | MFU 13.90% | FLOPs estimator: workload-aware | logical tokens/s: 199071.7 | valid tokens/s: 199071.7 | physical tokens/s: 199631.4 | loss tokens/s: 142864.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 5.999661E-01 | lm-global-supervised-tokens: 1.204022E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.259950E-02 | lm-token-buffer-mib: 5.808627E+01 | global_load_balancing_loss: 1.000017E+00 | loss scale: 1.0 | grad norm: 0.219 | params norm: 11457.014 | params rms: 0.062036 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 22:14:59.205640 | finish at 2026-10-10 13:03:58 + [2026-10-06 14:50:43] iteration 2830/ 6855 | consumed samples: 362240 | elapsed time per iteration (ms): 104600.1 | throughput per GPU (TFLOP/s/GPU): 34.4 | MFU 11.04% | FLOPs estimator: workload-aware | logical tokens/s: 160393.7 | valid tokens/s: 160393.6 | physical tokens/s: 160765.7 | loss tokens/s: 126074.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.649007E-01 | lm-global-supervised-tokens: 1.318740E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.638021E-02 | lm-token-buffer-mib: 5.813066E+01 | global_load_balancing_loss: 1.001290E+00 | loss scale: 1.0 | grad norm: 0.119 | params norm: 11456.950 | params rms: 0.062036 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 20:56:55.225130 | finish at 2026-10-11 11:47:39 + [2026-10-06 14:52:12] iteration 2831/ 6855 | consumed samples: 362368 | elapsed time per iteration (ms): 88239.9 | throughput per GPU (TFLOP/s/GPU): 41.7 | MFU 13.37% | FLOPs estimator: workload-aware | logical tokens/s: 190131.8 | valid tokens/s: 190131.8 | physical tokens/s: 190560.3 | loss tokens/s: 142813.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.837051E-01 | lm-global-supervised-tokens: 1.260185E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.477245E-02 | lm-token-buffer-mib: 5.807797E+01 | global_load_balancing_loss: 1.000718E+00 | loss scale: 1.0 | grad norm: 0.128 | params norm: 11456.887 | params rms: 0.062035 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 2:37:57.430965 | finish at 2026-10-10 17:30:09 + [2026-10-06 14:53:36] iteration 2832/ 6855 | consumed samples: 362496 | elapsed time per iteration (ms): 84167.5 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.68% | FLOPs estimator: workload-aware | logical tokens/s: 199330.6 | valid tokens/s: 199330.6 | physical tokens/s: 199887.4 | loss tokens/s: 143274.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.366411E-01 | lm-global-supervised-tokens: 1.205902E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.360998E-02 | lm-token-buffer-mib: 5.816582E+01 | global_load_balancing_loss: 1.000272E+00 | loss scale: 1.0 | grad norm: 0.120 | params norm: 11456.825 | params rms: 0.062035 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 22:03:25.697193 | finish at 2026-10-10 12:57:01 + [2026-10-06 14:55:04] iteration 2833/ 6855 | consumed samples: 362624 | elapsed time per iteration (ms): 87910.6 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.56% | FLOPs estimator: workload-aware | logical tokens/s: 190844.0 | valid tokens/s: 190844.0 | physical tokens/s: 191364.2 | loss tokens/s: 142682.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.918845E-01 | lm-global-supervised-tokens: 1.254327E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.637471E-02 | lm-token-buffer-mib: 5.810905E+01 | global_load_balancing_loss: 9.997353E-01 | loss scale: 1.0 | grad norm: 0.095 | params norm: 11456.763 | params rms: 0.062035 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 2:12:56.431069 | finish at 2026-10-10 17:08:00 + [2026-10-06 14:56:32] iteration 2834/ 6855 | consumed samples: 362752 | elapsed time per iteration (ms): 88378.5 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.69% | FLOPs estimator: workload-aware | logical tokens/s: 189833.3 | valid tokens/s: 189833.3 | physical tokens/s: 190406.7 | loss tokens/s: 139444.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.409556E-01 | lm-global-supervised-tokens: 1.232392E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.323765E-02 | lm-token-buffer-mib: 5.809255E+01 | global_load_balancing_loss: 9.999492E-01 | loss scale: 1.0 | grad norm: 0.071 | params norm: 11456.702 | params rms: 0.062034 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 2:42:49.969530 | finish at 2026-10-10 17:39:22 + [2026-10-06 14:57:59] iteration 2835/ 6855 | consumed samples: 362880 | elapsed time per iteration (ms): 86859.8 | throughput per GPU (TFLOP/s/GPU): 41.7 | MFU 13.36% | FLOPs estimator: workload-aware | logical tokens/s: 193152.4 | valid tokens/s: 193152.4 | physical tokens/s: 193710.8 | loss tokens/s: 146421.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.351903E-01 | lm-global-supervised-tokens: 1.271816E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.444799E-02 | lm-token-buffer-mib: 5.810324E+01 | global_load_balancing_loss: 1.000197E+00 | loss scale: 1.0 | grad norm: 0.086 | params norm: 11456.639 | params rms: 0.062034 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 0:59:36.451035 | finish at 2026-10-10 15:57:35 + [2026-10-06 14:59:25] iteration 2836/ 6855 | consumed samples: 363008 | elapsed time per iteration (ms): 86501.6 | throughput per GPU (TFLOP/s/GPU): 42.8 | MFU 13.71% | FLOPs estimator: workload-aware | logical tokens/s: 193952.4 | valid tokens/s: 193952.4 | physical tokens/s: 194272.9 | loss tokens/s: 141756.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.916884E-01 | lm-global-supervised-tokens: 1.226216E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.503608E-02 | lm-token-buffer-mib: 5.808032E+01 | global_load_balancing_loss: 1.000259E+00 | loss scale: 1.0 | grad norm: 0.036 | params norm: 11456.576 | params rms: 0.062034 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 0:34:09.796360 | finish at 2026-10-10 15:33:35 + [2026-10-06 15:00:52] iteration 2837/ 6855 | consumed samples: 363136 | elapsed time per iteration (ms): 86700.5 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.70% | FLOPs estimator: workload-aware | logical tokens/s: 193507.7 | valid tokens/s: 193507.7 | physical tokens/s: 193867.5 | loss tokens/s: 140889.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.500587E-01 | lm-global-supervised-tokens: 1.221514E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.194966E-02 | lm-token-buffer-mib: 5.815255E+01 | global_load_balancing_loss: 1.000186E+00 | loss scale: 1.0 | grad norm: 0.080 | params norm: 11456.513 | params rms: 0.062033 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 0:46:02.409789 | finish at 2026-10-10 15:46:55 + [2026-10-06 15:02:19] iteration 2838/ 6855 | consumed samples: 363264 | elapsed time per iteration (ms): 86719.1 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.77% | FLOPs estimator: workload-aware | logical tokens/s: 193465.8 | valid tokens/s: 193465.8 | physical tokens/s: 193918.6 | loss tokens/s: 149673.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.796665E-01 | lm-global-supervised-tokens: 1.297958E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.167521E-02 | lm-token-buffer-mib: 5.810361E+01 | global_load_balancing_loss: 1.000230E+00 | loss scale: 1.0 | grad norm: 0.088 | params norm: 11456.451 | params rms: 0.062033 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 0:45:50.598835 | finish at 2026-10-10 15:48:09 + [2026-10-06 15:03:45] iteration 2839/ 6855 | consumed samples: 363392 | elapsed time per iteration (ms): 85677.1 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.66% | FLOPs estimator: workload-aware | logical tokens/s: 195818.7 | valid tokens/s: 195818.7 | physical tokens/s: 196237.7 | loss tokens/s: 150214.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.528144E-01 | lm-global-supervised-tokens: 1.286996E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.803446E-02 | lm-token-buffer-mib: 5.821148E+01 | global_load_balancing_loss: 1.000492E+00 | loss scale: 1.0 | grad norm: 0.059 | params norm: 11456.385 | params rms: 0.062033 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 23:34:39.202732 | finish at 2026-10-10 14:38:24 + [2026-10-06 15:05:35] iteration 2840/ 6855 | consumed samples: 363520 | elapsed time per iteration (ms): 110628.9 | throughput per GPU (TFLOP/s/GPU): 34.0 | MFU 10.89% | FLOPs estimator: workload-aware | logical tokens/s: 151652.9 | valid tokens/s: 151652.9 | physical tokens/s: 152002.2 | loss tokens/s: 107449.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.063671E-01 | lm-global-supervised-tokens: 1.188702E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.255053E-02 | lm-token-buffer-mib: 5.809634E+01 | global_load_balancing_loss: 1.002263E+00 | loss scale: 1.0 | grad norm: 0.080 | params norm: 11456.320 | params rms: 0.062032 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 3:22:55.037534 | finish at 2026-10-11 18:28:30 +times across ranks (ms): + save-checkpoint: + rank 0: 24882.50 + rank 1: 24882.39 + rank 2: 24882.47 + rank 3: 24882.54 + rank 4: 24882.42 + rank 5: 24882.38 + rank 6: 24882.28 + rank 7: 24882.40 + rank 8: 24881.30 + rank 9: 24881.23 + rank 10: 24881.27 + rank 11: 24881.26 + rank 12: 24881.27 + rank 13: 24881.25 + rank 14: 24881.26 + rank 15: 24881.25 + rank 16: 24881.05 + rank 17: 24881.03 + rank 18: 24881.01 + rank 19: 24881.03 + rank 20: 24881.01 + rank 21: 24881.02 + rank 22: 24881.01 + rank 23: 24881.00 + rank 24: 24881.43 + rank 25: 24881.36 + rank 26: 24881.41 + rank 27: 24881.46 + rank 28: 24881.34 + rank 29: 24881.44 + rank 30: 24881.97 + rank 31: 24881.43 + rank 32: 24880.86 + rank 33: 24880.84 + rank 34: 24880.81 + rank 35: 24882.30 + rank 36: 24880.84 + rank 37: 24880.80 + rank 38: 24880.79 + rank 39: 24880.88 + rank 40: 24880.84 + rank 41: 24880.81 + rank 42: 24880.80 + rank 43: 24880.84 + rank 44: 24880.80 + rank 45: 24880.80 + rank 46: 24880.80 + rank 47: 24880.81 + rank 48: 24881.09 + rank 49: 24881.19 + rank 50: 24881.18 + rank 51: 24881.18 + rank 52: 24881.21 + rank 53: 24881.11 + rank 54: 24881.23 + rank 55: 24881.27 + rank 56: 24881.08 + rank 57: 24881.08 + rank 58: 24881.04 + rank 59: 24881.03 + rank 60: 24881.05 + rank 61: 24881.06 + rank 62: 24881.06 + rank 63: 24881.10 + [2026-10-06 15:07:45] iteration 2841/ 6855 | consumed samples: 363648 | elapsed time per iteration (ms): 105183.3 | throughput per GPU (TFLOP/s/GPU): 35.2 | MFU 11.28% | FLOPs estimator: workload-aware | logical tokens/s: 159503.5 | valid tokens/s: 159503.5 | physical tokens/s: 159910.1 | loss tokens/s: 117345.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.649399E-01 | lm-global-supervised-tokens: 1.234274E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 6.667349E-01 | lm-token-buffer-mib: 5.807072E+01 | global_load_balancing_loss: 1.000555E+00 | loss scale: 1.0 | grad norm: 0.097 | params norm: 11456.256 | params rms: 0.062032 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 21:16:45.821780 | finish at 2026-10-11 12:24:31 + [2026-10-06 15:09:21] iteration 2842/ 6855 | consumed samples: 363776 | elapsed time per iteration (ms): 95597.3 | throughput per GPU (TFLOP/s/GPU): 39.0 | MFU 12.50% | FLOPs estimator: workload-aware | logical tokens/s: 175498.8 | valid tokens/s: 175498.8 | physical tokens/s: 175831.8 | loss tokens/s: 128609.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.547334E-01 | lm-global-supervised-tokens: 1.229467E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.626405E-02 | lm-token-buffer-mib: 5.808966E+01 | global_load_balancing_loss: 9.991811E-01 | loss scale: 1.0 | grad norm: 0.164 | params norm: 11456.191 | params rms: 0.062032 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 10:33:51.925884 | finish at 2026-10-11 01:43:13 + [2026-10-06 15:10:49] iteration 2843/ 6855 | consumed samples: 363904 | elapsed time per iteration (ms): 88417.5 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.45% | FLOPs estimator: workload-aware | logical tokens/s: 189749.8 | valid tokens/s: 189749.8 | physical tokens/s: 190158.1 | loss tokens/s: 149087.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.582670E-01 | lm-global-supervised-tokens: 1.318189E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.158603E-02 | lm-token-buffer-mib: 5.815533E+01 | global_load_balancing_loss: 9.991854E-01 | loss scale: 1.0 | grad norm: 0.065 | params norm: 11456.126 | params rms: 0.062031 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 2:32:10.830248 | finish at 2026-10-10 17:43:00 + [2026-10-06 15:12:18] iteration 2844/ 6855 | consumed samples: 364032 | elapsed time per iteration (ms): 89001.2 | throughput per GPU (TFLOP/s/GPU): 41.6 | MFU 13.33% | FLOPs estimator: workload-aware | logical tokens/s: 188505.1 | valid tokens/s: 188505.0 | physical tokens/s: 188847.4 | loss tokens/s: 140461.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.082210E-01 | lm-global-supervised-tokens: 1.250121E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.211423E-02 | lm-token-buffer-mib: 5.813399E+01 | global_load_balancing_loss: 1.000690E+00 | loss scale: 1.0 | grad norm: 0.205 | params norm: 11456.061 | params rms: 0.062031 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 3:09:43.626565 | finish at 2026-10-10 18:22:02 + [2026-10-06 15:13:48] iteration 2845/ 6855 | consumed samples: 364160 | elapsed time per iteration (ms): 89304.0 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.44% | FLOPs estimator: workload-aware | logical tokens/s: 187865.4 | valid tokens/s: 187865.4 | physical tokens/s: 188232.7 | loss tokens/s: 135153.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.777729E-01 | lm-global-supervised-tokens: 1.206975E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.953408E-02 | lm-token-buffer-mib: 5.814326E+01 | global_load_balancing_loss: 1.000049E+00 | loss scale: 1.0 | grad norm: 0.041 | params norm: 11455.995 | params rms: 0.062030 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 3:28:29.050119 | finish at 2026-10-10 18:42:17 + [2026-10-06 15:15:12] iteration 2846/ 6855 | consumed samples: 364288 | elapsed time per iteration (ms): 84048.8 | throughput per GPU (TFLOP/s/GPU): 43.5 | MFU 13.94% | FLOPs estimator: workload-aware | logical tokens/s: 199612.0 | valid tokens/s: 199612.0 | physical tokens/s: 200124.4 | loss tokens/s: 150923.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.201102E-01 | lm-global-supervised-tokens: 1.268492E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.715301E-02 | lm-token-buffer-mib: 5.805354E+01 | global_load_balancing_loss: 9.996181E-01 | loss scale: 1.0 | grad norm: 0.209 | params norm: 11455.931 | params rms: 0.062030 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 21:35:51.756732 | finish at 2026-10-10 12:51:03 + [2026-10-06 15:16:38] iteration 2847/ 6855 | consumed samples: 364416 | elapsed time per iteration (ms): 86795.1 | throughput per GPU (TFLOP/s/GPU): 41.4 | MFU 13.28% | FLOPs estimator: workload-aware | logical tokens/s: 193296.3 | valid tokens/s: 193296.3 | physical tokens/s: 193877.7 | loss tokens/s: 151964.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.355750E-01 | lm-global-supervised-tokens: 1.318975E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.289838E-02 | lm-token-buffer-mib: 5.822043E+01 | global_load_balancing_loss: 9.984874E-01 | loss scale: 1.0 | grad norm: 0.049 | params norm: 11455.866 | params rms: 0.062030 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 0:37:54.770250 | finish at 2026-10-10 15:54:33 + [2026-10-06 15:18:06] iteration 2848/ 6855 | consumed samples: 364544 | elapsed time per iteration (ms): 87033.1 | throughput per GPU (TFLOP/s/GPU): 41.5 | MFU 13.29% | FLOPs estimator: workload-aware | logical tokens/s: 192767.7 | valid tokens/s: 192767.7 | physical tokens/s: 193421.1 | loss tokens/s: 146643.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.618622E-01 | lm-global-supervised-tokens: 1.276284E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.149969E-02 | lm-token-buffer-mib: 5.822755E+01 | global_load_balancing_loss: 1.000275E+00 | loss scale: 1.0 | grad norm: 0.312 | params norm: 11455.802 | params rms: 0.062029 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 0:52:21.581580 | finish at 2026-10-10 16:10:27 + [2026-10-06 15:19:31] iteration 2849/ 6855 | consumed samples: 364672 | elapsed time per iteration (ms): 85469.8 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.40% | FLOPs estimator: workload-aware | logical tokens/s: 196292.7 | valid tokens/s: 196292.7 | physical tokens/s: 196949.6 | loss tokens/s: 149248.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 5.761269E-01 | lm-global-supervised-tokens: 1.275622E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.286356E-02 | lm-token-buffer-mib: 5.820235E+01 | global_load_balancing_loss: 1.000909E+00 | loss scale: 1.0 | grad norm: 0.201 | params norm: 11455.737 | params rms: 0.062029 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 23:06:32.049345 | finish at 2026-10-10 14:26:03 + [2026-10-06 15:21:19] iteration 2850/ 6855 | consumed samples: 364800 | elapsed time per iteration (ms): 108380.0 | throughput per GPU (TFLOP/s/GPU): 33.0 | MFU 10.58% | FLOPs estimator: workload-aware | logical tokens/s: 154799.8 | valid tokens/s: 154799.8 | physical tokens/s: 155277.8 | loss tokens/s: 117509.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 5.981982E-01 | lm-global-supervised-tokens: 1.273568E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.334608E-02 | lm-token-buffer-mib: 5.817026E+01 | global_load_balancing_loss: 1.001693E+00 | loss scale: 1.0 | grad norm: 0.131 | params norm: 11455.672 | params rms: 0.062029 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 0:34:21.910007 | finish at 2026-10-11 15:55:41 + [2026-10-06 15:22:46] iteration 2851/ 6855 | consumed samples: 364928 | elapsed time per iteration (ms): 86393.3 | throughput per GPU (TFLOP/s/GPU): 41.7 | MFU 13.35% | FLOPs estimator: workload-aware | logical tokens/s: 194195.6 | valid tokens/s: 194195.6 | physical tokens/s: 194691.1 | loss tokens/s: 145688.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.381385E-01 | lm-global-supervised-tokens: 1.258653E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.143751E-02 | lm-token-buffer-mib: 5.830254E+01 | global_load_balancing_loss: 1.001744E+00 | loss scale: 1.0 | grad norm: 0.053 | params norm: 11455.608 | params rms: 0.062028 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 0:05:18.959579 | finish at 2026-10-10 15:28:05 + [2026-10-06 15:24:18] iteration 2852/ 6855 | consumed samples: 365056 | elapsed time per iteration (ms): 92114.5 | throughput per GPU (TFLOP/s/GPU): 40.6 | MFU 13.02% | FLOPs estimator: workload-aware | logical tokens/s: 182133.9 | valid tokens/s: 182133.9 | physical tokens/s: 182521.1 | loss tokens/s: 130435.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.274943E-01 | lm-global-supervised-tokens: 1.201496E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.739014E-02 | lm-token-buffer-mib: 5.806511E+01 | global_load_balancing_loss: 1.000142E+00 | loss scale: 1.0 | grad norm: 0.148 | params norm: 11455.543 | params rms: 0.062028 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 6:25:34.234883 | finish at 2026-10-10 21:49:52 + [2026-10-06 15:25:51] iteration 2853/ 6855 | consumed samples: 365184 | elapsed time per iteration (ms): 92989.6 | throughput per GPU (TFLOP/s/GPU): 40.6 | MFU 13.01% | FLOPs estimator: workload-aware | logical tokens/s: 180420.0 | valid tokens/s: 180420.0 | physical tokens/s: 180797.9 | loss tokens/s: 128601.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.209697E-01 | lm-global-supervised-tokens: 1.195862E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.450037E-02 | lm-token-buffer-mib: 5.804779E+01 | global_load_balancing_loss: 9.994613E-01 | loss scale: 1.0 | grad norm: 0.041 | params norm: 11455.476 | params rms: 0.062028 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 7:22:24.298824 | finish at 2026-10-10 22:48:15 + [2026-10-06 15:27:17] iteration 2854/ 6855 | consumed samples: 365312 | elapsed time per iteration (ms): 86579.3 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.57% | FLOPs estimator: workload-aware | logical tokens/s: 193778.6 | valid tokens/s: 193778.6 | physical tokens/s: 194192.2 | loss tokens/s: 139697.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.397418E-01 | lm-global-supervised-tokens: 1.209490E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.360871E-02 | lm-token-buffer-mib: 5.811542E+01 | global_load_balancing_loss: 9.997123E-01 | loss scale: 1.0 | grad norm: 0.121 | params norm: 11455.408 | params rms: 0.062027 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 0:13:23.732265 | finish at 2026-10-10 15:40:41 + [2026-10-06 15:28:44] iteration 2855/ 6855 | consumed samples: 365440 | elapsed time per iteration (ms): 86787.8 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.61% | FLOPs estimator: workload-aware | logical tokens/s: 193312.5 | valid tokens/s: 193312.5 | physical tokens/s: 194007.9 | loss tokens/s: 140153.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.524885E-01 | lm-global-supervised-tokens: 1.216364E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.085833E-02 | lm-token-buffer-mib: 5.823389E+01 | global_load_balancing_loss: 9.996492E-01 | loss scale: 1.0 | grad norm: 0.063 | params norm: 11455.339 | params rms: 0.062027 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 0:25:51.168823 | finish at 2026-10-10 15:54:35 + [2026-10-06 15:30:12] iteration 2856/ 6855 | consumed samples: 365568 | elapsed time per iteration (ms): 87560.3 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.50% | FLOPs estimator: workload-aware | logical tokens/s: 191607.3 | valid tokens/s: 191607.3 | physical tokens/s: 192056.6 | loss tokens/s: 146701.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.513580E-01 | lm-global-supervised-tokens: 1.284524E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.202635E-02 | lm-token-buffer-mib: 5.818171E+01 | global_load_balancing_loss: 9.998392E-01 | loss scale: 1.0 | grad norm: 0.045 | params norm: 11455.270 | params rms: 0.062027 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 1:15:53.832741 | finish at 2026-10-10 16:46:06 + [2026-10-06 15:31:42] iteration 2857/ 6855 | consumed samples: 365696 | elapsed time per iteration (ms): 89872.7 | throughput per GPU (TFLOP/s/GPU): 41.6 | MFU 13.34% | FLOPs estimator: workload-aware | logical tokens/s: 186677.1 | valid tokens/s: 186677.1 | physical tokens/s: 187211.8 | loss tokens/s: 139739.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.623507E-01 | lm-global-supervised-tokens: 1.255880E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.147762E-02 | lm-token-buffer-mib: 5.833727E+01 | global_load_balancing_loss: 9.992054E-01 | loss scale: 1.0 | grad norm: 0.116 | params norm: 11455.201 | params rms: 0.062026 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 3:48:31.191764 | finish at 2026-10-10 19:20:13 + [2026-10-06 15:33:12] iteration 2858/ 6855 | consumed samples: 365824 | elapsed time per iteration (ms): 90829.4 | throughput per GPU (TFLOP/s/GPU): 41.0 | MFU 13.15% | FLOPs estimator: workload-aware | logical tokens/s: 184710.8 | valid tokens/s: 184710.8 | physical tokens/s: 185049.0 | loss tokens/s: 132340.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.282368E-01 | lm-global-supervised-tokens: 1.202040E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.129936E-02 | lm-token-buffer-mib: 5.830893E+01 | global_load_balancing_loss: 9.999287E-01 | loss scale: 1.0 | grad norm: 0.033 | params norm: 11455.131 | params rms: 0.062026 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 4:50:45.118721 | finish at 2026-10-10 20:23:58 + [2026-10-06 15:34:38] iteration 2859/ 6855 | consumed samples: 365952 | elapsed time per iteration (ms): 85890.9 | throughput per GPU (TFLOP/s/GPU): 43.6 | MFU 13.97% | FLOPs estimator: workload-aware | logical tokens/s: 195331.4 | valid tokens/s: 195331.4 | physical tokens/s: 195647.5 | loss tokens/s: 145401.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.032445E-01 | lm-global-supervised-tokens: 1.248867E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.135852E-02 | lm-token-buffer-mib: 5.809280E+01 | global_load_balancing_loss: 1.001149E+00 | loss scale: 1.0 | grad norm: 0.096 | params norm: 11455.061 | params rms: 0.062025 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 23:20:19.974060 | finish at 2026-10-10 14:54:58 + [2026-10-06 15:36:25] iteration 2860/ 6855 | consumed samples: 366080 | elapsed time per iteration (ms): 106143.9 | throughput per GPU (TFLOP/s/GPU): 34.4 | MFU 11.02% | FLOPs estimator: workload-aware | logical tokens/s: 158060.6 | valid tokens/s: 158060.6 | physical tokens/s: 158456.7 | loss tokens/s: 119256.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.805363E-01 | lm-global-supervised-tokens: 1.265838E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.455977E-02 | lm-token-buffer-mib: 5.817588E+01 | global_load_balancing_loss: 1.001087E+00 | loss scale: 1.0 | grad norm: 0.060 | params norm: 11454.992 | params rms: 0.062025 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 21:47:24.861121 | finish at 2026-10-11 13:23:49 + [2026-10-06 15:37:51] iteration 2861/ 6855 | consumed samples: 366208 | elapsed time per iteration (ms): 86664.2 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.63% | FLOPs estimator: workload-aware | logical tokens/s: 193588.5 | valid tokens/s: 193588.5 | physical tokens/s: 194025.3 | loss tokens/s: 138706.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.554793E-01 | lm-global-supervised-tokens: 1.202086E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.529524E-02 | lm-token-buffer-mib: 5.825210E+01 | global_load_balancing_loss: 1.001153E+00 | loss scale: 1.0 | grad norm: 0.080 | params norm: 11454.923 | params rms: 0.062025 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 0:08:56.621764 | finish at 2026-10-10 15:46:48 + [2026-10-06 15:39:18] iteration 2862/ 6855 | consumed samples: 366336 | elapsed time per iteration (ms): 86728.2 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.60% | FLOPs estimator: workload-aware | logical tokens/s: 193445.5 | valid tokens/s: 193445.5 | physical tokens/s: 193754.2 | loss tokens/s: 148150.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.631590E-01 | lm-global-supervised-tokens: 1.284886E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.112405E-02 | lm-token-buffer-mib: 5.808315E+01 | global_load_balancing_loss: 1.001164E+00 | loss scale: 1.0 | grad norm: 0.061 | params norm: 11454.856 | params rms: 0.062024 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 0:11:45.678635 | finish at 2026-10-10 15:51:04 + [2026-10-06 15:40:44] iteration 2863/ 6855 | consumed samples: 366464 | elapsed time per iteration (ms): 85967.4 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.45% | FLOPs estimator: workload-aware | logical tokens/s: 195157.6 | valid tokens/s: 195157.6 | physical tokens/s: 195730.6 | loss tokens/s: 147897.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.385231E-01 | lm-global-supervised-tokens: 1.271432E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.307595E-02 | lm-token-buffer-mib: 5.827645E+01 | global_load_balancing_loss: 1.000402E+00 | loss scale: 1.0 | grad norm: 0.172 | params norm: 11454.787 | params rms: 0.062024 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 23:19:41.851578 | finish at 2026-10-10 15:00:26 + [2026-10-06 15:42:12] iteration 2864/ 6855 | consumed samples: 366592 | elapsed time per iteration (ms): 87932.1 | throughput per GPU (TFLOP/s/GPU): 40.9 | MFU 13.12% | FLOPs estimator: workload-aware | logical tokens/s: 190796.4 | valid tokens/s: 190796.4 | physical tokens/s: 191313.8 | loss tokens/s: 146987.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.446430E-01 | lm-global-supervised-tokens: 1.292495E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.148428E-02 | lm-token-buffer-mib: 5.807039E+01 | global_load_balancing_loss: 1.000015E+00 | loss scale: 1.0 | grad norm: 0.271 | params norm: 11454.717 | params rms: 0.062024 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 1:28:56.975172 | finish at 2026-10-10 17:11:09 + [2026-10-06 15:43:39] iteration 2865/ 6855 | consumed samples: 366720 | elapsed time per iteration (ms): 87285.2 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.68% | FLOPs estimator: workload-aware | logical tokens/s: 192210.9 | valid tokens/s: 192210.9 | physical tokens/s: 192774.0 | loss tokens/s: 144324.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.966195E-01 | lm-global-supervised-tokens: 1.259742E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.139960E-02 | lm-token-buffer-mib: 5.805772E+01 | global_load_balancing_loss: 9.991153E-01 | loss scale: 1.0 | grad norm: 0.107 | params norm: 11454.650 | params rms: 0.062023 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 0:44:28.000796 | finish at 2026-10-10 16:28:07 + [2026-10-06 15:45:08] iteration 2866/ 6855 | consumed samples: 366848 | elapsed time per iteration (ms): 88462.9 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.75% | FLOPs estimator: workload-aware | logical tokens/s: 189652.4 | valid tokens/s: 189652.3 | physical tokens/s: 190005.7 | loss tokens/s: 137366.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.596606E-01 | lm-global-supervised-tokens: 1.215184E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.207142E-02 | lm-token-buffer-mib: 5.811816E+01 | global_load_balancing_loss: 1.000104E+00 | loss scale: 1.0 | grad norm: 0.240 | params norm: 11454.582 | params rms: 0.062023 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 2:01:18.641892 | finish at 2026-10-10 17:46:26 + [2026-10-06 15:46:36] iteration 2867/ 6855 | consumed samples: 366976 | elapsed time per iteration (ms): 88238.2 | throughput per GPU (TFLOP/s/GPU): 41.3 | MFU 13.23% | FLOPs estimator: workload-aware | logical tokens/s: 190134.6 | valid tokens/s: 190134.6 | physical tokens/s: 190679.1 | loss tokens/s: 137853.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.244200E-01 | lm-global-supervised-tokens: 1.216396E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.508689E-02 | lm-token-buffer-mib: 5.805229E+01 | global_load_balancing_loss: 1.000473E+00 | loss scale: 1.0 | grad norm: 0.072 | params norm: 11454.513 | params rms: 0.062022 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 1:44:53.938545 | finish at 2026-10-10 17:31:30 + [2026-10-06 15:48:06] iteration 2868/ 6855 | consumed samples: 367104 | elapsed time per iteration (ms): 89751.7 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.52% | FLOPs estimator: workload-aware | logical tokens/s: 186928.7 | valid tokens/s: 186928.7 | physical tokens/s: 187517.3 | loss tokens/s: 135549.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.819969E-01 | lm-global-supervised-tokens: 1.216583E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 3.433357E-02 | lm-token-buffer-mib: 5.816161E+01 | global_load_balancing_loss: 9.999308E-01 | loss scale: 1.0 | grad norm: 0.320 | params norm: 11454.443 | params rms: 0.062022 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 3:23:59.840336 | finish at 2026-10-10 19:12:05 + [2026-10-06 15:49:37] iteration 2869/ 6855 | consumed samples: 367232 | elapsed time per iteration (ms): 91508.0 | throughput per GPU (TFLOP/s/GPU): 40.2 | MFU 12.89% | FLOPs estimator: workload-aware | logical tokens/s: 183341.2 | valid tokens/s: 183341.2 | physical tokens/s: 183787.8 | loss tokens/s: 135470.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.329136E-01 | lm-global-supervised-tokens: 1.239668E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.192594E-02 | lm-token-buffer-mib: 5.818594E+01 | global_load_balancing_loss: 9.996012E-01 | loss scale: 1.0 | grad norm: 0.053 | params norm: 11454.376 | params rms: 0.062022 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 5:19:11.011133 | finish at 2026-10-10 21:08:48 + [2026-10-06 15:51:23] iteration 2870/ 6855 | consumed samples: 367360 | elapsed time per iteration (ms): 106015.5 | throughput per GPU (TFLOP/s/GPU): 34.4 | MFU 11.03% | FLOPs estimator: workload-aware | logical tokens/s: 158251.8 | valid tokens/s: 158251.8 | physical tokens/s: 158738.5 | loss tokens/s: 114328.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.456573E-01 | lm-global-supervised-tokens: 1.212064E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.408815E-02 | lm-token-buffer-mib: 5.824400E+01 | global_load_balancing_loss: 1.000146E+00 | loss scale: 1.0 | grad norm: 0.414 | params norm: 11454.309 | params rms: 0.062021 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 21:21:11.914089 | finish at 2026-10-11 13:12:35 + [2026-10-06 15:52:47] iteration 2871/ 6855 | consumed samples: 367488 | elapsed time per iteration (ms): 84400.0 | throughput per GPU (TFLOP/s/GPU): 43.2 | MFU 13.84% | FLOPs estimator: workload-aware | logical tokens/s: 198781.6 | valid tokens/s: 198781.6 | physical tokens/s: 199289.1 | loss tokens/s: 144552.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.596937E-01 | lm-global-supervised-tokens: 1.220021E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.471957E-02 | lm-token-buffer-mib: 5.815371E+01 | global_load_balancing_loss: 1.001256E+00 | loss scale: 1.0 | grad norm: 0.104 | params norm: 11454.238 | params rms: 0.062021 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 21:24:09.648823 | finish at 2026-10-10 13:16:57 + [2026-10-06 15:54:17] iteration 2872/ 6855 | consumed samples: 367616 | elapsed time per iteration (ms): 89807.8 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.39% | FLOPs estimator: workload-aware | logical tokens/s: 186812.2 | valid tokens/s: 186812.2 | physical tokens/s: 187289.0 | loss tokens/s: 136164.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.097268E-01 | lm-global-supervised-tokens: 1.222861E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.204026E-02 | lm-token-buffer-mib: 5.810773E+01 | global_load_balancing_loss: 1.000741E+00 | loss scale: 1.0 | grad norm: 0.310 | params norm: 11454.167 | params rms: 0.062021 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 3:21:44.623355 | finish at 2026-10-10 19:16:02 + [2026-10-06 15:55:43] iteration 2873/ 6855 | consumed samples: 367744 | elapsed time per iteration (ms): 85884.3 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.55% | FLOPs estimator: workload-aware | logical tokens/s: 195346.3 | valid tokens/s: 195346.3 | physical tokens/s: 195743.3 | loss tokens/s: 145207.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.982180E-01 | lm-global-supervised-tokens: 1.247102E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 8.298871E-02 | lm-token-buffer-mib: 5.810907E+01 | global_load_balancing_loss: 1.000190E+00 | loss scale: 1.0 | grad norm: 0.063 | params norm: 11454.099 | params rms: 0.062020 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 22:59:51.102191 | finish at 2026-10-10 14:55:34 + [2026-10-06 15:57:12] iteration 2874/ 6855 | consumed samples: 367872 | elapsed time per iteration (ms): 89280.6 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.40% | FLOPs estimator: workload-aware | logical tokens/s: 187914.7 | valid tokens/s: 187914.7 | physical tokens/s: 188297.2 | loss tokens/s: 137464.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.639194E-01 | lm-global-supervised-tokens: 1.227291E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.590700E-02 | lm-token-buffer-mib: 5.836651E+01 | global_load_balancing_loss: 1.000282E+00 | loss scale: 1.0 | grad norm: 0.098 | params norm: 11454.031 | params rms: 0.062020 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 2:43:46.219796 | finish at 2026-10-10 18:40:59 + [2026-10-06 15:58:41] iteration 2875/ 6855 | consumed samples: 368000 | elapsed time per iteration (ms): 88632.8 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.75% | FLOPs estimator: workload-aware | logical tokens/s: 189288.0 | valid tokens/s: 189288.0 | physical tokens/s: 189530.0 | loss tokens/s: 136866.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.859390E-01 | lm-global-supervised-tokens: 1.213083E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.310626E-02 | lm-token-buffer-mib: 5.803225E+01 | global_load_balancing_loss: 9.998882E-01 | loss scale: 1.0 | grad norm: 0.138 | params norm: 11453.965 | params rms: 0.062019 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 1:59:18.567181 | finish at 2026-10-10 17:58:00 + [2026-10-06 16:00:06] iteration 2876/ 6855 | consumed samples: 368128 | elapsed time per iteration (ms): 85182.4 | throughput per GPU (TFLOP/s/GPU): 43.2 | MFU 13.85% | FLOPs estimator: workload-aware | logical tokens/s: 196956.0 | valid tokens/s: 196956.0 | physical tokens/s: 197464.1 | loss tokens/s: 144356.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.234329E-01 | lm-global-supervised-tokens: 1.229660E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.151099E-02 | lm-token-buffer-mib: 5.834686E+01 | global_load_balancing_loss: 1.001145E+00 | loss scale: 1.0 | grad norm: 0.192 | params norm: 11453.898 | params rms: 0.062019 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 22:09:00.672789 | finish at 2026-10-10 14:09:07 + [2026-10-06 16:01:33] iteration 2877/ 6855 | consumed samples: 368256 | elapsed time per iteration (ms): 86562.0 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.43% | FLOPs estimator: workload-aware | logical tokens/s: 193816.6 | valid tokens/s: 193816.6 | physical tokens/s: 194447.0 | loss tokens/s: 143313.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.001920E-01 | lm-global-supervised-tokens: 1.240552E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.483209E-02 | lm-token-buffer-mib: 5.834623E+01 | global_load_balancing_loss: 9.996941E-01 | loss scale: 1.0 | grad norm: 0.068 | params norm: 11453.831 | params rms: 0.062019 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 23:39:03.716761 | finish at 2026-10-10 15:40:37 + [2026-10-06 16:03:03] iteration 2878/ 6855 | consumed samples: 368384 | elapsed time per iteration (ms): 89867.6 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.49% | FLOPs estimator: workload-aware | logical tokens/s: 186688.0 | valid tokens/s: 186688.0 | physical tokens/s: 186985.0 | loss tokens/s: 137556.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.989149E-01 | lm-global-supervised-tokens: 1.236188E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.159372E-02 | lm-token-buffer-mib: 5.815866E+01 | global_load_balancing_loss: 9.989970E-01 | loss scale: 1.0 | grad norm: 0.071 | params norm: 11453.763 | params rms: 0.062018 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 3:16:43.619524 | finish at 2026-10-10 19:19:46 + [2026-10-06 16:04:32] iteration 2879/ 6855 | consumed samples: 368512 | elapsed time per iteration (ms): 89027.8 | throughput per GPU (TFLOP/s/GPU): 41.4 | MFU 13.26% | FLOPs estimator: workload-aware | logical tokens/s: 188448.9 | valid tokens/s: 188448.9 | physical tokens/s: 188922.6 | loss tokens/s: 137928.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.337506E-01 | lm-global-supervised-tokens: 1.227948E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.272844E-02 | lm-token-buffer-mib: 5.813429E+01 | global_load_balancing_loss: 1.000643E+00 | loss scale: 1.0 | grad norm: 0.173 | params norm: 11453.696 | params rms: 0.062018 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 2:19:34.386198 | finish at 2026-10-10 18:24:06 + [2026-10-06 16:06:18] iteration 2880/ 6855 | consumed samples: 368640 | elapsed time per iteration (ms): 106328.2 | throughput per GPU (TFLOP/s/GPU): 34.5 | MFU 11.05% | FLOPs estimator: workload-aware | logical tokens/s: 157787.1 | valid tokens/s: 157787.0 | physical tokens/s: 158031.4 | loss tokens/s: 121883.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.289999E-01 | lm-global-supervised-tokens: 1.295961E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.268515E-02 | lm-token-buffer-mib: 5.805279E+01 | global_load_balancing_loss: 1.000141E+00 | loss scale: 1.0 | grad norm: 0.070 | params norm: 11453.629 | params rms: 0.062018 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 21:24:14.406810 | finish at 2026-10-11 13:30:32 +times across ranks (ms): + save-checkpoint: + rank 0: 25713.21 + rank 1: 25713.18 + rank 2: 25713.23 + rank 3: 25713.20 + rank 4: 25713.20 + rank 5: 25713.16 + rank 6: 25713.17 + rank 7: 25713.20 + rank 8: 25713.54 + rank 9: 25713.46 + rank 10: 25713.54 + rank 11: 25713.52 + rank 12: 25713.52 + rank 13: 25713.48 + rank 14: 25713.50 + rank 15: 25713.48 + rank 16: 25714.03 + rank 17: 25714.00 + rank 18: 25714.01 + rank 19: 25714.02 + rank 20: 25713.98 + rank 21: 25714.03 + rank 22: 25713.97 + rank 23: 25713.96 + rank 24: 25713.68 + rank 25: 25713.65 + rank 26: 25713.71 + rank 27: 25713.66 + rank 28: 25713.65 + rank 29: 25713.70 + rank 30: 25713.64 + rank 31: 25713.43 + rank 32: 25713.19 + rank 33: 25713.36 + rank 34: 25713.37 + rank 35: 25713.37 + rank 36: 25713.34 + rank 37: 25713.26 + rank 38: 25713.30 + rank 39: 25713.32 + rank 40: 25713.23 + rank 41: 25713.22 + rank 42: 25713.21 + rank 43: 25713.24 + rank 44: 25713.20 + rank 45: 25713.28 + rank 46: 25713.22 + rank 47: 25713.19 + rank 48: 25714.35 + rank 49: 25714.51 + rank 50: 25714.41 + rank 51: 25714.44 + rank 52: 25714.43 + rank 53: 25714.46 + rank 54: 25714.46 + rank 55: 25714.47 + rank 56: 25713.23 + rank 57: 25713.21 + rank 58: 25713.17 + rank 59: 25713.17 + rank 60: 25713.26 + rank 61: 25713.20 + rank 62: 25713.20 + rank 63: 25713.20 + [2026-10-06 16:08:14] iteration 2881/ 6855 | consumed samples: 368768 | elapsed time per iteration (ms): 89724.5 | throughput per GPU (TFLOP/s/GPU): 41.2 | MFU 13.22% | FLOPs estimator: workload-aware | logical tokens/s: 186985.9 | valid tokens/s: 186985.9 | physical tokens/s: 187276.2 | loss tokens/s: 137274.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.606708E-01 | lm-global-supervised-tokens: 1.231691E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.828914E-01 | lm-token-buffer-mib: 5.810529E+01 | global_load_balancing_loss: 1.000520E+00 | loss scale: 1.0 | grad norm: 0.033 | params norm: 11453.561 | params rms: 0.062017 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 3:02:45.208244 | finish at 2026-10-10 19:10:59 + [2026-10-06 16:09:47] iteration 2882/ 6855 | consumed samples: 368896 | elapsed time per iteration (ms): 93684.4 | throughput per GPU (TFLOP/s/GPU): 40.4 | MFU 12.96% | FLOPs estimator: workload-aware | logical tokens/s: 179082.1 | valid tokens/s: 179082.0 | physical tokens/s: 179488.3 | loss tokens/s: 128903.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.941944E-01 | lm-global-supervised-tokens: 1.207628E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.618291E-02 | lm-token-buffer-mib: 5.828481E+01 | global_load_balancing_loss: 1.000928E+00 | loss scale: 1.0 | grad norm: 0.138 | params norm: 11453.496 | params rms: 0.062017 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 7:23:27.981333 | finish at 2026-10-10 23:33:15 + [2026-10-06 16:11:15] iteration 2883/ 6855 | consumed samples: 369024 | elapsed time per iteration (ms): 87978.0 | throughput per GPU (TFLOP/s/GPU): 42.8 | MFU 13.73% | FLOPs estimator: workload-aware | logical tokens/s: 190697.0 | valid tokens/s: 190697.0 | physical tokens/s: 191123.3 | loss tokens/s: 142200.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.771805E-01 | lm-global-supervised-tokens: 1.251050E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.180636E-02 | lm-token-buffer-mib: 5.803590E+01 | global_load_balancing_loss: 1.000018E+00 | loss scale: 1.0 | grad norm: 0.136 | params norm: 11453.429 | params rms: 0.062017 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 1:04:08.788058 | finish at 2026-10-10 17:15:24 + [2026-10-06 16:12:44] iteration 2884/ 6855 | consumed samples: 369152 | elapsed time per iteration (ms): 88344.5 | throughput per GPU (TFLOP/s/GPU): 41.3 | MFU 13.24% | FLOPs estimator: workload-aware | logical tokens/s: 189905.6 | valid tokens/s: 189905.6 | physical tokens/s: 190568.2 | loss tokens/s: 140017.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.321210E-01 | lm-global-supervised-tokens: 1.236977E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.551676E-02 | lm-token-buffer-mib: 5.816528E+01 | global_load_balancing_loss: 1.000437E+00 | loss scale: 1.0 | grad norm: 0.098 | params norm: 11453.363 | params rms: 0.062016 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 1:26:55.912241 | finish at 2026-10-10 17:39:40 + [2026-10-06 16:14:12] iteration 2885/ 6855 | consumed samples: 369280 | elapsed time per iteration (ms): 88646.3 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.40% | FLOPs estimator: workload-aware | logical tokens/s: 189260.1 | valid tokens/s: 189260.1 | physical tokens/s: 189723.4 | loss tokens/s: 141023.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.665522E-01 | lm-global-supervised-tokens: 1.250120E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.150162E-02 | lm-token-buffer-mib: 5.812028E+01 | global_load_balancing_loss: 1.000093E+00 | loss scale: 1.0 | grad norm: 0.176 | params norm: 11453.297 | params rms: 0.062016 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 1:45:25.958018 | finish at 2026-10-10 17:59:38 + [2026-10-06 16:15:40] iteration 2886/ 6855 | consumed samples: 369408 | elapsed time per iteration (ms): 88085.9 | throughput per GPU (TFLOP/s/GPU): 43.1 | MFU 13.82% | FLOPs estimator: workload-aware | logical tokens/s: 190464.1 | valid tokens/s: 190464.1 | physical tokens/s: 190877.8 | loss tokens/s: 137837.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.995521E-01 | lm-global-supervised-tokens: 1.214152E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.152515E-02 | lm-token-buffer-mib: 5.813821E+01 | global_load_balancing_loss: 1.000089E+00 | loss scale: 1.0 | grad norm: 0.087 | params norm: 11453.232 | params rms: 0.062016 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 1:06:53.132305 | finish at 2026-10-10 17:22:34 + [2026-10-06 16:17:07] iteration 2887/ 6855 | consumed samples: 369536 | elapsed time per iteration (ms): 86685.8 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.51% | FLOPs estimator: workload-aware | logical tokens/s: 193540.1 | valid tokens/s: 193540.1 | physical tokens/s: 194084.8 | loss tokens/s: 139860.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.440196E-01 | lm-global-supervised-tokens: 1.212392E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.131772E-02 | lm-token-buffer-mib: 5.812989E+01 | global_load_balancing_loss: 9.997854E-01 | loss scale: 1.0 | grad norm: 0.147 | params norm: 11453.167 | params rms: 0.062015 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 23:32:49.301056 | finish at 2026-10-10 15:49:56 + [2026-10-06 16:18:35] iteration 2888/ 6855 | consumed samples: 369664 | elapsed time per iteration (ms): 87837.1 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.38% | FLOPs estimator: workload-aware | logical tokens/s: 191003.2 | valid tokens/s: 191003.2 | physical tokens/s: 191580.2 | loss tokens/s: 137775.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.280640E-01 | lm-global-supervised-tokens: 1.210178E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.784524E-02 | lm-token-buffer-mib: 5.817414E+01 | global_load_balancing_loss: 1.000345E+00 | loss scale: 1.0 | grad norm: 0.100 | params norm: 11453.103 | params rms: 0.062015 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 0:47:29.893095 | finish at 2026-10-10 17:06:05 + [2026-10-06 16:20:02] iteration 2889/ 6855 | consumed samples: 369792 | elapsed time per iteration (ms): 86661.1 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.69% | FLOPs estimator: workload-aware | logical tokens/s: 193594.9 | valid tokens/s: 193594.8 | physical tokens/s: 194022.6 | loss tokens/s: 143660.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.162325E-01 | lm-global-supervised-tokens: 1.244975E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.105270E-02 | lm-token-buffer-mib: 5.814870E+01 | global_load_balancing_loss: 9.999842E-01 | loss scale: 1.0 | grad norm: 0.083 | params norm: 11453.039 | params rms: 0.062014 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 23:28:18.110414 | finish at 2026-10-10 15:48:20 + [2026-10-06 16:21:48] iteration 2890/ 6855 | consumed samples: 369920 | elapsed time per iteration (ms): 106481.7 | throughput per GPU (TFLOP/s/GPU): 34.1 | MFU 10.92% | FLOPs estimator: workload-aware | logical tokens/s: 157559.3 | valid tokens/s: 157559.3 | physical tokens/s: 157860.5 | loss tokens/s: 117819.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.951541E-01 | lm-global-supervised-tokens: 1.254563E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.931962E-02 | lm-token-buffer-mib: 5.817308E+01 | global_load_balancing_loss: 9.996886E-01 | loss scale: 1.0 | grad norm: 0.109 | params norm: 11452.975 | params rms: 0.062014 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 21:16:39.855196 | finish at 2026-10-11 13:38:28 + [2026-10-06 16:23:16] iteration 2891/ 6855 | consumed samples: 370048 | elapsed time per iteration (ms): 87448.6 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.42% | FLOPs estimator: workload-aware | logical tokens/s: 191851.4 | valid tokens/s: 191851.3 | physical tokens/s: 192392.7 | loss tokens/s: 136832.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.221159E-01 | lm-global-supervised-tokens: 1.196578E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.433657E-02 | lm-token-buffer-mib: 5.816219E+01 | global_load_balancing_loss: 1.000850E+00 | loss scale: 1.0 | grad norm: 0.136 | params norm: 11452.912 | params rms: 0.062014 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 0:17:26.436796 | finish at 2026-10-10 16:40:42 + [2026-10-06 16:24:47] iteration 2892/ 6855 | consumed samples: 370176 | elapsed time per iteration (ms): 90948.0 | throughput per GPU (TFLOP/s/GPU): 41.1 | MFU 13.16% | FLOPs estimator: workload-aware | logical tokens/s: 184470.4 | valid tokens/s: 184470.4 | physical tokens/s: 184959.7 | loss tokens/s: 135952.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.805557E-01 | lm-global-supervised-tokens: 1.236458E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.097058E-02 | lm-token-buffer-mib: 5.818902E+01 | global_load_balancing_loss: 9.995435E-01 | loss scale: 1.0 | grad norm: 0.099 | params norm: 11452.850 | params rms: 0.062013 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 4:07:06.803822 | finish at 2026-10-10 20:31:53 + [2026-10-06 16:26:13] iteration 2893/ 6855 | consumed samples: 370304 | elapsed time per iteration (ms): 86100.2 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.42% | FLOPs estimator: workload-aware | logical tokens/s: 194856.9 | valid tokens/s: 194856.8 | physical tokens/s: 195288.4 | loss tokens/s: 147940.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.518593E-01 | lm-global-supervised-tokens: 1.273768E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.235573E-02 | lm-token-buffer-mib: 5.817609E+01 | global_load_balancing_loss: 1.000379E+00 | loss scale: 1.0 | grad norm: 0.179 | params norm: 11452.786 | params rms: 0.062013 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 22:45:28.956258 | finish at 2026-10-10 15:11:42 + [2026-10-06 16:27:40] iteration 2894/ 6855 | consumed samples: 370432 | elapsed time per iteration (ms): 87599.3 | throughput per GPU (TFLOP/s/GPU): 41.5 | MFU 13.30% | FLOPs estimator: workload-aware | logical tokens/s: 191521.8 | valid tokens/s: 191521.8 | physical tokens/s: 191929.8 | loss tokens/s: 142934.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.676146E-01 | lm-global-supervised-tokens: 1.252100E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.151384E-02 | lm-token-buffer-mib: 5.812134E+01 | global_load_balancing_loss: 1.000910E+00 | loss scale: 1.0 | grad norm: 0.061 | params norm: 11452.722 | params rms: 0.062013 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 0:23:01.021476 | finish at 2026-10-10 16:50:41 + [2026-10-06 16:29:07] iteration 2895/ 6855 | consumed samples: 370560 | elapsed time per iteration (ms): 86931.9 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.63% | FLOPs estimator: workload-aware | logical tokens/s: 192992.5 | valid tokens/s: 192992.5 | physical tokens/s: 193396.3 | loss tokens/s: 139208.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.527625E-01 | lm-global-supervised-tokens: 1.210162E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.878124E-02 | lm-token-buffer-mib: 5.817496E+01 | global_load_balancing_loss: 1.000721E+00 | loss scale: 1.0 | grad norm: 0.235 | params norm: 11452.657 | params rms: 0.062012 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 23:37:30.272169 | finish at 2026-10-10 16:06:37 + [2026-10-06 16:30:33] iteration 2896/ 6855 | consumed samples: 370688 | elapsed time per iteration (ms): 85381.9 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.47% | FLOPs estimator: workload-aware | logical tokens/s: 196495.3 | valid tokens/s: 196495.3 | physical tokens/s: 196897.1 | loss tokens/s: 150029.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.491066E-01 | lm-global-supervised-tokens: 1.280984E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.158911E-02 | lm-token-buffer-mib: 5.805125E+01 | global_load_balancing_loss: 9.999794E-01 | loss scale: 1.0 | grad norm: 0.082 | params norm: 11452.594 | params rms: 0.062012 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 21:53:47.105680 | finish at 2026-10-10 14:24:20 + [2026-10-06 16:31:58] iteration 2897/ 6855 | consumed samples: 370816 | elapsed time per iteration (ms): 85119.3 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.58% | FLOPs estimator: workload-aware | logical tokens/s: 197101.5 | valid tokens/s: 197101.5 | physical tokens/s: 197682.7 | loss tokens/s: 153540.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.722169E-01 | lm-global-supervised-tokens: 1.306922E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.810548E-02 | lm-token-buffer-mib: 5.812934E+01 | global_load_balancing_loss: 9.994599E-01 | loss scale: 1.0 | grad norm: 0.212 | params norm: 11452.531 | params rms: 0.062012 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 21:35:02.124790 | finish at 2026-10-10 14:07:00 + [2026-10-06 16:33:27] iteration 2898/ 6855 | consumed samples: 370944 | elapsed time per iteration (ms): 89248.4 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.56% | FLOPs estimator: workload-aware | logical tokens/s: 187982.8 | valid tokens/s: 187982.8 | physical tokens/s: 188430.9 | loss tokens/s: 136575.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.037604E-01 | lm-global-supervised-tokens: 1.218917E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.270236E-02 | lm-token-buffer-mib: 5.811523E+01 | global_load_balancing_loss: 9.996067E-01 | loss scale: 1.0 | grad norm: 0.106 | params norm: 11452.467 | params rms: 0.062011 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 2:05:55.823407 | finish at 2026-10-10 18:39:23 + [2026-10-06 16:34:57] iteration 2899/ 6855 | consumed samples: 371072 | elapsed time per iteration (ms): 89804.3 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.47% | FLOPs estimator: workload-aware | logical tokens/s: 186819.5 | valid tokens/s: 186819.5 | physical tokens/s: 187396.6 | loss tokens/s: 133011.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 5.739188E-01 | lm-global-supervised-tokens: 1.194499E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.222042E-02 | lm-token-buffer-mib: 5.808050E+01 | global_load_balancing_loss: 1.001034E+00 | loss scale: 1.0 | grad norm: 0.171 | params norm: 11452.403 | params rms: 0.062011 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 2:41:05.854463 | finish at 2026-10-10 19:16:03 + [2026-10-06 16:36:45] iteration 2900/ 6855 | consumed samples: 371200 | elapsed time per iteration (ms): 108000.7 | throughput per GPU (TFLOP/s/GPU): 34.2 | MFU 10.95% | FLOPs estimator: workload-aware | logical tokens/s: 155343.3 | valid tokens/s: 155343.3 | physical tokens/s: 155736.7 | loss tokens/s: 115461.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.422133E-01 | lm-global-supervised-tokens: 1.246989E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.399950E-02 | lm-token-buffer-mib: 5.820752E+01 | global_load_balancing_loss: 1.000658E+00 | loss scale: 1.0 | grad norm: 0.047 | params norm: 11452.340 | params rms: 0.062011 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 22:39:02.641190 | finish at 2026-10-11 15:15:47 + [2026-10-06 16:38:10] iteration 2901/ 6855 | consumed samples: 371328 | elapsed time per iteration (ms): 84978.6 | throughput per GPU (TFLOP/s/GPU): 43.4 | MFU 13.92% | FLOPs estimator: workload-aware | logical tokens/s: 197428.3 | valid tokens/s: 197428.3 | physical tokens/s: 198099.3 | loss tokens/s: 149552.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.773807E-01 | lm-global-supervised-tokens: 1.270873E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.844494E-02 | lm-token-buffer-mib: 5.833011E+01 | global_load_balancing_loss: 9.998463E-01 | loss scale: 1.0 | grad norm: 0.122 | params norm: 11452.276 | params rms: 0.062010 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 21:20:05.218583 | finish at 2026-10-10 13:58:15 + [2026-10-06 16:39:37] iteration 2902/ 6855 | consumed samples: 371456 | elapsed time per iteration (ms): 87212.6 | throughput per GPU (TFLOP/s/GPU): 43.2 | MFU 13.86% | FLOPs estimator: workload-aware | logical tokens/s: 192371.2 | valid tokens/s: 192371.2 | physical tokens/s: 192802.8 | loss tokens/s: 138935.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.538741E-01 | lm-global-supervised-tokens: 1.211696E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.342794E-02 | lm-token-buffer-mib: 5.834151E+01 | global_load_balancing_loss: 9.992580E-01 | loss scale: 1.0 | grad norm: 0.127 | params norm: 11452.212 | params rms: 0.062010 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 23:45:51.379497 | finish at 2026-10-10 16:25:28 + [2026-10-06 16:41:04] iteration 2903/ 6855 | consumed samples: 371584 | elapsed time per iteration (ms): 87066.9 | throughput per GPU (TFLOP/s/GPU): 43.1 | MFU 13.82% | FLOPs estimator: workload-aware | logical tokens/s: 192692.8 | valid tokens/s: 192692.8 | physical tokens/s: 193077.5 | loss tokens/s: 145272.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.123283E-01 | lm-global-supervised-tokens: 1.264845E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.534672E-02 | lm-token-buffer-mib: 5.824320E+01 | global_load_balancing_loss: 9.995247E-01 | loss scale: 1.0 | grad norm: 0.054 | params norm: 11452.149 | params rms: 0.062010 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 23:34:48.374725 | finish at 2026-10-10 16:15:52 + [2026-10-06 16:42:34] iteration 2904/ 6855 | consumed samples: 371712 | elapsed time per iteration (ms): 89602.3 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.50% | FLOPs estimator: workload-aware | logical tokens/s: 187240.5 | valid tokens/s: 187240.5 | physical tokens/s: 187554.0 | loss tokens/s: 139587.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.011850E-01 | lm-global-supervised-tokens: 1.250733E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.150294E-02 | lm-token-buffer-mib: 5.816103E+01 | global_load_balancing_loss: 1.000624E+00 | loss scale: 1.0 | grad norm: 0.101 | params norm: 11452.085 | params rms: 0.062009 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 2:20:18.638035 | finish at 2026-10-10 19:02:52 + [2026-10-06 16:44:02] iteration 2905/ 6855 | consumed samples: 371840 | elapsed time per iteration (ms): 88237.5 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.40% | FLOPs estimator: workload-aware | logical tokens/s: 190137.1 | valid tokens/s: 190137.1 | physical tokens/s: 190700.8 | loss tokens/s: 136857.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.571036E-01 | lm-global-supervised-tokens: 1.207595E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.521209E-02 | lm-token-buffer-mib: 5.805310E+01 | global_load_balancing_loss: 1.000338E+00 | loss scale: 1.0 | grad norm: 0.129 | params norm: 11452.022 | params rms: 0.062009 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 0:48:58.071132 | finish at 2026-10-10 17:33:00 + [2026-10-06 16:45:26] iteration 2906/ 6855 | consumed samples: 371968 | elapsed time per iteration (ms): 84608.0 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.73% | FLOPs estimator: workload-aware | logical tokens/s: 198293.3 | valid tokens/s: 198293.3 | physical tokens/s: 198834.7 | loss tokens/s: 151807.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.494954E-01 | lm-global-supervised-tokens: 1.284416E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.176131E-02 | lm-token-buffer-mib: 5.831009E+01 | global_load_balancing_loss: 1.000293E+00 | loss scale: 1.0 | grad norm: 0.041 | params norm: 11451.958 | params rms: 0.062009 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 20:48:37.105140 | finish at 2026-10-10 13:34:04 + [2026-10-06 16:46:55] iteration 2907/ 6855 | consumed samples: 372096 | elapsed time per iteration (ms): 88162.1 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.74% | FLOPs estimator: workload-aware | logical tokens/s: 190299.0 | valid tokens/s: 190299.0 | physical tokens/s: 190818.1 | loss tokens/s: 145027.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.728839E-01 | lm-global-supervised-tokens: 1.278591E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.468844E-02 | lm-token-buffer-mib: 5.841405E+01 | global_load_balancing_loss: 9.999809E-01 | loss scale: 1.0 | grad norm: 0.139 | params norm: 11451.896 | params rms: 0.062008 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 0:41:03.966396 | finish at 2026-10-10 17:27:59 + [2026-10-06 16:48:25] iteration 2908/ 6855 | consumed samples: 372224 | elapsed time per iteration (ms): 90400.6 | throughput per GPU (TFLOP/s/GPU): 41.2 | MFU 13.20% | FLOPs estimator: workload-aware | logical tokens/s: 185586.5 | valid tokens/s: 185586.5 | physical tokens/s: 186147.4 | loss tokens/s: 132689.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.860185E-01 | lm-global-supervised-tokens: 1.199518E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.293584E-02 | lm-token-buffer-mib: 5.812028E+01 | global_load_balancing_loss: 1.001105E+00 | loss scale: 1.0 | grad norm: 0.167 | params norm: 11451.834 | params rms: 0.062008 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 3:06:51.272484 | finish at 2026-10-10 19:55:16 + [2026-10-06 16:49:54] iteration 2909/ 6855 | consumed samples: 372352 | elapsed time per iteration (ms): 89462.5 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.46% | FLOPs estimator: workload-aware | logical tokens/s: 187533.2 | valid tokens/s: 187533.2 | physical tokens/s: 188162.3 | loss tokens/s: 140504.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.369570E-01 | lm-global-supervised-tokens: 1.256986E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.173572E-02 | lm-token-buffer-mib: 5.811942E+01 | global_load_balancing_loss: 1.000859E+00 | loss scale: 1.0 | grad norm: 0.150 | params norm: 11451.772 | params rms: 0.062008 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 2:03:39.122279 | finish at 2026-10-10 18:53:34 + [2026-10-06 16:51:45] iteration 2910/ 6855 | consumed samples: 372480 | elapsed time per iteration (ms): 110270.6 | throughput per GPU (TFLOP/s/GPU): 34.1 | MFU 10.92% | FLOPs estimator: workload-aware | logical tokens/s: 152145.6 | valid tokens/s: 152145.6 | physical tokens/s: 152438.2 | loss tokens/s: 108903.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.367082E-01 | lm-global-supervised-tokens: 1.200888E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.601841E-02 | lm-token-buffer-mib: 5.803525E+01 | global_load_balancing_loss: 9.998888E-01 | loss scale: 1.0 | grad norm: 0.152 | params norm: 11451.712 | params rms: 0.062007 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 0:50:17.654639 | finish at 2026-10-11 17:42:02 + [2026-10-06 16:53:13] iteration 2911/ 6855 | consumed samples: 372608 | elapsed time per iteration (ms): 88212.9 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.59% | FLOPs estimator: workload-aware | logical tokens/s: 190190.0 | valid tokens/s: 190190.0 | physical tokens/s: 190564.8 | loss tokens/s: 144221.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.606751E-01 | lm-global-supervised-tokens: 1.272223E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.890057E-02 | lm-token-buffer-mib: 5.816206E+01 | global_load_balancing_loss: 9.988990E-01 | loss scale: 1.0 | grad norm: 0.051 | params norm: 11451.650 | params rms: 0.062007 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 0:38:31.654730 | finish at 2026-10-10 17:31:45 + [2026-10-06 16:54:41] iteration 2912/ 6855 | consumed samples: 372736 | elapsed time per iteration (ms): 87775.9 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.50% | FLOPs estimator: workload-aware | logical tokens/s: 191136.1 | valid tokens/s: 191136.1 | physical tokens/s: 191614.2 | loss tokens/s: 143030.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.193151E-01 | lm-global-supervised-tokens: 1.255466E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.376209E-02 | lm-token-buffer-mib: 5.820736E+01 | global_load_balancing_loss: 1.000788E+00 | loss scale: 1.0 | grad norm: 0.231 | params norm: 11451.587 | params rms: 0.062007 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 0:08:20.238823 | finish at 2026-10-10 17:03:01 + [2026-10-06 16:56:09] iteration 2913/ 6855 | consumed samples: 372864 | elapsed time per iteration (ms): 88244.4 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.48% | FLOPs estimator: workload-aware | logical tokens/s: 190120.6 | valid tokens/s: 190120.6 | physical tokens/s: 190705.5 | loss tokens/s: 142249.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.840975E-01 | lm-global-supervised-tokens: 1.255271E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.144226E-02 | lm-token-buffer-mib: 5.813927E+01 | global_load_balancing_loss: 1.000313E+00 | loss scale: 1.0 | grad norm: 0.044 | params norm: 11451.523 | params rms: 0.062006 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 0:37:39.609106 | finish at 2026-10-10 17:33:49 + [2026-10-06 16:57:36] iteration 2914/ 6855 | consumed samples: 372992 | elapsed time per iteration (ms): 87456.1 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.63% | FLOPs estimator: workload-aware | logical tokens/s: 191835.3 | valid tokens/s: 191835.3 | physical tokens/s: 192383.0 | loss tokens/s: 138180.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.428761E-01 | lm-global-supervised-tokens: 1.208474E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.133334E-02 | lm-token-buffer-mib: 5.815900E+01 | global_load_balancing_loss: 9.998173E-01 | loss scale: 1.0 | grad norm: 0.170 | params norm: 11451.460 | params rms: 0.062006 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 23:44:24.337832 | finish at 2026-10-10 16:42:01 + [2026-10-06 16:59:08] iteration 2915/ 6855 | consumed samples: 373120 | elapsed time per iteration (ms): 91403.6 | throughput per GPU (TFLOP/s/GPU): 41.1 | MFU 13.16% | FLOPs estimator: workload-aware | logical tokens/s: 183550.7 | valid tokens/s: 183550.7 | physical tokens/s: 183969.2 | loss tokens/s: 132291.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.545774E-01 | lm-global-supervised-tokens: 1.209194E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.654278E-02 | lm-token-buffer-mib: 5.818173E+01 | global_load_balancing_loss: 1.000425E+00 | loss scale: 1.0 | grad norm: 0.062 | params norm: 11451.397 | params rms: 0.062006 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 4:02:10.352998 | finish at 2026-10-10 21:01:18 + [2026-10-06 17:00:33] iteration 2916/ 6855 | consumed samples: 373248 | elapsed time per iteration (ms): 85377.5 | throughput per GPU (TFLOP/s/GPU): 43.7 | MFU 14.01% | FLOPs estimator: workload-aware | logical tokens/s: 196505.9 | valid tokens/s: 196505.9 | physical tokens/s: 196971.0 | loss tokens/s: 141413.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.869106E-01 | lm-global-supervised-tokens: 1.207352E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.169790E-02 | lm-token-buffer-mib: 5.808254E+01 | global_load_balancing_loss: 1.000958E+00 | loss scale: 1.0 | grad norm: 0.084 | params norm: 11451.331 | params rms: 0.062005 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 21:25:02.018743 | finish at 2026-10-10 14:25:35 + [2026-10-06 17:02:01] iteration 2917/ 6855 | consumed samples: 373376 | elapsed time per iteration (ms): 87709.7 | throughput per GPU (TFLOP/s/GPU): 41.3 | MFU 13.25% | FLOPs estimator: workload-aware | logical tokens/s: 191280.4 | valid tokens/s: 191280.3 | physical tokens/s: 191779.6 | loss tokens/s: 143167.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.828369E-01 | lm-global-supervised-tokens: 1.255714E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.642053E-02 | lm-token-buffer-mib: 5.814375E+01 | global_load_balancing_loss: 9.997898E-01 | loss scale: 1.0 | grad norm: 0.088 | params norm: 11451.266 | params rms: 0.062005 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 23:56:40.992435 | finish at 2026-10-10 16:58:42 + [2026-10-06 17:03:29] iteration 2918/ 6855 | consumed samples: 373504 | elapsed time per iteration (ms): 88534.5 | throughput per GPU (TFLOP/s/GPU): 43.5 | MFU 13.95% | FLOPs estimator: workload-aware | logical tokens/s: 189498.9 | valid tokens/s: 189498.9 | physical tokens/s: 189841.0 | loss tokens/s: 134431.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.695575E-01 | lm-global-supervised-tokens: 1.190183E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.171228E-02 | lm-token-buffer-mib: 5.802604E+01 | global_load_balancing_loss: 9.996142E-01 | loss scale: 1.0 | grad norm: 0.036 | params norm: 11451.201 | params rms: 0.062005 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 0:49:20.178673 | finish at 2026-10-10 17:52:50 + [2026-10-06 17:04:57] iteration 2919/ 6855 | consumed samples: 373632 | elapsed time per iteration (ms): 87642.7 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.77% | FLOPs estimator: workload-aware | logical tokens/s: 191427.4 | valid tokens/s: 191427.4 | physical tokens/s: 191813.0 | loss tokens/s: 137152.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.726686E-01 | lm-global-supervised-tokens: 1.202042E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.238316E-02 | lm-token-buffer-mib: 5.803313E+01 | global_load_balancing_loss: 9.995325E-01 | loss scale: 1.0 | grad norm: 0.042 | params norm: 11451.136 | params rms: 0.062004 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 23:49:21.644508 | finish at 2026-10-10 16:54:19 + [2026-10-06 17:06:47] iteration 2920/ 6855 | consumed samples: 373760 | elapsed time per iteration (ms): 109873.0 | throughput per GPU (TFLOP/s/GPU): 34.0 | MFU 10.89% | FLOPs estimator: workload-aware | logical tokens/s: 152696.5 | valid tokens/s: 152696.5 | physical tokens/s: 153093.6 | loss tokens/s: 111814.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.905180E-01 | lm-global-supervised-tokens: 1.228536E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.161309E-02 | lm-token-buffer-mib: 5.821284E+01 | global_load_balancing_loss: 9.999486E-01 | loss scale: 1.0 | grad norm: 0.079 | params norm: 11451.071 | params rms: 0.062004 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 0:05:50.096080 | finish at 2026-10-11 17:12:37 +times across ranks (ms): + save-checkpoint: + rank 0: 25151.22 + rank 1: 25151.21 + rank 2: 25151.26 + rank 3: 25151.23 + rank 4: 25151.23 + rank 5: 25151.22 + rank 6: 25151.20 + rank 7: 25151.21 + rank 8: 25151.45 + rank 9: 25151.44 + rank 10: 25151.45 + rank 11: 25151.43 + rank 12: 25151.44 + rank 13: 25151.44 + rank 14: 25151.44 + rank 15: 25151.43 + rank 16: 25151.11 + rank 17: 25151.03 + rank 18: 25151.04 + rank 19: 25151.04 + rank 20: 25151.03 + rank 21: 25151.04 + rank 22: 25151.03 + rank 23: 25151.02 + rank 24: 25151.26 + rank 25: 25151.15 + rank 26: 25151.31 + rank 27: 25151.04 + rank 28: 25151.14 + rank 29: 25151.53 + rank 30: 25151.19 + rank 31: 25151.17 + rank 32: 25151.22 + rank 33: 25151.28 + rank 34: 25151.38 + rank 35: 25151.23 + rank 36: 25151.19 + rank 37: 25151.18 + rank 38: 25151.18 + rank 39: 25151.28 + rank 40: 25150.26 + rank 41: 25150.25 + rank 42: 25150.26 + rank 43: 25150.28 + rank 44: 25150.24 + rank 45: 25150.25 + rank 46: 25150.28 + rank 47: 25150.25 + rank 48: 25150.49 + rank 49: 25150.38 + rank 50: 25150.44 + rank 51: 25150.37 + rank 52: 25150.28 + rank 53: 25150.23 + rank 54: 25150.39 + rank 55: 25150.42 + rank 56: 25152.20 + rank 57: 25152.17 + rank 58: 25152.16 + rank 59: 25152.16 + rank 60: 25152.16 + rank 61: 25152.16 + rank 62: 25152.16 + rank 63: 25152.19 + [2026-10-06 17:08:50] iteration 2921/ 6855 | consumed samples: 373888 | elapsed time per iteration (ms): 97239.9 | throughput per GPU (TFLOP/s/GPU): 37.6 | MFU 12.05% | FLOPs estimator: workload-aware | logical tokens/s: 172534.3 | valid tokens/s: 172534.3 | physical tokens/s: 172920.8 | loss tokens/s: 125494.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.620229E-01 | lm-global-supervised-tokens: 1.220307E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.294636E-02 | lm-token-buffer-mib: 5.806001E+01 | global_load_balancing_loss: 1.000524E+00 | loss scale: 1.0 | grad norm: 0.044 | params norm: 11451.007 | params rms: 0.062003 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 10:15:41.762352 | finish at 2026-10-11 03:24:31 + [2026-10-06 17:10:22] iteration 2922/ 6855 | consumed samples: 374016 | elapsed time per iteration (ms): 92547.4 | throughput per GPU (TFLOP/s/GPU): 40.7 | MFU 13.04% | FLOPs estimator: workload-aware | logical tokens/s: 181282.4 | valid tokens/s: 181282.4 | physical tokens/s: 181670.3 | loss tokens/s: 133628.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.434345E-01 | lm-global-supervised-tokens: 1.236699E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.644196E-02 | lm-token-buffer-mib: 5.803154E+01 | global_load_balancing_loss: 1.000763E+00 | loss scale: 1.0 | grad norm: 0.065 | params norm: 11450.943 | params rms: 0.062003 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 5:06:28.808854 | finish at 2026-10-10 22:16:51 + [2026-10-06 17:11:48] iteration 2923/ 6855 | consumed samples: 374144 | elapsed time per iteration (ms): 85797.1 | throughput per GPU (TFLOP/s/GPU): 43.2 | MFU 13.83% | FLOPs estimator: workload-aware | logical tokens/s: 195544.6 | valid tokens/s: 195544.6 | physical tokens/s: 195932.4 | loss tokens/s: 145024.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.013443E-01 | lm-global-supervised-tokens: 1.244265E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.453345E-02 | lm-token-buffer-mib: 5.811584E+01 | global_load_balancing_loss: 9.996598E-01 | loss scale: 1.0 | grad norm: 0.064 | params norm: 11450.878 | params rms: 0.062003 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 21:42:34.273398 | finish at 2026-10-10 14:54:22 + [2026-10-06 17:13:16] iteration 2924/ 6855 | consumed samples: 374272 | elapsed time per iteration (ms): 87838.5 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.61% | FLOPs estimator: workload-aware | logical tokens/s: 190999.5 | valid tokens/s: 190999.5 | physical tokens/s: 191414.1 | loss tokens/s: 139228.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.876063E-01 | lm-global-supervised-tokens: 1.222959E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.298678E-02 | lm-token-buffer-mib: 5.813148E+01 | global_load_balancing_loss: 9.998854E-01 | loss scale: 1.0 | grad norm: 0.074 | params norm: 11450.814 | params rms: 0.062002 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 23:54:52.990823 | finish at 2026-10-10 17:08:09 + [2026-10-06 17:14:45] iteration 2925/ 6855 | consumed samples: 374400 | elapsed time per iteration (ms): 89104.3 | throughput per GPU (TFLOP/s/GPU): 40.7 | MFU 13.06% | FLOPs estimator: workload-aware | logical tokens/s: 188286.3 | valid tokens/s: 188286.3 | physical tokens/s: 188742.2 | loss tokens/s: 136479.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.469896E-01 | lm-global-supervised-tokens: 1.216092E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.700714E-02 | lm-token-buffer-mib: 5.817219E+01 | global_load_balancing_loss: 1.000686E+00 | loss scale: 1.0 | grad norm: 0.051 | params norm: 11450.750 | params rms: 0.062002 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 1:16:19.978731 | finish at 2026-10-10 18:31:05 + [2026-10-06 17:16:11] iteration 2926/ 6855 | consumed samples: 374528 | elapsed time per iteration (ms): 86125.1 | throughput per GPU (TFLOP/s/GPU): 43.5 | MFU 13.95% | FLOPs estimator: workload-aware | logical tokens/s: 194800.4 | valid tokens/s: 194800.4 | physical tokens/s: 195325.4 | loss tokens/s: 141897.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.928754E-01 | lm-global-supervised-tokens: 1.222097E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.234421E-02 | lm-token-buffer-mib: 5.808943E+01 | global_load_balancing_loss: 1.000682E+00 | loss scale: 1.0 | grad norm: 0.058 | params norm: 11450.689 | params rms: 0.062002 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 21:59:45.470659 | finish at 2026-10-10 15:15:56 + [2026-10-06 17:17:35] iteration 2927/ 6855 | consumed samples: 374656 | elapsed time per iteration (ms): 84303.4 | throughput per GPU (TFLOP/s/GPU): 43.7 | MFU 14.00% | FLOPs estimator: workload-aware | logical tokens/s: 199009.3 | valid tokens/s: 199009.3 | physical tokens/s: 199397.8 | loss tokens/s: 149327.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.667513E-01 | lm-global-supervised-tokens: 1.258882E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.160462E-02 | lm-token-buffer-mib: 5.804569E+01 | global_load_balancing_loss: 1.000038E+00 | loss scale: 1.0 | grad norm: 0.039 | params norm: 11450.627 | params rms: 0.062001 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 19:59:03.892086 | finish at 2026-10-10 13:16:39 + [2026-10-06 17:19:01] iteration 2928/ 6855 | consumed samples: 374784 | elapsed time per iteration (ms): 86131.5 | throughput per GPU (TFLOP/s/GPU): 42.8 | MFU 13.71% | FLOPs estimator: workload-aware | logical tokens/s: 194785.4 | valid tokens/s: 194785.4 | physical tokens/s: 195248.9 | loss tokens/s: 141862.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.329675E-01 | lm-global-supervised-tokens: 1.221886E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.150624E-02 | lm-token-buffer-mib: 5.823845E+01 | global_load_balancing_loss: 9.997318E-01 | loss scale: 1.0 | grad norm: 0.037 | params norm: 11450.564 | params rms: 0.062001 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 21:57:18.304087 | finish at 2026-10-10 15:16:20 + [2026-10-06 17:20:30] iteration 2929/ 6855 | consumed samples: 374912 | elapsed time per iteration (ms): 88333.4 | throughput per GPU (TFLOP/s/GPU): 41.6 | MFU 13.34% | FLOPs estimator: workload-aware | logical tokens/s: 189930.0 | valid tokens/s: 189930.0 | physical tokens/s: 190482.5 | loss tokens/s: 144788.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.602435E-01 | lm-global-supervised-tokens: 1.278961E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.506873E-02 | lm-token-buffer-mib: 5.825193E+01 | global_load_balancing_loss: 1.000083E+00 | loss scale: 1.0 | grad norm: 0.045 | params norm: 11450.502 | params rms: 0.062001 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 0:19:56.916275 | finish at 2026-10-10 17:40:27 + [2026-10-06 17:22:21] iteration 2930/ 6855 | consumed samples: 375040 | elapsed time per iteration (ms): 111035.1 | throughput per GPU (TFLOP/s/GPU): 33.7 | MFU 10.80% | FLOPs estimator: workload-aware | logical tokens/s: 151098.0 | valid tokens/s: 151098.0 | physical tokens/s: 151467.4 | loss tokens/s: 109120.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.082208E-01 | lm-global-supervised-tokens: 1.211622E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.211977E-02 | lm-token-buffer-mib: 5.802731E+01 | global_load_balancing_loss: 1.000467E+00 | loss scale: 1.0 | grad norm: 0.039 | params norm: 11450.442 | params rms: 0.062000 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 1:03:32.734681 | finish at 2026-10-11 18:25:53 + [2026-10-06 17:23:47] iteration 2931/ 6855 | consumed samples: 375168 | elapsed time per iteration (ms): 85842.5 | throughput per GPU (TFLOP/s/GPU): 42.8 | MFU 13.72% | FLOPs estimator: workload-aware | logical tokens/s: 195441.4 | valid tokens/s: 195441.4 | physical tokens/s: 195942.9 | loss tokens/s: 147716.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.656560E-01 | lm-global-supervised-tokens: 1.268034E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.321556E-02 | lm-token-buffer-mib: 5.832956E+01 | global_load_balancing_loss: 9.996704E-01 | loss scale: 1.0 | grad norm: 0.045 | params norm: 11450.379 | params rms: 0.062000 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 21:34:05.886623 | finish at 2026-10-10 14:57:52 + [2026-10-06 17:25:16] iteration 2932/ 6855 | consumed samples: 375296 | elapsed time per iteration (ms): 89382.1 | throughput per GPU (TFLOP/s/GPU): 41.7 | MFU 13.38% | FLOPs estimator: workload-aware | logical tokens/s: 187702.1 | valid tokens/s: 187702.1 | physical tokens/s: 188233.8 | loss tokens/s: 135246.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 5.988350E-01 | lm-global-supervised-tokens: 1.208858E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.371243E-02 | lm-token-buffer-mib: 5.804960E+01 | global_load_balancing_loss: 1.001461E+00 | loss scale: 1.0 | grad norm: 0.066 | params norm: 11450.317 | params rms: 0.062000 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 1:24:06.105916 | finish at 2026-10-10 18:49:22 + [2026-10-06 17:26:44] iteration 2933/ 6855 | consumed samples: 375424 | elapsed time per iteration (ms): 88440.1 | throughput per GPU (TFLOP/s/GPU): 41.5 | MFU 13.29% | FLOPs estimator: workload-aware | logical tokens/s: 189700.8 | valid tokens/s: 189700.8 | physical tokens/s: 190105.9 | loss tokens/s: 134639.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.636969E-01 | lm-global-supervised-tokens: 1.190754E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.434315E-02 | lm-token-buffer-mib: 5.817935E+01 | global_load_balancing_loss: 9.994435E-01 | loss scale: 1.0 | grad norm: 0.091 | params norm: 11450.255 | params rms: 0.061999 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 0:21:01.916799 | finish at 2026-10-10 17:47:46 + [2026-10-06 17:28:10] iteration 2934/ 6855 | consumed samples: 375552 | elapsed time per iteration (ms): 85443.1 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.54% | FLOPs estimator: workload-aware | logical tokens/s: 196355.2 | valid tokens/s: 196355.2 | physical tokens/s: 196872.0 | loss tokens/s: 151452.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.076271E-01 | lm-global-supervised-tokens: 1.294060E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.278771E-02 | lm-token-buffer-mib: 5.813283E+01 | global_load_balancing_loss: 9.994429E-01 | loss scale: 1.0 | grad norm: 0.089 | params norm: 11450.194 | params rms: 0.061999 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 21:03:42.540839 | finish at 2026-10-10 14:31:52 + [2026-10-06 17:29:36] iteration 2935/ 6855 | consumed samples: 375680 | elapsed time per iteration (ms): 86530.1 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.48% | FLOPs estimator: workload-aware | logical tokens/s: 193888.5 | valid tokens/s: 193888.5 | physical tokens/s: 194325.8 | loss tokens/s: 144680.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.061117E-01 | lm-global-supervised-tokens: 1.251918E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.267458E-02 | lm-token-buffer-mib: 5.818151E+01 | global_load_balancing_loss: 1.000329E+00 | loss scale: 1.0 | grad norm: 0.073 | params norm: 11450.130 | params rms: 0.061999 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 22:13:18.153172 | finish at 2026-10-10 15:42:55 + [2026-10-06 17:31:03] iteration 2936/ 6855 | consumed samples: 375808 | elapsed time per iteration (ms): 86691.4 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.44% | FLOPs estimator: workload-aware | logical tokens/s: 193526.9 | valid tokens/s: 193526.9 | physical tokens/s: 193934.3 | loss tokens/s: 137426.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.574062E-01 | lm-global-supervised-tokens: 1.191368E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.326588E-02 | lm-token-buffer-mib: 5.820084E+01 | global_load_balancing_loss: 1.001248E+00 | loss scale: 1.0 | grad norm: 0.041 | params norm: 11450.066 | params rms: 0.061998 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 22:22:23.492152 | finish at 2026-10-10 15:53:27 + [2026-10-06 17:32:30] iteration 2937/ 6855 | consumed samples: 375936 | elapsed time per iteration (ms): 87137.8 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.58% | FLOPs estimator: workload-aware | logical tokens/s: 192535.6 | valid tokens/s: 192535.6 | physical tokens/s: 192893.5 | loss tokens/s: 138149.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.669224E-01 | lm-global-supervised-tokens: 1.203801E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.492386E-02 | lm-token-buffer-mib: 5.812568E+01 | global_load_balancing_loss: 1.000990E+00 | loss scale: 1.0 | grad norm: 0.058 | params norm: 11450.002 | params rms: 0.061998 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 22:50:05.912458 | finish at 2026-10-10 16:22:36 + [2026-10-06 17:34:03] iteration 2938/ 6855 | consumed samples: 376064 | elapsed time per iteration (ms): 92724.6 | throughput per GPU (TFLOP/s/GPU): 41.1 | MFU 13.19% | FLOPs estimator: workload-aware | logical tokens/s: 180935.5 | valid tokens/s: 180935.5 | physical tokens/s: 181302.9 | loss tokens/s: 128836.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.629570E-01 | lm-global-supervised-tokens: 1.194631E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.434297E-02 | lm-token-buffer-mib: 5.817839E+01 | global_load_balancing_loss: 9.993116E-01 | loss scale: 1.0 | grad norm: 0.035 | params norm: 11449.937 | params rms: 0.061998 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 4:53:22.406054 | finish at 2026-10-10 22:27:25 + [2026-10-06 17:35:30] iteration 2939/ 6855 | consumed samples: 376192 | elapsed time per iteration (ms): 87465.9 | throughput per GPU (TFLOP/s/GPU): 43.1 | MFU 13.83% | FLOPs estimator: workload-aware | logical tokens/s: 191814.2 | valid tokens/s: 191814.2 | physical tokens/s: 192212.7 | loss tokens/s: 132795.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.664355E-01 | lm-global-supervised-tokens: 1.161512E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.443762E-02 | lm-token-buffer-mib: 5.821626E+01 | global_load_balancing_loss: 9.997950E-01 | loss scale: 1.0 | grad norm: 0.047 | params norm: 11449.872 | params rms: 0.061997 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 23:08:36.535139 | finish at 2026-10-10 16:44:07 + [2026-10-06 17:37:20] iteration 2940/ 6855 | consumed samples: 376320 | elapsed time per iteration (ms): 109149.1 | throughput per GPU (TFLOP/s/GPU): 33.5 | MFU 10.74% | FLOPs estimator: workload-aware | logical tokens/s: 153708.9 | valid tokens/s: 153708.9 | physical tokens/s: 154121.5 | loss tokens/s: 113704.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.454427E-01 | lm-global-supervised-tokens: 1.241078E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.227966E-02 | lm-token-buffer-mib: 5.809903E+01 | global_load_balancing_loss: 9.989230E-01 | loss scale: 1.0 | grad norm: 0.076 | params norm: 11449.805 | params rms: 0.061997 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 22:41:58.659550 | finish at 2026-10-11 16:19:18 + [2026-10-06 17:38:47] iteration 2941/ 6855 | consumed samples: 376448 | elapsed time per iteration (ms): 87008.4 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.76% | FLOPs estimator: workload-aware | logical tokens/s: 192821.6 | valid tokens/s: 192821.6 | physical tokens/s: 193444.8 | loss tokens/s: 139386.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.825208E-01 | lm-global-supervised-tokens: 1.212780E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.176113E-02 | lm-token-buffer-mib: 5.827898E+01 | global_load_balancing_loss: 1.000055E+00 | loss scale: 1.0 | grad norm: 0.176 | params norm: 11449.737 | params rms: 0.061997 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 22:35:50.724417 | finish at 2026-10-10 16:14:37 + [2026-10-06 17:40:14] iteration 2942/ 6855 | consumed samples: 376576 | elapsed time per iteration (ms): 87048.3 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.69% | FLOPs estimator: workload-aware | logical tokens/s: 192734.6 | valid tokens/s: 192734.6 | physical tokens/s: 193264.3 | loss tokens/s: 145968.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.531673E-01 | lm-global-supervised-tokens: 1.270627E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.849007E-02 | lm-token-buffer-mib: 5.821498E+01 | global_load_balancing_loss: 1.000388E+00 | loss scale: 1.0 | grad norm: 0.189 | params norm: 11449.669 | params rms: 0.061996 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 22:36:59.987747 | finish at 2026-10-10 16:17:14 + [2026-10-06 17:41:40] iteration 2943/ 6855 | consumed samples: 376704 | elapsed time per iteration (ms): 86058.2 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.47% | FLOPs estimator: workload-aware | logical tokens/s: 194951.8 | valid tokens/s: 194951.8 | physical tokens/s: 195509.9 | loss tokens/s: 148522.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.508251E-01 | lm-global-supervised-tokens: 1.278159E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.279116E-02 | lm-token-buffer-mib: 5.820506E+01 | global_load_balancing_loss: 1.001325E+00 | loss scale: 1.0 | grad norm: 0.041 | params norm: 11449.604 | params rms: 0.061996 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 21:30:59.761883 | finish at 2026-10-10 15:12:39 + [2026-10-06 17:43:07] iteration 2944/ 6855 | consumed samples: 376832 | elapsed time per iteration (ms): 86856.7 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.44% | FLOPs estimator: workload-aware | logical tokens/s: 193159.3 | valid tokens/s: 193159.3 | physical tokens/s: 193620.5 | loss tokens/s: 145989.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.513390E-01 | lm-global-supervised-tokens: 1.268015E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.368642E-02 | lm-token-buffer-mib: 5.802985E+01 | global_load_balancing_loss: 1.000836E+00 | loss scale: 1.0 | grad norm: 0.120 | params norm: 11449.538 | params rms: 0.061996 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 22:21:36.688685 | finish at 2026-10-10 16:04:43 + [2026-10-06 17:44:34] iteration 2945/ 6855 | consumed samples: 376960 | elapsed time per iteration (ms): 87553.4 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.73% | FLOPs estimator: workload-aware | logical tokens/s: 191622.2 | valid tokens/s: 191622.2 | physical tokens/s: 192101.4 | loss tokens/s: 141823.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.667783E-01 | lm-global-supervised-tokens: 1.241712E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.449702E-02 | lm-token-buffer-mib: 5.825126E+01 | global_load_balancing_loss: 1.000491E+00 | loss scale: 1.0 | grad norm: 0.055 | params norm: 11449.468 | params rms: 0.061995 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 23:05:33.949835 | finish at 2026-10-10 16:50:08 + [2026-10-06 17:46:06] iteration 2946/ 6855 | consumed samples: 377088 | elapsed time per iteration (ms): 91472.0 | throughput per GPU (TFLOP/s/GPU): 40.5 | MFU 12.97% | FLOPs estimator: workload-aware | logical tokens/s: 183412.8 | valid tokens/s: 183412.8 | physical tokens/s: 183889.5 | loss tokens/s: 138419.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.964669E-01 | lm-global-supervised-tokens: 1.266150E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.554978E-02 | lm-token-buffer-mib: 5.812765E+01 | global_load_balancing_loss: 1.000116E+00 | loss scale: 1.0 | grad norm: 0.113 | params norm: 11449.400 | params rms: 0.061995 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 3:19:24.203186 | finish at 2026-10-10 21:05:30 + [2026-10-06 17:47:32] iteration 2947/ 6855 | consumed samples: 377216 | elapsed time per iteration (ms): 86868.4 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.65% | FLOPs estimator: workload-aware | logical tokens/s: 193133.3 | valid tokens/s: 193133.3 | physical tokens/s: 193605.2 | loss tokens/s: 143686.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.760903E-01 | lm-global-supervised-tokens: 1.248179E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.350413E-02 | lm-token-buffer-mib: 5.804488E+01 | global_load_balancing_loss: 9.998779E-01 | loss scale: 1.0 | grad norm: 0.079 | params norm: 11449.330 | params rms: 0.061994 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 22:18:01.813797 | finish at 2026-10-10 16:05:34 + [2026-10-06 17:48:59] iteration 2948/ 6855 | consumed samples: 377344 | elapsed time per iteration (ms): 86950.0 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.49% | FLOPs estimator: workload-aware | logical tokens/s: 192951.3 | valid tokens/s: 192951.3 | physical tokens/s: 193386.6 | loss tokens/s: 149631.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.874719E-01 | lm-global-supervised-tokens: 1.301044E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.544051E-02 | lm-token-buffer-mib: 5.815164E+01 | global_load_balancing_loss: 9.992256E-01 | loss scale: 1.0 | grad norm: 0.079 | params norm: 11449.260 | params rms: 0.061994 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 22:21:53.557036 | finish at 2026-10-10 16:10:53 + [2026-10-06 17:50:22] iteration 2949/ 6855 | consumed samples: 377472 | elapsed time per iteration (ms): 82734.4 | throughput per GPU (TFLOP/s/GPU): 43.8 | MFU 14.05% | FLOPs estimator: workload-aware | logical tokens/s: 202784.0 | valid tokens/s: 202784.0 | physical tokens/s: 203263.9 | loss tokens/s: 150326.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.745812E-01 | lm-global-supervised-tokens: 1.243713E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.606243E-02 | lm-token-buffer-mib: 5.810675E+01 | global_load_balancing_loss: 9.997633E-01 | loss scale: 1.0 | grad norm: 0.167 | params norm: 11449.191 | params rms: 0.061994 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 17:46:00.402630 | finish at 2026-10-10 11:36:22 + [2026-10-06 17:52:11] iteration 2950/ 6855 | consumed samples: 377600 | elapsed time per iteration (ms): 108476.8 | throughput per GPU (TFLOP/s/GPU): 34.2 | MFU 10.98% | FLOPs estimator: workload-aware | logical tokens/s: 154661.4 | valid tokens/s: 154661.4 | physical tokens/s: 154932.8 | loss tokens/s: 114759.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.515660E-01 | lm-global-supervised-tokens: 1.244870E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.871553E-02 | lm-token-buffer-mib: 5.805751E+01 | global_load_balancing_loss: 1.000661E+00 | loss scale: 1.0 | grad norm: 0.053 | params norm: 11449.121 | params rms: 0.061993 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 21:40:02.097516 | finish at 2026-10-11 15:32:13 + [2026-10-06 17:53:37] iteration 2951/ 6855 | consumed samples: 377728 | elapsed time per iteration (ms): 86368.4 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.61% | FLOPs estimator: workload-aware | logical tokens/s: 194251.7 | valid tokens/s: 194251.7 | physical tokens/s: 194789.2 | loss tokens/s: 143939.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.456677E-01 | lm-global-supervised-tokens: 1.243180E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.268659E-02 | lm-token-buffer-mib: 5.808956E+01 | global_load_balancing_loss: 1.000721E+00 | loss scale: 1.0 | grad norm: 0.123 | params norm: 11449.052 | params rms: 0.061993 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 21:39:42.286102 | finish at 2026-10-10 15:33:19 + [2026-10-06 17:55:01] iteration 2952/ 6855 | consumed samples: 377856 | elapsed time per iteration (ms): 83705.5 | throughput per GPU (TFLOP/s/GPU): 43.1 | MFU 13.82% | FLOPs estimator: workload-aware | logical tokens/s: 200430.8 | valid tokens/s: 200430.8 | physical tokens/s: 201099.6 | loss tokens/s: 151563.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.842337E-01 | lm-global-supervised-tokens: 1.268671E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.227152E-02 | lm-token-buffer-mib: 5.805318E+01 | global_load_balancing_loss: 1.001917E+00 | loss scale: 1.0 | grad norm: 0.118 | params norm: 11448.984 | params rms: 0.061993 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 18:45:02.720532 | finish at 2026-10-10 12:40:03 + [2026-10-06 17:56:29] iteration 2953/ 6855 | consumed samples: 377984 | elapsed time per iteration (ms): 88121.6 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.62% | FLOPs estimator: workload-aware | logical tokens/s: 190386.5 | valid tokens/s: 190386.5 | physical tokens/s: 190715.3 | loss tokens/s: 139656.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.632104E-01 | lm-global-supervised-tokens: 1.230672E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.107666E-02 | lm-token-buffer-mib: 5.809924E+01 | global_load_balancing_loss: 1.000253E+00 | loss scale: 1.0 | grad norm: 0.133 | params norm: 11448.917 | params rms: 0.061992 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 23:30:50.296797 | finish at 2026-10-10 17:27:19 + [2026-10-06 17:57:54] iteration 2954/ 6855 | consumed samples: 378112 | elapsed time per iteration (ms): 84942.5 | throughput per GPU (TFLOP/s/GPU): 43.5 | MFU 13.93% | FLOPs estimator: workload-aware | logical tokens/s: 197511.5 | valid tokens/s: 197511.5 | physical tokens/s: 197966.8 | loss tokens/s: 145640.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.166529E-01 | lm-global-supervised-tokens: 1.237107E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.575072E-02 | lm-token-buffer-mib: 5.810641E+01 | global_load_balancing_loss: 9.998370E-01 | loss scale: 1.0 | grad norm: 0.250 | params norm: 11448.848 | params rms: 0.061992 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 20:02:40.597149 | finish at 2026-10-10 14:00:34 + [2026-10-06 17:59:21] iteration 2955/ 6855 | consumed samples: 378240 | elapsed time per iteration (ms): 86829.4 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.62% | FLOPs estimator: workload-aware | logical tokens/s: 193220.4 | valid tokens/s: 193220.4 | physical tokens/s: 193610.0 | loss tokens/s: 142161.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.683283E-01 | lm-global-supervised-tokens: 1.234379E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.166009E-02 | lm-token-buffer-mib: 5.811110E+01 | global_load_balancing_loss: 9.997290E-01 | loss scale: 1.0 | grad norm: 0.157 | params norm: 11448.779 | params rms: 0.061991 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 22:03:54.738350 | finish at 2026-10-10 16:03:15 + [2026-10-06 18:00:47] iteration 2956/ 6855 | consumed samples: 378368 | elapsed time per iteration (ms): 86344.9 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.64% | FLOPs estimator: workload-aware | logical tokens/s: 194303.9 | valid tokens/s: 194303.9 | physical tokens/s: 194757.8 | loss tokens/s: 139735.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.317942E-01 | lm-global-supervised-tokens: 1.206547E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.674476E-02 | lm-token-buffer-mib: 5.812686E+01 | global_load_balancing_loss: 9.993419E-01 | loss scale: 1.0 | grad norm: 0.200 | params norm: 11448.711 | params rms: 0.061991 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 21:30:58.653131 | finish at 2026-10-10 15:31:46 + [2026-10-06 18:02:11] iteration 2957/ 6855 | consumed samples: 378496 | elapsed time per iteration (ms): 84336.7 | throughput per GPU (TFLOP/s/GPU): 43.3 | MFU 13.89% | FLOPs estimator: workload-aware | logical tokens/s: 198930.6 | valid tokens/s: 198930.6 | physical tokens/s: 199477.6 | loss tokens/s: 149703.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.923389E-01 | lm-global-supervised-tokens: 1.262552E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.748811E-02 | lm-token-buffer-mib: 5.808120E+01 | global_load_balancing_loss: 9.995084E-01 | loss scale: 1.0 | grad norm: 0.082 | params norm: 11448.643 | params rms: 0.061991 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 19:19:04.419280 | finish at 2026-10-10 13:21:16 + [2026-10-06 18:03:40] iteration 2958/ 6855 | consumed samples: 378624 | elapsed time per iteration (ms): 89231.8 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.55% | FLOPs estimator: workload-aware | logical tokens/s: 188018.3 | valid tokens/s: 188018.3 | physical tokens/s: 188423.1 | loss tokens/s: 138825.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.164778E-01 | lm-global-supervised-tokens: 1.238760E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.137860E-02 | lm-token-buffer-mib: 5.810849E+01 | global_load_balancing_loss: 1.000292E+00 | loss scale: 1.0 | grad norm: 0.152 | params norm: 11448.575 | params rms: 0.061990 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 0:35:36.229210 | finish at 2026-10-10 18:39:17 + [2026-10-06 18:05:09] iteration 2959/ 6855 | consumed samples: 378752 | elapsed time per iteration (ms): 88785.0 | throughput per GPU (TFLOP/s/GPU): 41.7 | MFU 13.36% | FLOPs estimator: workload-aware | logical tokens/s: 188963.7 | valid tokens/s: 188963.7 | physical tokens/s: 189525.4 | loss tokens/s: 143578.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.715196E-01 | lm-global-supervised-tokens: 1.274759E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.344404E-02 | lm-token-buffer-mib: 5.820557E+01 | global_load_balancing_loss: 1.000514E+00 | loss scale: 1.0 | grad norm: 0.139 | params norm: 11448.506 | params rms: 0.061990 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 0:05:06.205212 | finish at 2026-10-10 18:10:15 + [2026-10-06 18:06:59] iteration 2960/ 6855 | consumed samples: 378880 | elapsed time per iteration (ms): 109764.7 | throughput per GPU (TFLOP/s/GPU): 34.6 | MFU 11.10% | FLOPs estimator: workload-aware | logical tokens/s: 152847.0 | valid tokens/s: 152847.0 | physical tokens/s: 153121.9 | loss tokens/s: 112762.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.751352E-01 | lm-global-supervised-tokens: 1.237729E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.142632E-02 | lm-token-buffer-mib: 5.814837E+01 | global_load_balancing_loss: 1.000098E+00 | loss scale: 1.0 | grad norm: 0.070 | params norm: 11448.438 | params rms: 0.061990 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 22:45:33.553611 | finish at 2026-10-11 16:52:33 +times across ranks (ms): + save-checkpoint: + rank 0: 25460.71 + rank 1: 25460.68 + rank 2: 25460.72 + rank 3: 25460.71 + rank 4: 25460.68 + rank 5: 25460.66 + rank 6: 25460.65 + rank 7: 25460.71 + rank 8: 25460.08 + rank 9: 25460.06 + rank 10: 25460.06 + rank 11: 25460.06 + rank 12: 25460.07 + rank 13: 25460.06 + rank 14: 25460.09 + rank 15: 25460.10 + rank 16: 25460.84 + rank 17: 25460.86 + rank 18: 25460.82 + rank 19: 25460.85 + rank 20: 25460.82 + rank 21: 25460.85 + rank 22: 25460.82 + rank 23: 25460.81 + rank 24: 25460.75 + rank 25: 25461.04 + rank 26: 25460.82 + rank 27: 25460.80 + rank 28: 25460.65 + rank 29: 25460.82 + rank 30: 25460.71 + rank 31: 25460.74 + rank 32: 25460.60 + rank 33: 25460.71 + rank 34: 25460.69 + rank 35: 25460.54 + rank 36: 25460.59 + rank 37: 25460.58 + rank 38: 25460.66 + rank 39: 25460.60 + rank 40: 25460.79 + rank 41: 25460.79 + rank 42: 25460.80 + rank 43: 25460.81 + rank 44: 25460.80 + rank 45: 25460.81 + rank 46: 25460.79 + rank 47: 25460.80 + rank 48: 25460.83 + rank 49: 25460.79 + rank 50: 25460.64 + rank 51: 25460.59 + rank 52: 25460.84 + rank 53: 25460.77 + rank 54: 25460.61 + rank 55: 25460.82 + rank 56: 25460.77 + rank 57: 25460.75 + rank 58: 25460.73 + rank 59: 25460.73 + rank 60: 25460.73 + rank 61: 25460.74 + rank 62: 25460.73 + rank 63: 25460.59 + [2026-10-06 18:09:05] iteration 2961/ 6855 | consumed samples: 379008 | elapsed time per iteration (ms): 100481.1 | throughput per GPU (TFLOP/s/GPU): 36.2 | MFU 11.61% | FLOPs estimator: workload-aware | logical tokens/s: 166968.0 | valid tokens/s: 166968.0 | physical tokens/s: 167413.4 | loss tokens/s: 122897.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 5.910417E-01 | lm-global-supervised-tokens: 1.234891E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.377322E+00 | lm-token-buffer-mib: 5.813485E+01 | global_load_balancing_loss: 1.001035E+00 | loss scale: 1.0 | grad norm: 0.190 | params norm: 11448.371 | params rms: 0.061989 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 12:41:13.558764 | finish at 2026-10-11 06:50:19 + [2026-10-06 18:10:37] iteration 2962/ 6855 | consumed samples: 379136 | elapsed time per iteration (ms): 91809.6 | throughput per GPU (TFLOP/s/GPU): 40.4 | MFU 12.94% | FLOPs estimator: workload-aware | logical tokens/s: 182739.1 | valid tokens/s: 182739.1 | physical tokens/s: 183082.3 | loss tokens/s: 135110.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.232821E-01 | lm-global-supervised-tokens: 1.240445E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.491774E-02 | lm-token-buffer-mib: 5.811897E+01 | global_load_balancing_loss: 1.001742E+00 | loss scale: 1.0 | grad norm: 0.067 | params norm: 11448.304 | params rms: 0.061989 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 3:16:54.956095 | finish at 2026-10-10 21:27:32 + [2026-10-06 18:12:00] iteration 2963/ 6855 | consumed samples: 379264 | elapsed time per iteration (ms): 83111.9 | throughput per GPU (TFLOP/s/GPU): 43.5 | MFU 13.95% | FLOPs estimator: workload-aware | logical tokens/s: 201861.7 | valid tokens/s: 201861.7 | physical tokens/s: 202275.3 | loss tokens/s: 147362.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.703248E-01 | lm-global-supervised-tokens: 1.224756E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.181645E-02 | lm-token-buffer-mib: 5.815668E+01 | global_load_balancing_loss: 1.000127E+00 | loss scale: 1.0 | grad norm: 0.179 | params norm: 11448.241 | params rms: 0.061988 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 17:51:11.616299 | finish at 2026-10-10 12:03:12 + [2026-10-06 18:13:26] iteration 2964/ 6855 | consumed samples: 379392 | elapsed time per iteration (ms): 85527.7 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.74% | FLOPs estimator: workload-aware | logical tokens/s: 196160.8 | valid tokens/s: 196160.8 | physical tokens/s: 196531.4 | loss tokens/s: 149127.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.348222E-01 | lm-global-supervised-tokens: 1.275456E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.152434E-02 | lm-token-buffer-mib: 5.811166E+01 | global_load_balancing_loss: 9.999806E-01 | loss scale: 1.0 | grad norm: 0.099 | params norm: 11448.177 | params rms: 0.061988 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 20:26:28.291627 | finish at 2026-10-10 14:39:54 + [2026-10-06 18:14:53] iteration 2965/ 6855 | consumed samples: 379520 | elapsed time per iteration (ms): 87292.5 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.47% | FLOPs estimator: workload-aware | logical tokens/s: 192195.3 | valid tokens/s: 192195.3 | physical tokens/s: 192633.0 | loss tokens/s: 140756.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.325937E-01 | lm-global-supervised-tokens: 1.228701E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.377511E-02 | lm-token-buffer-mib: 5.820037E+01 | global_load_balancing_loss: 9.986281E-01 | loss scale: 1.0 | grad norm: 0.134 | params norm: 11448.115 | params rms: 0.061988 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 22:19:27.721200 | finish at 2026-10-10 16:34:21 + [2026-10-06 18:16:19] iteration 2966/ 6855 | consumed samples: 379648 | elapsed time per iteration (ms): 86052.0 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.43% | FLOPs estimator: workload-aware | logical tokens/s: 194965.7 | valid tokens/s: 194965.7 | physical tokens/s: 195639.5 | loss tokens/s: 152614.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.620604E-01 | lm-global-supervised-tokens: 1.313273E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.159061E-02 | lm-token-buffer-mib: 5.804549E+01 | global_load_balancing_loss: 9.982963E-01 | loss scale: 1.0 | grad norm: 0.310 | params norm: 11448.052 | params rms: 0.061987 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 20:57:36.173473 | finish at 2026-10-10 15:13:55 + [2026-10-06 18:17:47] iteration 2967/ 6855 | consumed samples: 379776 | elapsed time per iteration (ms): 87521.1 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.39% | FLOPs estimator: workload-aware | logical tokens/s: 191692.5 | valid tokens/s: 191692.5 | physical tokens/s: 192194.3 | loss tokens/s: 138742.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.351418E-01 | lm-global-supervised-tokens: 1.214290E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.243629E-02 | lm-token-buffer-mib: 5.830013E+01 | global_load_balancing_loss: 9.998716E-01 | loss scale: 1.0 | grad norm: 0.130 | params norm: 11447.989 | params rms: 0.061987 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 22:31:22.067562 | finish at 2026-10-10 16:49:09 + [2026-10-06 18:19:15] iteration 2968/ 6855 | consumed samples: 379904 | elapsed time per iteration (ms): 88823.3 | throughput per GPU (TFLOP/s/GPU): 40.7 | MFU 13.03% | FLOPs estimator: workload-aware | logical tokens/s: 188883.0 | valid tokens/s: 188883.0 | physical tokens/s: 189390.7 | loss tokens/s: 144374.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.880385E-01 | lm-global-supervised-tokens: 1.282380E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.371722E-02 | lm-token-buffer-mib: 5.804915E+01 | global_load_balancing_loss: 1.000473E+00 | loss scale: 1.0 | grad norm: 0.059 | params norm: 11447.923 | params rms: 0.061987 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 23:54:16.120316 | finish at 2026-10-10 18:13:31 + [2026-10-06 18:20:44] iteration 2969/ 6855 | consumed samples: 380032 | elapsed time per iteration (ms): 88961.7 | throughput per GPU (TFLOP/s/GPU): 41.2 | MFU 13.20% | FLOPs estimator: workload-aware | logical tokens/s: 188588.5 | valid tokens/s: 188588.5 | physical tokens/s: 189073.6 | loss tokens/s: 136104.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.226808E-01 | lm-global-supervised-tokens: 1.210807E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.030138E-01 | lm-token-buffer-mib: 5.824812E+01 | global_load_balancing_loss: 1.001406E+00 | loss scale: 1.0 | grad norm: 0.127 | params norm: 11447.858 | params rms: 0.061986 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 0:01:45.082670 | finish at 2026-10-10 18:22:29 + [2026-10-06 18:22:37] iteration 2970/ 6855 | consumed samples: 380160 | elapsed time per iteration (ms): 112706.8 | throughput per GPU (TFLOP/s/GPU): 32.6 | MFU 10.45% | FLOPs estimator: workload-aware | logical tokens/s: 148857.1 | valid tokens/s: 148857.1 | physical tokens/s: 149224.0 | loss tokens/s: 112096.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.626170E-01 | lm-global-supervised-tokens: 1.263405E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.175590E-02 | lm-token-buffer-mib: 5.827837E+01 | global_load_balancing_loss: 1.002017E+00 | loss scale: 1.0 | grad norm: 0.039 | params norm: 11447.792 | params rms: 0.061986 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 1:37:45.872551 | finish at 2026-10-11 20:00:23 + [2026-10-06 18:24:03] iteration 2971/ 6855 | consumed samples: 380288 | elapsed time per iteration (ms): 85678.4 | throughput per GPU (TFLOP/s/GPU): 42.8 | MFU 13.72% | FLOPs estimator: workload-aware | logical tokens/s: 195816.1 | valid tokens/s: 195816.1 | physical tokens/s: 196227.6 | loss tokens/s: 148968.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.420580E-01 | lm-global-supervised-tokens: 1.276342E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.233110E-02 | lm-token-buffer-mib: 5.806870E+01 | global_load_balancing_loss: 1.000852E+00 | loss scale: 1.0 | grad norm: 0.055 | params norm: 11447.727 | params rms: 0.061986 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 20:26:14.799262 | finish at 2026-10-10 14:50:17 + [2026-10-06 18:25:32] iteration 2972/ 6855 | consumed samples: 380416 | elapsed time per iteration (ms): 89172.9 | throughput per GPU (TFLOP/s/GPU): 41.2 | MFU 13.22% | FLOPs estimator: workload-aware | logical tokens/s: 188141.6 | valid tokens/s: 188141.6 | physical tokens/s: 188553.4 | loss tokens/s: 141767.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.565225E-01 | lm-global-supervised-tokens: 1.264178E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.336732E-02 | lm-token-buffer-mib: 5.806499E+01 | global_load_balancing_loss: 1.000757E+00 | loss scale: 1.0 | grad norm: 0.112 | params norm: 11447.661 | params rms: 0.061985 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 0:10:58.418691 | finish at 2026-10-10 18:36:30 + [2026-10-06 18:26:58] iteration 2973/ 6855 | consumed samples: 380544 | elapsed time per iteration (ms): 85699.0 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.68% | FLOPs estimator: workload-aware | logical tokens/s: 195769.1 | valid tokens/s: 195769.1 | physical tokens/s: 196194.6 | loss tokens/s: 142759.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.173245E-01 | lm-global-supervised-tokens: 1.223436E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.162454E-02 | lm-token-buffer-mib: 5.803113E+01 | global_load_balancing_loss: 1.000671E+00 | loss scale: 1.0 | grad norm: 0.091 | params norm: 11447.594 | params rms: 0.061985 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 20:24:43.426838 | finish at 2026-10-10 14:51:41 + [2026-10-06 18:28:30] iteration 2974/ 6855 | consumed samples: 380672 | elapsed time per iteration (ms): 92233.6 | throughput per GPU (TFLOP/s/GPU): 41.3 | MFU 13.23% | FLOPs estimator: workload-aware | logical tokens/s: 181897.8 | valid tokens/s: 181897.8 | physical tokens/s: 182374.9 | loss tokens/s: 126392.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.490250E-01 | lm-global-supervised-tokens: 1.165768E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.397683E-02 | lm-token-buffer-mib: 5.811230E+01 | global_load_balancing_loss: 9.992171E-01 | loss scale: 1.0 | grad norm: 0.078 | params norm: 11447.528 | params rms: 0.061985 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 3:25:58.734460 | finish at 2026-10-10 21:54:29 + [2026-10-06 18:29:58] iteration 2975/ 6855 | consumed samples: 380800 | elapsed time per iteration (ms): 88390.4 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.41% | FLOPs estimator: workload-aware | logical tokens/s: 189807.2 | valid tokens/s: 189807.2 | physical tokens/s: 190267.1 | loss tokens/s: 142320.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.438369E-01 | lm-global-supervised-tokens: 1.257972E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.374949E-02 | lm-token-buffer-mib: 5.833879E+01 | global_load_balancing_loss: 9.991512E-01 | loss scale: 1.0 | grad norm: 0.064 | params norm: 11447.461 | params rms: 0.061984 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 23:15:54.636106 | finish at 2026-10-10 17:45:53 + [2026-10-06 18:31:27] iteration 2976/ 6855 | consumed samples: 380928 | elapsed time per iteration (ms): 88993.1 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.55% | FLOPs estimator: workload-aware | logical tokens/s: 188521.8 | valid tokens/s: 188521.8 | physical tokens/s: 188966.6 | loss tokens/s: 133993.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.714642E-01 | lm-global-supervised-tokens: 1.192451E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.147070E-02 | lm-token-buffer-mib: 5.815815E+01 | global_load_balancing_loss: 9.986548E-01 | loss scale: 1.0 | grad norm: 0.102 | params norm: 11447.395 | params rms: 0.061984 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 23:53:24.090348 | finish at 2026-10-10 18:24:51 + [2026-10-06 18:32:52] iteration 2977/ 6855 | consumed samples: 381056 | elapsed time per iteration (ms): 85138.9 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.47% | FLOPs estimator: workload-aware | logical tokens/s: 197057.0 | valid tokens/s: 197057.0 | physical tokens/s: 197560.9 | loss tokens/s: 150684.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.500056E-01 | lm-global-supervised-tokens: 1.282907E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.304758E-02 | lm-token-buffer-mib: 5.806847E+01 | global_load_balancing_loss: 9.991658E-01 | loss scale: 1.0 | grad norm: 0.126 | params norm: 11447.327 | params rms: 0.061984 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 19:42:48.546185 | finish at 2026-10-10 14:15:41 + [2026-10-06 18:34:17] iteration 2978/ 6855 | consumed samples: 381184 | elapsed time per iteration (ms): 84717.1 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.75% | FLOPs estimator: workload-aware | logical tokens/s: 198038.0 | valid tokens/s: 198038.0 | physical tokens/s: 198570.3 | loss tokens/s: 153094.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.494437E-01 | lm-global-supervised-tokens: 1.296973E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.242342E-02 | lm-token-buffer-mib: 5.817873E+01 | global_load_balancing_loss: 1.000605E+00 | loss scale: 1.0 | grad norm: 0.215 | params norm: 11447.259 | params rms: 0.061983 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 19:14:08.348849 | finish at 2026-10-10 13:48:25 + [2026-10-06 18:35:44] iteration 2979/ 6855 | consumed samples: 381312 | elapsed time per iteration (ms): 86635.1 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.66% | FLOPs estimator: workload-aware | logical tokens/s: 193653.3 | valid tokens/s: 193653.3 | physical tokens/s: 194061.5 | loss tokens/s: 143874.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.746897E-01 | lm-global-supervised-tokens: 1.246457E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.375912E-02 | lm-token-buffer-mib: 5.813167E+01 | global_load_balancing_loss: 1.000591E+00 | loss scale: 1.0 | grad norm: 0.209 | params norm: 11447.190 | params rms: 0.061983 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 21:16:37.720170 | finish at 2026-10-10 15:52:21 + [2026-10-06 18:37:33] iteration 2980/ 6855 | consumed samples: 381440 | elapsed time per iteration (ms): 109527.3 | throughput per GPU (TFLOP/s/GPU): 33.7 | MFU 10.81% | FLOPs estimator: workload-aware | logical tokens/s: 153177.9 | valid tokens/s: 153177.9 | physical tokens/s: 153638.0 | loss tokens/s: 109245.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.886808E-01 | lm-global-supervised-tokens: 1.196534E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.155029E-02 | lm-token-buffer-mib: 5.842156E+01 | global_load_balancing_loss: 1.001398E+00 | loss scale: 1.0 | grad norm: 0.223 | params norm: 11447.123 | params rms: 0.061982 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 21:53:38.234378 | finish at 2026-10-11 16:31:11 + [2026-10-06 18:38:56] iteration 2981/ 6855 | consumed samples: 381568 | elapsed time per iteration (ms): 82872.4 | throughput per GPU (TFLOP/s/GPU): 44.3 | MFU 14.20% | FLOPs estimator: workload-aware | logical tokens/s: 202446.4 | valid tokens/s: 202446.4 | physical tokens/s: 202816.4 | loss tokens/s: 149440.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.707916E-01 | lm-global-supervised-tokens: 1.238450E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.221824E-02 | lm-token-buffer-mib: 5.802955E+01 | global_load_balancing_loss: 1.000342E+00 | loss scale: 1.0 | grad norm: 0.142 | params norm: 11447.054 | params rms: 0.061982 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 17:10:47.521682 | finish at 2026-10-10 11:49:44 + [2026-10-06 18:40:23] iteration 2982/ 6855 | consumed samples: 381696 | elapsed time per iteration (ms): 86661.5 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.69% | FLOPs estimator: workload-aware | logical tokens/s: 193594.1 | valid tokens/s: 193594.1 | physical tokens/s: 194058.5 | loss tokens/s: 146034.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.869223E-01 | lm-global-supervised-tokens: 1.265554E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.483572E-02 | lm-token-buffer-mib: 5.809793E+01 | global_load_balancing_loss: 1.000315E+00 | loss scale: 1.0 | grad norm: 0.096 | params norm: 11446.986 | params rms: 0.061982 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 21:13:59.867523 | finish at 2026-10-10 15:54:23 + [2026-10-06 18:41:50] iteration 2983/ 6855 | consumed samples: 381824 | elapsed time per iteration (ms): 87382.9 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.51% | FLOPs estimator: workload-aware | logical tokens/s: 191995.9 | valid tokens/s: 191995.9 | physical tokens/s: 192488.6 | loss tokens/s: 138323.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.775888E-01 | lm-global-supervised-tokens: 1.208712E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.172652E-02 | lm-token-buffer-mib: 5.806226E+01 | global_load_balancing_loss: 1.001345E+00 | loss scale: 1.0 | grad norm: 0.274 | params norm: 11446.917 | params rms: 0.061981 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 21:59:06.745735 | finish at 2026-10-10 16:40:57 + [2026-10-06 18:43:16] iteration 2984/ 6855 | consumed samples: 381952 | elapsed time per iteration (ms): 85983.0 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.45% | FLOPs estimator: workload-aware | logical tokens/s: 195122.3 | valid tokens/s: 195122.3 | physical tokens/s: 195493.7 | loss tokens/s: 147336.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.329777E-01 | lm-global-supervised-tokens: 1.266841E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.249781E-02 | lm-token-buffer-mib: 5.808951E+01 | global_load_balancing_loss: 1.000190E+00 | loss scale: 1.0 | grad norm: 0.207 | params norm: 11446.850 | params rms: 0.061981 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 20:27:20.140549 | finish at 2026-10-10 15:10:36 + [2026-10-06 18:44:44] iteration 2985/ 6855 | consumed samples: 382080 | elapsed time per iteration (ms): 88231.6 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.52% | FLOPs estimator: workload-aware | logical tokens/s: 190149.7 | valid tokens/s: 190149.7 | physical tokens/s: 190729.8 | loss tokens/s: 136238.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.602481E-01 | lm-global-supervised-tokens: 1.202055E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.473715E-02 | lm-token-buffer-mib: 5.821877E+01 | global_load_balancing_loss: 1.000095E+00 | loss scale: 1.0 | grad norm: 0.192 | params norm: 11446.784 | params rms: 0.061981 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 22:50:56.142704 | finish at 2026-10-10 17:35:40 + [2026-10-06 18:46:08] iteration 2986/ 6855 | consumed samples: 382208 | elapsed time per iteration (ms): 84110.8 | throughput per GPU (TFLOP/s/GPU): 42.8 | MFU 13.73% | FLOPs estimator: workload-aware | logical tokens/s: 199465.2 | valid tokens/s: 199465.2 | physical tokens/s: 199920.0 | loss tokens/s: 152527.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.024413E-01 | lm-global-supervised-tokens: 1.282924E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.623032E-02 | lm-token-buffer-mib: 5.805895E+01 | global_load_balancing_loss: 9.991471E-01 | loss scale: 1.0 | grad norm: 0.310 | params norm: 11446.718 | params rms: 0.061980 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 18:23:44.842123 | finish at 2026-10-10 13:09:53 + [2026-10-06 18:47:34] iteration 2987/ 6855 | consumed samples: 382336 | elapsed time per iteration (ms): 85571.3 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.76% | FLOPs estimator: workload-aware | logical tokens/s: 196060.5 | valid tokens/s: 196060.5 | physical tokens/s: 196576.9 | loss tokens/s: 146064.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.725020E-01 | lm-global-supervised-tokens: 1.249893E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.156496E-02 | lm-token-buffer-mib: 5.830669E+01 | global_load_balancing_loss: 1.000005E+00 | loss scale: 1.0 | grad norm: 0.056 | params norm: 11446.652 | params rms: 0.061980 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 19:56:29.609608 | finish at 2026-10-10 14:44:04 + [2026-10-06 18:49:00] iteration 2988/ 6855 | consumed samples: 382464 | elapsed time per iteration (ms): 85753.3 | throughput per GPU (TFLOP/s/GPU): 42.8 | MFU 13.70% | FLOPs estimator: workload-aware | logical tokens/s: 195644.6 | valid tokens/s: 195644.6 | physical tokens/s: 196353.0 | loss tokens/s: 145475.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.963635E-01 | lm-global-supervised-tokens: 1.247497E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.233842E-02 | lm-token-buffer-mib: 5.811346E+01 | global_load_balancing_loss: 9.998904E-01 | loss scale: 1.0 | grad norm: 0.224 | params norm: 11446.588 | params rms: 0.061980 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 20:06:48.000771 | finish at 2026-10-10 14:55:48 + [2026-10-06 18:50:26] iteration 2989/ 6855 | consumed samples: 382592 | elapsed time per iteration (ms): 85767.5 | throughput per GPU (TFLOP/s/GPU): 42.8 | MFU 13.70% | FLOPs estimator: workload-aware | logical tokens/s: 195612.2 | valid tokens/s: 195612.2 | physical tokens/s: 196031.5 | loss tokens/s: 148121.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.855459E-01 | lm-global-supervised-tokens: 1.270399E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.877633E-02 | lm-token-buffer-mib: 5.810358E+01 | global_load_balancing_loss: 9.992389E-01 | loss scale: 1.0 | grad norm: 0.037 | params norm: 11446.524 | params rms: 0.061979 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 20:06:16.977734 | finish at 2026-10-10 14:56:43 + [2026-10-06 18:52:15] iteration 2990/ 6855 | consumed samples: 382720 | elapsed time per iteration (ms): 109757.4 | throughput per GPU (TFLOP/s/GPU): 33.3 | MFU 10.68% | FLOPs estimator: workload-aware | logical tokens/s: 152856.7 | valid tokens/s: 152856.7 | physical tokens/s: 153302.8 | loss tokens/s: 115663.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.220585E-01 | lm-global-supervised-tokens: 1.269489E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.155858E-02 | lm-token-buffer-mib: 5.818403E+01 | global_load_balancing_loss: 1.001062E+00 | loss scale: 1.0 | grad norm: 0.090 | params norm: 11446.458 | params rms: 0.061979 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 21:50:12.176977 | finish at 2026-10-11 16:42:27 + [2026-10-06 18:53:43] iteration 2991/ 6855 | consumed samples: 382848 | elapsed time per iteration (ms): 87431.7 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.57% | FLOPs estimator: workload-aware | logical tokens/s: 191889.0 | valid tokens/s: 191889.0 | physical tokens/s: 192356.0 | loss tokens/s: 140953.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.510622E-01 | lm-global-supervised-tokens: 1.232377E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.156332E-02 | lm-token-buffer-mib: 5.822235E+01 | global_load_balancing_loss: 1.001375E+00 | loss scale: 1.0 | grad norm: 0.101 | params norm: 11446.393 | params rms: 0.061978 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 21:50:35.910965 | finish at 2026-10-10 16:44:19 + [2026-10-06 18:55:09] iteration 2992/ 6855 | consumed samples: 382976 | elapsed time per iteration (ms): 85898.5 | throughput per GPU (TFLOP/s/GPU): 43.1 | MFU 13.80% | FLOPs estimator: workload-aware | logical tokens/s: 195313.7 | valid tokens/s: 195313.7 | physical tokens/s: 195685.2 | loss tokens/s: 144329.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.523055E-01 | lm-global-supervised-tokens: 1.239768E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.323240E-02 | lm-token-buffer-mib: 5.809001E+01 | global_load_balancing_loss: 9.996057E-01 | loss scale: 1.0 | grad norm: 0.047 | params norm: 11446.329 | params rms: 0.061978 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 20:10:25.978127 | finish at 2026-10-10 15:05:35 + [2026-10-06 18:56:33] iteration 2993/ 6855 | consumed samples: 383104 | elapsed time per iteration (ms): 83977.6 | throughput per GPU (TFLOP/s/GPU): 43.4 | MFU 13.92% | FLOPs estimator: workload-aware | logical tokens/s: 199781.6 | valid tokens/s: 199781.6 | physical tokens/s: 200188.3 | loss tokens/s: 143439.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.781401E-01 | lm-global-supervised-tokens: 1.204567E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.358113E-02 | lm-token-buffer-mib: 5.806547E+01 | global_load_balancing_loss: 1.000291E+00 | loss scale: 1.0 | grad norm: 0.092 | params norm: 11446.262 | params rms: 0.061978 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 18:05:21.462112 | finish at 2026-10-10 13:01:54 + [2026-10-06 18:58:01] iteration 2994/ 6855 | consumed samples: 383232 | elapsed time per iteration (ms): 87901.4 | throughput per GPU (TFLOP/s/GPU): 42.8 | MFU 13.73% | FLOPs estimator: workload-aware | logical tokens/s: 190863.8 | valid tokens/s: 190863.8 | physical tokens/s: 191421.3 | loss tokens/s: 137676.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.419398E-01 | lm-global-supervised-tokens: 1.210198E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.719909E-02 | lm-token-buffer-mib: 5.806227E+01 | global_load_balancing_loss: 1.000712E+00 | loss scale: 1.0 | grad norm: 0.055 | params norm: 11446.195 | params rms: 0.061977 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 22:16:27.488010 | finish at 2026-10-10 17:14:28 + [2026-10-06 18:59:23] iteration 2995/ 6855 | consumed samples: 383360 | elapsed time per iteration (ms): 82625.1 | throughput per GPU (TFLOP/s/GPU): 43.6 | MFU 13.97% | FLOPs estimator: workload-aware | logical tokens/s: 203051.7 | valid tokens/s: 203051.7 | physical tokens/s: 203768.1 | loss tokens/s: 152604.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.538981E-01 | lm-global-supervised-tokens: 1.260900E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.156655E-02 | lm-token-buffer-mib: 5.836767E+01 | global_load_balancing_loss: 9.990274E-01 | loss scale: 1.0 | grad norm: 0.044 | params norm: 11446.128 | params rms: 0.061977 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 16:35:32.816582 | finish at 2026-10-10 11:34:56 + [2026-10-06 19:00:49] iteration 2996/ 6855 | consumed samples: 383488 | elapsed time per iteration (ms): 85864.0 | throughput per GPU (TFLOP/s/GPU): 44.1 | MFU 14.14% | FLOPs estimator: workload-aware | logical tokens/s: 195392.9 | valid tokens/s: 195392.9 | physical tokens/s: 195724.2 | loss tokens/s: 136763.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.114779E-01 | lm-global-supervised-tokens: 1.174307E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.281208E-02 | lm-token-buffer-mib: 5.808041E+01 | global_load_balancing_loss: 9.994296E-01 | loss scale: 1.0 | grad norm: 0.082 | params norm: 11446.064 | params rms: 0.061977 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 20:02:29.200238 | finish at 2026-10-10 15:03:18 + [2026-10-06 19:02:14] iteration 2997/ 6855 | consumed samples: 383616 | elapsed time per iteration (ms): 85187.1 | throughput per GPU (TFLOP/s/GPU): 43.1 | MFU 13.80% | FLOPs estimator: workload-aware | logical tokens/s: 196944.9 | valid tokens/s: 196944.9 | physical tokens/s: 197492.4 | loss tokens/s: 148748.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.025524E-01 | lm-global-supervised-tokens: 1.267146E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.314911E-02 | lm-token-buffer-mib: 5.814500E+01 | global_load_balancing_loss: 1.000305E+00 | loss scale: 1.0 | grad norm: 0.052 | params norm: 11445.999 | params rms: 0.061976 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 19:17:31.975036 | finish at 2026-10-10 14:19:46 + [2026-10-06 19:03:42] iteration 2998/ 6855 | consumed samples: 383744 | elapsed time per iteration (ms): 87772.3 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.46% | FLOPs estimator: workload-aware | logical tokens/s: 191144.3 | valid tokens/s: 191144.3 | physical tokens/s: 191687.9 | loss tokens/s: 138169.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.858381E-01 | lm-global-supervised-tokens: 1.212750E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.154270E-02 | lm-token-buffer-mib: 5.806231E+01 | global_load_balancing_loss: 1.000566E+00 | loss scale: 1.0 | grad norm: 0.055 | params norm: 11445.935 | params rms: 0.061976 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 22:02:17.857675 | finish at 2026-10-10 17:06:00 + [2026-10-06 19:05:09] iteration 2999/ 6855 | consumed samples: 383872 | elapsed time per iteration (ms): 86640.7 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.41% | FLOPs estimator: workload-aware | logical tokens/s: 193640.4 | valid tokens/s: 193640.4 | physical tokens/s: 194302.9 | loss tokens/s: 141320.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.168134E-01 | lm-global-supervised-tokens: 1.224411E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.173032E-02 | lm-token-buffer-mib: 5.807854E+01 | global_load_balancing_loss: 1.001303E+00 | loss scale: 1.0 | grad norm: 0.107 | params norm: 11445.872 | params rms: 0.061976 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 20:48:06.379627 | finish at 2026-10-10 15:53:15 + [2026-10-06 19:07:00] iteration 3000/ 6855 | consumed samples: 384000 | elapsed time per iteration (ms): 111203.6 | throughput per GPU (TFLOP/s/GPU): 33.7 | MFU 10.81% | FLOPs estimator: workload-aware | logical tokens/s: 150869.0 | valid tokens/s: 150869.0 | physical tokens/s: 151157.4 | loss tokens/s: 113242.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.073686E-01 | lm-global-supervised-tokens: 1.259296E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.347304E-02 | lm-token-buffer-mib: 5.815038E+01 | global_load_balancing_loss: 9.990969E-01 | loss scale: 1.0 | grad norm: 0.066 | params norm: 11445.810 | params rms: 0.061975 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 23:04:50.027543 | finish at 2026-10-11 18:11:50 +times across ranks (ms): + save-checkpoint: + rank 0: 23907.26 + rank 1: 23907.26 + rank 2: 23907.29 + rank 3: 23907.26 + rank 4: 23907.26 + rank 5: 23907.24 + rank 6: 23907.24 + rank 7: 23907.32 + rank 8: 23909.07 + rank 9: 23909.04 + rank 10: 23909.05 + rank 11: 23909.05 + rank 12: 23909.04 + rank 13: 23909.04 + rank 14: 23909.03 + rank 15: 23909.03 + rank 16: 23907.94 + rank 17: 23907.90 + rank 18: 23907.89 + rank 19: 23907.91 + rank 20: 23907.93 + rank 21: 23907.94 + rank 22: 23907.89 + rank 23: 23907.89 + rank 24: 23907.95 + rank 25: 23907.95 + rank 26: 23907.96 + rank 27: 23907.95 + rank 28: 23907.94 + rank 29: 23907.91 + rank 30: 23907.95 + rank 31: 23907.91 + rank 32: 23907.61 + rank 33: 23907.64 + rank 34: 23907.62 + rank 35: 23907.59 + rank 36: 23907.60 + rank 37: 23907.60 + rank 38: 23907.62 + rank 39: 23907.61 + rank 40: 23907.52 + rank 41: 23907.50 + rank 42: 23907.50 + rank 43: 23907.52 + rank 44: 23907.51 + rank 45: 23907.50 + rank 46: 23907.53 + rank 47: 23907.50 + rank 48: 23907.88 + rank 49: 23907.86 + rank 50: 23907.85 + rank 51: 23907.89 + rank 52: 23908.04 + rank 53: 23907.86 + rank 54: 23907.86 + rank 55: 23907.86 + rank 56: 23907.65 + rank 57: 23907.64 + rank 58: 23907.63 + rank 59: 23907.64 + rank 60: 23907.64 + rank 61: 23907.63 + rank 62: 23907.68 + rank 63: 23907.65 + [2026-10-06 19:08:51] iteration 3001/ 6855 | consumed samples: 384128 | elapsed time per iteration (ms): 86656.5 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.58% | FLOPs estimator: workload-aware | logical tokens/s: 193605.3 | valid tokens/s: 193605.3 | physical tokens/s: 194066.2 | loss tokens/s: 145572.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.208963E-01 | lm-global-supervised-tokens: 1.261479E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 9.995437E-01 | lm-token-buffer-mib: 5.836626E+01 | global_load_balancing_loss: 9.991577E-01 | loss scale: 1.0 | grad norm: 0.085 | params norm: 11445.744 | params rms: 0.061975 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 20:46:14.192739 | finish at 2026-10-10 15:55:05 +times across ranks (ms): + forward-backward: + rank 0: 84893.18 + rank 1: 84894.32 + rank 2: 84893.68 + rank 3: 84895.87 + rank 4: 84896.68 + rank 5: 84896.96 + rank 6: 84894.80 + rank 7: 84893.82 + rank 8: 84898.05 + rank 9: 84899.01 + rank 10: 84898.69 + rank 11: 84900.94 + rank 12: 84901.78 + rank 13: 84902.77 + rank 14: 84895.86 + rank 15: 84898.77 + rank 16: 84888.04 + rank 17: 84900.08 + rank 18: 84887.88 + rank 19: 84900.98 + rank 20: 84891.64 + rank 21: 84892.95 + rank 22: 84886.24 + rank 23: 84888.61 + rank 24: 84886.32 + rank 25: 84889.93 + rank 26: 84887.34 + rank 27: 84889.07 + rank 28: 84892.52 + rank 29: 84891.35 + rank 30: 84885.27 + rank 31: 84887.03 + rank 32: 84882.02 + rank 33: 84882.89 + rank 34: 84882.33 + rank 35: 84883.76 + rank 36: 84885.69 + rank 37: 84890.77 + rank 38: 84882.29 + rank 39: 84884.15 + rank 40: 84885.62 + rank 41: 84885.70 + rank 42: 84884.02 + rank 43: 84885.93 + rank 44: 84885.82 + rank 45: 84885.92 + rank 46: 84886.30 + rank 47: 84886.92 + rank 48: 84896.09 + rank 49: 84892.80 + rank 50: 84890.17 + rank 51: 84895.15 + rank 52: 84892.89 + rank 53: 84892.64 + rank 54: 84892.37 + rank 55: 84898.68 + rank 56: 84894.99 + rank 57: 84894.84 + rank 58: 84893.52 + rank 59: 84894.41 + rank 60: 84894.82 + rank 61: 84894.52 + rank 62: 84894.88 + rank 63: 84895.82 + forward-compute: + rank 0: 30795.02 + rank 1: 30800.45 + rank 2: 30796.23 + rank 3: 30796.79 + rank 4: 30797.88 + rank 5: 30796.69 + rank 6: 30794.53 + rank 7: 30801.64 + rank 8: 32104.80 + rank 9: 32105.31 + rank 10: 32107.89 + rank 11: 32110.16 + rank 12: 32111.91 + rank 13: 32116.90 + rank 14: 32107.60 + rank 15: 32110.07 + rank 16: 30347.23 + rank 17: 30341.04 + rank 18: 30339.13 + rank 19: 30343.87 + rank 20: 30341.81 + rank 21: 30345.40 + rank 22: 30340.97 + rank 23: 30344.49 + rank 24: 32044.36 + rank 25: 32045.04 + rank 26: 32044.52 + rank 27: 32047.14 + rank 28: 32048.02 + rank 29: 32048.35 + rank 30: 32044.95 + rank 31: 32046.09 + rank 32: 29864.89 + rank 33: 29866.52 + rank 34: 29867.48 + rank 35: 29872.80 + rank 36: 29879.10 + rank 37: 29885.38 + rank 38: 29881.27 + rank 39: 29873.22 + rank 40: 30183.97 + rank 41: 30182.37 + rank 42: 30181.53 + rank 43: 30184.43 + rank 44: 30187.13 + rank 45: 30188.66 + rank 46: 30185.85 + rank 47: 30187.64 + rank 48: 32338.83 + rank 49: 32346.25 + rank 50: 32337.01 + rank 51: 32338.01 + rank 52: 32345.67 + rank 53: 32350.58 + rank 54: 32336.30 + rank 55: 32335.69 + rank 56: 32847.38 + rank 57: 32849.50 + rank 58: 32850.14 + rank 59: 32850.00 + rank 60: 32852.59 + rank 61: 32853.32 + rank 62: 32848.74 + rank 63: 32853.97 + backward-compute: + rank 0: 54063.79 + rank 1: 54062.76 + rank 2: 54060.00 + rank 3: 54064.85 + rank 4: 54066.69 + rank 5: 54066.66 + rank 6: 54063.20 + rank 7: 54057.94 + rank 8: 52759.46 + rank 9: 52758.11 + rank 10: 52756.81 + rank 11: 52755.90 + rank 12: 52752.95 + rank 13: 52749.63 + rank 14: 52754.40 + rank 15: 52753.00 + rank 16: 54506.50 + rank 17: 54514.07 + rank 18: 54515.46 + rank 19: 54512.74 + rank 20: 54517.42 + rank 21: 54513.56 + rank 22: 54511.09 + rank 23: 54510.64 + rank 24: 52799.46 + rank 25: 52802.14 + rank 26: 52800.48 + rank 27: 52800.42 + rank 28: 52797.77 + rank 29: 52800.50 + rank 30: 52796.85 + rank 31: 52799.43 + rank 32: 54978.70 + rank 33: 54976.18 + rank 34: 54975.88 + rank 35: 54973.50 + rank 36: 54966.60 + rank 37: 54962.21 + rank 38: 54962.99 + rank 39: 54972.80 + rank 40: 54667.51 + rank 41: 54670.49 + rank 42: 54669.54 + rank 43: 54667.25 + rank 44: 54663.22 + rank 45: 54660.51 + rank 46: 54665.04 + rank 47: 54665.97 + rank 48: 52517.57 + rank 49: 52508.90 + rank 50: 52518.50 + rank 51: 52518.64 + rank 52: 52506.17 + rank 53: 52505.86 + rank 54: 52515.92 + rank 55: 52522.55 + rank 56: 52011.50 + rank 57: 52008.02 + rank 58: 52008.75 + rank 59: 52009.00 + rank 60: 52006.92 + rank 61: 52005.04 + rank 62: 52011.57 + rank 63: 52002.13 + batch-generator: + rank 0: 26.04 + rank 1: 22.48 + rank 2: 25.93 + rank 3: 25.71 + rank 4: 23.54 + rank 5: 24.00 + rank 6: 24.31 + rank 7: 25.26 + rank 8: 25.17 + rank 9: 23.83 + rank 10: 23.26 + rank 11: 25.60 + rank 12: 25.25 + rank 13: 24.03 + rank 14: 25.22 + rank 15: 24.26 + rank 16: 23.45 + rank 17: 22.93 + rank 18: 24.99 + rank 19: 24.08 + rank 20: 23.04 + rank 21: 25.02 + rank 22: 23.46 + rank 23: 24.03 + rank 24: 33.72 + rank 25: 31.89 + rank 26: 35.07 + rank 27: 34.51 + rank 28: 35.40 + rank 29: 33.87 + rank 30: 33.25 + rank 31: 33.79 + rank 32: 35.42 + rank 33: 33.31 + rank 34: 34.86 + rank 35: 24.22 + rank 36: 32.38 + rank 37: 32.95 + rank 38: 33.57 + rank 39: 33.76 + rank 40: 24.41 + rank 41: 24.62 + rank 42: 24.84 + rank 43: 24.40 + rank 44: 24.06 + rank 45: 23.56 + rank 46: 23.81 + rank 47: 24.06 + rank 48: 32.44 + rank 49: 30.67 + rank 50: 24.37 + rank 51: 32.22 + rank 52: 36.62 + rank 53: 27.89 + rank 54: 32.64 + rank 55: 32.76 + rank 56: 25.07 + rank 57: 24.77 + rank 58: 24.18 + rank 59: 24.04 + rank 60: 23.89 + rank 61: 24.60 + rank 62: 25.41 + rank 63: 26.48 + embedding-grads-all-reduce: + rank 0: 0.03 + rank 1: 0.02 + rank 2: 0.03 + rank 3: 0.03 + rank 4: 0.02 + rank 5: 0.02 + rank 6: 0.04 + rank 7: 0.03 + rank 8: 0.02 + rank 9: 0.03 + rank 10: 0.03 + rank 11: 0.03 + rank 12: 0.03 + rank 13: 0.02 + rank 14: 0.03 + rank 15: 0.02 + rank 16: 0.03 + rank 17: 0.04 + rank 18: 0.03 + rank 19: 0.04 + rank 20: 0.03 + rank 21: 0.03 + rank 22: 0.03 + rank 23: 0.02 + rank 24: 0.04 + rank 25: 0.04 + rank 26: 0.04 + rank 27: 0.05 + rank 28: 0.04 + rank 29: 0.03 + rank 30: 0.04 + rank 31: 0.04 + rank 32: 0.03 + rank 33: 0.03 + rank 34: 0.04 + rank 35: 0.03 + rank 36: 0.03 + rank 37: 0.03 + rank 38: 0.03 + rank 39: 0.03 + rank 40: 0.03 + rank 41: 0.03 + rank 42: 0.02 + rank 43: 0.03 + rank 44: 0.03 + rank 45: 0.03 + rank 46: 0.03 + rank 47: 0.04 + rank 48: 0.03 + rank 49: 0.05 + rank 50: 0.03 + rank 51: 0.03 + rank 52: 0.03 + rank 53: 0.03 + rank 54: 0.03 + rank 55: 0.03 + rank 56: 0.03 + rank 57: 0.03 + rank 58: 0.03 + rank 59: 0.03 + rank 60: 0.03 + rank 61: 0.03 + rank 62: 0.03 + rank 63: 0.03 + all-grads-sync: + rank 0: 0.80 + rank 1: 0.77 + rank 2: 0.83 + rank 3: 0.78 + rank 4: 0.78 + rank 5: 0.80 + rank 6: 1.30 + rank 7: 0.80 + rank 8: 0.77 + rank 9: 0.80 + rank 10: 0.96 + rank 11: 0.78 + rank 12: 0.79 + rank 13: 0.99 + rank 14: 0.78 + rank 15: 1.16 + rank 16: 0.89 + rank 17: 1.08 + rank 18: 0.75 + rank 19: 1.24 + rank 20: 0.78 + rank 21: 0.76 + rank 22: 0.78 + rank 23: 0.76 + rank 24: 1.43 + rank 25: 1.61 + rank 26: 1.26 + rank 27: 1.32 + rank 28: 1.51 + rank 29: 1.20 + rank 30: 1.35 + rank 31: 1.34 + rank 32: 1.25 + rank 33: 1.22 + rank 34: 1.14 + rank 35: 1.09 + rank 36: 1.21 + rank 37: 1.37 + rank 38: 1.25 + rank 39: 1.20 + rank 40: 0.79 + rank 41: 0.82 + rank 42: 0.98 + rank 43: 1.02 + rank 44: 0.88 + rank 45: 1.34 + rank 46: 0.92 + rank 47: 0.81 + rank 48: 1.08 + rank 49: 0.98 + rank 50: 0.92 + rank 51: 0.92 + rank 52: 1.11 + rank 53: 0.98 + rank 54: 0.84 + rank 55: 1.00 + rank 56: 0.75 + rank 57: 0.89 + rank 58: 0.78 + rank 59: 0.83 + rank 60: 0.76 + rank 61: 0.75 + rank 62: 0.83 + rank 63: 1.10 + params-all-gather: + rank 0: 0.02 + rank 1: 0.02 + rank 2: 0.02 + rank 3: 0.02 + rank 4: 0.02 + rank 5: 0.02 + rank 6: 0.02 + rank 7: 0.02 + rank 8: 0.02 + rank 9: 0.02 + rank 10: 0.02 + rank 11: 0.02 + rank 12: 0.02 + rank 13: 0.02 + rank 14: 0.02 + rank 15: 0.02 + rank 16: 0.02 + rank 17: 0.04 + rank 18: 0.02 + rank 19: 0.04 + rank 20: 0.02 + rank 21: 0.02 + rank 22: 0.02 + rank 23: 0.02 + rank 24: 0.04 + rank 25: 0.03 + rank 26: 0.04 + rank 27: 0.03 + rank 28: 0.03 + rank 29: 0.03 + rank 30: 0.03 + rank 31: 0.04 + rank 32: 0.03 + rank 33: 0.02 + rank 34: 0.02 + rank 35: 0.02 + rank 36: 0.02 + rank 37: 0.02 + rank 38: 0.02 + rank 39: 0.03 + rank 40: 0.02 + rank 41: 0.02 + rank 42: 0.02 + rank 43: 0.02 + rank 44: 0.02 + rank 45: 0.02 + rank 46: 0.02 + rank 47: 0.02 + rank 48: 0.03 + rank 49: 0.03 + rank 50: 0.03 + rank 51: 0.03 + rank 52: 0.03 + rank 53: 0.02 + rank 54: 0.02 + rank 55: 0.03 + rank 56: 0.02 + rank 57: 0.02 + rank 58: 0.02 + rank 59: 0.02 + rank 60: 0.02 + rank 61: 0.02 + rank 62: 0.02 + rank 63: 0.02 + optimizer-copy-to-main-grad: + rank 0: 2.30 + rank 1: 1.77 + rank 2: 1.74 + rank 3: 2.20 + rank 4: 1.51 + rank 5: 1.79 + rank 6: 1.91 + rank 7: 1.98 + rank 8: 1.78 + rank 9: 2.07 + rank 10: 1.77 + rank 11: 1.73 + rank 12: 1.76 + rank 13: 1.75 + rank 14: 1.82 + rank 15: 1.98 + rank 16: 2.47 + rank 17: 3.54 + rank 18: 2.14 + rank 19: 3.54 + rank 20: 2.14 + rank 21: 2.33 + rank 22: 2.55 + rank 23: 2.34 + rank 24: 1.92 + rank 25: 2.21 + rank 26: 2.05 + rank 27: 2.05 + rank 28: 2.32 + rank 29: 2.18 + rank 30: 2.20 + rank 31: 1.98 + rank 32: 2.06 + rank 33: 1.88 + rank 34: 2.01 + rank 35: 2.45 + rank 36: 2.11 + rank 37: 2.51 + rank 38: 2.29 + rank 39: 1.91 + rank 40: 1.82 + rank 41: 1.97 + rank 42: 1.95 + rank 43: 1.91 + rank 44: 2.15 + rank 45: 2.90 + rank 46: 2.50 + rank 47: 1.95 + rank 48: 2.66 + rank 49: 1.99 + rank 50: 2.24 + rank 51: 2.46 + rank 52: 1.97 + rank 53: 1.70 + rank 54: 1.77 + rank 55: 2.35 + rank 56: 1.75 + rank 57: 2.35 + rank 58: 1.83 + rank 59: 1.69 + rank 60: 1.80 + rank 61: 1.66 + rank 62: 1.91 + rank 63: 1.84 + optimizer-inner-step: + rank 0: 42.98 + rank 1: 42.89 + rank 2: 42.66 + rank 3: 42.78 + rank 4: 42.58 + rank 5: 42.71 + rank 6: 43.05 + rank 7: 43.05 + rank 8: 42.79 + rank 9: 42.79 + rank 10: 42.72 + rank 11: 42.81 + rank 12: 42.81 + rank 13: 43.03 + rank 14: 42.78 + rank 15: 42.64 + rank 16: 43.08 + rank 17: 50.90 + rank 18: 42.94 + rank 19: 52.48 + rank 20: 42.88 + rank 21: 42.73 + rank 22: 43.26 + rank 23: 42.91 + rank 24: 42.53 + rank 25: 43.63 + rank 26: 42.73 + rank 27: 43.72 + rank 28: 48.06 + rank 29: 42.76 + rank 30: 42.66 + rank 31: 43.17 + rank 32: 42.87 + rank 33: 42.67 + rank 34: 42.76 + rank 35: 50.40 + rank 36: 42.85 + rank 37: 42.89 + rank 38: 42.74 + rank 39: 42.70 + rank 40: 42.79 + rank 41: 42.93 + rank 42: 42.97 + rank 43: 42.90 + rank 44: 42.58 + rank 45: 43.13 + rank 46: 42.83 + rank 47: 42.42 + rank 48: 53.42 + rank 49: 42.66 + rank 50: 54.40 + rank 51: 45.31 + rank 52: 42.83 + rank 53: 42.64 + rank 54: 42.91 + rank 55: 49.29 + rank 56: 42.45 + rank 57: 47.38 + rank 58: 42.78 + rank 59: 42.63 + rank 60: 42.97 + rank 61: 42.61 + rank 62: 42.71 + rank 63: 42.54 + optimizer-copy-main-to-model-params: + rank 0: 4.39 + rank 1: 3.89 + rank 2: 3.76 + rank 3: 4.33 + rank 4: 3.72 + rank 5: 3.90 + rank 6: 3.96 + rank 7: 4.04 + rank 8: 3.84 + rank 9: 4.45 + rank 10: 3.79 + rank 11: 3.73 + rank 12: 3.91 + rank 13: 3.76 + rank 14: 4.02 + rank 15: 4.23 + rank 16: 4.41 + rank 17: 5.32 + rank 18: 4.08 + rank 19: 5.75 + rank 20: 4.00 + rank 21: 4.24 + rank 22: 4.52 + rank 23: 4.18 + rank 24: 3.88 + rank 25: 4.20 + rank 26: 3.91 + rank 27: 4.09 + rank 28: 4.44 + rank 29: 4.41 + rank 30: 4.30 + rank 31: 3.99 + rank 32: 4.15 + rank 33: 4.08 + rank 34: 4.12 + rank 35: 4.69 + rank 36: 4.29 + rank 37: 4.33 + rank 38: 4.53 + rank 39: 3.88 + rank 40: 3.78 + rank 41: 4.03 + rank 42: 3.98 + rank 43: 4.06 + rank 44: 4.04 + rank 45: 4.22 + rank 46: 3.83 + rank 47: 3.78 + rank 48: 4.07 + rank 49: 4.00 + rank 50: 4.86 + rank 51: 4.75 + rank 52: 4.24 + rank 53: 3.95 + rank 54: 3.97 + rank 55: 4.67 + rank 56: 3.77 + rank 57: 4.70 + rank 58: 3.85 + rank 59: 3.75 + rank 60: 3.72 + rank 61: 3.65 + rank 62: 3.87 + rank 63: 3.73 + optimizer: + rank 0: 56.57 + rank 1: 55.80 + rank 2: 55.36 + rank 3: 56.29 + rank 4: 55.26 + rank 5: 55.62 + rank 6: 56.09 + rank 7: 56.15 + rank 8: 55.64 + rank 9: 56.48 + rank 10: 55.51 + rank 11: 55.50 + rank 12: 55.72 + rank 13: 55.79 + rank 14: 55.86 + rank 15: 56.01 + rank 16: 56.99 + rank 17: 66.29 + rank 18: 56.31 + rank 19: 68.35 + rank 20: 56.13 + rank 21: 56.35 + rank 22: 57.28 + rank 23: 56.45 + rank 24: 55.41 + rank 25: 56.91 + rank 26: 55.64 + rank 27: 56.86 + rank 28: 61.65 + rank 29: 56.30 + rank 30: 56.09 + rank 31: 56.19 + rank 32: 56.04 + rank 33: 55.66 + rank 34: 55.83 + rank 35: 64.29 + rank 36: 56.20 + rank 37: 56.22 + rank 38: 56.41 + rank 39: 55.49 + rank 40: 55.55 + rank 41: 56.02 + rank 42: 56.02 + rank 43: 56.05 + rank 44: 55.67 + rank 45: 56.53 + rank 46: 55.68 + rank 47: 55.17 + rank 48: 66.89 + rank 49: 55.69 + rank 50: 68.61 + rank 51: 59.36 + rank 52: 56.14 + rank 53: 55.52 + rank 54: 55.81 + rank 55: 63.23 + rank 56: 55.21 + rank 57: 61.44 + rank 58: 55.64 + rank 59: 55.31 + rank 60: 55.65 + rank 61: 55.18 + rank 62: 55.56 + rank 63: 55.26 + interval-time: + rank 0: 0.91 + rank 1: 0.80 + rank 2: 0.91 + rank 3: 0.86 + rank 4: 0.94 + rank 5: 0.93 + rank 6: 0.94 + rank 7: 0.85 + rank 8: 0.75 + rank 9: 0.75 + rank 10: 0.83 + rank 11: 0.73 + rank 12: 0.82 + rank 13: 0.72 + rank 14: 0.73 + rank 15: 0.79 + rank 16: 0.79 + rank 17: 0.75 + rank 18: 0.81 + rank 19: 0.76 + rank 20: 0.78 + rank 21: 0.76 + rank 22: 0.83 + rank 23: 0.72 + rank 24: 0.88 + rank 25: 0.86 + rank 26: 0.87 + rank 27: 0.89 + rank 28: 0.85 + rank 29: 0.94 + rank 30: 0.92 + rank 31: 0.87 + rank 32: 1.32 + rank 33: 1.21 + rank 34: 1.17 + rank 35: 1.18 + rank 36: 1.27 + rank 37: 1.25 + rank 38: 1.28 + rank 39: 1.10 + rank 40: 0.82 + rank 41: 0.83 + rank 42: 0.81 + rank 43: 0.76 + rank 44: 0.73 + rank 45: 0.81 + rank 46: 0.81 + rank 47: 0.77 + rank 48: 0.96 + rank 49: 1.00 + rank 50: 0.81 + rank 51: 0.82 + rank 52: 0.85 + rank 53: 0.83 + rank 54: 0.96 + rank 55: 0.91 + rank 56: 1.00 + rank 57: 0.88 + rank 58: 0.90 + rank 59: 0.95 + rank 60: 1.00 + rank 61: 0.99 + rank 62: 0.98 + rank 63: 3.75 + [2026-10-06 19:10:23] iteration 3002/ 6855 | consumed samples: 384256 | elapsed time per iteration (ms): 91846.7 | throughput per GPU (TFLOP/s/GPU): 39.2 | MFU 12.57% | FLOPs estimator: workload-aware | logical tokens/s: 182665.3 | valid tokens/s: 182665.3 | physical tokens/s: 183153.4 | loss tokens/s: 134420.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.513680E-01 | lm-global-supervised-tokens: 1.234607E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.383227E-02 | lm-token-buffer-mib: 5.818897E+01 | global_load_balancing_loss: 1.000608E+00 | loss scale: 1.0 | grad norm: 0.048 | params norm: 11445.681 | params rms: 0.061975 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 2:18:05.197125 | finish at 2026-10-10 21:28:28 +times across ranks (ms): + forward-backward: + rank 0: 85328.52 + rank 1: 85447.09 + rank 2: 85443.18 + rank 3: 85447.29 + rank 4: 85447.12 + rank 5: 85448.03 + rank 6: 85445.29 + rank 7: 85445.34 + rank 8: 85341.88 + rank 9: 85451.29 + rank 10: 85446.65 + rank 11: 85450.82 + rank 12: 85461.51 + rank 13: 85451.55 + rank 14: 85449.00 + rank 15: 85449.30 + rank 16: 85324.81 + rank 17: 85441.98 + rank 18: 85437.52 + rank 19: 85441.84 + rank 20: 85442.23 + rank 21: 85443.32 + rank 22: 85439.62 + rank 23: 85441.02 + rank 24: 85320.75 + rank 25: 85442.02 + rank 26: 85437.77 + rank 27: 85439.78 + rank 28: 85443.63 + rank 29: 85441.13 + rank 30: 85438.49 + rank 31: 85438.23 + rank 32: 85317.41 + rank 33: 85436.14 + rank 34: 85432.45 + rank 35: 85435.14 + rank 36: 85435.98 + rank 37: 85434.90 + rank 38: 85436.22 + rank 39: 85435.97 + rank 40: 85320.74 + rank 41: 85438.15 + rank 42: 85433.82 + rank 43: 85436.84 + rank 44: 85436.59 + rank 45: 85435.34 + rank 46: 85439.68 + rank 47: 85438.06 + rank 48: 85327.99 + rank 49: 85444.70 + rank 50: 85438.84 + rank 51: 85444.09 + rank 52: 85443.16 + rank 53: 85442.91 + rank 54: 85446.51 + rank 55: 85446.64 + rank 56: 85330.08 + rank 57: 85447.32 + rank 58: 85442.76 + rank 59: 85445.36 + rank 60: 85445.98 + rank 61: 85444.86 + rank 62: 85448.62 + rank 63: 85448.22 + forward-compute: + rank 0: 34042.93 + rank 1: 34160.16 + rank 2: 34157.59 + rank 3: 34159.30 + rank 4: 34157.94 + rank 5: 34156.21 + rank 6: 34155.83 + rank 7: 34161.53 + rank 8: 31666.26 + rank 9: 31791.87 + rank 10: 31786.39 + rank 11: 31789.05 + rank 12: 31789.55 + rank 13: 31790.62 + rank 14: 31783.58 + rank 15: 31787.35 + rank 16: 32079.82 + rank 17: 32194.45 + rank 18: 32195.83 + rank 19: 32200.82 + rank 20: 32194.37 + rank 21: 32196.82 + rank 22: 32191.36 + rank 23: 32194.96 + rank 24: 32377.17 + rank 25: 32505.67 + rank 26: 32502.26 + rank 27: 32503.65 + rank 28: 32515.46 + rank 29: 32513.29 + rank 30: 32508.97 + rank 31: 32507.26 + rank 32: 31775.69 + rank 33: 31899.63 + rank 34: 31899.20 + rank 35: 31908.30 + rank 36: 31917.42 + rank 37: 31914.26 + rank 38: 31916.49 + rank 39: 31911.43 + rank 40: 33109.61 + rank 41: 33223.82 + rank 42: 33216.09 + rank 43: 33224.09 + rank 44: 33222.48 + rank 45: 33221.68 + rank 46: 33220.76 + rank 47: 33217.84 + rank 48: 29412.71 + rank 49: 29531.07 + rank 50: 29523.89 + rank 51: 29526.55 + rank 52: 29538.95 + rank 53: 29535.97 + rank 54: 29529.79 + rank 55: 29529.88 + rank 56: 32576.34 + rank 57: 32699.69 + rank 58: 32697.23 + rank 59: 32694.88 + rank 60: 32703.55 + rank 61: 32706.26 + rank 62: 32701.83 + rank 63: 32708.88 + backward-compute: + rank 0: 51249.46 + rank 1: 51255.55 + rank 2: 51247.26 + rank 3: 51254.30 + rank 4: 51255.32 + rank 5: 51257.71 + rank 6: 51255.22 + rank 7: 51249.75 + rank 8: 53631.75 + rank 9: 53626.64 + rank 10: 53627.99 + rank 11: 53627.74 + rank 12: 53628.85 + rank 13: 53627.27 + rank 14: 53631.53 + rank 15: 53627.82 + rank 16: 53210.44 + rank 17: 53214.34 + rank 18: 53208.60 + rank 19: 53208.54 + rank 20: 53215.63 + rank 21: 53213.44 + rank 22: 53214.54 + rank 23: 53212.61 + rank 24: 52903.04 + rank 25: 52896.05 + rank 26: 52893.96 + rank 27: 52896.24 + rank 28: 52884.45 + rank 29: 52879.27 + rank 30: 52889.59 + rank 31: 52889.93 + rank 32: 53500.87 + rank 33: 53496.93 + rank 34: 53493.04 + rank 35: 53489.41 + rank 36: 53481.56 + rank 37: 53485.29 + rank 38: 53482.34 + rank 39: 53486.33 + rank 40: 52175.91 + rank 41: 52179.87 + rank 42: 52182.75 + rank 43: 52176.73 + rank 44: 52178.89 + rank 45: 52178.02 + rank 46: 52183.74 + rank 47: 52187.12 + rank 48: 55878.70 + rank 49: 55879.63 + rank 50: 55884.76 + rank 51: 55883.32 + rank 52: 55867.69 + rank 53: 55871.45 + rank 54: 55882.30 + rank 55: 55879.22 + rank 56: 52719.82 + rank 57: 52713.59 + rank 58: 52710.99 + rank 59: 52716.43 + rank 60: 52708.14 + rank 61: 52704.13 + rank 62: 52714.45 + rank 63: 52702.91 + batch-generator: + rank 0: 23.57 + rank 1: 21.56 + rank 2: 26.49 + rank 3: 24.40 + rank 4: 23.74 + rank 5: 22.82 + rank 6: 24.51 + rank 7: 24.49 + rank 8: 23.12 + rank 9: 22.14 + rank 10: 22.47 + rank 11: 22.78 + rank 12: 22.94 + rank 13: 22.36 + rank 14: 22.71 + rank 15: 21.67 + rank 16: 23.25 + rank 17: 22.00 + rank 18: 23.08 + rank 19: 22.65 + rank 20: 22.24 + rank 21: 22.00 + rank 22: 22.00 + rank 23: 22.36 + rank 24: 31.63 + rank 25: 34.00 + rank 26: 34.02 + rank 27: 36.60 + rank 28: 36.44 + rank 29: 34.92 + rank 30: 35.08 + rank 31: 32.92 + rank 32: 34.73 + rank 33: 31.33 + rank 34: 35.45 + rank 35: 33.50 + rank 36: 31.55 + rank 37: 28.41 + rank 38: 33.99 + rank 39: 33.32 + rank 40: 24.18 + rank 41: 22.80 + rank 42: 23.86 + rank 43: 24.27 + rank 44: 23.72 + rank 45: 21.64 + rank 46: 22.83 + rank 47: 23.19 + rank 48: 30.50 + rank 49: 27.71 + rank 50: 21.27 + rank 51: 26.54 + rank 52: 32.35 + rank 53: 30.39 + rank 54: 24.02 + rank 55: 31.98 + rank 56: 24.13 + rank 57: 22.54 + rank 58: 22.35 + rank 59: 22.61 + rank 60: 23.20 + rank 61: 22.40 + rank 62: 23.28 + rank 63: 23.34 + embedding-grads-all-reduce: + rank 0: 0.03 + rank 1: 0.02 + rank 2: 0.03 + rank 3: 0.03 + rank 4: 0.02 + rank 5: 0.03 + rank 6: 0.02 + rank 7: 0.03 + rank 8: 0.03 + rank 9: 0.03 + rank 10: 0.03 + rank 11: 0.03 + rank 12: 0.04 + rank 13: 0.02 + rank 14: 0.02 + rank 15: 0.02 + rank 16: 0.03 + rank 17: 0.03 + rank 18: 0.03 + rank 19: 0.02 + rank 20: 0.03 + rank 21: 0.02 + rank 22: 0.02 + rank 23: 0.03 + rank 24: 0.04 + rank 25: 0.04 + rank 26: 0.04 + rank 27: 0.04 + rank 28: 0.04 + rank 29: 0.04 + rank 30: 0.04 + rank 31: 0.04 + rank 32: 0.04 + rank 33: 0.04 + rank 34: 0.04 + rank 35: 0.03 + rank 36: 0.03 + rank 37: 0.03 + rank 38: 0.03 + rank 39: 0.03 + rank 40: 0.03 + rank 41: 0.02 + rank 42: 0.02 + rank 43: 0.03 + rank 44: 0.03 + rank 45: 0.02 + rank 46: 0.03 + rank 47: 0.03 + rank 48: 0.04 + rank 49: 0.03 + rank 50: 0.03 + rank 51: 0.03 + rank 52: 0.03 + rank 53: 0.03 + rank 54: 0.03 + rank 55: 0.03 + rank 56: 0.03 + rank 57: 0.03 + rank 58: 0.02 + rank 59: 0.02 + rank 60: 0.03 + rank 61: 0.03 + rank 62: 0.02 + rank 63: 0.03 + all-grads-sync: + rank 0: 0.83 + rank 1: 0.82 + rank 2: 0.82 + rank 3: 0.89 + rank 4: 0.83 + rank 5: 1.31 + rank 6: 0.77 + rank 7: 0.86 + rank 8: 1.19 + rank 9: 0.96 + rank 10: 0.77 + rank 11: 0.77 + rank 12: 1.17 + rank 13: 0.84 + rank 14: 1.08 + rank 15: 0.86 + rank 16: 0.77 + rank 17: 0.73 + rank 18: 0.78 + rank 19: 0.97 + rank 20: 0.84 + rank 21: 0.80 + rank 22: 0.80 + rank 23: 1.18 + rank 24: 1.29 + rank 25: 1.28 + rank 26: 1.28 + rank 27: 1.17 + rank 28: 1.80 + rank 29: 1.29 + rank 30: 1.09 + rank 31: 1.22 + rank 32: 1.25 + rank 33: 1.24 + rank 34: 1.25 + rank 35: 1.22 + rank 36: 1.17 + rank 37: 1.21 + rank 38: 1.24 + rank 39: 1.21 + rank 40: 0.75 + rank 41: 0.77 + rank 42: 0.79 + rank 43: 1.13 + rank 44: 0.77 + rank 45: 0.80 + rank 46: 0.98 + rank 47: 0.94 + rank 48: 1.04 + rank 49: 1.00 + rank 50: 0.80 + rank 51: 1.01 + rank 52: 1.03 + rank 53: 1.01 + rank 54: 1.06 + rank 55: 1.14 + rank 56: 0.78 + rank 57: 0.77 + rank 58: 0.76 + rank 59: 0.76 + rank 60: 0.79 + rank 61: 0.81 + rank 62: 0.79 + rank 63: 1.21 + params-all-gather: + rank 0: 0.02 + rank 1: 0.02 + rank 2: 0.02 + rank 3: 0.02 + rank 4: 0.02 + rank 5: 0.02 + rank 6: 0.02 + rank 7: 0.02 + rank 8: 0.04 + rank 9: 0.02 + rank 10: 0.02 + rank 11: 0.02 + rank 12: 0.04 + rank 13: 0.02 + rank 14: 0.02 + rank 15: 0.02 + rank 16: 0.03 + rank 17: 0.02 + rank 18: 0.02 + rank 19: 0.02 + rank 20: 0.03 + rank 21: 0.02 + rank 22: 0.03 + rank 23: 0.02 + rank 24: 0.04 + rank 25: 0.03 + rank 26: 0.04 + rank 27: 0.03 + rank 28: 0.03 + rank 29: 0.03 + rank 30: 0.03 + rank 31: 0.04 + rank 32: 0.03 + rank 33: 0.03 + rank 34: 0.02 + rank 35: 0.02 + rank 36: 0.02 + rank 37: 0.03 + rank 38: 0.02 + rank 39: 0.02 + rank 40: 0.02 + rank 41: 0.02 + rank 42: 0.02 + rank 43: 0.02 + rank 44: 0.02 + rank 45: 0.02 + rank 46: 0.02 + rank 47: 0.02 + rank 48: 0.03 + rank 49: 0.03 + rank 50: 0.03 + rank 51: 0.03 + rank 52: 0.03 + rank 53: 0.03 + rank 54: 0.03 + rank 55: 0.02 + rank 56: 0.02 + rank 57: 0.02 + rank 58: 0.02 + rank 59: 0.02 + rank 60: 0.02 + rank 61: 0.02 + rank 62: 0.02 + rank 63: 0.02 + optimizer-copy-to-main-grad: + rank 0: 2.22 + rank 1: 1.78 + rank 2: 1.73 + rank 3: 2.11 + rank 4: 1.68 + rank 5: 1.80 + rank 6: 1.86 + rank 7: 1.90 + rank 8: 2.89 + rank 9: 2.10 + rank 10: 1.69 + rank 11: 1.73 + rank 12: 2.93 + rank 13: 1.80 + rank 14: 1.86 + rank 15: 1.99 + rank 16: 2.44 + rank 17: 2.08 + rank 18: 2.13 + rank 19: 2.11 + rank 20: 2.13 + rank 21: 2.37 + rank 22: 2.52 + rank 23: 2.30 + rank 24: 1.93 + rank 25: 2.24 + rank 26: 2.01 + rank 27: 2.12 + rank 28: 2.37 + rank 29: 2.47 + rank 30: 2.31 + rank 31: 1.97 + rank 32: 2.01 + rank 33: 1.88 + rank 34: 2.20 + rank 35: 2.15 + rank 36: 2.10 + rank 37: 1.95 + rank 38: 2.42 + rank 39: 1.94 + rank 40: 1.70 + rank 41: 1.83 + rank 42: 1.91 + rank 43: 1.87 + rank 44: 1.85 + rank 45: 2.04 + rank 46: 1.73 + rank 47: 1.65 + rank 48: 1.95 + rank 49: 1.82 + rank 50: 2.30 + rank 51: 1.81 + rank 52: 1.98 + rank 53: 1.73 + rank 54: 1.81 + rank 55: 1.80 + rank 56: 1.77 + rank 57: 2.34 + rank 58: 1.82 + rank 59: 1.69 + rank 60: 1.71 + rank 61: 1.71 + rank 62: 1.79 + rank 63: 1.80 + optimizer-inner-step: + rank 0: 44.65 + rank 1: 44.55 + rank 2: 44.30 + rank 3: 44.46 + rank 4: 44.29 + rank 5: 44.46 + rank 6: 44.77 + rank 7: 44.75 + rank 8: 56.42 + rank 9: 44.47 + rank 10: 44.37 + rank 11: 44.49 + rank 12: 57.01 + rank 13: 44.67 + rank 14: 44.44 + rank 15: 44.29 + rank 16: 44.83 + rank 17: 44.56 + rank 18: 44.60 + rank 19: 44.58 + rank 20: 44.61 + rank 21: 44.47 + rank 22: 44.94 + rank 23: 44.61 + rank 24: 44.16 + rank 25: 44.47 + rank 26: 44.38 + rank 27: 44.64 + rank 28: 45.86 + rank 29: 44.39 + rank 30: 44.37 + rank 31: 44.61 + rank 32: 44.57 + rank 33: 44.38 + rank 34: 44.47 + rank 35: 47.45 + rank 36: 44.54 + rank 37: 44.54 + rank 38: 44.41 + rank 39: 44.38 + rank 40: 44.50 + rank 41: 44.57 + rank 42: 44.61 + rank 43: 44.66 + rank 44: 44.27 + rank 45: 44.85 + rank 46: 44.50 + rank 47: 44.17 + rank 48: 44.54 + rank 49: 44.35 + rank 50: 53.34 + rank 51: 44.16 + rank 52: 44.56 + rank 53: 44.36 + rank 54: 44.54 + rank 55: 44.40 + rank 56: 44.13 + rank 57: 46.95 + rank 58: 44.43 + rank 59: 44.30 + rank 60: 44.67 + rank 61: 44.18 + rank 62: 44.46 + rank 63: 44.19 + optimizer-copy-main-to-model-params: + rank 0: 4.54 + rank 1: 3.86 + rank 2: 3.82 + rank 3: 4.38 + rank 4: 3.74 + rank 5: 3.91 + rank 6: 3.99 + rank 7: 4.00 + rank 8: 5.36 + rank 9: 4.41 + rank 10: 3.82 + rank 11: 3.78 + rank 12: 5.53 + rank 13: 3.79 + rank 14: 4.00 + rank 15: 4.20 + rank 16: 4.45 + rank 17: 4.07 + rank 18: 3.98 + rank 19: 4.09 + rank 20: 4.02 + rank 21: 4.32 + rank 22: 4.58 + rank 23: 4.22 + rank 24: 3.90 + rank 25: 4.26 + rank 26: 4.01 + rank 27: 4.15 + rank 28: 4.32 + rank 29: 4.40 + rank 30: 4.30 + rank 31: 4.03 + rank 32: 4.44 + rank 33: 4.52 + rank 34: 4.22 + rank 35: 4.54 + rank 36: 4.20 + rank 37: 4.07 + rank 38: 4.53 + rank 39: 4.13 + rank 40: 3.74 + rank 41: 3.98 + rank 42: 4.06 + rank 43: 4.00 + rank 44: 3.98 + rank 45: 4.21 + rank 46: 3.78 + rank 47: 3.70 + rank 48: 4.09 + rank 49: 3.97 + rank 50: 4.89 + rank 51: 3.93 + rank 52: 4.22 + rank 53: 3.89 + rank 54: 3.88 + rank 55: 3.87 + rank 56: 3.82 + rank 57: 4.78 + rank 58: 3.87 + rank 59: 3.76 + rank 60: 3.74 + rank 61: 3.73 + rank 62: 3.90 + rank 63: 3.70 + optimizer: + rank 0: 56.72 + rank 1: 55.71 + rank 2: 55.40 + rank 3: 56.34 + rank 4: 55.31 + rank 5: 55.68 + rank 6: 56.12 + rank 7: 56.12 + rank 8: 69.69 + rank 9: 56.38 + rank 10: 55.45 + rank 11: 55.49 + rank 12: 70.48 + rank 13: 55.72 + rank 14: 55.75 + rank 15: 55.90 + rank 16: 56.99 + rank 17: 56.16 + rank 18: 56.14 + rank 19: 56.21 + rank 20: 56.12 + rank 21: 56.46 + rank 22: 57.32 + rank 23: 56.41 + rank 24: 55.35 + rank 25: 56.07 + rank 26: 55.66 + rank 27: 56.10 + rank 28: 57.60 + rank 29: 56.18 + rank 30: 56.05 + rank 31: 55.93 + rank 32: 56.31 + rank 33: 56.13 + rank 34: 55.92 + rank 35: 59.42 + rank 36: 56.05 + rank 37: 55.89 + rank 38: 56.36 + rank 39: 55.71 + rank 40: 55.47 + rank 41: 55.85 + rank 42: 56.00 + rank 43: 55.96 + rank 44: 55.54 + rank 45: 56.48 + rank 46: 55.50 + rank 47: 55.09 + rank 48: 55.98 + rank 49: 55.56 + rank 50: 65.85 + rank 51: 55.36 + rank 52: 56.14 + rank 53: 55.47 + rank 54: 55.65 + rank 55: 55.50 + rank 56: 55.20 + rank 57: 59.36 + rank 58: 55.57 + rank 59: 55.28 + rank 60: 55.64 + rank 61: 55.10 + rank 62: 55.60 + rank 63: 55.09 + interval-time: + rank 0: 0.82 + rank 1: 0.74 + rank 2: 0.85 + rank 3: 0.75 + rank 4: 0.82 + rank 5: 0.80 + rank 6: 0.81 + rank 7: 0.74 + rank 8: 0.75 + rank 9: 0.76 + rank 10: 0.83 + rank 11: 0.73 + rank 12: 0.83 + rank 13: 0.74 + rank 14: 0.74 + rank 15: 0.81 + rank 16: 0.76 + rank 17: 0.74 + rank 18: 0.84 + rank 19: 0.76 + rank 20: 0.81 + rank 21: 0.76 + rank 22: 0.74 + rank 23: 0.73 + rank 24: 0.90 + rank 25: 0.95 + rank 26: 0.89 + rank 27: 0.89 + rank 28: 0.84 + rank 29: 0.94 + rank 30: 0.92 + rank 31: 1.08 + rank 32: 0.93 + rank 33: 0.94 + rank 34: 0.83 + rank 35: 0.80 + rank 36: 0.88 + rank 37: 0.89 + rank 38: 0.90 + rank 39: 0.88 + rank 40: 1.04 + rank 41: 0.96 + rank 42: 1.06 + rank 43: 1.08 + rank 44: 0.86 + rank 45: 1.03 + rank 46: 0.91 + rank 47: 0.91 + rank 48: 0.92 + rank 49: 0.96 + rank 50: 0.79 + rank 51: 0.79 + rank 52: 0.83 + rank 53: 0.83 + rank 54: 0.91 + rank 55: 0.86 + rank 56: 0.82 + rank 57: 0.74 + rank 58: 0.74 + rank 59: 0.75 + rank 60: 0.72 + rank 61: 0.80 + rank 62: 0.80 + rank 63: 3.31 + [2026-10-06 19:11:50] iteration 3003/ 6855 | consumed samples: 384384 | elapsed time per iteration (ms): 87186.4 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.60% | FLOPs estimator: workload-aware | logical tokens/s: 192429.1 | valid tokens/s: 192429.1 | physical tokens/s: 193054.8 | loss tokens/s: 144019.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.657853E-01 | lm-global-supervised-tokens: 1.255654E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.355403E-02 | lm-token-buffer-mib: 5.807832E+01 | global_load_balancing_loss: 1.000602E+00 | loss scale: 1.0 | grad norm: 0.070 | params norm: 11445.618 | params rms: 0.061974 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 21:17:21.866532 | finish at 2026-10-10 16:29:12 +times across ranks (ms): + forward-backward: + rank 0: 86977.95 + rank 1: 86978.98 + rank 2: 86978.68 + rank 3: 86979.16 + rank 4: 86979.66 + rank 5: 86979.35 + rank 6: 86977.00 + rank 7: 86978.51 + rank 8: 86981.23 + rank 9: 86981.48 + rank 10: 86981.88 + rank 11: 86983.71 + rank 12: 86983.06 + rank 13: 86983.19 + rank 14: 86980.36 + rank 15: 86980.73 + rank 16: 86972.01 + rank 17: 86973.38 + rank 18: 86973.87 + rank 19: 86973.65 + rank 20: 86974.04 + rank 21: 86975.75 + rank 22: 86971.95 + rank 23: 86971.97 + rank 24: 86970.68 + rank 25: 86972.00 + rank 26: 86973.80 + rank 27: 86972.73 + rank 28: 86974.76 + rank 29: 86973.44 + rank 30: 86970.78 + rank 31: 86969.86 + rank 32: 86966.80 + rank 33: 86966.45 + rank 34: 86967.09 + rank 35: 86967.30 + rank 36: 86968.48 + rank 37: 86967.86 + rank 38: 86967.74 + rank 39: 86969.52 + rank 40: 86973.17 + rank 41: 86973.05 + rank 42: 86972.89 + rank 43: 86973.17 + rank 44: 86973.16 + rank 45: 86972.98 + rank 46: 86973.34 + rank 47: 86973.02 + rank 48: 86977.69 + rank 49: 86976.05 + rank 50: 86974.98 + rank 51: 86977.07 + rank 52: 86974.91 + rank 53: 86975.46 + rank 54: 86977.51 + rank 55: 86978.68 + rank 56: 86979.48 + rank 57: 86978.43 + rank 58: 86978.20 + rank 59: 86977.16 + rank 60: 86977.88 + rank 61: 86976.83 + rank 62: 86980.11 + rank 63: 86980.34 + forward-compute: + rank 0: 32713.53 + rank 1: 32711.68 + rank 2: 32717.51 + rank 3: 32717.09 + rank 4: 32718.33 + rank 5: 32715.36 + rank 6: 32716.00 + rank 7: 32722.29 + rank 8: 33764.60 + rank 9: 33763.55 + rank 10: 33768.36 + rank 11: 33769.55 + rank 12: 33772.06 + rank 13: 33775.93 + rank 14: 33771.35 + rank 15: 33771.76 + rank 16: 34199.22 + rank 17: 34198.57 + rank 18: 34198.71 + rank 19: 34200.68 + rank 20: 34200.65 + rank 21: 34199.70 + rank 22: 34198.70 + rank 23: 34200.43 + rank 24: 33764.69 + rank 25: 33766.44 + rank 26: 33767.85 + rank 27: 33766.13 + rank 28: 33775.54 + rank 29: 33772.39 + rank 30: 33772.65 + rank 31: 33767.47 + rank 32: 31726.51 + rank 33: 31740.37 + rank 34: 31742.03 + rank 35: 31743.68 + rank 36: 31749.09 + rank 37: 31754.79 + rank 38: 31754.86 + rank 39: 31744.14 + rank 40: 35344.21 + rank 41: 35341.08 + rank 42: 35342.73 + rank 43: 35343.94 + rank 44: 35343.40 + rank 45: 35345.47 + rank 46: 35342.78 + rank 47: 35342.77 + rank 48: 35230.62 + rank 49: 35241.36 + rank 50: 35244.23 + rank 51: 35243.46 + rank 52: 35252.08 + rank 53: 35252.27 + rank 54: 35249.24 + rank 55: 35248.16 + rank 56: 33461.14 + rank 57: 33461.59 + rank 58: 33461.70 + rank 59: 33459.73 + rank 60: 33462.91 + rank 61: 33464.68 + rank 62: 33461.14 + rank 63: 33465.04 + backward-compute: + rank 0: 54230.87 + rank 1: 54234.34 + rank 2: 54225.56 + rank 3: 54228.33 + rank 4: 54227.54 + rank 5: 54231.31 + rank 6: 54226.52 + rank 7: 54223.25 + rank 8: 53182.21 + rank 9: 53184.94 + rank 10: 53181.14 + rank 11: 53177.96 + rank 12: 53175.92 + rank 13: 53174.54 + rank 14: 53175.74 + rank 15: 53175.71 + rank 16: 52739.05 + rank 17: 52741.23 + rank 18: 52741.47 + rank 19: 52737.39 + rank 20: 52742.03 + rank 21: 52741.71 + rank 22: 52737.86 + rank 23: 52738.44 + rank 24: 53169.32 + rank 25: 53169.77 + rank 26: 53163.75 + rank 27: 53163.73 + rank 28: 53155.14 + rank 29: 53157.93 + rank 30: 53156.45 + rank 31: 53163.86 + rank 32: 55202.80 + rank 33: 55186.07 + rank 34: 55186.05 + rank 35: 55186.02 + rank 36: 55181.88 + rank 37: 55176.25 + rank 38: 55174.42 + rank 39: 55187.02 + rank 40: 51589.96 + rank 41: 51593.42 + rank 42: 51592.28 + rank 43: 51587.91 + rank 44: 51590.16 + rank 45: 51586.46 + rank 46: 51591.85 + rank 47: 51593.25 + rank 48: 51712.18 + rank 49: 51698.96 + rank 50: 51698.19 + rank 51: 51698.61 + rank 52: 51685.64 + rank 53: 51686.63 + rank 54: 51693.58 + rank 55: 51694.05 + rank 56: 53483.07 + rank 57: 53481.80 + rank 58: 53483.90 + rank 59: 53483.92 + rank 60: 53482.17 + rank 61: 53479.29 + rank 62: 53484.32 + rank 63: 53478.92 + batch-generator: + rank 0: 22.64 + rank 1: 22.07 + rank 2: 23.96 + rank 3: 22.71 + rank 4: 22.72 + rank 5: 21.96 + rank 6: 22.93 + rank 7: 23.30 + rank 8: 23.45 + rank 9: 21.84 + rank 10: 21.71 + rank 11: 21.95 + rank 12: 21.99 + rank 13: 22.04 + rank 14: 23.04 + rank 15: 22.02 + rank 16: 23.15 + rank 17: 22.03 + rank 18: 22.68 + rank 19: 21.87 + rank 20: 22.19 + rank 21: 23.09 + rank 22: 21.71 + rank 23: 23.14 + rank 24: 30.69 + rank 25: 31.61 + rank 26: 37.41 + rank 27: 35.24 + rank 28: 35.18 + rank 29: 35.76 + rank 30: 33.49 + rank 31: 33.70 + rank 32: 34.73 + rank 33: 31.68 + rank 34: 33.35 + rank 35: 31.02 + rank 36: 31.02 + rank 37: 30.94 + rank 38: 34.78 + rank 39: 32.05 + rank 40: 28.37 + rank 41: 26.43 + rank 42: 28.65 + rank 43: 28.50 + rank 44: 27.67 + rank 45: 27.48 + rank 46: 28.12 + rank 47: 27.59 + rank 48: 24.83 + rank 49: 30.93 + rank 50: 21.86 + rank 51: 29.25 + rank 52: 30.80 + rank 53: 29.51 + rank 54: 28.34 + rank 55: 28.69 + rank 56: 23.10 + rank 57: 21.84 + rank 58: 22.40 + rank 59: 22.08 + rank 60: 24.22 + rank 61: 22.11 + rank 62: 22.18 + rank 63: 22.57 + embedding-grads-all-reduce: + rank 0: 0.03 + rank 1: 0.02 + rank 2: 0.02 + rank 3: 0.02 + rank 4: 0.02 + rank 5: 0.02 + rank 6: 0.02 + rank 7: 0.03 + rank 8: 0.02 + rank 9: 0.03 + rank 10: 0.02 + rank 11: 0.03 + rank 12: 0.03 + rank 13: 0.02 + rank 14: 0.03 + rank 15: 0.02 + rank 16: 0.03 + rank 17: 0.03 + rank 18: 0.03 + rank 19: 0.02 + rank 20: 0.03 + rank 21: 0.03 + rank 22: 0.03 + rank 23: 0.03 + rank 24: 0.04 + rank 25: 0.03 + rank 26: 0.04 + rank 27: 0.04 + rank 28: 0.03 + rank 29: 0.04 + rank 30: 0.04 + rank 31: 0.03 + rank 32: 0.03 + rank 33: 0.03 + rank 34: 0.03 + rank 35: 0.03 + rank 36: 0.03 + rank 37: 0.03 + rank 38: 0.03 + rank 39: 0.03 + rank 40: 0.03 + rank 41: 0.02 + rank 42: 0.02 + rank 43: 0.03 + rank 44: 0.03 + rank 45: 0.03 + rank 46: 0.03 + rank 47: 0.03 + rank 48: 0.04 + rank 49: 0.03 + rank 50: 0.03 + rank 51: 0.03 + rank 52: 0.03 + rank 53: 0.03 + rank 54: 0.03 + rank 55: 0.03 + rank 56: 0.03 + rank 57: 0.03 + rank 58: 0.02 + rank 59: 0.02 + rank 60: 0.03 + rank 61: 0.02 + rank 62: 0.02 + rank 63: 0.03 + all-grads-sync: + rank 0: 0.94 + rank 1: 0.95 + rank 2: 0.78 + rank 3: 0.84 + rank 4: 0.76 + rank 5: 0.79 + rank 6: 0.81 + rank 7: 0.80 + rank 8: 0.93 + rank 9: 0.79 + rank 10: 0.78 + rank 11: 1.08 + rank 12: 0.86 + rank 13: 0.79 + rank 14: 0.81 + rank 15: 0.98 + rank 16: 0.84 + rank 17: 0.78 + rank 18: 0.80 + rank 19: 0.77 + rank 20: 0.81 + rank 21: 0.76 + rank 22: 0.81 + rank 23: 0.78 + rank 24: 1.35 + rank 25: 1.10 + rank 26: 1.66 + rank 27: 1.46 + rank 28: 1.21 + rank 29: 1.32 + rank 30: 1.29 + rank 31: 1.20 + rank 32: 1.50 + rank 33: 1.33 + rank 34: 1.21 + rank 35: 1.22 + rank 36: 1.32 + rank 37: 1.24 + rank 38: 1.33 + rank 39: 1.87 + rank 40: 0.82 + rank 41: 0.78 + rank 42: 0.94 + rank 43: 0.82 + rank 44: 0.83 + rank 45: 0.77 + rank 46: 0.91 + rank 47: 1.01 + rank 48: 1.06 + rank 49: 0.95 + rank 50: 0.88 + rank 51: 1.32 + rank 52: 0.98 + rank 53: 0.99 + rank 54: 1.02 + rank 55: 1.08 + rank 56: 0.81 + rank 57: 0.78 + rank 58: 0.80 + rank 59: 0.78 + rank 60: 0.78 + rank 61: 0.91 + rank 62: 0.79 + rank 63: 1.12 + params-all-gather: + rank 0: 0.02 + rank 1: 0.02 + rank 2: 0.02 + rank 3: 0.02 + rank 4: 0.02 + rank 5: 0.02 + rank 6: 0.02 + rank 7: 0.02 + rank 8: 0.02 + rank 9: 0.02 + rank 10: 0.02 + rank 11: 0.02 + rank 12: 0.02 + rank 13: 0.02 + rank 14: 0.02 + rank 15: 0.02 + rank 16: 0.02 + rank 17: 0.02 + rank 18: 0.02 + rank 19: 0.02 + rank 20: 0.02 + rank 21: 0.02 + rank 22: 0.02 + rank 23: 0.02 + rank 24: 0.03 + rank 25: 0.03 + rank 26: 0.03 + rank 27: 0.03 + rank 28: 0.04 + rank 29: 0.03 + rank 30: 0.03 + rank 31: 0.04 + rank 32: 0.03 + rank 33: 0.02 + rank 34: 0.03 + rank 35: 0.03 + rank 36: 0.02 + rank 37: 0.02 + rank 38: 0.02 + rank 39: 0.03 + rank 40: 0.02 + rank 41: 0.02 + rank 42: 0.02 + rank 43: 0.02 + rank 44: 0.02 + rank 45: 0.02 + rank 46: 0.02 + rank 47: 0.02 + rank 48: 0.03 + rank 49: 0.03 + rank 50: 0.02 + rank 51: 0.02 + rank 52: 0.03 + rank 53: 0.03 + rank 54: 0.03 + rank 55: 0.03 + rank 56: 0.02 + rank 57: 0.02 + rank 58: 0.02 + rank 59: 0.02 + rank 60: 0.02 + rank 61: 0.02 + rank 62: 0.02 + rank 63: 0.02 + optimizer-copy-to-main-grad: + rank 0: 2.14 + rank 1: 1.79 + rank 2: 1.66 + rank 3: 2.06 + rank 4: 1.70 + rank 5: 1.78 + rank 6: 1.81 + rank 7: 1.91 + rank 8: 1.78 + rank 9: 2.07 + rank 10: 1.74 + rank 11: 1.74 + rank 12: 1.81 + rank 13: 1.74 + rank 14: 1.87 + rank 15: 1.99 + rank 16: 2.33 + rank 17: 2.03 + rank 18: 2.10 + rank 19: 2.05 + rank 20: 2.15 + rank 21: 2.28 + rank 22: 2.49 + rank 23: 2.22 + rank 24: 1.93 + rank 25: 2.28 + rank 26: 2.03 + rank 27: 2.09 + rank 28: 2.39 + rank 29: 2.43 + rank 30: 2.32 + rank 31: 2.02 + rank 32: 1.97 + rank 33: 1.85 + rank 34: 1.88 + rank 35: 2.20 + rank 36: 2.15 + rank 37: 1.98 + rank 38: 2.28 + rank 39: 1.93 + rank 40: 1.79 + rank 41: 1.82 + rank 42: 1.84 + rank 43: 1.87 + rank 44: 1.84 + rank 45: 2.03 + rank 46: 1.85 + rank 47: 1.69 + rank 48: 1.97 + rank 49: 1.77 + rank 50: 2.33 + rank 51: 1.81 + rank 52: 1.95 + rank 53: 1.75 + rank 54: 1.77 + rank 55: 1.82 + rank 56: 1.74 + rank 57: 2.31 + rank 58: 1.81 + rank 59: 1.69 + rank 60: 1.72 + rank 61: 1.66 + rank 62: 1.86 + rank 63: 1.73 + optimizer-inner-step: + rank 0: 42.18 + rank 1: 42.12 + rank 2: 41.81 + rank 3: 41.98 + rank 4: 41.79 + rank 5: 41.89 + rank 6: 42.24 + rank 7: 42.20 + rank 8: 42.00 + rank 9: 42.04 + rank 10: 41.94 + rank 11: 42.03 + rank 12: 41.97 + rank 13: 42.18 + rank 14: 41.97 + rank 15: 41.88 + rank 16: 42.32 + rank 17: 42.09 + rank 18: 42.16 + rank 19: 42.16 + rank 20: 42.06 + rank 21: 41.92 + rank 22: 42.44 + rank 23: 42.11 + rank 24: 41.75 + rank 25: 42.12 + rank 26: 42.08 + rank 27: 42.28 + rank 28: 46.04 + rank 29: 41.99 + rank 30: 41.95 + rank 31: 42.36 + rank 32: 42.07 + rank 33: 41.86 + rank 34: 42.01 + rank 35: 47.45 + rank 36: 42.13 + rank 37: 42.07 + rank 38: 41.99 + rank 39: 41.92 + rank 40: 42.05 + rank 41: 42.09 + rank 42: 42.21 + rank 43: 42.22 + rank 44: 41.60 + rank 45: 42.45 + rank 46: 42.05 + rank 47: 41.72 + rank 48: 42.54 + rank 49: 41.93 + rank 50: 53.74 + rank 51: 41.75 + rank 52: 42.19 + rank 53: 41.94 + rank 54: 42.18 + rank 55: 42.05 + rank 56: 41.65 + rank 57: 47.07 + rank 58: 42.02 + rank 59: 41.89 + rank 60: 42.16 + rank 61: 41.76 + rank 62: 41.92 + rank 63: 41.84 + optimizer-copy-main-to-model-params: + rank 0: 4.42 + rank 1: 3.83 + rank 2: 3.75 + rank 3: 4.32 + rank 4: 3.70 + rank 5: 3.87 + rank 6: 3.89 + rank 7: 4.02 + rank 8: 3.92 + rank 9: 4.44 + rank 10: 3.82 + rank 11: 3.79 + rank 12: 3.95 + rank 13: 3.79 + rank 14: 4.09 + rank 15: 4.23 + rank 16: 4.33 + rank 17: 4.00 + rank 18: 4.06 + rank 19: 4.02 + rank 20: 4.03 + rank 21: 4.23 + rank 22: 4.55 + rank 23: 4.16 + rank 24: 3.88 + rank 25: 4.17 + rank 26: 3.94 + rank 27: 4.07 + rank 28: 4.40 + rank 29: 4.64 + rank 30: 4.33 + rank 31: 4.03 + rank 32: 4.06 + rank 33: 3.89 + rank 34: 3.99 + rank 35: 4.54 + rank 36: 4.20 + rank 37: 4.06 + rank 38: 4.47 + rank 39: 3.90 + rank 40: 3.75 + rank 41: 3.97 + rank 42: 3.88 + rank 43: 3.99 + rank 44: 4.01 + rank 45: 4.22 + rank 46: 3.75 + rank 47: 3.70 + rank 48: 4.10 + rank 49: 3.89 + rank 50: 4.92 + rank 51: 3.93 + rank 52: 4.24 + rank 53: 3.88 + rank 54: 3.85 + rank 55: 3.86 + rank 56: 3.76 + rank 57: 4.98 + rank 58: 3.88 + rank 59: 3.71 + rank 60: 3.76 + rank 61: 3.63 + rank 62: 3.91 + rank 63: 3.66 + optimizer: + rank 0: 53.67 + rank 1: 52.81 + rank 2: 52.41 + rank 3: 53.40 + rank 4: 52.30 + rank 5: 52.66 + rank 6: 53.09 + rank 7: 53.14 + rank 8: 52.77 + rank 9: 53.55 + rank 10: 52.58 + rank 11: 52.63 + rank 12: 52.79 + rank 13: 52.82 + rank 14: 52.96 + rank 15: 53.08 + rank 16: 53.99 + rank 17: 53.20 + rank 18: 53.37 + rank 19: 53.24 + rank 20: 53.24 + rank 21: 53.41 + rank 22: 54.37 + rank 23: 53.50 + rank 24: 52.46 + rank 25: 53.19 + rank 26: 52.85 + rank 27: 53.20 + rank 28: 57.41 + rank 29: 53.60 + rank 30: 53.23 + rank 31: 53.22 + rank 32: 52.98 + rank 33: 52.51 + rank 34: 52.79 + rank 35: 58.98 + rank 36: 53.22 + rank 37: 52.97 + rank 38: 53.44 + rank 39: 52.59 + rank 40: 52.60 + rank 41: 52.92 + rank 42: 52.99 + rank 43: 53.09 + rank 44: 52.69 + rank 45: 53.67 + rank 46: 52.60 + rank 47: 52.21 + rank 48: 53.54 + rank 49: 52.63 + rank 50: 65.84 + rank 51: 52.49 + rank 52: 53.34 + rank 53: 52.61 + rank 54: 52.81 + rank 55: 52.70 + rank 56: 52.22 + rank 57: 59.26 + rank 58: 52.73 + rank 59: 52.35 + rank 60: 52.71 + rank 61: 52.14 + rank 62: 52.66 + rank 63: 52.25 + interval-time: + rank 0: 0.74 + rank 1: 0.74 + rank 2: 0.84 + rank 3: 0.74 + rank 4: 0.83 + rank 5: 0.81 + rank 6: 0.82 + rank 7: 0.75 + rank 8: 0.75 + rank 9: 0.74 + rank 10: 0.81 + rank 11: 0.73 + rank 12: 0.83 + rank 13: 0.76 + rank 14: 0.74 + rank 15: 0.72 + rank 16: 0.72 + rank 17: 0.74 + rank 18: 0.81 + rank 19: 0.73 + rank 20: 0.80 + rank 21: 0.75 + rank 22: 0.72 + rank 23: 0.72 + rank 24: 0.87 + rank 25: 0.93 + rank 26: 0.87 + rank 27: 0.86 + rank 28: 0.83 + rank 29: 0.96 + rank 30: 0.92 + rank 31: 0.96 + rank 32: 0.92 + rank 33: 0.95 + rank 34: 0.87 + rank 35: 0.80 + rank 36: 0.89 + rank 37: 0.89 + rank 38: 0.89 + rank 39: 0.88 + rank 40: 0.82 + rank 41: 0.81 + rank 42: 0.80 + rank 43: 0.82 + rank 44: 0.72 + rank 45: 0.80 + rank 46: 0.83 + rank 47: 0.78 + rank 48: 0.93 + rank 49: 0.92 + rank 50: 0.80 + rank 51: 0.80 + rank 52: 0.83 + rank 53: 0.82 + rank 54: 0.90 + rank 55: 0.83 + rank 56: 0.82 + rank 57: 0.74 + rank 58: 0.74 + rank 59: 0.76 + rank 60: 0.81 + rank 61: 0.81 + rank 62: 0.79 + rank 63: 2.43 + [2026-10-06 19:13:20] iteration 3004/ 6855 | consumed samples: 384512 | elapsed time per iteration (ms): 90038.2 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.62% | FLOPs estimator: workload-aware | logical tokens/s: 186333.7 | valid tokens/s: 186333.7 | physical tokens/s: 186694.3 | loss tokens/s: 139828.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.167833E-01 | lm-global-supervised-tokens: 1.258988E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.785884E-02 | lm-token-buffer-mib: 5.809418E+01 | global_load_balancing_loss: 1.001306E+00 | loss scale: 1.0 | grad norm: 0.051 | params norm: 11445.556 | params rms: 0.061974 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 0:18:57.138120 | finish at 2026-10-10 19:32:17 +times across ranks (ms): + forward-backward: + rank 0: 89819.73 + rank 1: 89819.88 + rank 2: 89821.32 + rank 3: 89821.27 + rank 4: 89821.32 + rank 5: 89821.81 + rank 6: 89820.04 + rank 7: 89820.46 + rank 8: 89822.44 + rank 9: 89822.20 + rank 10: 89822.92 + rank 11: 89823.84 + rank 12: 89823.65 + rank 13: 89824.48 + rank 14: 89821.12 + rank 15: 89821.73 + rank 16: 89814.31 + rank 17: 89814.96 + rank 18: 89815.26 + rank 19: 89815.45 + rank 20: 89816.29 + rank 21: 89817.09 + rank 22: 89813.80 + rank 23: 89814.00 + rank 24: 89811.63 + rank 25: 89813.20 + rank 26: 89814.31 + rank 27: 89814.41 + rank 28: 89816.08 + rank 29: 89816.12 + rank 30: 89812.40 + rank 31: 89812.25 + rank 32: 89808.28 + rank 33: 89809.12 + rank 34: 89808.17 + rank 35: 89808.76 + rank 36: 89809.98 + rank 37: 89808.62 + rank 38: 89809.06 + rank 39: 89810.27 + rank 40: 89811.70 + rank 41: 89812.04 + rank 42: 89811.06 + rank 43: 89810.59 + rank 44: 89810.02 + rank 45: 89809.65 + rank 46: 89813.29 + rank 47: 89812.00 + rank 48: 89818.24 + rank 49: 89818.46 + rank 50: 89817.45 + rank 51: 89818.09 + rank 52: 89816.37 + rank 53: 89817.24 + rank 54: 89820.02 + rank 55: 89819.96 + rank 56: 89821.16 + rank 57: 89820.22 + rank 58: 89819.57 + rank 59: 89819.65 + rank 60: 89818.99 + rank 61: 89818.83 + rank 62: 89821.66 + rank 63: 89821.66 + forward-compute: + rank 0: 39728.32 + rank 1: 39728.83 + rank 2: 39730.17 + rank 3: 39728.91 + rank 4: 39727.98 + rank 5: 39729.14 + rank 6: 39727.02 + rank 7: 39725.59 + rank 8: 30170.95 + rank 9: 30166.46 + rank 10: 30165.90 + rank 11: 30167.11 + rank 12: 30169.31 + rank 13: 30173.10 + rank 14: 30164.62 + rank 15: 30166.94 + rank 16: 34421.96 + rank 17: 34422.19 + rank 18: 34424.14 + rank 19: 34424.52 + rank 20: 34424.95 + rank 21: 34424.29 + rank 22: 34425.58 + rank 23: 34429.89 + rank 24: 37014.93 + rank 25: 37021.05 + rank 26: 37022.41 + rank 27: 37022.41 + rank 28: 37027.89 + rank 29: 37023.91 + rank 30: 37024.49 + rank 31: 37027.78 + rank 32: 35073.50 + rank 33: 35066.37 + rank 34: 35074.16 + rank 35: 35075.83 + rank 36: 35078.69 + rank 37: 35081.47 + rank 38: 35076.17 + rank 39: 35071.40 + rank 40: 32271.64 + rank 41: 32276.45 + rank 42: 32280.50 + rank 43: 32281.90 + rank 44: 32276.79 + rank 45: 32276.17 + rank 46: 32282.96 + rank 47: 32286.23 + rank 48: 36421.28 + rank 49: 36425.13 + rank 50: 36423.41 + rank 51: 36425.56 + rank 52: 36426.16 + rank 53: 36425.66 + rank 54: 36421.96 + rank 55: 36421.93 + rank 56: 31396.77 + rank 57: 31395.57 + rank 58: 31395.29 + rank 59: 31389.26 + rank 60: 31391.90 + rank 61: 31386.56 + rank 62: 31379.49 + rank 63: 31391.01 + backward-compute: + rank 0: 50052.04 + rank 1: 50054.23 + rank 2: 50050.52 + rank 3: 50052.45 + rank 4: 50057.04 + rank 5: 50057.92 + rank 6: 50056.23 + rank 7: 50057.68 + rank 8: 59615.88 + rank 9: 59624.96 + rank 10: 59624.87 + rank 11: 59623.70 + rank 12: 59621.64 + rank 13: 59619.05 + rank 14: 59624.49 + rank 15: 59621.78 + rank 16: 55358.64 + rank 17: 55358.32 + rank 18: 55355.46 + rank 19: 55356.28 + rank 20: 55357.23 + rank 21: 55356.95 + rank 22: 55354.32 + rank 23: 55350.08 + rank 24: 52754.97 + rank 25: 52751.02 + rank 26: 52749.21 + rank 27: 52746.75 + rank 28: 52745.82 + rank 29: 52746.19 + rank 30: 52747.18 + rank 31: 52743.95 + rank 32: 54693.25 + rank 33: 54702.93 + rank 34: 54694.01 + rank 35: 54694.52 + rank 36: 54695.16 + rank 37: 54690.05 + rank 38: 54694.43 + rank 39: 54698.05 + rank 40: 57506.40 + rank 41: 57502.04 + rank 42: 57496.16 + rank 43: 57494.50 + rank 44: 57498.75 + rank 45: 57500.44 + rank 46: 57496.68 + rank 47: 57493.65 + rank 48: 53361.24 + rank 49: 53356.83 + rank 50: 53359.77 + rank 51: 53356.81 + rank 52: 53349.62 + rank 53: 53353.88 + rank 54: 53361.28 + rank 55: 53361.60 + rank 56: 58390.41 + rank 57: 58391.76 + rank 58: 58392.79 + rank 59: 58397.55 + rank 60: 58392.80 + rank 61: 58399.34 + rank 62: 58409.55 + rank 63: 58396.77 + batch-generator: + rank 0: 22.09 + rank 1: 22.50 + rank 2: 23.77 + rank 3: 22.73 + rank 4: 23.80 + rank 5: 21.97 + rank 6: 24.04 + rank 7: 23.05 + rank 8: 22.39 + rank 9: 22.03 + rank 10: 20.57 + rank 11: 22.94 + rank 12: 21.89 + rank 13: 21.31 + rank 14: 21.98 + rank 15: 21.47 + rank 16: 20.70 + rank 17: 21.58 + rank 18: 21.37 + rank 19: 21.64 + rank 20: 21.54 + rank 21: 21.86 + rank 22: 21.44 + rank 23: 21.63 + rank 24: 34.23 + rank 25: 31.94 + rank 26: 31.00 + rank 27: 34.89 + rank 28: 30.63 + rank 29: 38.88 + rank 30: 28.95 + rank 31: 31.23 + rank 32: 33.10 + rank 33: 33.08 + rank 34: 31.99 + rank 35: 32.17 + rank 36: 27.63 + rank 37: 28.65 + rank 38: 32.52 + rank 39: 32.58 + rank 40: 23.53 + rank 41: 21.44 + rank 42: 23.10 + rank 43: 23.41 + rank 44: 21.10 + rank 45: 21.55 + rank 46: 23.57 + rank 47: 21.33 + rank 48: 22.19 + rank 49: 31.36 + rank 50: 21.77 + rank 51: 27.37 + rank 52: 31.28 + rank 53: 28.74 + rank 54: 23.06 + rank 55: 30.46 + rank 56: 23.35 + rank 57: 22.44 + rank 58: 23.19 + rank 59: 21.21 + rank 60: 22.49 + rank 61: 22.09 + rank 62: 22.63 + rank 63: 23.34 + embedding-grads-all-reduce: + rank 0: 0.03 + rank 1: 0.02 + rank 2: 0.03 + rank 3: 0.03 + rank 4: 0.03 + rank 5: 0.02 + rank 6: 0.02 + rank 7: 0.02 + rank 8: 0.02 + rank 9: 0.03 + rank 10: 0.02 + rank 11: 0.03 + rank 12: 0.03 + rank 13: 0.02 + rank 14: 0.02 + rank 15: 0.02 + rank 16: 0.03 + rank 17: 0.03 + rank 18: 0.03 + rank 19: 0.02 + rank 20: 0.03 + rank 21: 0.02 + rank 22: 0.02 + rank 23: 0.03 + rank 24: 0.04 + rank 25: 0.04 + rank 26: 0.04 + rank 27: 0.04 + rank 28: 0.04 + rank 29: 0.04 + rank 30: 0.04 + rank 31: 0.04 + rank 32: 0.04 + rank 33: 0.03 + rank 34: 0.04 + rank 35: 0.03 + rank 36: 0.03 + rank 37: 0.03 + rank 38: 0.03 + rank 39: 0.03 + rank 40: 0.03 + rank 41: 0.03 + rank 42: 0.02 + rank 43: 0.03 + rank 44: 0.02 + rank 45: 0.03 + rank 46: 0.03 + rank 47: 0.03 + rank 48: 0.03 + rank 49: 0.03 + rank 50: 0.03 + rank 51: 0.03 + rank 52: 0.03 + rank 53: 0.03 + rank 54: 0.03 + rank 55: 0.03 + rank 56: 0.03 + rank 57: 0.03 + rank 58: 0.02 + rank 59: 0.02 + rank 60: 0.03 + rank 61: 0.03 + rank 62: 0.02 + rank 63: 0.03 + all-grads-sync: + rank 0: 0.77 + rank 1: 0.79 + rank 2: 0.82 + rank 3: 0.79 + rank 4: 0.80 + rank 5: 0.97 + rank 6: 1.14 + rank 7: 1.30 + rank 8: 0.92 + rank 9: 0.77 + rank 10: 0.93 + rank 11: 0.77 + rank 12: 0.79 + rank 13: 0.85 + rank 14: 0.78 + rank 15: 0.82 + rank 16: 0.85 + rank 17: 0.76 + rank 18: 0.76 + rank 19: 0.76 + rank 20: 0.80 + rank 21: 0.90 + rank 22: 0.76 + rank 23: 0.77 + rank 24: 1.19 + rank 25: 1.17 + rank 26: 1.27 + rank 27: 1.28 + rank 28: 1.22 + rank 29: 1.22 + rank 30: 1.17 + rank 31: 1.36 + rank 32: 1.29 + rank 33: 1.21 + rank 34: 1.28 + rank 35: 1.34 + rank 36: 1.54 + rank 37: 0.93 + rank 38: 1.18 + rank 39: 1.54 + rank 40: 0.79 + rank 41: 0.84 + rank 42: 0.83 + rank 43: 0.83 + rank 44: 0.79 + rank 45: 0.75 + rank 46: 0.84 + rank 47: 0.90 + rank 48: 0.84 + rank 49: 1.08 + rank 50: 1.18 + rank 51: 1.02 + rank 52: 0.86 + rank 53: 1.08 + rank 54: 0.99 + rank 55: 1.24 + rank 56: 0.86 + rank 57: 0.78 + rank 58: 0.84 + rank 59: 0.89 + rank 60: 0.77 + rank 61: 0.85 + rank 62: 0.79 + rank 63: 0.98 + params-all-gather: + rank 0: 0.03 + rank 1: 0.02 + rank 2: 0.02 + rank 3: 0.02 + rank 4: 0.02 + rank 5: 0.02 + rank 6: 0.02 + rank 7: 0.02 + rank 8: 0.02 + rank 9: 0.02 + rank 10: 0.02 + rank 11: 0.02 + rank 12: 0.02 + rank 13: 0.02 + rank 14: 0.02 + rank 15: 0.02 + rank 16: 0.02 + rank 17: 0.02 + rank 18: 0.02 + rank 19: 0.02 + rank 20: 0.02 + rank 21: 0.02 + rank 22: 0.02 + rank 23: 0.02 + rank 24: 0.03 + rank 25: 0.03 + rank 26: 0.03 + rank 27: 0.03 + rank 28: 0.03 + rank 29: 0.03 + rank 30: 0.03 + rank 31: 0.04 + rank 32: 0.03 + rank 33: 0.02 + rank 34: 0.02 + rank 35: 0.02 + rank 36: 0.02 + rank 37: 0.03 + rank 38: 0.02 + rank 39: 0.03 + rank 40: 0.02 + rank 41: 0.02 + rank 42: 0.02 + rank 43: 0.02 + rank 44: 0.02 + rank 45: 0.03 + rank 46: 0.02 + rank 47: 0.03 + rank 48: 0.03 + rank 49: 0.03 + rank 50: 0.03 + rank 51: 0.02 + rank 52: 0.03 + rank 53: 0.02 + rank 54: 0.03 + rank 55: 0.02 + rank 56: 0.02 + rank 57: 0.02 + rank 58: 0.02 + rank 59: 0.02 + rank 60: 0.02 + rank 61: 0.02 + rank 62: 0.02 + rank 63: 0.02 + optimizer-copy-to-main-grad: + rank 0: 2.17 + rank 1: 1.73 + rank 2: 1.75 + rank 3: 2.09 + rank 4: 1.70 + rank 5: 1.80 + rank 6: 1.94 + rank 7: 2.05 + rank 8: 1.74 + rank 9: 2.06 + rank 10: 1.72 + rank 11: 1.69 + rank 12: 1.77 + rank 13: 1.72 + rank 14: 1.85 + rank 15: 1.97 + rank 16: 2.35 + rank 17: 2.05 + rank 18: 2.11 + rank 19: 2.11 + rank 20: 2.12 + rank 21: 2.31 + rank 22: 2.50 + rank 23: 2.28 + rank 24: 1.89 + rank 25: 2.14 + rank 26: 1.99 + rank 27: 2.07 + rank 28: 2.34 + rank 29: 2.28 + rank 30: 2.14 + rank 31: 1.93 + rank 32: 1.95 + rank 33: 1.91 + rank 34: 1.90 + rank 35: 2.19 + rank 36: 2.06 + rank 37: 1.88 + rank 38: 2.25 + rank 39: 1.84 + rank 40: 1.73 + rank 41: 1.87 + rank 42: 1.85 + rank 43: 1.79 + rank 44: 1.88 + rank 45: 2.01 + rank 46: 1.74 + rank 47: 1.66 + rank 48: 1.94 + rank 49: 1.81 + rank 50: 2.33 + rank 51: 1.84 + rank 52: 1.97 + rank 53: 1.78 + rank 54: 1.78 + rank 55: 1.79 + rank 56: 1.74 + rank 57: 2.32 + rank 58: 1.89 + rank 59: 1.75 + rank 60: 1.71 + rank 61: 1.64 + rank 62: 1.85 + rank 63: 1.80 + optimizer-inner-step: + rank 0: 42.33 + rank 1: 42.15 + rank 2: 41.91 + rank 3: 42.08 + rank 4: 41.94 + rank 5: 42.08 + rank 6: 42.41 + rank 7: 42.28 + rank 8: 42.08 + rank 9: 42.03 + rank 10: 41.95 + rank 11: 42.08 + rank 12: 42.06 + rank 13: 42.29 + rank 14: 42.05 + rank 15: 41.87 + rank 16: 42.40 + rank 17: 42.20 + rank 18: 42.26 + rank 19: 42.20 + rank 20: 42.19 + rank 21: 42.10 + rank 22: 42.52 + rank 23: 42.24 + rank 24: 41.86 + rank 25: 44.08 + rank 26: 42.10 + rank 27: 42.24 + rank 28: 46.19 + rank 29: 42.88 + rank 30: 42.02 + rank 31: 42.29 + rank 32: 42.18 + rank 33: 41.96 + rank 34: 42.10 + rank 35: 47.43 + rank 36: 42.19 + rank 37: 42.17 + rank 38: 42.07 + rank 39: 42.02 + rank 40: 42.08 + rank 41: 42.20 + rank 42: 42.25 + rank 43: 42.26 + rank 44: 41.98 + rank 45: 42.45 + rank 46: 42.01 + rank 47: 41.83 + rank 48: 42.30 + rank 49: 41.91 + rank 50: 53.97 + rank 51: 41.82 + rank 52: 42.08 + rank 53: 42.00 + rank 54: 42.21 + rank 55: 42.07 + rank 56: 41.78 + rank 57: 46.88 + rank 58: 42.14 + rank 59: 41.93 + rank 60: 42.26 + rank 61: 41.89 + rank 62: 42.01 + rank 63: 41.86 + optimizer-copy-main-to-model-params: + rank 0: 4.39 + rank 1: 3.84 + rank 2: 3.80 + rank 3: 4.37 + rank 4: 3.69 + rank 5: 3.88 + rank 6: 3.92 + rank 7: 3.97 + rank 8: 3.86 + rank 9: 4.44 + rank 10: 3.81 + rank 11: 3.81 + rank 12: 3.92 + rank 13: 3.78 + rank 14: 4.02 + rank 15: 4.22 + rank 16: 4.36 + rank 17: 4.05 + rank 18: 4.00 + rank 19: 4.02 + rank 20: 4.04 + rank 21: 4.21 + rank 22: 4.52 + rank 23: 4.19 + rank 24: 3.93 + rank 25: 4.28 + rank 26: 4.05 + rank 27: 4.24 + rank 28: 4.45 + rank 29: 4.55 + rank 30: 4.37 + rank 31: 3.96 + rank 32: 4.13 + rank 33: 3.89 + rank 34: 3.96 + rank 35: 4.50 + rank 36: 4.24 + rank 37: 4.03 + rank 38: 4.47 + rank 39: 3.86 + rank 40: 3.74 + rank 41: 4.03 + rank 42: 3.95 + rank 43: 4.01 + rank 44: 4.03 + rank 45: 4.25 + rank 46: 3.80 + rank 47: 4.98 + rank 48: 4.07 + rank 49: 3.90 + rank 50: 4.93 + rank 51: 3.97 + rank 52: 4.24 + rank 53: 3.87 + rank 54: 3.88 + rank 55: 3.85 + rank 56: 3.83 + rank 57: 4.71 + rank 58: 3.86 + rank 59: 3.74 + rank 60: 3.73 + rank 61: 3.66 + rank 62: 3.92 + rank 63: 3.62 + optimizer: + rank 0: 53.77 + rank 1: 52.84 + rank 2: 52.52 + rank 3: 53.53 + rank 4: 52.44 + rank 5: 52.80 + rank 6: 53.25 + rank 7: 53.22 + rank 8: 52.85 + rank 9: 53.60 + rank 10: 52.65 + rank 11: 52.77 + rank 12: 52.93 + rank 13: 52.98 + rank 14: 53.01 + rank 15: 53.15 + rank 16: 54.10 + rank 17: 53.38 + rank 18: 53.43 + rank 19: 53.35 + rank 20: 53.34 + rank 21: 53.58 + rank 22: 54.42 + rank 23: 53.64 + rank 24: 52.64 + rank 25: 55.29 + rank 26: 52.98 + rank 27: 53.40 + rank 28: 57.64 + rank 29: 54.42 + rank 30: 53.36 + rank 31: 53.11 + rank 32: 53.19 + rank 33: 52.64 + rank 34: 52.88 + rank 35: 58.94 + rank 36: 53.34 + rank 37: 53.05 + rank 38: 53.55 + rank 39: 52.67 + rank 40: 52.68 + rank 41: 53.16 + rank 42: 53.12 + rank 43: 53.20 + rank 44: 52.93 + rank 45: 53.76 + rank 46: 52.72 + rank 47: 53.73 + rank 48: 53.30 + rank 49: 52.73 + rank 50: 66.17 + rank 51: 52.65 + rank 52: 53.32 + rank 53: 52.68 + rank 54: 52.93 + rank 55: 52.74 + rank 56: 52.43 + rank 57: 58.81 + rank 58: 52.85 + rank 59: 52.48 + rank 60: 52.81 + rank 61: 52.35 + rank 62: 52.79 + rank 63: 52.26 + interval-time: + rank 0: 0.73 + rank 1: 0.73 + rank 2: 0.86 + rank 3: 0.75 + rank 4: 0.82 + rank 5: 0.79 + rank 6: 0.81 + rank 7: 0.74 + rank 8: 1.82 + rank 9: 1.11 + rank 10: 1.67 + rank 11: 1.78 + rank 12: 1.79 + rank 13: 1.68 + rank 14: 0.87 + rank 15: 1.81 + rank 16: 0.73 + rank 17: 0.74 + rank 18: 0.82 + rank 19: 0.73 + rank 20: 0.79 + rank 21: 0.75 + rank 22: 0.72 + rank 23: 0.72 + rank 24: 0.90 + rank 25: 0.96 + rank 26: 0.87 + rank 27: 0.92 + rank 28: 0.85 + rank 29: 0.94 + rank 30: 0.93 + rank 31: 0.95 + rank 32: 0.98 + rank 33: 0.96 + rank 34: 0.87 + rank 35: 0.82 + rank 36: 0.90 + rank 37: 0.87 + rank 38: 0.89 + rank 39: 0.88 + rank 40: 0.83 + rank 41: 0.82 + rank 42: 0.81 + rank 43: 0.83 + rank 44: 0.73 + rank 45: 0.72 + rank 46: 0.75 + rank 47: 1.04 + rank 48: 0.94 + rank 49: 0.92 + rank 50: 0.82 + rank 51: 0.81 + rank 52: 0.84 + rank 53: 0.84 + rank 54: 0.88 + rank 55: 0.84 + rank 56: 0.82 + rank 57: 0.74 + rank 58: 0.72 + rank 59: 0.73 + rank 60: 0.81 + rank 61: 0.81 + rank 62: 0.79 + rank 63: 2.46 + [2026-10-06 19:14:45] iteration 3005/ 6855 | consumed samples: 384640 | elapsed time per iteration (ms): 85425.1 | throughput per GPU (TFLOP/s/GPU): 43.4 | MFU 13.90% | FLOPs estimator: workload-aware | logical tokens/s: 196396.1 | valid tokens/s: 196396.1 | physical tokens/s: 196716.8 | loss tokens/s: 149602.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.929041E-01 | lm-global-supervised-tokens: 1.277978E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.147179E-02 | lm-token-buffer-mib: 5.807834E+01 | global_load_balancing_loss: 9.995968E-01 | loss scale: 1.0 | grad norm: 0.044 | params norm: 11445.493 | params rms: 0.061974 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 19:21:26.743653 | finish at 2026-10-10 14:36:12 +times across ranks (ms): + forward-backward: + rank 0: 85218.77 + rank 1: 85218.38 + rank 2: 85219.34 + rank 3: 85220.23 + rank 4: 85220.49 + rank 5: 85219.93 + rank 6: 85217.85 + rank 7: 85218.41 + rank 8: 85220.43 + rank 9: 85221.39 + rank 10: 85220.63 + rank 11: 85221.95 + rank 12: 85222.09 + rank 13: 85223.38 + rank 14: 85220.08 + rank 15: 85219.60 + rank 16: 85211.90 + rank 17: 85213.37 + rank 18: 85212.86 + rank 19: 85214.13 + rank 20: 85214.45 + rank 21: 85217.11 + rank 22: 85212.85 + rank 23: 85213.05 + rank 24: 85211.88 + rank 25: 85212.09 + rank 26: 85212.95 + rank 27: 85212.60 + rank 28: 85216.26 + rank 29: 85213.67 + rank 30: 85214.46 + rank 31: 85212.35 + rank 32: 85208.44 + rank 33: 85209.26 + rank 34: 85206.59 + rank 35: 85209.70 + rank 36: 85209.33 + rank 37: 85208.71 + rank 38: 85209.21 + rank 39: 85208.55 + rank 40: 85209.86 + rank 41: 85209.17 + rank 42: 85209.15 + rank 43: 85208.71 + rank 44: 85208.44 + rank 45: 85207.04 + rank 46: 85211.58 + rank 47: 85210.45 + rank 48: 85216.36 + rank 49: 85216.52 + rank 50: 85215.30 + rank 51: 85216.73 + rank 52: 85215.61 + rank 53: 85215.15 + rank 54: 85217.38 + rank 55: 85218.04 + rank 56: 85219.34 + rank 57: 85218.53 + rank 58: 85218.45 + rank 59: 85218.00 + rank 60: 85218.06 + rank 61: 85218.09 + rank 62: 85220.66 + rank 63: 85219.94 + forward-compute: + rank 0: 30938.83 + rank 1: 30938.60 + rank 2: 30944.25 + rank 3: 30942.63 + rank 4: 30945.96 + rank 5: 30941.50 + rank 6: 30940.44 + rank 7: 30946.87 + rank 8: 31626.19 + rank 9: 31632.47 + rank 10: 31632.54 + rank 11: 31634.22 + rank 12: 31636.36 + rank 13: 31636.99 + rank 14: 31632.04 + rank 15: 31635.62 + rank 16: 29500.08 + rank 17: 29503.43 + rank 18: 29509.94 + rank 19: 29506.67 + rank 20: 29508.30 + rank 21: 29512.52 + rank 22: 29501.05 + rank 23: 29505.38 + rank 24: 32443.21 + rank 25: 32445.23 + rank 26: 32445.74 + rank 27: 32444.72 + rank 28: 32521.04 + rank 29: 32449.10 + rank 30: 32447.50 + rank 31: 32447.60 + rank 32: 32725.70 + rank 33: 32728.91 + rank 34: 32730.30 + rank 35: 32732.00 + rank 36: 32726.78 + rank 37: 32737.34 + rank 38: 32736.27 + rank 39: 32731.45 + rank 40: 31627.86 + rank 41: 31621.72 + rank 42: 31623.38 + rank 43: 31625.05 + rank 44: 31627.88 + rank 45: 31624.48 + rank 46: 31625.67 + rank 47: 31624.46 + rank 48: 32551.08 + rank 49: 32551.82 + rank 50: 32552.95 + rank 51: 32551.35 + rank 52: 32553.67 + rank 53: 32555.08 + rank 54: 32553.83 + rank 55: 32553.49 + rank 56: 31016.19 + rank 57: 31019.04 + rank 58: 31023.38 + rank 59: 31022.11 + rank 60: 31027.43 + rank 61: 31029.63 + rank 62: 31023.05 + rank 63: 31033.39 + backward-compute: + rank 0: 54245.59 + rank 1: 54236.92 + rank 2: 54238.93 + rank 3: 54244.13 + rank 4: 54239.51 + rank 5: 54245.38 + rank 6: 54243.27 + rank 7: 54238.82 + rank 8: 53556.89 + rank 9: 53550.57 + rank 10: 53555.64 + rank 11: 53553.94 + rank 12: 53551.57 + rank 13: 53553.05 + rank 14: 53553.97 + rank 15: 53550.79 + rank 16: 55680.84 + rank 17: 55678.54 + rank 18: 55672.29 + rank 19: 55675.86 + rank 20: 55674.56 + rank 21: 55670.17 + rank 22: 55678.71 + rank 23: 55675.39 + rank 24: 52713.95 + rank 25: 52729.59 + rank 26: 52709.49 + rank 27: 52711.20 + rank 28: 52653.18 + rank 29: 52706.98 + rank 30: 52705.99 + rank 31: 52711.44 + rank 32: 52444.15 + rank 33: 52443.20 + rank 34: 52439.09 + rank 35: 52436.92 + rank 36: 52436.12 + rank 37: 52432.30 + rank 38: 52433.81 + rank 39: 52439.66 + rank 40: 53547.41 + rank 41: 53556.26 + rank 42: 53552.06 + rank 43: 53549.83 + rank 44: 53547.46 + rank 45: 53550.18 + rank 46: 53551.41 + rank 47: 53554.98 + rank 48: 52629.26 + rank 49: 52629.41 + rank 50: 52626.88 + rank 51: 52628.97 + rank 52: 52626.02 + rank 53: 52622.75 + rank 54: 52629.60 + rank 55: 52628.79 + rank 56: 54168.61 + rank 57: 54165.86 + rank 58: 54161.31 + rank 59: 54162.72 + rank 60: 54157.02 + rank 61: 54153.80 + rank 62: 54163.76 + rank 63: 54150.89 + batch-generator: + rank 0: 25.85 + rank 1: 28.12 + rank 2: 25.63 + rank 3: 25.96 + rank 4: 25.13 + rank 5: 25.06 + rank 6: 25.56 + rank 7: 26.02 + rank 8: 22.74 + rank 9: 22.33 + rank 10: 21.39 + rank 11: 22.42 + rank 12: 22.53 + rank 13: 20.76 + rank 14: 21.97 + rank 15: 21.44 + rank 16: 21.20 + rank 17: 21.20 + rank 18: 23.14 + rank 19: 21.41 + rank 20: 22.00 + rank 21: 22.24 + rank 22: 22.22 + rank 23: 21.29 + rank 24: 33.01 + rank 25: 24.05 + rank 26: 34.00 + rank 27: 33.81 + rank 28: 32.45 + rank 29: 34.87 + rank 30: 33.50 + rank 31: 29.55 + rank 32: 31.60 + rank 33: 31.23 + rank 34: 33.87 + rank 35: 31.80 + rank 36: 32.06 + rank 37: 27.85 + rank 38: 34.29 + rank 39: 32.69 + rank 40: 21.95 + rank 41: 21.80 + rank 42: 20.99 + rank 43: 23.23 + rank 44: 22.37 + rank 45: 21.43 + rank 46: 22.06 + rank 47: 22.16 + rank 48: 22.05 + rank 49: 28.65 + rank 50: 29.07 + rank 51: 28.57 + rank 52: 34.46 + rank 53: 30.58 + rank 54: 25.96 + rank 55: 28.25 + rank 56: 24.69 + rank 57: 22.42 + rank 58: 21.52 + rank 59: 22.11 + rank 60: 22.43 + rank 61: 22.16 + rank 62: 22.10 + rank 63: 22.20 + embedding-grads-all-reduce: + rank 0: 0.03 + rank 1: 0.02 + rank 2: 0.03 + rank 3: 0.03 + rank 4: 0.02 + rank 5: 0.02 + rank 6: 0.03 + rank 7: 0.03 + rank 8: 0.02 + rank 9: 0.03 + rank 10: 0.02 + rank 11: 0.03 + rank 12: 0.03 + rank 13: 0.03 + rank 14: 0.02 + rank 15: 0.02 + rank 16: 0.03 + rank 17: 0.02 + rank 18: 0.03 + rank 19: 0.02 + rank 20: 0.03 + rank 21: 0.03 + rank 22: 0.03 + rank 23: 0.03 + rank 24: 0.04 + rank 25: 0.03 + rank 26: 0.04 + rank 27: 0.04 + rank 28: 0.04 + rank 29: 0.04 + rank 30: 0.04 + rank 31: 0.04 + rank 32: 0.04 + rank 33: 0.03 + rank 34: 0.04 + rank 35: 0.03 + rank 36: 0.03 + rank 37: 0.03 + rank 38: 0.03 + rank 39: 0.03 + rank 40: 0.03 + rank 41: 0.03 + rank 42: 0.02 + rank 43: 0.03 + rank 44: 0.03 + rank 45: 0.03 + rank 46: 0.03 + rank 47: 0.03 + rank 48: 0.03 + rank 49: 0.04 + rank 50: 0.03 + rank 51: 0.03 + rank 52: 0.03 + rank 53: 0.03 + rank 54: 0.03 + rank 55: 0.03 + rank 56: 0.03 + rank 57: 0.02 + rank 58: 0.02 + rank 59: 0.03 + rank 60: 0.02 + rank 61: 0.03 + rank 62: 0.02 + rank 63: 0.03 + all-grads-sync: + rank 0: 1.45 + rank 1: 0.73 + rank 2: 0.94 + rank 3: 0.77 + rank 4: 0.77 + rank 5: 0.76 + rank 6: 0.98 + rank 7: 0.87 + rank 8: 0.79 + rank 9: 0.78 + rank 10: 0.77 + rank 11: 0.81 + rank 12: 0.98 + rank 13: 1.04 + rank 14: 0.79 + rank 15: 0.85 + rank 16: 0.76 + rank 17: 0.85 + rank 18: 0.79 + rank 19: 0.75 + rank 20: 0.78 + rank 21: 1.37 + rank 22: 1.24 + rank 23: 0.77 + rank 24: 1.38 + rank 25: 0.97 + rank 26: 1.43 + rank 27: 1.44 + rank 28: 1.64 + rank 29: 1.40 + rank 30: 1.97 + rank 31: 1.08 + rank 32: 1.28 + rank 33: 1.31 + rank 34: 1.36 + rank 35: 1.24 + rank 36: 1.14 + rank 37: 1.21 + rank 38: 1.58 + rank 39: 1.46 + rank 40: 0.81 + rank 41: 0.81 + rank 42: 0.80 + rank 43: 0.79 + rank 44: 0.78 + rank 45: 0.76 + rank 46: 0.76 + rank 47: 1.02 + rank 48: 1.01 + rank 49: 1.01 + rank 50: 1.12 + rank 51: 1.05 + rank 52: 1.17 + rank 53: 0.83 + rank 54: 0.96 + rank 55: 0.99 + rank 56: 0.80 + rank 57: 0.77 + rank 58: 0.83 + rank 59: 0.78 + rank 60: 0.91 + rank 61: 1.23 + rank 62: 0.81 + rank 63: 0.75 + params-all-gather: + rank 0: 0.02 + rank 1: 0.02 + rank 2: 0.02 + rank 3: 0.02 + rank 4: 0.02 + rank 5: 0.02 + rank 6: 0.02 + rank 7: 0.02 + rank 8: 0.03 + rank 9: 0.02 + rank 10: 0.02 + rank 11: 0.02 + rank 12: 0.02 + rank 13: 0.02 + rank 14: 0.02 + rank 15: 0.02 + rank 16: 0.02 + rank 17: 0.02 + rank 18: 0.02 + rank 19: 0.02 + rank 20: 0.02 + rank 21: 0.02 + rank 22: 0.02 + rank 23: 0.02 + rank 24: 0.03 + rank 25: 0.03 + rank 26: 0.04 + rank 27: 0.04 + rank 28: 0.03 + rank 29: 0.03 + rank 30: 0.03 + rank 31: 0.03 + rank 32: 0.02 + rank 33: 0.03 + rank 34: 0.02 + rank 35: 0.02 + rank 36: 0.02 + rank 37: 0.03 + rank 38: 0.02 + rank 39: 0.02 + rank 40: 0.02 + rank 41: 0.02 + rank 42: 0.02 + rank 43: 0.02 + rank 44: 0.02 + rank 45: 0.02 + rank 46: 0.02 + rank 47: 0.02 + rank 48: 0.03 + rank 49: 0.03 + rank 50: 0.02 + rank 51: 0.02 + rank 52: 0.03 + rank 53: 0.03 + rank 54: 0.03 + rank 55: 0.03 + rank 56: 0.03 + rank 57: 0.02 + rank 58: 0.02 + rank 59: 0.02 + rank 60: 0.02 + rank 61: 0.02 + rank 62: 0.02 + rank 63: 0.02 + optimizer-copy-to-main-grad: + rank 0: 2.19 + rank 1: 1.74 + rank 2: 1.69 + rank 3: 2.07 + rank 4: 1.72 + rank 5: 1.75 + rank 6: 1.83 + rank 7: 1.87 + rank 8: 1.79 + rank 9: 2.16 + rank 10: 1.78 + rank 11: 1.72 + rank 12: 1.81 + rank 13: 1.74 + rank 14: 1.86 + rank 15: 1.96 + rank 16: 2.40 + rank 17: 2.08 + rank 18: 2.12 + rank 19: 2.13 + rank 20: 2.16 + rank 21: 2.42 + rank 22: 2.58 + rank 23: 2.31 + rank 24: 1.93 + rank 25: 2.17 + rank 26: 2.13 + rank 27: 2.01 + rank 28: 2.28 + rank 29: 2.37 + rank 30: 2.26 + rank 31: 1.87 + rank 32: 2.07 + rank 33: 1.85 + rank 34: 2.02 + rank 35: 2.24 + rank 36: 2.08 + rank 37: 1.99 + rank 38: 2.26 + rank 39: 1.95 + rank 40: 1.75 + rank 41: 1.77 + rank 42: 1.82 + rank 43: 1.85 + rank 44: 1.87 + rank 45: 2.01 + rank 46: 1.79 + rank 47: 1.63 + rank 48: 1.93 + rank 49: 1.84 + rank 50: 2.28 + rank 51: 1.82 + rank 52: 1.94 + rank 53: 1.77 + rank 54: 1.76 + rank 55: 1.77 + rank 56: 1.79 + rank 57: 2.32 + rank 58: 1.81 + rank 59: 1.74 + rank 60: 1.73 + rank 61: 1.69 + rank 62: 1.85 + rank 63: 1.72 + optimizer-inner-step: + rank 0: 41.62 + rank 1: 41.58 + rank 2: 41.35 + rank 3: 41.55 + rank 4: 41.35 + rank 5: 41.52 + rank 6: 41.83 + rank 7: 41.78 + rank 8: 41.55 + rank 9: 41.57 + rank 10: 41.38 + rank 11: 41.58 + rank 12: 41.52 + rank 13: 41.74 + rank 14: 41.59 + rank 15: 41.40 + rank 16: 41.88 + rank 17: 41.64 + rank 18: 41.72 + rank 19: 41.70 + rank 20: 41.65 + rank 21: 41.51 + rank 22: 41.99 + rank 23: 41.66 + rank 24: 40.91 + rank 25: 41.46 + rank 26: 41.43 + rank 27: 43.26 + rank 28: 47.40 + rank 29: 41.84 + rank 30: 41.17 + rank 31: 41.68 + rank 32: 41.57 + rank 33: 41.36 + rank 34: 41.49 + rank 35: 47.19 + rank 36: 41.61 + rank 37: 41.65 + rank 38: 41.49 + rank 39: 41.46 + rank 40: 41.60 + rank 41: 41.66 + rank 42: 41.72 + rank 43: 41.75 + rank 44: 41.38 + rank 45: 41.95 + rank 46: 41.56 + rank 47: 41.27 + rank 48: 42.56 + rank 49: 41.49 + rank 50: 52.92 + rank 51: 41.26 + rank 52: 41.61 + rank 53: 41.46 + rank 54: 41.64 + rank 55: 41.56 + rank 56: 41.09 + rank 57: 46.75 + rank 58: 41.54 + rank 59: 41.42 + rank 60: 41.59 + rank 61: 41.34 + rank 62: 41.49 + rank 63: 41.34 + optimizer-copy-main-to-model-params: + rank 0: 4.42 + rank 1: 3.81 + rank 2: 3.73 + rank 3: 4.31 + rank 4: 3.70 + rank 5: 3.88 + rank 6: 3.90 + rank 7: 3.96 + rank 8: 3.87 + rank 9: 4.42 + rank 10: 3.76 + rank 11: 3.78 + rank 12: 3.86 + rank 13: 3.76 + rank 14: 3.96 + rank 15: 4.19 + rank 16: 4.42 + rank 17: 3.95 + rank 18: 4.06 + rank 19: 4.01 + rank 20: 4.09 + rank 21: 4.22 + rank 22: 4.63 + rank 23: 4.25 + rank 24: 3.98 + rank 25: 4.24 + rank 26: 4.05 + rank 27: 4.19 + rank 28: 4.48 + rank 29: 4.43 + rank 30: 4.33 + rank 31: 3.96 + rank 32: 4.08 + rank 33: 3.86 + rank 34: 3.97 + rank 35: 4.56 + rank 36: 4.25 + rank 37: 4.05 + rank 38: 4.49 + rank 39: 3.88 + rank 40: 3.76 + rank 41: 3.93 + rank 42: 3.91 + rank 43: 4.02 + rank 44: 3.93 + rank 45: 4.19 + rank 46: 3.74 + rank 47: 3.67 + rank 48: 4.12 + rank 49: 3.94 + rank 50: 4.91 + rank 51: 4.03 + rank 52: 4.21 + rank 53: 3.86 + rank 54: 3.85 + rank 55: 3.88 + rank 56: 3.77 + rank 57: 4.70 + rank 58: 3.84 + rank 59: 3.75 + rank 60: 3.71 + rank 61: 3.69 + rank 62: 3.89 + rank 63: 3.64 + optimizer: + rank 0: 54.25 + rank 1: 53.29 + rank 2: 52.95 + rank 3: 53.97 + rank 4: 52.88 + rank 5: 53.32 + rank 6: 53.70 + rank 7: 53.72 + rank 8: 53.29 + rank 9: 54.14 + rank 10: 53.08 + rank 11: 53.20 + rank 12: 53.30 + rank 13: 53.38 + rank 14: 53.48 + rank 15: 53.62 + rank 16: 54.66 + rank 17: 53.73 + rank 18: 53.96 + rank 19: 53.86 + rank 20: 53.93 + rank 21: 54.04 + rank 22: 55.07 + rank 23: 54.20 + rank 24: 53.17 + rank 25: 53.82 + rank 26: 53.45 + rank 27: 55.99 + rank 28: 60.49 + rank 29: 54.79 + rank 30: 53.99 + rank 31: 53.59 + rank 32: 53.54 + rank 33: 53.06 + rank 34: 53.33 + rank 35: 59.80 + rank 36: 53.85 + rank 37: 53.59 + rank 38: 54.02 + rank 39: 53.17 + rank 40: 53.21 + rank 41: 53.51 + rank 42: 53.57 + rank 43: 53.71 + rank 44: 53.24 + rank 45: 54.20 + rank 46: 53.17 + rank 47: 52.79 + rank 48: 54.66 + rank 49: 53.30 + rank 50: 66.08 + rank 51: 53.16 + rank 52: 53.79 + rank 53: 53.16 + rank 54: 53.34 + rank 55: 53.28 + rank 56: 52.82 + rank 57: 59.72 + rank 58: 53.29 + rank 59: 53.00 + rank 60: 53.30 + rank 61: 52.85 + rank 62: 53.30 + rank 63: 52.80 + interval-time: + rank 0: 0.73 + rank 1: 0.74 + rank 2: 0.83 + rank 3: 0.73 + rank 4: 0.82 + rank 5: 0.80 + rank 6: 0.74 + rank 7: 0.84 + rank 8: 0.74 + rank 9: 0.74 + rank 10: 0.82 + rank 11: 0.73 + rank 12: 0.82 + rank 13: 0.73 + rank 14: 0.73 + rank 15: 0.73 + rank 16: 0.73 + rank 17: 0.74 + rank 18: 0.81 + rank 19: 0.73 + rank 20: 0.82 + rank 21: 0.74 + rank 22: 0.72 + rank 23: 0.73 + rank 24: 1.16 + rank 25: 1.18 + rank 26: 1.32 + rank 27: 1.31 + rank 28: 1.22 + rank 29: 1.29 + rank 30: 1.25 + rank 31: 1.37 + rank 32: 0.91 + rank 33: 0.92 + rank 34: 0.84 + rank 35: 0.82 + rank 36: 0.88 + rank 37: 0.89 + rank 38: 0.89 + rank 39: 0.89 + rank 40: 0.83 + rank 41: 0.82 + rank 42: 0.80 + rank 43: 0.82 + rank 44: 0.72 + rank 45: 0.77 + rank 46: 0.74 + rank 47: 0.80 + rank 48: 0.92 + rank 49: 0.94 + rank 50: 0.82 + rank 51: 0.81 + rank 52: 0.84 + rank 53: 0.81 + rank 54: 0.92 + rank 55: 0.84 + rank 56: 0.83 + rank 57: 0.76 + rank 58: 0.72 + rank 59: 0.73 + rank 60: 0.78 + rank 61: 0.80 + rank 62: 0.79 + rank 63: 3.89 + [2026-10-06 19:16:13] iteration 3006/ 6855 | consumed samples: 384768 | elapsed time per iteration (ms): 87496.1 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.78% | FLOPs estimator: workload-aware | logical tokens/s: 191748.1 | valid tokens/s: 191748.1 | physical tokens/s: 192167.4 | loss tokens/s: 137789.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.279992E-01 | lm-global-supervised-tokens: 1.205606E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.607740E-02 | lm-token-buffer-mib: 5.809727E+01 | global_load_balancing_loss: 1.000585E+00 | loss scale: 1.0 | grad norm: 0.049 | params norm: 11445.429 | params rms: 0.061973 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 21:32:52.590565 | finish at 2026-10-10 16:49:05 + [2026-10-06 19:17:40] iteration 3007/ 6855 | consumed samples: 384896 | elapsed time per iteration (ms): 87031.2 | throughput per GPU (TFLOP/s/GPU): 41.6 | MFU 13.34% | FLOPs estimator: workload-aware | logical tokens/s: 192771.8 | valid tokens/s: 192771.8 | physical tokens/s: 193295.8 | loss tokens/s: 143599.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.435719E-01 | lm-global-supervised-tokens: 1.249760E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.206810E-02 | lm-token-buffer-mib: 5.804369E+01 | global_load_balancing_loss: 1.000018E+00 | loss scale: 1.0 | grad norm: 0.032 | params norm: 11445.367 | params rms: 0.061973 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 21:01:36.172018 | finish at 2026-10-10 16:19:16 + [2026-10-06 19:19:11] iteration 3008/ 6855 | consumed samples: 385024 | elapsed time per iteration (ms): 91236.7 | throughput per GPU (TFLOP/s/GPU): 41.7 | MFU 13.36% | FLOPs estimator: workload-aware | logical tokens/s: 183885.9 | valid tokens/s: 183885.9 | physical tokens/s: 184222.7 | loss tokens/s: 134361.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.328016E-01 | lm-global-supervised-tokens: 1.225872E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.665586E-02 | lm-token-buffer-mib: 5.809374E+01 | global_load_balancing_loss: 9.994312E-01 | loss scale: 1.0 | grad norm: 0.086 | params norm: 11445.304 | params rms: 0.061973 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 1:29:47.645658 | finish at 2026-10-10 20:48:59 + [2026-10-06 19:20:40] iteration 3009/ 6855 | consumed samples: 385152 | elapsed time per iteration (ms): 89353.7 | throughput per GPU (TFLOP/s/GPU): 41.3 | MFU 13.24% | FLOPs estimator: workload-aware | logical tokens/s: 187761.2 | valid tokens/s: 187761.2 | physical tokens/s: 188179.0 | loss tokens/s: 137436.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.618155E-01 | lm-global-supervised-tokens: 1.228041E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.319039E-02 | lm-token-buffer-mib: 5.815986E+01 | global_load_balancing_loss: 1.000178E+00 | loss scale: 1.0 | grad norm: 0.057 | params norm: 11445.242 | params rms: 0.061972 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 23:27:34.187774 | finish at 2026-10-10 18:48:14 + [2026-10-06 19:22:29] iteration 3010/ 6855 | consumed samples: 385280 | elapsed time per iteration (ms): 108341.4 | throughput per GPU (TFLOP/s/GPU): 34.6 | MFU 11.07% | FLOPs estimator: workload-aware | logical tokens/s: 154855.0 | valid tokens/s: 154855.0 | physical tokens/s: 155159.8 | loss tokens/s: 112063.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.744777E-01 | lm-global-supervised-tokens: 1.214109E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.430894E-02 | lm-token-buffer-mib: 5.810928E+01 | global_load_balancing_loss: 1.000595E+00 | loss scale: 1.0 | grad norm: 0.046 | params norm: 11445.180 | params rms: 0.061972 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 19:42:52.813737 | finish at 2026-10-11 15:05:21 + [2026-10-06 19:23:59] iteration 3011/ 6855 | consumed samples: 385408 | elapsed time per iteration (ms): 89973.4 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.47% | FLOPs estimator: workload-aware | logical tokens/s: 186468.7 | valid tokens/s: 186468.7 | physical tokens/s: 186802.7 | loss tokens/s: 139575.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.948767E-01 | lm-global-supervised-tokens: 1.255805E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.230382E-02 | lm-token-buffer-mib: 5.814942E+01 | global_load_balancing_loss: 1.000077E+00 | loss scale: 1.0 | grad norm: 0.048 | params norm: 11445.119 | params rms: 0.061972 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 0:04:17.590578 | finish at 2026-10-10 19:28:16 + [2026-10-06 19:25:27] iteration 3012/ 6855 | consumed samples: 385536 | elapsed time per iteration (ms): 88871.6 | throughput per GPU (TFLOP/s/GPU): 41.4 | MFU 13.27% | FLOPs estimator: workload-aware | logical tokens/s: 188780.1 | valid tokens/s: 188780.1 | physical tokens/s: 189325.4 | loss tokens/s: 129262.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 5.826639E-01 | lm-global-supervised-tokens: 1.148772E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.790193E-02 | lm-token-buffer-mib: 5.814487E+01 | global_load_balancing_loss: 1.001090E+00 | loss scale: 1.0 | grad norm: 0.048 | params norm: 11445.053 | params rms: 0.061971 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 22:52:13.498909 | finish at 2026-10-10 18:17:41 + [2026-10-06 19:26:56] iteration 3013/ 6855 | consumed samples: 385664 | elapsed time per iteration (ms): 88744.6 | throughput per GPU (TFLOP/s/GPU): 41.1 | MFU 13.17% | FLOPs estimator: workload-aware | logical tokens/s: 189050.4 | valid tokens/s: 189050.4 | physical tokens/s: 189595.0 | loss tokens/s: 143280.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.719589E-01 | lm-global-supervised-tokens: 1.271532E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.314370E-02 | lm-token-buffer-mib: 5.822626E+01 | global_load_balancing_loss: 9.997495E-01 | loss scale: 1.0 | grad norm: 0.053 | params norm: 11444.988 | params rms: 0.061971 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 22:42:36.586709 | finish at 2026-10-10 18:09:33 + [2026-10-06 19:28:25] iteration 3014/ 6855 | consumed samples: 385792 | elapsed time per iteration (ms): 88422.3 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.68% | FLOPs estimator: workload-aware | logical tokens/s: 189739.3 | valid tokens/s: 189739.3 | physical tokens/s: 190092.9 | loss tokens/s: 136106.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.623775E-01 | lm-global-supervised-tokens: 1.203483E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.422481E-02 | lm-token-buffer-mib: 5.817097E+01 | global_load_balancing_loss: 1.000565E+00 | loss scale: 1.0 | grad norm: 0.047 | params norm: 11444.924 | params rms: 0.061971 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 22:20:29.941130 | finish at 2026-10-10 17:48:55 + [2026-10-06 19:29:51] iteration 3015/ 6855 | consumed samples: 385920 | elapsed time per iteration (ms): 86143.9 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.64% | FLOPs estimator: workload-aware | logical tokens/s: 194757.3 | valid tokens/s: 194757.3 | physical tokens/s: 195175.4 | loss tokens/s: 145998.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.962580E-01 | lm-global-supervised-tokens: 1.257688E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.735310E-02 | lm-token-buffer-mib: 5.830624E+01 | global_load_balancing_loss: 1.000157E+00 | loss scale: 1.0 | grad norm: 0.043 | params norm: 11444.861 | params rms: 0.061970 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 19:53:12.471313 | finish at 2026-10-10 15:23:03 + [2026-10-06 19:31:20] iteration 3016/ 6855 | consumed samples: 386048 | elapsed time per iteration (ms): 88722.7 | throughput per GPU (TFLOP/s/GPU): 41.4 | MFU 13.26% | FLOPs estimator: workload-aware | logical tokens/s: 189096.8 | valid tokens/s: 189096.8 | physical tokens/s: 189653.5 | loss tokens/s: 141491.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.302125E-01 | lm-global-supervised-tokens: 1.255347E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.333228E-02 | lm-token-buffer-mib: 5.827492E+01 | global_load_balancing_loss: 1.000053E+00 | loss scale: 1.0 | grad norm: 0.037 | params norm: 11444.799 | params rms: 0.061970 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 22:36:46.633391 | finish at 2026-10-10 18:08:06 + [2026-10-06 19:32:49] iteration 3017/ 6855 | consumed samples: 386176 | elapsed time per iteration (ms): 89153.8 | throughput per GPU (TFLOP/s/GPU): 40.9 | MFU 13.10% | FLOPs estimator: workload-aware | logical tokens/s: 188182.1 | valid tokens/s: 188182.1 | physical tokens/s: 188707.0 | loss tokens/s: 139767.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.294730E-01 | lm-global-supervised-tokens: 1.246078E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.349562E-02 | lm-token-buffer-mib: 5.817406E+01 | global_load_balancing_loss: 1.000268E+00 | loss scale: 1.0 | grad norm: 0.060 | params norm: 11444.737 | params rms: 0.061970 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 23:02:52.350325 | finish at 2026-10-10 18:35:41 + [2026-10-06 19:34:15] iteration 3018/ 6855 | consumed samples: 386304 | elapsed time per iteration (ms): 86130.1 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.56% | FLOPs estimator: workload-aware | logical tokens/s: 194788.2 | valid tokens/s: 194788.2 | physical tokens/s: 195215.4 | loss tokens/s: 145155.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.808116E-01 | lm-global-supervised-tokens: 1.250227E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.133431E-02 | lm-token-buffer-mib: 5.807754E+01 | global_load_balancing_loss: 9.993610E-01 | loss scale: 1.0 | grad norm: 0.055 | params norm: 11444.675 | params rms: 0.061969 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 19:48:01.110497 | finish at 2026-10-10 15:22:16 + [2026-10-06 19:35:39] iteration 3019/ 6855 | consumed samples: 386432 | elapsed time per iteration (ms): 83894.7 | throughput per GPU (TFLOP/s/GPU): 43.6 | MFU 13.97% | FLOPs estimator: workload-aware | logical tokens/s: 199979.4 | valid tokens/s: 199979.4 | physical tokens/s: 200468.0 | loss tokens/s: 149457.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.577694E-01 | lm-global-supervised-tokens: 1.253864E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.581456E-02 | lm-token-buffer-mib: 5.817018E+01 | global_load_balancing_loss: 1.000178E+00 | loss scale: 1.0 | grad norm: 0.090 | params norm: 11444.614 | params rms: 0.061969 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 17:23:39.910257 | finish at 2026-10-10 12:59:19 + [2026-10-06 19:37:23] iteration 3020/ 6855 | consumed samples: 386560 | elapsed time per iteration (ms): 104633.3 | throughput per GPU (TFLOP/s/GPU): 35.3 | MFU 11.30% | FLOPs estimator: workload-aware | logical tokens/s: 160342.7 | valid tokens/s: 160342.7 | physical tokens/s: 160747.8 | loss tokens/s: 120260.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.651460E-01 | lm-global-supervised-tokens: 1.258325E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.143827E-02 | lm-token-buffer-mib: 5.824784E+01 | global_load_balancing_loss: 9.995612E-01 | loss scale: 1.0 | grad norm: 0.054 | params norm: 11444.552 | params rms: 0.061969 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 15:27:48.601519 | finish at 2026-10-11 11:05:12 + [2026-10-06 19:38:48] iteration 3021/ 6855 | consumed samples: 386688 | elapsed time per iteration (ms): 85063.0 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.68% | FLOPs estimator: workload-aware | logical tokens/s: 197232.0 | valid tokens/s: 197232.0 | physical tokens/s: 197710.2 | loss tokens/s: 145791.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.545112E-01 | lm-global-supervised-tokens: 1.240143E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.153846E-02 | lm-token-buffer-mib: 5.804045E+01 | global_load_balancing_loss: 1.000564E+00 | loss scale: 1.0 | grad norm: 0.048 | params norm: 11444.490 | params rms: 0.061968 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 18:35:31.648810 | finish at 2026-10-10 14:14:20 + [2026-10-06 19:40:13] iteration 3022/ 6855 | consumed samples: 386816 | elapsed time per iteration (ms): 84654.9 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.63% | FLOPs estimator: workload-aware | logical tokens/s: 198183.1 | valid tokens/s: 198183.1 | physical tokens/s: 198950.3 | loss tokens/s: 154414.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.792253E-01 | lm-global-supervised-tokens: 1.307192E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.146172E-02 | lm-token-buffer-mib: 5.831687E+01 | global_load_balancing_loss: 1.001072E+00 | loss scale: 1.0 | grad norm: 0.152 | params norm: 11444.427 | params rms: 0.061968 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 18:08:02.358096 | finish at 2026-10-10 13:48:15 + [2026-10-06 19:41:44] iteration 3023/ 6855 | consumed samples: 386944 | elapsed time per iteration (ms): 90471.8 | throughput per GPU (TFLOP/s/GPU): 41.4 | MFU 13.26% | FLOPs estimator: workload-aware | logical tokens/s: 185440.1 | valid tokens/s: 185440.1 | physical tokens/s: 185780.0 | loss tokens/s: 140754.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.293122E-01 | lm-global-supervised-tokens: 1.273428E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.524080E-02 | lm-token-buffer-mib: 5.812266E+01 | global_load_balancing_loss: 1.000089E+00 | loss scale: 1.0 | grad norm: 0.297 | params norm: 11444.365 | params rms: 0.061968 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 0:18:07.997326 | finish at 2026-10-10 19:59:52 + [2026-10-06 19:43:14] iteration 3024/ 6855 | consumed samples: 387072 | elapsed time per iteration (ms): 90205.6 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.42% | FLOPs estimator: workload-aware | logical tokens/s: 185988.2 | valid tokens/s: 185988.2 | physical tokens/s: 186376.8 | loss tokens/s: 135610.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.611599E-01 | lm-global-supervised-tokens: 1.223280E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.916862E-02 | lm-token-buffer-mib: 5.817616E+01 | global_load_balancing_loss: 1.001421E+00 | loss scale: 1.0 | grad norm: 0.207 | params norm: 11444.300 | params rms: 0.061967 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 23:59:37.542257 | finish at 2026-10-10 19:42:51 + [2026-10-06 19:44:44] iteration 3025/ 6855 | consumed samples: 387200 | elapsed time per iteration (ms): 90584.5 | throughput per GPU (TFLOP/s/GPU): 41.2 | MFU 13.21% | FLOPs estimator: workload-aware | logical tokens/s: 185210.7 | valid tokens/s: 185210.7 | physical tokens/s: 185568.5 | loss tokens/s: 133286.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.135423E-01 | lm-global-supervised-tokens: 1.207370E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.648676E-02 | lm-token-buffer-mib: 5.804062E+01 | global_load_balancing_loss: 1.000530E+00 | loss scale: 1.0 | grad norm: 0.075 | params norm: 11444.236 | params rms: 0.061967 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 0:22:18.589628 | finish at 2026-10-10 20:07:03 + [2026-10-06 19:46:12] iteration 3026/ 6855 | consumed samples: 387328 | elapsed time per iteration (ms): 87211.6 | throughput per GPU (TFLOP/s/GPU): 41.7 | MFU 13.37% | FLOPs estimator: workload-aware | logical tokens/s: 192373.4 | valid tokens/s: 192373.4 | physical tokens/s: 192969.8 | loss tokens/s: 140216.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.484013E-01 | lm-global-supervised-tokens: 1.222853E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.457048E-02 | lm-token-buffer-mib: 5.807584E+01 | global_load_balancing_loss: 9.993084E-01 | loss scale: 1.0 | grad norm: 0.163 | params norm: 11444.172 | params rms: 0.061966 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 20:45:33.173743 | finish at 2026-10-10 16:31:45 + [2026-10-06 19:47:38] iteration 3027/ 6855 | consumed samples: 387456 | elapsed time per iteration (ms): 86456.0 | throughput per GPU (TFLOP/s/GPU): 43.3 | MFU 13.89% | FLOPs estimator: workload-aware | logical tokens/s: 194055.0 | valid tokens/s: 194055.0 | physical tokens/s: 194477.4 | loss tokens/s: 139174.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.013447E-01 | lm-global-supervised-tokens: 1.203246E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.263780E-02 | lm-token-buffer-mib: 5.813846E+01 | global_load_balancing_loss: 9.991988E-01 | loss scale: 1.0 | grad norm: 0.043 | params norm: 11444.108 | params rms: 0.061966 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 19:55:53.410452 | finish at 2026-10-10 15:43:31 + [2026-10-06 19:49:06] iteration 3028/ 6855 | consumed samples: 387584 | elapsed time per iteration (ms): 88229.4 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.54% | FLOPs estimator: workload-aware | logical tokens/s: 190153.6 | valid tokens/s: 190153.5 | physical tokens/s: 190777.3 | loss tokens/s: 139543.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.595908E-01 | lm-global-supervised-tokens: 1.231180E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.543803E-02 | lm-token-buffer-mib: 5.829380E+01 | global_load_balancing_loss: 9.991142E-01 | loss scale: 1.0 | grad norm: 0.147 | params norm: 11444.045 | params rms: 0.061966 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 21:47:33.745605 | finish at 2026-10-10 17:36:40 + [2026-10-06 19:50:34] iteration 3029/ 6855 | consumed samples: 387712 | elapsed time per iteration (ms): 87956.0 | throughput per GPU (TFLOP/s/GPU): 43.2 | MFU 13.86% | FLOPs estimator: workload-aware | logical tokens/s: 190744.9 | valid tokens/s: 190744.9 | physical tokens/s: 191051.6 | loss tokens/s: 135347.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.491009E-01 | lm-global-supervised-tokens: 1.190459E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.213226E-02 | lm-token-buffer-mib: 5.805437E+01 | global_load_balancing_loss: 1.000347E+00 | loss scale: 1.0 | grad norm: 0.038 | params norm: 11443.982 | params rms: 0.061965 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 21:28:39.782225 | finish at 2026-10-10 17:19:14 + [2026-10-06 19:52:24] iteration 3030/ 6855 | consumed samples: 387840 | elapsed time per iteration (ms): 109396.6 | throughput per GPU (TFLOP/s/GPU): 32.8 | MFU 10.52% | FLOPs estimator: workload-aware | logical tokens/s: 153361.2 | valid tokens/s: 153361.2 | physical tokens/s: 153902.7 | loss tokens/s: 115409.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.656993E-01 | lm-global-supervised-tokens: 1.262536E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.447636E-02 | lm-token-buffer-mib: 5.827739E+01 | global_load_balancing_loss: 1.000639E+00 | loss scale: 1.0 | grad norm: 0.091 | params norm: 11443.920 | params rms: 0.061965 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 20:14:01.986823 | finish at 2026-10-11 16:06:26 + [2026-10-06 19:53:51] iteration 3031/ 6855 | consumed samples: 387968 | elapsed time per iteration (ms): 87569.0 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.50% | FLOPs estimator: workload-aware | logical tokens/s: 191588.5 | valid tokens/s: 191588.5 | physical tokens/s: 191951.8 | loss tokens/s: 143977.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.106980E-01 | lm-global-supervised-tokens: 1.260800E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.272669E-02 | lm-token-buffer-mib: 5.811565E+01 | global_load_balancing_loss: 1.000409E+00 | loss scale: 1.0 | grad norm: 0.065 | params norm: 11443.857 | params rms: 0.061965 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 21:01:03.792202 | finish at 2026-10-10 16:54:55 + [2026-10-06 19:55:20] iteration 3032/ 6855 | consumed samples: 388096 | elapsed time per iteration (ms): 89269.1 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.70% | FLOPs estimator: workload-aware | logical tokens/s: 187939.8 | valid tokens/s: 187939.8 | physical tokens/s: 188264.5 | loss tokens/s: 132533.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.323295E-01 | lm-global-supervised-tokens: 1.183117E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.131266E-02 | lm-token-buffer-mib: 5.809798E+01 | global_load_balancing_loss: 1.001063E+00 | loss scale: 1.0 | grad norm: 0.032 | params norm: 11443.794 | params rms: 0.061964 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 22:47:55.619630 | finish at 2026-10-10 18:43:16 + [2026-10-06 19:56:48] iteration 3033/ 6855 | consumed samples: 388224 | elapsed time per iteration (ms): 87354.9 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.56% | FLOPs estimator: workload-aware | logical tokens/s: 192057.4 | valid tokens/s: 192057.4 | physical tokens/s: 192526.7 | loss tokens/s: 146688.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.384137E-01 | lm-global-supervised-tokens: 1.281391E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.154234E-02 | lm-token-buffer-mib: 5.803948E+01 | global_load_balancing_loss: 9.993409E-01 | loss scale: 1.0 | grad norm: 0.096 | params norm: 11443.729 | params rms: 0.061964 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 20:44:30.326207 | finish at 2026-10-10 16:41:18 + [2026-10-06 19:58:16] iteration 3034/ 6855 | consumed samples: 388352 | elapsed time per iteration (ms): 87877.9 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.60% | FLOPs estimator: workload-aware | logical tokens/s: 190914.7 | valid tokens/s: 190914.7 | physical tokens/s: 191353.5 | loss tokens/s: 137959.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.331973E-01 | lm-global-supervised-tokens: 1.212356E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.163727E-02 | lm-token-buffer-mib: 5.813696E+01 | global_load_balancing_loss: 1.000668E+00 | loss scale: 1.0 | grad norm: 0.050 | params norm: 11443.666 | params rms: 0.061964 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 21:16:21.454550 | finish at 2026-10-10 17:14:37 + [2026-10-06 19:59:46] iteration 3035/ 6855 | consumed samples: 388480 | elapsed time per iteration (ms): 90350.0 | throughput per GPU (TFLOP/s/GPU): 41.7 | MFU 13.36% | FLOPs estimator: workload-aware | logical tokens/s: 185691.0 | valid tokens/s: 185691.0 | physical tokens/s: 186129.1 | loss tokens/s: 133209.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.714277E-01 | lm-global-supervised-tokens: 1.203548E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.258828E-02 | lm-token-buffer-mib: 5.809906E+01 | global_load_balancing_loss: 9.998877E-01 | loss scale: 1.0 | grad norm: 0.077 | params norm: 11443.604 | params rms: 0.061963 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 23:52:17.090712 | finish at 2026-10-10 19:52:03 + [2026-10-06 20:01:12] iteration 3036/ 6855 | consumed samples: 388608 | elapsed time per iteration (ms): 85769.9 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.66% | FLOPs estimator: workload-aware | logical tokens/s: 195607.0 | valid tokens/s: 195607.0 | physical tokens/s: 196363.8 | loss tokens/s: 144448.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.558036E-01 | lm-global-supervised-tokens: 1.238934E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.162243E-02 | lm-token-buffer-mib: 5.828435E+01 | global_load_balancing_loss: 1.000182E+00 | loss scale: 1.0 | grad norm: 0.150 | params norm: 11443.541 | params rms: 0.061963 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 18:59:15.100004 | finish at 2026-10-10 15:00:27 + [2026-10-06 20:02:43] iteration 3037/ 6855 | consumed samples: 388736 | elapsed time per iteration (ms): 91342.4 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.59% | FLOPs estimator: workload-aware | logical tokens/s: 183673.4 | valid tokens/s: 183673.4 | physical tokens/s: 183885.9 | loss tokens/s: 130841.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.282676E-01 | lm-global-supervised-tokens: 1.195139E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.441380E-02 | lm-token-buffer-mib: 5.803381E+01 | global_load_balancing_loss: 9.995826E-01 | loss scale: 1.0 | grad norm: 0.205 | params norm: 11443.478 | params rms: 0.061963 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 0:52:25.244340 | finish at 2026-10-10 20:55:08 + [2026-10-06 20:04:10] iteration 3038/ 6855 | consumed samples: 388864 | elapsed time per iteration (ms): 87400.5 | throughput per GPU (TFLOP/s/GPU): 41.7 | MFU 13.36% | FLOPs estimator: workload-aware | logical tokens/s: 191957.8 | valid tokens/s: 191957.8 | physical tokens/s: 192443.5 | loss tokens/s: 147320.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.435016E-01 | lm-global-supervised-tokens: 1.287592E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.156235E-02 | lm-token-buffer-mib: 5.811143E+01 | global_load_balancing_loss: 1.000786E+00 | loss scale: 1.0 | grad norm: 0.106 | params norm: 11443.415 | params rms: 0.061962 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 20:40:07.794270 | finish at 2026-10-10 16:44:18 + [2026-10-06 20:05:36] iteration 3039/ 6855 | consumed samples: 388992 | elapsed time per iteration (ms): 85238.9 | throughput per GPU (TFLOP/s/GPU): 42.8 | MFU 13.73% | FLOPs estimator: workload-aware | logical tokens/s: 196824.9 | valid tokens/s: 196824.9 | physical tokens/s: 197408.6 | loss tokens/s: 146903.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.391176E-01 | lm-global-supervised-tokens: 1.252189E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.596931E-02 | lm-token-buffer-mib: 5.814126E+01 | global_load_balancing_loss: 9.992833E-01 | loss scale: 1.0 | grad norm: 0.255 | params norm: 11443.352 | params rms: 0.061962 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 18:21:11.672218 | finish at 2026-10-10 14:26:47 + [2026-10-06 20:07:26] iteration 3040/ 6855 | consumed samples: 389120 | elapsed time per iteration (ms): 110030.0 | throughput per GPU (TFLOP/s/GPU): 33.5 | MFU 10.75% | FLOPs estimator: workload-aware | logical tokens/s: 152478.2 | valid tokens/s: 152478.2 | physical tokens/s: 152801.8 | loss tokens/s: 110174.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 5.848502E-01 | lm-global-supervised-tokens: 1.212245E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.129876E-02 | lm-token-buffer-mib: 5.804005E+01 | global_load_balancing_loss: 9.997482E-01 | loss scale: 1.0 | grad norm: 0.049 | params norm: 11443.289 | params rms: 0.061962 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 20:36:04.435338 | finish at 2026-10-11 16:43:30 +times across ranks (ms): + save-checkpoint: + rank 0: 26110.09 + rank 1: 26110.08 + rank 2: 26110.13 + rank 3: 26110.09 + rank 4: 26110.32 + rank 5: 26110.05 + rank 6: 26110.09 + rank 7: 26110.09 + rank 8: 26110.25 + rank 9: 26110.23 + rank 10: 26110.24 + rank 11: 26110.24 + rank 12: 26110.24 + rank 13: 26110.09 + rank 14: 26110.26 + rank 15: 26110.23 + rank 16: 26110.30 + rank 17: 26110.28 + rank 18: 26110.27 + rank 19: 26110.27 + rank 20: 26110.25 + rank 21: 26110.32 + rank 22: 26110.26 + rank 23: 26110.25 + rank 24: 26109.99 + rank 25: 26110.54 + rank 26: 26109.88 + rank 27: 26109.97 + rank 28: 26109.88 + rank 29: 26109.90 + rank 30: 26110.47 + rank 31: 26109.91 + rank 32: 26110.73 + rank 33: 26110.73 + rank 34: 26110.75 + rank 35: 26110.69 + rank 36: 26111.92 + rank 37: 26110.70 + rank 38: 26110.57 + rank 39: 26110.67 + rank 40: 26109.88 + rank 41: 26109.85 + rank 42: 26109.87 + rank 43: 26109.89 + rank 44: 26109.85 + rank 45: 26109.86 + rank 46: 26109.87 + rank 47: 26109.88 + rank 48: 26109.99 + rank 49: 26110.20 + rank 50: 26110.08 + rank 51: 26110.06 + rank 52: 26110.03 + rank 53: 26110.07 + rank 54: 26110.12 + rank 55: 26110.11 + rank 56: 26110.03 + rank 57: 26110.00 + rank 58: 26109.98 + rank 59: 26109.97 + rank 60: 26109.98 + rank 61: 26110.03 + rank 62: 26109.98 + rank 63: 26110.01 + [2026-10-06 20:09:40] iteration 3041/ 6855 | consumed samples: 389248 | elapsed time per iteration (ms): 107900.0 | throughput per GPU (TFLOP/s/GPU): 33.9 | MFU 10.86% | FLOPs estimator: workload-aware | logical tokens/s: 155488.0 | valid tokens/s: 155488.0 | physical tokens/s: 156028.3 | loss tokens/s: 115691.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.917480E-01 | lm-global-supervised-tokens: 1.248306E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.026838E+00 | lm-token-buffer-mib: 5.854656E+01 | global_load_balancing_loss: 1.000620E+00 | loss scale: 1.0 | grad norm: 0.178 | params norm: 11443.226 | params rms: 0.061961 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 18:18:50.648558 | finish at 2026-10-11 14:28:31 + [2026-10-06 20:11:12] iteration 3042/ 6855 | consumed samples: 389376 | elapsed time per iteration (ms): 91713.4 | throughput per GPU (TFLOP/s/GPU): 40.1 | MFU 12.85% | FLOPs estimator: workload-aware | logical tokens/s: 182930.4 | valid tokens/s: 182930.4 | physical tokens/s: 183301.2 | loss tokens/s: 139041.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.159369E-01 | lm-global-supervised-tokens: 1.275196E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.974277E-02 | lm-token-buffer-mib: 5.808780E+01 | global_load_balancing_loss: 1.001125E+00 | loss scale: 1.0 | grad norm: 0.047 | params norm: 11443.163 | params rms: 0.061961 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 1:08:23.377406 | finish at 2026-10-10 21:19:35 + [2026-10-06 20:12:43] iteration 3043/ 6855 | consumed samples: 389504 | elapsed time per iteration (ms): 90885.8 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.59% | FLOPs estimator: workload-aware | logical tokens/s: 184596.6 | valid tokens/s: 184596.6 | physical tokens/s: 185081.4 | loss tokens/s: 130094.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.756720E-01 | lm-global-supervised-tokens: 1.182376E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.430561E-02 | lm-token-buffer-mib: 5.804661E+01 | global_load_balancing_loss: 1.001277E+00 | loss scale: 1.0 | grad norm: 0.054 | params norm: 11443.099 | params rms: 0.061961 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 0:14:16.815486 | finish at 2026-10-10 20:27:00 + [2026-10-06 20:14:07] iteration 3044/ 6855 | consumed samples: 389632 | elapsed time per iteration (ms): 84290.6 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.77% | FLOPs estimator: workload-aware | logical tokens/s: 199040.0 | valid tokens/s: 199040.0 | physical tokens/s: 199644.5 | loss tokens/s: 147757.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.495456E-01 | lm-global-supervised-tokens: 1.245460E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.147956E-02 | lm-token-buffer-mib: 5.836314E+01 | global_load_balancing_loss: 1.000168E+00 | loss scale: 1.0 | grad norm: 0.084 | params norm: 11443.031 | params rms: 0.061960 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 17:13:51.288751 | finish at 2026-10-10 13:27:58 + [2026-10-06 20:15:36] iteration 3045/ 6855 | consumed samples: 389760 | elapsed time per iteration (ms): 89170.0 | throughput per GPU (TFLOP/s/GPU): 41.3 | MFU 13.25% | FLOPs estimator: workload-aware | logical tokens/s: 188148.5 | valid tokens/s: 188148.5 | physical tokens/s: 188613.3 | loss tokens/s: 138871.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.234847E-01 | lm-global-supervised-tokens: 1.238316E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.604208E-02 | lm-token-buffer-mib: 5.814875E+01 | global_load_balancing_loss: 9.996420E-01 | loss scale: 1.0 | grad norm: 0.065 | params norm: 11442.966 | params rms: 0.061960 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 22:22:17.596736 | finish at 2026-10-10 18:37:54 + [2026-10-06 20:17:06] iteration 3046/ 6855 | consumed samples: 389888 | elapsed time per iteration (ms): 89766.8 | throughput per GPU (TFLOP/s/GPU): 41.7 | MFU 13.36% | FLOPs estimator: workload-aware | logical tokens/s: 186897.4 | valid tokens/s: 186897.4 | physical tokens/s: 187175.9 | loss tokens/s: 136423.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.711073E-01 | lm-global-supervised-tokens: 1.224626E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.233874E-02 | lm-token-buffer-mib: 5.806248E+01 | global_load_balancing_loss: 9.997687E-01 | loss scale: 1.0 | grad norm: 0.035 | params norm: 11442.902 | params rms: 0.061960 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 22:58:41.690065 | finish at 2026-10-10 19:15:48 + [2026-10-06 20:18:34] iteration 3047/ 6855 | consumed samples: 390016 | elapsed time per iteration (ms): 88154.1 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.76% | FLOPs estimator: workload-aware | logical tokens/s: 190315.9 | valid tokens/s: 190315.9 | physical tokens/s: 190938.1 | loss tokens/s: 136344.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.750371E-01 | lm-global-supervised-tokens: 1.201929E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.478806E-02 | lm-token-buffer-mib: 5.836230E+01 | global_load_balancing_loss: 9.989941E-01 | loss scale: 1.0 | grad norm: 0.073 | params norm: 11442.839 | params rms: 0.061959 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 21:14:50.924248 | finish at 2026-10-10 17:33:25 + [2026-10-06 20:20:01] iteration 3048/ 6855 | consumed samples: 390144 | elapsed time per iteration (ms): 87312.6 | throughput per GPU (TFLOP/s/GPU): 43.1 | MFU 13.81% | FLOPs estimator: workload-aware | logical tokens/s: 192150.8 | valid tokens/s: 192150.8 | physical tokens/s: 192475.4 | loss tokens/s: 141374.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.941770E-01 | lm-global-supervised-tokens: 1.234380E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.106490E-02 | lm-token-buffer-mib: 5.812086E+01 | global_load_balancing_loss: 1.000129E+00 | loss scale: 1.0 | grad norm: 0.041 | params norm: 11442.775 | params rms: 0.061959 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 20:19:58.938922 | finish at 2026-10-10 16:40:00 + [2026-10-06 20:21:31] iteration 3049/ 6855 | consumed samples: 390272 | elapsed time per iteration (ms): 89434.0 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.46% | FLOPs estimator: workload-aware | logical tokens/s: 187593.1 | valid tokens/s: 187593.1 | physical tokens/s: 188018.8 | loss tokens/s: 136235.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.478677E-01 | lm-global-supervised-tokens: 1.218407E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.168973E-02 | lm-token-buffer-mib: 5.804879E+01 | global_load_balancing_loss: 1.000661E+00 | loss scale: 1.0 | grad norm: 0.076 | params norm: 11442.708 | params rms: 0.061959 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 22:33:05.980098 | finish at 2026-10-10 18:54:37 + [2026-10-06 20:23:18] iteration 3050/ 6855 | consumed samples: 390400 | elapsed time per iteration (ms): 107188.2 | throughput per GPU (TFLOP/s/GPU): 34.9 | MFU 11.18% | FLOPs estimator: workload-aware | logical tokens/s: 156520.0 | valid tokens/s: 156520.0 | physical tokens/s: 156925.8 | loss tokens/s: 113469.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.527288E-01 | lm-global-supervised-tokens: 1.216258E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.170251E-02 | lm-token-buffer-mib: 5.805159E+01 | global_load_balancing_loss: 1.000191E+00 | loss scale: 1.0 | grad norm: 0.077 | params norm: 11442.643 | params rms: 0.061958 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 17:17:31.106431 | finish at 2026-10-11 13:40:49 + [2026-10-06 20:24:50] iteration 3051/ 6855 | consumed samples: 390528 | elapsed time per iteration (ms): 91492.7 | throughput per GPU (TFLOP/s/GPU): 40.9 | MFU 13.12% | FLOPs estimator: workload-aware | logical tokens/s: 183371.8 | valid tokens/s: 183371.8 | physical tokens/s: 183883.1 | loss tokens/s: 134234.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.977237E-01 | lm-global-supervised-tokens: 1.228143E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.018355E-02 | lm-token-buffer-mib: 5.802219E+01 | global_load_balancing_loss: 1.000511E+00 | loss scale: 1.0 | grad norm: 0.063 | params norm: 11442.578 | params rms: 0.061958 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 0:40:38.353618 | finish at 2026-10-10 21:05:28 + [2026-10-06 20:26:19] iteration 3052/ 6855 | consumed samples: 390656 | elapsed time per iteration (ms): 89022.7 | throughput per GPU (TFLOP/s/GPU): 41.7 | MFU 13.37% | FLOPs estimator: workload-aware | logical tokens/s: 188459.6 | valid tokens/s: 188459.6 | physical tokens/s: 188783.6 | loss tokens/s: 139909.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.169541E-01 | lm-global-supervised-tokens: 1.245512E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.301734E-02 | lm-token-buffer-mib: 5.806769E+01 | global_load_balancing_loss: 9.994614E-01 | loss scale: 1.0 | grad norm: 0.057 | params norm: 11442.512 | params rms: 0.061957 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 22:02:33.461657 | finish at 2026-10-10 18:28:52 + [2026-10-06 20:27:48] iteration 3053/ 6855 | consumed samples: 390784 | elapsed time per iteration (ms): 88981.8 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.40% | FLOPs estimator: workload-aware | logical tokens/s: 188545.7 | valid tokens/s: 188545.7 | physical tokens/s: 188888.3 | loss tokens/s: 143055.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.604125E-01 | lm-global-supervised-tokens: 1.272937E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.469317E-02 | lm-token-buffer-mib: 5.804109E+01 | global_load_balancing_loss: 9.995421E-01 | loss scale: 1.0 | grad norm: 0.047 | params norm: 11442.446 | params rms: 0.061957 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 21:58:28.982476 | finish at 2026-10-10 18:26:17 + [2026-10-06 20:29:14] iteration 3054/ 6855 | consumed samples: 390912 | elapsed time per iteration (ms): 86276.0 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.56% | FLOPs estimator: workload-aware | logical tokens/s: 194458.8 | valid tokens/s: 194458.8 | physical tokens/s: 194850.9 | loss tokens/s: 146709.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.816354E-01 | lm-global-supervised-tokens: 1.265749E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.435624E-02 | lm-token-buffer-mib: 5.805912E+01 | global_load_balancing_loss: 1.000368E+00 | loss scale: 1.0 | grad norm: 0.057 | params norm: 11442.382 | params rms: 0.061957 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 19:05:35.069743 | finish at 2026-10-10 15:34:49 + [2026-10-06 20:30:41] iteration 3055/ 6855 | consumed samples: 391040 | elapsed time per iteration (ms): 87256.8 | throughput per GPU (TFLOP/s/GPU): 41.2 | MFU 13.21% | FLOPs estimator: workload-aware | logical tokens/s: 192273.1 | valid tokens/s: 192273.1 | physical tokens/s: 192969.2 | loss tokens/s: 139018.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.151499E-01 | lm-global-supervised-tokens: 1.213031E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.161833E-02 | lm-token-buffer-mib: 5.826929E+01 | global_load_balancing_loss: 1.001242E+00 | loss scale: 1.0 | grad norm: 0.053 | params norm: 11442.318 | params rms: 0.061956 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 20:06:15.782681 | finish at 2026-10-10 16:36:57 + [2026-10-06 20:32:09] iteration 3056/ 6855 | consumed samples: 391168 | elapsed time per iteration (ms): 88213.6 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.75% | FLOPs estimator: workload-aware | logical tokens/s: 190188.5 | valid tokens/s: 190188.4 | physical tokens/s: 190762.7 | loss tokens/s: 131908.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.614807E-01 | lm-global-supervised-tokens: 1.163613E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.171165E-02 | lm-token-buffer-mib: 5.819501E+01 | global_load_balancing_loss: 1.000725E+00 | loss scale: 1.0 | grad norm: 0.054 | params norm: 11442.256 | params rms: 0.061956 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 21:05:23.285852 | finish at 2026-10-10 17:37:33 + [2026-10-06 20:33:35] iteration 3057/ 6855 | consumed samples: 391296 | elapsed time per iteration (ms): 85650.1 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.61% | FLOPs estimator: workload-aware | logical tokens/s: 195880.3 | valid tokens/s: 195880.3 | physical tokens/s: 196303.8 | loss tokens/s: 149411.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.815401E-01 | lm-global-supervised-tokens: 1.279713E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.348921E-02 | lm-token-buffer-mib: 5.819398E+01 | global_load_balancing_loss: 9.999838E-01 | loss scale: 1.0 | grad norm: 0.032 | params norm: 11442.191 | params rms: 0.061956 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 18:21:39.099694 | finish at 2026-10-10 14:55:14 + [2026-10-06 20:35:00] iteration 3058/ 6855 | consumed samples: 391424 | elapsed time per iteration (ms): 84760.9 | throughput per GPU (TFLOP/s/GPU): 43.3 | MFU 13.86% | FLOPs estimator: workload-aware | logical tokens/s: 197935.6 | valid tokens/s: 197935.6 | physical tokens/s: 198359.0 | loss tokens/s: 151580.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.813453E-01 | lm-global-supervised-tokens: 1.284810E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.531041E-02 | lm-token-buffer-mib: 5.813535E+01 | global_load_balancing_loss: 9.986113E-01 | loss scale: 1.0 | grad norm: 0.041 | params norm: 11442.126 | params rms: 0.061955 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 17:23:57.147336 | finish at 2026-10-10 13:58:57 + [2026-10-06 20:36:30] iteration 3059/ 6855 | consumed samples: 391552 | elapsed time per iteration (ms): 90196.4 | throughput per GPU (TFLOP/s/GPU): 40.7 | MFU 13.04% | FLOPs estimator: workload-aware | logical tokens/s: 186007.6 | valid tokens/s: 186007.6 | physical tokens/s: 186437.4 | loss tokens/s: 133643.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.287255E-01 | lm-global-supervised-tokens: 1.205418E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.598962E-02 | lm-token-buffer-mib: 5.818672E+01 | global_load_balancing_loss: 9.996611E-01 | loss scale: 1.0 | grad norm: 0.044 | params norm: 11442.061 | params rms: 0.061955 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 23:06:25.538649 | finish at 2026-10-10 19:42:55 + [2026-10-06 20:38:16] iteration 3060/ 6855 | consumed samples: 391680 | elapsed time per iteration (ms): 106011.9 | throughput per GPU (TFLOP/s/GPU): 34.3 | MFU 10.98% | FLOPs estimator: workload-aware | logical tokens/s: 158257.5 | valid tokens/s: 158257.5 | physical tokens/s: 158574.9 | loss tokens/s: 116519.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.407364E-01 | lm-global-supervised-tokens: 1.235241E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.382584E-02 | lm-token-buffer-mib: 5.816125E+01 | global_load_balancing_loss: 1.001122E+00 | loss scale: 1.0 | grad norm: 0.123 | params norm: 11441.996 | params rms: 0.061955 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 15:45:15.322262 | finish at 2026-10-11 12:23:31 + [2026-10-06 20:39:45] iteration 3061/ 6855 | consumed samples: 391808 | elapsed time per iteration (ms): 88904.8 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.48% | FLOPs estimator: workload-aware | logical tokens/s: 188709.9 | valid tokens/s: 188709.9 | physical tokens/s: 189085.1 | loss tokens/s: 144598.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.728549E-01 | lm-global-supervised-tokens: 1.285553E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.850447E-02 | lm-token-buffer-mib: 5.811922E+01 | global_load_balancing_loss: 9.999552E-01 | loss scale: 1.0 | grad norm: 0.077 | params norm: 11441.932 | params rms: 0.061954 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 21:41:44.816393 | finish at 2026-10-10 18:21:30 + [2026-10-06 20:41:08] iteration 3062/ 6855 | consumed samples: 391936 | elapsed time per iteration (ms): 83144.7 | throughput per GPU (TFLOP/s/GPU): 43.9 | MFU 14.06% | FLOPs estimator: workload-aware | logical tokens/s: 201783.3 | valid tokens/s: 201783.3 | physical tokens/s: 202257.8 | loss tokens/s: 147861.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.496359E-01 | lm-global-supervised-tokens: 1.229393E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.420327E-02 | lm-token-buffer-mib: 5.821214E+01 | global_load_balancing_loss: 1.000652E+00 | loss scale: 1.0 | grad norm: 0.044 | params norm: 11441.868 | params rms: 0.061954 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 15:36:07.924158 | finish at 2026-10-10 12:17:16 + [2026-10-06 20:42:37] iteration 3063/ 6855 | consumed samples: 392064 | elapsed time per iteration (ms): 89273.4 | throughput per GPU (TFLOP/s/GPU): 41.3 | MFU 13.25% | FLOPs estimator: workload-aware | logical tokens/s: 187930.3 | valid tokens/s: 187930.3 | physical tokens/s: 188364.2 | loss tokens/s: 142247.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.814932E-01 | lm-global-supervised-tokens: 1.269890E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.256081E-02 | lm-token-buffer-mib: 5.812550E+01 | global_load_balancing_loss: 1.001143E+00 | loss scale: 1.0 | grad norm: 0.046 | params norm: 11441.804 | params rms: 0.061954 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 22:02:04.857822 | finish at 2026-10-10 18:44:42 + [2026-10-06 20:44:03] iteration 3064/ 6855 | consumed samples: 392192 | elapsed time per iteration (ms): 85428.1 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.52% | FLOPs estimator: workload-aware | logical tokens/s: 196389.9 | valid tokens/s: 196389.9 | physical tokens/s: 196909.6 | loss tokens/s: 150222.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.295314E-01 | lm-global-supervised-tokens: 1.283320E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.153560E-02 | lm-token-buffer-mib: 5.834838E+01 | global_load_balancing_loss: 1.000131E+00 | loss scale: 1.0 | grad norm: 0.035 | params norm: 11441.738 | params rms: 0.061953 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 17:57:37.940167 | finish at 2026-10-10 14:41:41 + [2026-10-06 20:45:31] iteration 3065/ 6855 | consumed samples: 392320 | elapsed time per iteration (ms): 88585.5 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.48% | FLOPs estimator: workload-aware | logical tokens/s: 189389.7 | valid tokens/s: 189389.6 | physical tokens/s: 189724.9 | loss tokens/s: 140809.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.535977E-01 | lm-global-supervised-tokens: 1.247367E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.481434E-02 | lm-token-buffer-mib: 5.810051E+01 | global_load_balancing_loss: 1.000666E+00 | loss scale: 1.0 | grad norm: 0.034 | params norm: 11441.674 | params rms: 0.061953 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 21:15:39.225729 | finish at 2026-10-10 18:01:10 + [2026-10-06 20:46:58] iteration 3066/ 6855 | consumed samples: 392448 | elapsed time per iteration (ms): 87075.9 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.41% | FLOPs estimator: workload-aware | logical tokens/s: 192673.1 | valid tokens/s: 192673.1 | physical tokens/s: 193176.6 | loss tokens/s: 150547.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.954164E-01 | lm-global-supervised-tokens: 1.310907E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.479626E-02 | lm-token-buffer-mib: 5.810149E+01 | global_load_balancing_loss: 9.996198E-01 | loss scale: 1.0 | grad norm: 0.046 | params norm: 11441.609 | params rms: 0.061953 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 19:38:50.609786 | finish at 2026-10-10 16:25:49 + [2026-10-06 20:48:28] iteration 3067/ 6855 | consumed samples: 392576 | elapsed time per iteration (ms): 89885.9 | throughput per GPU (TFLOP/s/GPU): 41.7 | MFU 13.38% | FLOPs estimator: workload-aware | logical tokens/s: 186649.7 | valid tokens/s: 186649.7 | physical tokens/s: 186974.7 | loss tokens/s: 139000.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.962081E-01 | lm-global-supervised-tokens: 1.249416E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.871604E-02 | lm-token-buffer-mib: 5.806223E+01 | global_load_balancing_loss: 9.997258E-01 | loss scale: 1.0 | grad norm: 0.033 | params norm: 11441.543 | params rms: 0.061952 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 22:34:47.927457 | finish at 2026-10-10 19:23:16 + [2026-10-06 20:49:57] iteration 3068/ 6855 | consumed samples: 392704 | elapsed time per iteration (ms): 88442.4 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.61% | FLOPs estimator: workload-aware | logical tokens/s: 189695.6 | valid tokens/s: 189695.6 | physical tokens/s: 190021.4 | loss tokens/s: 136308.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.347960E-01 | lm-global-supervised-tokens: 1.205549E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.668164E-02 | lm-token-buffer-mib: 5.806227E+01 | global_load_balancing_loss: 1.000229E+00 | loss scale: 1.0 | grad norm: 0.042 | params norm: 11441.477 | params rms: 0.061952 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 21:02:11.510473 | finish at 2026-10-10 17:52:08 + [2026-10-06 20:51:21] iteration 3069/ 6855 | consumed samples: 392832 | elapsed time per iteration (ms): 83974.5 | throughput per GPU (TFLOP/s/GPU): 43.4 | MFU 13.92% | FLOPs estimator: workload-aware | logical tokens/s: 199788.9 | valid tokens/s: 199788.9 | physical tokens/s: 200249.3 | loss tokens/s: 147257.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.478726E-01 | lm-global-supervised-tokens: 1.236592E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.271122E-02 | lm-token-buffer-mib: 5.817942E+01 | global_load_balancing_loss: 9.997196E-01 | loss scale: 1.0 | grad norm: 0.037 | params norm: 11441.412 | params rms: 0.061952 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 16:18:47.586758 | finish at 2026-10-10 13:10:08 + [2026-10-06 20:53:14] iteration 3070/ 6855 | consumed samples: 392960 | elapsed time per iteration (ms): 113753.8 | throughput per GPU (TFLOP/s/GPU): 33.6 | MFU 10.77% | FLOPs estimator: workload-aware | logical tokens/s: 147486.4 | valid tokens/s: 147486.4 | physical tokens/s: 147800.1 | loss tokens/s: 108779.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.189991E-01 | lm-global-supervised-tokens: 1.237409E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.256243E-02 | lm-token-buffer-mib: 5.806956E+01 | global_load_balancing_loss: 9.994993E-01 | loss scale: 1.0 | grad norm: 0.057 | params norm: 11441.346 | params rms: 0.061951 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 23:35:58.096583 | finish at 2026-10-11 20:29:12 + [2026-10-06 20:54:42] iteration 3071/ 6855 | consumed samples: 393088 | elapsed time per iteration (ms): 87456.8 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.45% | FLOPs estimator: workload-aware | logical tokens/s: 191834.1 | valid tokens/s: 191834.1 | physical tokens/s: 192300.3 | loss tokens/s: 142297.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.435854E-01 | lm-global-supervised-tokens: 1.244490E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.132800E-02 | lm-token-buffer-mib: 5.823190E+01 | global_load_balancing_loss: 1.000446E+00 | loss scale: 1.0 | grad norm: 0.072 | params norm: 11441.279 | params rms: 0.061951 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 19:55:36.498661 | finish at 2026-10-10 16:50:18 + [2026-10-06 20:56:13] iteration 3072/ 6855 | consumed samples: 393216 | elapsed time per iteration (ms): 90875.3 | throughput per GPU (TFLOP/s/GPU): 40.8 | MFU 13.08% | FLOPs estimator: workload-aware | logical tokens/s: 184617.8 | valid tokens/s: 184617.8 | physical tokens/s: 185016.7 | loss tokens/s: 136214.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.833497E-01 | lm-global-supervised-tokens: 1.237852E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.538844E-02 | lm-token-buffer-mib: 5.807241E+01 | global_load_balancing_loss: 1.001307E+00 | loss scale: 1.0 | grad norm: 0.066 | params norm: 11441.213 | params rms: 0.061950 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 23:29:41.116229 | finish at 2026-10-10 20:25:54 + [2026-10-06 20:57:41] iteration 3073/ 6855 | consumed samples: 393344 | elapsed time per iteration (ms): 88285.3 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.65% | FLOPs estimator: workload-aware | logical tokens/s: 190033.9 | valid tokens/s: 190033.9 | physical tokens/s: 190506.9 | loss tokens/s: 139869.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.767255E-01 | lm-global-supervised-tokens: 1.234840E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.191549E-02 | lm-token-buffer-mib: 5.819505E+01 | global_load_balancing_loss: 1.001005E+00 | loss scale: 1.0 | grad norm: 0.050 | params norm: 11441.149 | params rms: 0.061950 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 20:44:54.946953 | finish at 2026-10-10 17:42:36 + [2026-10-06 20:59:07] iteration 3074/ 6855 | consumed samples: 393472 | elapsed time per iteration (ms): 85837.0 | throughput per GPU (TFLOP/s/GPU): 42.8 | MFU 13.70% | FLOPs estimator: workload-aware | logical tokens/s: 195454.2 | valid tokens/s: 195454.2 | physical tokens/s: 196144.7 | loss tokens/s: 147090.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.696115E-01 | lm-global-supervised-tokens: 1.262582E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.777328E-02 | lm-token-buffer-mib: 5.832553E+01 | global_load_balancing_loss: 1.000467E+00 | loss scale: 1.0 | grad norm: 0.066 | params norm: 11441.085 | params rms: 0.061950 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 18:09:09.820998 | finish at 2026-10-10 15:08:17 + [2026-10-06 21:00:35] iteration 3075/ 6855 | consumed samples: 393600 | elapsed time per iteration (ms): 88173.5 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.47% | FLOPs estimator: workload-aware | logical tokens/s: 190274.4 | valid tokens/s: 190274.4 | physical tokens/s: 190659.6 | loss tokens/s: 140945.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.854884E-01 | lm-global-supervised-tokens: 1.242764E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.150706E-02 | lm-token-buffer-mib: 5.807919E+01 | global_load_balancing_loss: 1.000801E+00 | loss scale: 1.0 | grad norm: 0.083 | params norm: 11441.019 | params rms: 0.061949 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 20:34:55.981636 | finish at 2026-10-10 17:35:31 + [2026-10-06 21:02:09] iteration 3076/ 6855 | consumed samples: 393728 | elapsed time per iteration (ms): 93535.4 | throughput per GPU (TFLOP/s/GPU): 40.4 | MFU 12.95% | FLOPs estimator: workload-aware | logical tokens/s: 179366.5 | valid tokens/s: 179366.5 | physical tokens/s: 179721.2 | loss tokens/s: 131541.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.704585E-01 | lm-global-supervised-tokens: 1.230378E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.456966E-02 | lm-token-buffer-mib: 5.813305E+01 | global_load_balancing_loss: 1.000188E+00 | loss scale: 1.0 | grad norm: 0.049 | params norm: 11440.954 | params rms: 0.061949 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 2:11:10.409620 | finish at 2026-10-10 23:13:19 + [2026-10-06 21:03:38] iteration 3077/ 6855 | consumed samples: 393856 | elapsed time per iteration (ms): 89014.7 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.55% | FLOPs estimator: workload-aware | logical tokens/s: 188476.5 | valid tokens/s: 188476.5 | physical tokens/s: 188926.1 | loss tokens/s: 141456.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.278556E-01 | lm-global-supervised-tokens: 1.259175E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.179039E-02 | lm-token-buffer-mib: 5.805116E+01 | global_load_balancing_loss: 9.992111E-01 | loss scale: 1.0 | grad norm: 0.196 | params norm: 11440.891 | params rms: 0.061949 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 21:24:57.607517 | finish at 2026-10-10 18:28:35 + [2026-10-06 21:05:03] iteration 3078/ 6855 | consumed samples: 393984 | elapsed time per iteration (ms): 85318.0 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.67% | FLOPs estimator: workload-aware | logical tokens/s: 196643.3 | valid tokens/s: 196643.3 | physical tokens/s: 197422.3 | loss tokens/s: 151097.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.511377E-01 | lm-global-supervised-tokens: 1.289132E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.473946E-02 | lm-token-buffer-mib: 5.817972E+01 | global_load_balancing_loss: 9.990768E-01 | loss scale: 1.0 | grad norm: 0.427 | params norm: 11440.825 | params rms: 0.061948 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 17:30:46.105206 | finish at 2026-10-10 14:35:49 + [2026-10-06 21:06:29] iteration 3079/ 6855 | consumed samples: 394112 | elapsed time per iteration (ms): 85647.1 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.60% | FLOPs estimator: workload-aware | logical tokens/s: 195887.3 | valid tokens/s: 195887.3 | physical tokens/s: 196576.7 | loss tokens/s: 148615.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.367949E-01 | lm-global-supervised-tokens: 1.272845E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.902072E-02 | lm-token-buffer-mib: 5.809750E+01 | global_load_balancing_loss: 1.000392E+00 | loss scale: 1.0 | grad norm: 0.643 | params norm: 11440.760 | params rms: 0.061948 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 17:50:03.390076 | finish at 2026-10-10 14:56:32 + [2026-10-06 21:08:19] iteration 3080/ 6855 | consumed samples: 394240 | elapsed time per iteration (ms): 110374.2 | throughput per GPU (TFLOP/s/GPU): 33.7 | MFU 10.80% | FLOPs estimator: workload-aware | logical tokens/s: 152002.8 | valid tokens/s: 152002.8 | physical tokens/s: 152408.6 | loss tokens/s: 111960.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.606948E-01 | lm-global-supervised-tokens: 1.235756E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.228272E-02 | lm-token-buffer-mib: 5.809347E+01 | global_load_balancing_loss: 1.001190E+00 | loss scale: 1.0 | grad norm: 0.226 | params norm: 11440.693 | params rms: 0.061948 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 19:44:22.775505 | finish at 2026-10-11 16:52:42 +times across ranks (ms): + save-checkpoint: + rank 0: 23995.99 + rank 1: 23996.00 + rank 2: 23996.05 + rank 3: 23996.02 + rank 4: 23996.02 + rank 5: 23996.01 + rank 6: 23995.99 + rank 7: 23995.98 + rank 8: 23995.52 + rank 9: 23995.46 + rank 10: 23995.46 + rank 11: 23995.47 + rank 12: 23995.46 + rank 13: 23995.47 + rank 14: 23995.47 + rank 15: 23995.46 + rank 16: 23995.79 + rank 17: 23995.77 + rank 18: 23995.78 + rank 19: 23995.76 + rank 20: 23995.78 + rank 21: 23995.79 + rank 22: 23995.78 + rank 23: 23995.75 + rank 24: 23996.93 + rank 25: 23996.87 + rank 26: 23996.89 + rank 27: 23997.04 + rank 28: 23996.87 + rank 29: 23997.00 + rank 30: 23996.86 + rank 31: 23996.86 + rank 32: 23997.87 + rank 33: 23997.90 + rank 34: 23997.88 + rank 35: 23997.86 + rank 36: 23997.85 + rank 37: 23997.84 + rank 38: 23997.86 + rank 39: 23997.85 + rank 40: 23995.49 + rank 41: 23995.49 + rank 42: 23995.49 + rank 43: 23995.49 + rank 44: 23995.51 + rank 45: 23995.47 + rank 46: 23995.52 + rank 47: 23995.47 + rank 48: 23995.16 + rank 49: 23995.15 + rank 50: 23995.12 + rank 51: 23995.14 + rank 52: 23995.16 + rank 53: 23995.12 + rank 54: 23995.14 + rank 55: 23995.14 + rank 56: 23995.99 + rank 57: 23995.99 + rank 58: 23995.96 + rank 59: 23995.94 + rank 60: 23995.96 + rank 61: 23995.96 + rank 62: 23995.95 + rank 63: 23995.95 + [2026-10-06 21:10:13] iteration 3081/ 6855 | consumed samples: 394368 | elapsed time per iteration (ms): 89631.4 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.46% | FLOPs estimator: workload-aware | logical tokens/s: 187179.5 | valid tokens/s: 187179.5 | physical tokens/s: 187517.5 | loss tokens/s: 141100.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.246460E-01 | lm-global-supervised-tokens: 1.264700E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 4.732132E-01 | lm-token-buffer-mib: 5.819917E+01 | global_load_balancing_loss: 9.997075E-01 | loss scale: 1.0 | grad norm: 0.155 | params norm: 11440.627 | params rms: 0.061947 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 21:57:48.902209 | finish at 2026-10-10 19:08:01 + [2026-10-06 21:11:45] iteration 3082/ 6855 | consumed samples: 394496 | elapsed time per iteration (ms): 92533.7 | throughput per GPU (TFLOP/s/GPU): 40.1 | MFU 12.87% | FLOPs estimator: workload-aware | logical tokens/s: 181308.8 | valid tokens/s: 181308.8 | physical tokens/s: 181661.3 | loss tokens/s: 135405.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.226064E-01 | lm-global-supervised-tokens: 1.252958E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.239678E-02 | lm-token-buffer-mib: 5.810115E+01 | global_load_balancing_loss: 1.000520E+00 | loss scale: 1.0 | grad norm: 0.381 | params norm: 11440.558 | params rms: 0.061947 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 0:58:49.637466 | finish at 2026-10-10 22:10:35 + [2026-10-06 21:13:13] iteration 3083/ 6855 | consumed samples: 394624 | elapsed time per iteration (ms): 87783.7 | throughput per GPU (TFLOP/s/GPU): 43.2 | MFU 13.83% | FLOPs estimator: workload-aware | logical tokens/s: 191119.3 | valid tokens/s: 191119.3 | physical tokens/s: 191638.8 | loss tokens/s: 131914.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.642470E-01 | lm-global-supervised-tokens: 1.157996E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.246511E-02 | lm-token-buffer-mib: 5.804311E+01 | global_load_balancing_loss: 9.999764E-01 | loss scale: 1.0 | grad norm: 0.107 | params norm: 11440.490 | params rms: 0.061947 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 19:58:40.182010 | finish at 2026-10-10 17:11:53 + [2026-10-06 21:14:43] iteration 3084/ 6855 | consumed samples: 394752 | elapsed time per iteration (ms): 89700.6 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.41% | FLOPs estimator: workload-aware | logical tokens/s: 187034.0 | valid tokens/s: 187034.0 | physical tokens/s: 187501.1 | loss tokens/s: 137018.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.732848E-01 | lm-global-supervised-tokens: 1.229067E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.162820E-02 | lm-token-buffer-mib: 5.815512E+01 | global_load_balancing_loss: 1.000723E+00 | loss scale: 1.0 | grad norm: 0.262 | params norm: 11440.424 | params rms: 0.061946 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 21:57:40.920000 | finish at 2026-10-10 19:12:24 + [2026-10-06 21:16:12] iteration 3085/ 6855 | consumed samples: 394880 | elapsed time per iteration (ms): 89609.5 | throughput per GPU (TFLOP/s/GPU): 41.7 | MFU 13.35% | FLOPs estimator: workload-aware | logical tokens/s: 187225.4 | valid tokens/s: 187225.4 | physical tokens/s: 187728.8 | loss tokens/s: 147219.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.218950E-01 | lm-global-supervised-tokens: 1.319228E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.542461E-02 | lm-token-buffer-mib: 5.815651E+01 | global_load_balancing_loss: 9.999924E-01 | loss scale: 1.0 | grad norm: 0.153 | params norm: 11440.357 | params rms: 0.061946 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 21:50:27.637670 | finish at 2026-10-10 19:06:40 + [2026-10-06 21:17:36] iteration 3086/ 6855 | consumed samples: 395008 | elapsed time per iteration (ms): 83834.7 | throughput per GPU (TFLOP/s/GPU): 43.3 | MFU 13.88% | FLOPs estimator: workload-aware | logical tokens/s: 200122.1 | valid tokens/s: 200122.1 | physical tokens/s: 200516.4 | loss tokens/s: 151757.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.352298E-01 | lm-global-supervised-tokens: 1.272252E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.168902E-02 | lm-token-buffer-mib: 5.808392E+01 | global_load_balancing_loss: 1.000563E+00 | loss scale: 1.0 | grad norm: 0.241 | params norm: 11440.293 | params rms: 0.061945 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 15:46:12.865192 | finish at 2026-10-10 13:03:49 + [2026-10-06 21:19:01] iteration 3087/ 6855 | consumed samples: 395136 | elapsed time per iteration (ms): 84888.8 | throughput per GPU (TFLOP/s/GPU): 43.2 | MFU 13.83% | FLOPs estimator: workload-aware | logical tokens/s: 197637.4 | valid tokens/s: 197637.4 | physical tokens/s: 198016.2 | loss tokens/s: 148340.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.807606E-01 | lm-global-supervised-tokens: 1.259245E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.696169E-02 | lm-token-buffer-mib: 5.808379E+01 | global_load_balancing_loss: 1.000740E+00 | loss scale: 1.0 | grad norm: 0.160 | params norm: 11440.229 | params rms: 0.061945 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 16:51:01.072609 | finish at 2026-10-10 14:10:02 + [2026-10-06 21:20:28] iteration 3088/ 6855 | consumed samples: 395264 | elapsed time per iteration (ms): 86945.5 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.67% | FLOPs estimator: workload-aware | logical tokens/s: 192962.0 | valid tokens/s: 192962.0 | physical tokens/s: 193383.9 | loss tokens/s: 143427.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.874251E-01 | lm-global-supervised-tokens: 1.247040E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.441343E-02 | lm-token-buffer-mib: 5.817800E+01 | global_load_balancing_loss: 1.000091E+00 | loss scale: 1.0 | grad norm: 0.133 | params norm: 11440.167 | params rms: 0.061945 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 18:58:43.637040 | finish at 2026-10-10 16:19:12 + [2026-10-06 21:21:53] iteration 3089/ 6855 | consumed samples: 395392 | elapsed time per iteration (ms): 85385.2 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.79% | FLOPs estimator: workload-aware | logical tokens/s: 196488.6 | valid tokens/s: 196488.6 | physical tokens/s: 196899.1 | loss tokens/s: 144452.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.284248E-01 | lm-global-supervised-tokens: 1.233414E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.140609E-02 | lm-token-buffer-mib: 5.808513E+01 | global_load_balancing_loss: 9.997107E-01 | loss scale: 1.0 | grad norm: 0.103 | params norm: 11440.103 | params rms: 0.061944 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 17:19:20.686634 | finish at 2026-10-10 14:41:14 + [2026-10-06 21:23:42] iteration 3090/ 6855 | consumed samples: 395520 | elapsed time per iteration (ms): 108268.9 | throughput per GPU (TFLOP/s/GPU): 33.6 | MFU 10.77% | FLOPs estimator: workload-aware | logical tokens/s: 154958.7 | valid tokens/s: 154958.7 | physical tokens/s: 155395.6 | loss tokens/s: 118237.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.360461E-01 | lm-global-supervised-tokens: 1.280138E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.154942E-02 | lm-token-buffer-mib: 5.804644E+01 | global_load_balancing_loss: 1.000046E+00 | loss scale: 1.0 | grad norm: 0.130 | params norm: 11440.038 | params rms: 0.061944 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 17:13:52.324709 | finish at 2026-10-11 14:37:34 + [2026-10-06 21:25:11] iteration 3091/ 6855 | consumed samples: 395648 | elapsed time per iteration (ms): 89853.2 | throughput per GPU (TFLOP/s/GPU): 40.6 | MFU 13.02% | FLOPs estimator: workload-aware | logical tokens/s: 186717.4 | valid tokens/s: 186717.4 | physical tokens/s: 187162.1 | loss tokens/s: 142479.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.404140E-01 | lm-global-supervised-tokens: 1.280221E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.563570E-02 | lm-token-buffer-mib: 5.808578E+01 | global_load_balancing_loss: 1.000161E+00 | loss scale: 1.0 | grad norm: 0.036 | params norm: 11439.975 | params rms: 0.061944 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 21:56:47.524514 | finish at 2026-10-10 19:21:59 + [2026-10-06 21:26:37] iteration 3092/ 6855 | consumed samples: 395776 | elapsed time per iteration (ms): 85681.9 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.74% | FLOPs estimator: workload-aware | logical tokens/s: 195808.0 | valid tokens/s: 195808.0 | physical tokens/s: 196272.2 | loss tokens/s: 140854.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.750672E-01 | lm-global-supervised-tokens: 1.206869E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.136588E-02 | lm-token-buffer-mib: 5.811555E+01 | global_load_balancing_loss: 1.001186E+00 | loss scale: 1.0 | grad norm: 0.066 | params norm: 11439.913 | params rms: 0.061943 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 17:33:41.022090 | finish at 2026-10-10 15:00:18 + [2026-10-06 21:28:03] iteration 3093/ 6855 | consumed samples: 395904 | elapsed time per iteration (ms): 85837.7 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.63% | FLOPs estimator: workload-aware | logical tokens/s: 195452.7 | valid tokens/s: 195452.7 | physical tokens/s: 195932.2 | loss tokens/s: 149867.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.395847E-01 | lm-global-supervised-tokens: 1.286426E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.326899E-02 | lm-token-buffer-mib: 5.812338E+01 | global_load_balancing_loss: 9.997674E-01 | loss scale: 1.0 | grad norm: 0.058 | params norm: 11439.851 | params rms: 0.061943 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 17:42:01.436853 | finish at 2026-10-10 15:10:04 + [2026-10-06 21:29:30] iteration 3094/ 6855 | consumed samples: 396032 | elapsed time per iteration (ms): 87240.1 | throughput per GPU (TFLOP/s/GPU): 41.1 | MFU 13.19% | FLOPs estimator: workload-aware | logical tokens/s: 192310.9 | valid tokens/s: 192310.9 | physical tokens/s: 192645.2 | loss tokens/s: 147767.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.333768E-01 | lm-global-supervised-tokens: 1.289121E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.428666E-02 | lm-token-buffer-mib: 5.816901E+01 | global_load_balancing_loss: 9.995762E-01 | loss scale: 1.0 | grad norm: 0.049 | params norm: 11439.788 | params rms: 0.061943 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 19:08:29.899180 | finish at 2026-10-10 16:38:00 + [2026-10-06 21:30:55] iteration 3095/ 6855 | consumed samples: 396160 | elapsed time per iteration (ms): 85222.1 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.76% | FLOPs estimator: workload-aware | logical tokens/s: 196864.4 | valid tokens/s: 196864.4 | physical tokens/s: 197305.3 | loss tokens/s: 143121.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.304250E-01 | lm-global-supervised-tokens: 1.219713E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.330817E-02 | lm-token-buffer-mib: 5.809577E+01 | global_load_balancing_loss: 1.000231E+00 | loss scale: 1.0 | grad norm: 0.039 | params norm: 11439.724 | params rms: 0.061942 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 17:00:35.027943 | finish at 2026-10-10 14:31:30 + [2026-10-06 21:32:22] iteration 3096/ 6855 | consumed samples: 396288 | elapsed time per iteration (ms): 86195.1 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.52% | FLOPs estimator: workload-aware | logical tokens/s: 194641.8 | valid tokens/s: 194641.8 | physical tokens/s: 195173.1 | loss tokens/s: 148091.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.587621E-01 | lm-global-supervised-tokens: 1.276478E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.557960E-02 | lm-token-buffer-mib: 5.806818E+01 | global_load_balancing_loss: 9.999768E-01 | loss scale: 1.0 | grad norm: 0.063 | params norm: 11439.661 | params rms: 0.061942 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 18:00:07.292434 | finish at 2026-10-10 15:32:29 + [2026-10-06 21:33:49] iteration 3097/ 6855 | consumed samples: 396416 | elapsed time per iteration (ms): 87639.9 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.74% | FLOPs estimator: workload-aware | logical tokens/s: 191433.5 | valid tokens/s: 191433.5 | physical tokens/s: 191781.4 | loss tokens/s: 147313.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.623865E-01 | lm-global-supervised-tokens: 1.291057E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.188667E-02 | lm-token-buffer-mib: 5.818203E+01 | global_load_balancing_loss: 9.991992E-01 | loss scale: 1.0 | grad norm: 0.052 | params norm: 11439.595 | params rms: 0.061942 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 19:29:10.635671 | finish at 2026-10-10 17:03:00 + [2026-10-06 21:35:15] iteration 3098/ 6855 | consumed samples: 396544 | elapsed time per iteration (ms): 85396.8 | throughput per GPU (TFLOP/s/GPU): 42.8 | MFU 13.73% | FLOPs estimator: workload-aware | logical tokens/s: 196461.2 | valid tokens/s: 196461.2 | physical tokens/s: 197031.0 | loss tokens/s: 145191.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.424785E-01 | lm-global-supervised-tokens: 1.239884E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.146430E-02 | lm-token-buffer-mib: 5.811535E+01 | global_load_balancing_loss: 1.000679E+00 | loss scale: 1.0 | grad norm: 0.056 | params norm: 11439.528 | params rms: 0.061941 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 17:07:15.649664 | finish at 2026-10-10 14:42:30 + [2026-10-06 21:36:40] iteration 3099/ 6855 | consumed samples: 396672 | elapsed time per iteration (ms): 85801.5 | throughput per GPU (TFLOP/s/GPU): 43.2 | MFU 13.83% | FLOPs estimator: workload-aware | logical tokens/s: 195534.7 | valid tokens/s: 195534.7 | physical tokens/s: 196031.7 | loss tokens/s: 146922.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.316922E-01 | lm-global-supervised-tokens: 1.260616E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.586411E-02 | lm-token-buffer-mib: 5.811111E+01 | global_load_balancing_loss: 1.000436E+00 | loss scale: 1.0 | grad norm: 0.063 | params norm: 11439.463 | params rms: 0.061941 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 17:31:10.312520 | finish at 2026-10-10 15:07:51 + [2026-10-06 21:38:27] iteration 3100/ 6855 | consumed samples: 396800 | elapsed time per iteration (ms): 106380.0 | throughput per GPU (TFLOP/s/GPU): 34.6 | MFU 11.10% | FLOPs estimator: workload-aware | logical tokens/s: 157709.5 | valid tokens/s: 157709.5 | physical tokens/s: 158277.0 | loss tokens/s: 119223.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.642258E-01 | lm-global-supervised-tokens: 1.268296E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.649269E-02 | lm-token-buffer-mib: 5.824036E+01 | global_load_balancing_loss: 1.001194E+00 | loss scale: 1.0 | grad norm: 0.042 | params norm: 11439.398 | params rms: 0.061941 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 14:57:36.757188 | finish at 2026-10-11 12:36:04 + [2026-10-06 21:39:56] iteration 3101/ 6855 | consumed samples: 396928 | elapsed time per iteration (ms): 89617.3 | throughput per GPU (TFLOP/s/GPU): 41.3 | MFU 13.25% | FLOPs estimator: workload-aware | logical tokens/s: 187209.0 | valid tokens/s: 187209.0 | physical tokens/s: 187698.2 | loss tokens/s: 142567.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.962252E-01 | lm-global-supervised-tokens: 1.277650E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.138455E-02 | lm-token-buffer-mib: 5.823777E+01 | global_load_balancing_loss: 1.000174E+00 | loss scale: 1.0 | grad norm: 0.062 | params norm: 11439.332 | params rms: 0.061940 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 21:27:03.392657 | finish at 2026-10-10 19:07:00 + [2026-10-06 21:41:26] iteration 3102/ 6855 | consumed samples: 397056 | elapsed time per iteration (ms): 89398.2 | throughput per GPU (TFLOP/s/GPU): 41.0 | MFU 13.14% | FLOPs estimator: workload-aware | logical tokens/s: 187668.3 | valid tokens/s: 187668.3 | physical tokens/s: 188031.9 | loss tokens/s: 144748.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.950662E-01 | lm-global-supervised-tokens: 1.294024E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.339189E-02 | lm-token-buffer-mib: 5.816911E+01 | global_load_balancing_loss: 1.000853E+00 | loss scale: 1.0 | grad norm: 0.091 | params norm: 11439.266 | params rms: 0.061940 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 21:11:51.469786 | finish at 2026-10-10 18:53:17 + [2026-10-06 21:42:52] iteration 3103/ 6855 | consumed samples: 397184 | elapsed time per iteration (ms): 85959.6 | throughput per GPU (TFLOP/s/GPU): 43.1 | MFU 13.80% | FLOPs estimator: workload-aware | logical tokens/s: 195175.6 | valid tokens/s: 195175.6 | physical tokens/s: 195595.0 | loss tokens/s: 142091.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.205111E-01 | lm-global-supervised-tokens: 1.221412E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.147955E-02 | lm-token-buffer-mib: 5.811053E+01 | global_load_balancing_loss: 9.996907E-01 | loss scale: 1.0 | grad norm: 0.049 | params norm: 11439.201 | params rms: 0.061940 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 17:35:20.381523 | finish at 2026-10-10 15:18:12 + [2026-10-06 21:44:22] iteration 3104/ 6855 | consumed samples: 397312 | elapsed time per iteration (ms): 90254.0 | throughput per GPU (TFLOP/s/GPU): 41.2 | MFU 13.19% | FLOPs estimator: workload-aware | logical tokens/s: 185888.8 | valid tokens/s: 185888.8 | physical tokens/s: 186336.6 | loss tokens/s: 135036.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.741237E-01 | lm-global-supervised-tokens: 1.218757E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.156758E-02 | lm-token-buffer-mib: 5.803994E+01 | global_load_balancing_loss: 9.999939E-01 | loss scale: 1.0 | grad norm: 0.043 | params norm: 11439.135 | params rms: 0.061939 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 22:02:22.901368 | finish at 2026-10-10 19:46:45 + [2026-10-06 21:45:51] iteration 3105/ 6855 | consumed samples: 397440 | elapsed time per iteration (ms): 88586.8 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.39% | FLOPs estimator: workload-aware | logical tokens/s: 189387.2 | valid tokens/s: 189387.2 | physical tokens/s: 190012.8 | loss tokens/s: 142124.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.695894E-01 | lm-global-supervised-tokens: 1.259036E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.150785E-02 | lm-token-buffer-mib: 5.836752E+01 | global_load_balancing_loss: 9.997974E-01 | loss scale: 1.0 | grad norm: 0.051 | params norm: 11439.069 | params rms: 0.061939 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 20:16:40.416327 | finish at 2026-10-10 18:02:31 + [2026-10-06 21:47:19] iteration 3106/ 6855 | consumed samples: 397568 | elapsed time per iteration (ms): 87991.5 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.56% | FLOPs estimator: workload-aware | logical tokens/s: 190668.1 | valid tokens/s: 190668.1 | physical tokens/s: 191060.8 | loss tokens/s: 141588.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.509723E-01 | lm-global-supervised-tokens: 1.245859E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.690310E-02 | lm-token-buffer-mib: 5.818398E+01 | global_load_balancing_loss: 9.997689E-01 | loss scale: 1.0 | grad norm: 0.042 | params norm: 11439.003 | params rms: 0.061938 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 19:38:00.209104 | finish at 2026-10-10 17:25:19 + [2026-10-06 21:48:40] iteration 3107/ 6855 | consumed samples: 397696 | elapsed time per iteration (ms): 81766.2 | throughput per GPU (TFLOP/s/GPU): 43.8 | MFU 14.04% | FLOPs estimator: workload-aware | logical tokens/s: 205184.5 | valid tokens/s: 205184.5 | physical tokens/s: 205672.4 | loss tokens/s: 162838.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.959522E-01 | lm-global-supervised-tokens: 1.331465E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.215677E-02 | lm-token-buffer-mib: 5.807448E+01 | global_load_balancing_loss: 1.000403E+00 | loss scale: 1.0 | grad norm: 0.094 | params norm: 11438.940 | params rms: 0.061938 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 13:07:39.833119 | finish at 2026-10-10 10:56:20 + [2026-10-06 21:50:04] iteration 3108/ 6855 | consumed samples: 397824 | elapsed time per iteration (ms): 83732.2 | throughput per GPU (TFLOP/s/GPU): 43.4 | MFU 13.92% | FLOPs estimator: workload-aware | logical tokens/s: 200367.6 | valid tokens/s: 200367.6 | physical tokens/s: 200860.5 | loss tokens/s: 152201.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.482393E-01 | lm-global-supervised-tokens: 1.274416E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.229314E-02 | lm-token-buffer-mib: 5.828210E+01 | global_load_balancing_loss: 1.001611E+00 | loss scale: 1.0 | grad norm: 0.085 | params norm: 11438.877 | params rms: 0.061938 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 15:09:04.381529 | finish at 2026-10-10 12:59:08 + [2026-10-06 21:51:31] iteration 3109/ 6855 | consumed samples: 397952 | elapsed time per iteration (ms): 87005.2 | throughput per GPU (TFLOP/s/GPU): 43.3 | MFU 13.87% | FLOPs estimator: workload-aware | logical tokens/s: 192829.0 | valid tokens/s: 192828.9 | physical tokens/s: 193308.3 | loss tokens/s: 142916.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.053688E-01 | lm-global-supervised-tokens: 1.243448E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.341914E-02 | lm-token-buffer-mib: 5.804491E+01 | global_load_balancing_loss: 1.000984E+00 | loss scale: 1.0 | grad norm: 0.052 | params norm: 11438.812 | params rms: 0.061937 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 18:32:01.624438 | finish at 2026-10-10 16:23:33 + [2026-10-06 21:53:19] iteration 3110/ 6855 | consumed samples: 398080 | elapsed time per iteration (ms): 108157.8 | throughput per GPU (TFLOP/s/GPU): 34.4 | MFU 11.03% | FLOPs estimator: workload-aware | logical tokens/s: 155117.6 | valid tokens/s: 155117.6 | physical tokens/s: 155795.5 | loss tokens/s: 116078.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.048913E-01 | lm-global-supervised-tokens: 1.255475E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.384374E-02 | lm-token-buffer-mib: 5.812421E+01 | global_load_balancing_loss: 1.000486E+00 | loss scale: 1.0 | grad norm: 0.106 | params norm: 11438.748 | params rms: 0.061937 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 16:30:50.893881 | finish at 2026-10-11 14:24:10 + [2026-10-06 21:54:48] iteration 3111/ 6855 | consumed samples: 398208 | elapsed time per iteration (ms): 88485.6 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.49% | FLOPs estimator: workload-aware | logical tokens/s: 189603.4 | valid tokens/s: 189603.4 | physical tokens/s: 190162.1 | loss tokens/s: 135941.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.495181E-01 | lm-global-supervised-tokens: 1.202884E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.177659E-02 | lm-token-buffer-mib: 5.812148E+01 | global_load_balancing_loss: 9.997146E-01 | loss scale: 1.0 | grad norm: 0.089 | params norm: 11438.686 | params rms: 0.061937 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 20:01:30.025681 | finish at 2026-10-10 17:56:18 + [2026-10-06 21:56:13] iteration 3112/ 6855 | consumed samples: 398336 | elapsed time per iteration (ms): 85438.5 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.55% | FLOPs estimator: workload-aware | logical tokens/s: 196365.7 | valid tokens/s: 196365.7 | physical tokens/s: 196798.8 | loss tokens/s: 147079.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.177386E-01 | lm-global-supervised-tokens: 1.256625E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.375210E-02 | lm-token-buffer-mib: 5.813697E+01 | global_load_balancing_loss: 9.988719E-01 | loss scale: 1.0 | grad norm: 0.035 | params norm: 11438.624 | params rms: 0.061936 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 16:49:56.234729 | finish at 2026-10-10 14:46:09 + [2026-10-06 21:57:43] iteration 3113/ 6855 | consumed samples: 398464 | elapsed time per iteration (ms): 89768.7 | throughput per GPU (TFLOP/s/GPU): 41.6 | MFU 13.34% | FLOPs estimator: workload-aware | logical tokens/s: 186892.9 | valid tokens/s: 186892.9 | physical tokens/s: 187386.6 | loss tokens/s: 139139.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.137023E-01 | lm-global-supervised-tokens: 1.249035E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.852919E-02 | lm-token-buffer-mib: 5.810724E+01 | global_load_balancing_loss: 9.985408E-01 | loss scale: 1.0 | grad norm: 0.072 | params norm: 11438.561 | params rms: 0.061936 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 21:18:34.479428 | finish at 2026-10-10 19:16:17 + [2026-10-06 21:59:09] iteration 3114/ 6855 | consumed samples: 398592 | elapsed time per iteration (ms): 86307.1 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.59% | FLOPs estimator: workload-aware | logical tokens/s: 194389.1 | valid tokens/s: 194389.0 | physical tokens/s: 194931.2 | loss tokens/s: 144949.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.882061E-01 | lm-global-supervised-tokens: 1.251019E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.330853E-02 | lm-token-buffer-mib: 5.812783E+01 | global_load_balancing_loss: 9.998388E-01 | loss scale: 1.0 | grad norm: 0.089 | params norm: 11438.498 | params rms: 0.061936 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 17:41:15.012943 | finish at 2026-10-10 15:40:24 + [2026-10-06 22:00:34] iteration 3115/ 6855 | consumed samples: 398720 | elapsed time per iteration (ms): 85172.8 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.43% | FLOPs estimator: workload-aware | logical tokens/s: 196977.3 | valid tokens/s: 196977.3 | physical tokens/s: 197529.1 | loss tokens/s: 151893.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.378480E-01 | lm-global-supervised-tokens: 1.293717E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.151460E-02 | lm-token-buffer-mib: 5.807868E+01 | global_load_balancing_loss: 1.000342E+00 | loss scale: 1.0 | grad norm: 0.048 | params norm: 11438.434 | params rms: 0.061935 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 16:29:06.451468 | finish at 2026-10-10 14:29:41 + [2026-10-06 22:02:03] iteration 3116/ 6855 | consumed samples: 398848 | elapsed time per iteration (ms): 88286.0 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.49% | FLOPs estimator: workload-aware | logical tokens/s: 190032.2 | valid tokens/s: 190032.2 | physical tokens/s: 190533.1 | loss tokens/s: 139956.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.882008E-01 | lm-global-supervised-tokens: 1.235621E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.217917E-02 | lm-token-buffer-mib: 5.803114E+01 | global_load_balancing_loss: 1.001812E+00 | loss scale: 1.0 | grad norm: 0.058 | params norm: 11438.370 | params rms: 0.061935 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 19:41:41.458349 | finish at 2026-10-10 17:43:44 + [2026-10-06 22:03:30] iteration 3117/ 6855 | consumed samples: 398976 | elapsed time per iteration (ms): 87012.1 | throughput per GPU (TFLOP/s/GPU): 43.1 | MFU 13.81% | FLOPs estimator: workload-aware | logical tokens/s: 192814.2 | valid tokens/s: 192814.2 | physical tokens/s: 193185.5 | loss tokens/s: 139194.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.628863E-01 | lm-global-supervised-tokens: 1.211161E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.253608E-02 | lm-token-buffer-mib: 5.808900E+01 | global_load_balancing_loss: 1.002086E+00 | loss scale: 1.0 | grad norm: 0.058 | params norm: 11438.308 | params rms: 0.061935 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 18:20:51.145065 | finish at 2026-10-10 16:24:21 + [2026-10-06 22:04:59] iteration 3118/ 6855 | consumed samples: 399104 | elapsed time per iteration (ms): 89237.0 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.55% | FLOPs estimator: workload-aware | logical tokens/s: 188006.7 | valid tokens/s: 188006.7 | physical tokens/s: 188398.9 | loss tokens/s: 133896.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.330853E-01 | lm-global-supervised-tokens: 1.194852E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.336367E-02 | lm-token-buffer-mib: 5.811599E+01 | global_load_balancing_loss: 1.000709E+00 | loss scale: 1.0 | grad norm: 0.057 | params norm: 11438.246 | params rms: 0.061934 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 20:37:58.527293 | finish at 2026-10-10 18:42:58 + [2026-10-06 22:06:24] iteration 3119/ 6855 | consumed samples: 399232 | elapsed time per iteration (ms): 84700.5 | throughput per GPU (TFLOP/s/GPU): 43.1 | MFU 13.83% | FLOPs estimator: workload-aware | logical tokens/s: 198076.8 | valid tokens/s: 198076.8 | physical tokens/s: 198484.4 | loss tokens/s: 146560.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.910121E-01 | lm-global-supervised-tokens: 1.241377E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.457722E-02 | lm-token-buffer-mib: 5.815231E+01 | global_load_balancing_loss: 9.997187E-01 | loss scale: 1.0 | grad norm: 0.063 | params norm: 11438.184 | params rms: 0.061934 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 15:54:01.125940 | finish at 2026-10-10 14:00:25 + [2026-10-06 22:08:10] iteration 3120/ 6855 | consumed samples: 399360 | elapsed time per iteration (ms): 105846.0 | throughput per GPU (TFLOP/s/GPU): 34.6 | MFU 11.10% | FLOPs estimator: workload-aware | logical tokens/s: 158505.4 | valid tokens/s: 158505.4 | physical tokens/s: 158979.8 | loss tokens/s: 122979.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.281732E-01 | lm-global-supervised-tokens: 1.301693E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.462041E-02 | lm-token-buffer-mib: 5.824452E+01 | global_load_balancing_loss: 9.995972E-01 | loss scale: 1.0 | grad norm: 0.061 | params norm: 11438.122 | params rms: 0.061934 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 13:48:54.723458 | finish at 2026-10-11 11:57:04 +times across ranks (ms): + save-checkpoint: + rank 0: 24567.61 + rank 1: 24567.60 + rank 2: 24567.64 + rank 3: 24567.63 + rank 4: 24567.60 + rank 5: 24567.59 + rank 6: 24567.58 + rank 7: 24567.60 + rank 8: 24567.80 + rank 9: 24567.75 + rank 10: 24567.75 + rank 11: 24567.76 + rank 12: 24567.79 + rank 13: 24567.75 + rank 14: 24567.77 + rank 15: 24567.74 + rank 16: 24567.63 + rank 17: 24567.59 + rank 18: 24567.57 + rank 19: 24567.66 + rank 20: 24567.57 + rank 21: 24567.61 + rank 22: 24567.57 + rank 23: 24567.57 + rank 24: 24567.74 + rank 25: 24567.69 + rank 26: 24567.68 + rank 27: 24567.69 + rank 28: 24567.69 + rank 29: 24567.67 + rank 30: 24567.65 + rank 31: 24567.68 + rank 32: 24567.68 + rank 33: 24567.68 + rank 34: 24567.67 + rank 35: 24567.65 + rank 36: 24567.65 + rank 37: 24567.62 + rank 38: 24567.65 + rank 39: 24567.65 + rank 40: 24568.06 + rank 41: 24568.04 + rank 42: 24568.07 + rank 43: 24568.06 + rank 44: 24568.02 + rank 45: 24568.02 + rank 46: 24568.04 + rank 47: 24568.02 + rank 48: 24568.04 + rank 49: 24568.08 + rank 50: 24567.99 + rank 51: 24567.96 + rank 52: 24567.95 + rank 53: 24567.95 + rank 54: 24567.99 + rank 55: 24567.98 + rank 56: 24566.14 + rank 57: 24566.13 + rank 58: 24566.14 + rank 59: 24566.13 + rank 60: 24566.15 + rank 61: 24566.12 + rank 62: 24566.19 + rank 63: 24566.15 + [2026-10-06 22:10:05] iteration 3121/ 6855 | consumed samples: 399488 | elapsed time per iteration (ms): 90436.1 | throughput per GPU (TFLOP/s/GPU): 41.5 | MFU 13.30% | FLOPs estimator: workload-aware | logical tokens/s: 185514.2 | valid tokens/s: 185514.2 | physical tokens/s: 186005.1 | loss tokens/s: 139322.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.503124E-01 | lm-global-supervised-tokens: 1.259981E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 3.006505E-01 | lm-token-buffer-mib: 5.809088E+01 | global_load_balancing_loss: 9.994955E-01 | loss scale: 1.0 | grad norm: 0.076 | params norm: 11438.057 | params rms: 0.061933 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 21:48:08.552327 | finish at 2026-10-10 19:58:13 + [2026-10-06 22:11:46] iteration 3122/ 6855 | consumed samples: 399616 | elapsed time per iteration (ms): 100971.6 | throughput per GPU (TFLOP/s/GPU): 35.9 | MFU 11.50% | FLOPs estimator: workload-aware | logical tokens/s: 166157.3 | valid tokens/s: 166157.3 | physical tokens/s: 166486.3 | loss tokens/s: 133804.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.155328E-01 | lm-global-supervised-tokens: 1.351048E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.897111E-02 | lm-token-buffer-mib: 5.818418E+01 | global_load_balancing_loss: 1.000313E+00 | loss scale: 1.0 | grad norm: 0.032 | params norm: 11437.991 | params rms: 0.061933 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 8:42:06.814795 | finish at 2026-10-11 06:53:53 + [2026-10-06 22:13:12] iteration 3123/ 6855 | consumed samples: 399744 | elapsed time per iteration (ms): 86425.6 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.74% | FLOPs estimator: workload-aware | logical tokens/s: 194122.2 | valid tokens/s: 194122.2 | physical tokens/s: 194537.7 | loss tokens/s: 143298.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.528911E-01 | lm-global-supervised-tokens: 1.238462E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.135921E-02 | lm-token-buffer-mib: 5.816848E+01 | global_load_balancing_loss: 9.993733E-01 | loss scale: 1.0 | grad norm: 0.070 | params norm: 11437.926 | params rms: 0.061933 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 17:35:40.161438 | finish at 2026-10-10 15:48:52 + [2026-10-06 22:14:38] iteration 3124/ 6855 | consumed samples: 399872 | elapsed time per iteration (ms): 86195.1 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.77% | FLOPs estimator: workload-aware | logical tokens/s: 194642.1 | valid tokens/s: 194642.1 | physical tokens/s: 195040.5 | loss tokens/s: 145825.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.366564E-01 | lm-global-supervised-tokens: 1.256948E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.162410E-02 | lm-token-buffer-mib: 5.807782E+01 | global_load_balancing_loss: 1.000002E+00 | loss scale: 1.0 | grad norm: 0.104 | params norm: 11437.862 | params rms: 0.061932 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 17:19:53.970840 | finish at 2026-10-10 15:34:32 + [2026-10-06 22:16:04] iteration 3125/ 6855 | consumed samples: 400000 | elapsed time per iteration (ms): 85586.3 | throughput per GPU (TFLOP/s/GPU): 43.4 | MFU 13.91% | FLOPs estimator: workload-aware | logical tokens/s: 196026.8 | valid tokens/s: 196026.8 | physical tokens/s: 196381.5 | loss tokens/s: 148017.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.551859E-01 | lm-global-supervised-tokens: 1.266831E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.161145E-02 | lm-token-buffer-mib: 5.816002E+01 | global_load_balancing_loss: 9.997061E-01 | loss scale: 1.0 | grad norm: 0.092 | params norm: 11437.800 | params rms: 0.061932 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 16:40:37.007108 | finish at 2026-10-10 14:56:41 + [2026-10-06 22:17:31] iteration 3126/ 6855 | consumed samples: 400128 | elapsed time per iteration (ms): 87360.4 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.77% | FLOPs estimator: workload-aware | logical tokens/s: 192045.9 | valid tokens/s: 192045.9 | physical tokens/s: 192503.3 | loss tokens/s: 139930.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.672097E-01 | lm-global-supervised-tokens: 1.222437E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.644728E-02 | lm-token-buffer-mib: 5.825798E+01 | global_load_balancing_loss: 1.000654E+00 | loss scale: 1.0 | grad norm: 0.031 | params norm: 11437.736 | params rms: 0.061932 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 18:29:27.007916 | finish at 2026-10-10 16:46:58 + [2026-10-06 22:18:56] iteration 3127/ 6855 | consumed samples: 400256 | elapsed time per iteration (ms): 84508.0 | throughput per GPU (TFLOP/s/GPU): 43.3 | MFU 13.88% | FLOPs estimator: workload-aware | logical tokens/s: 198528.1 | valid tokens/s: 198528.1 | physical tokens/s: 199123.7 | loss tokens/s: 144838.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.146204E-01 | lm-global-supervised-tokens: 1.224000E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.712411E-02 | lm-token-buffer-mib: 5.827899E+01 | global_load_balancing_loss: 1.000827E+00 | loss scale: 1.0 | grad norm: 0.101 | params norm: 11437.671 | params rms: 0.061931 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 15:30:45.871613 | finish at 2026-10-10 13:49:42 + [2026-10-06 22:20:20] iteration 3128/ 6855 | consumed samples: 400384 | elapsed time per iteration (ms): 83790.9 | throughput per GPU (TFLOP/s/GPU): 43.1 | MFU 13.83% | FLOPs estimator: workload-aware | logical tokens/s: 200226.7 | valid tokens/s: 200226.7 | physical tokens/s: 200813.1 | loss tokens/s: 148575.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.610961E-01 | lm-global-supervised-tokens: 1.244930E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.591736E-02 | lm-token-buffer-mib: 5.823978E+01 | global_load_balancing_loss: 1.000877E+00 | loss scale: 1.0 | grad norm: 0.109 | params norm: 11437.607 | params rms: 0.061931 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 14:44:48.810449 | finish at 2026-10-10 13:05:09 + [2026-10-06 22:21:50] iteration 3129/ 6855 | consumed samples: 400512 | elapsed time per iteration (ms): 90135.6 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.48% | FLOPs estimator: workload-aware | logical tokens/s: 186132.6 | valid tokens/s: 186132.6 | physical tokens/s: 186507.8 | loss tokens/s: 139777.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.707325E-01 | lm-global-supervised-tokens: 1.259894E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.428623E-02 | lm-token-buffer-mib: 5.808114E+01 | global_load_balancing_loss: 1.000018E+00 | loss scale: 1.0 | grad norm: 0.051 | params norm: 11437.542 | params rms: 0.061931 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 21:17:25.286799 | finish at 2026-10-10 19:39:15 + [2026-10-06 22:23:40] iteration 3130/ 6855 | consumed samples: 400640 | elapsed time per iteration (ms): 110276.5 | throughput per GPU (TFLOP/s/GPU): 34.2 | MFU 10.95% | FLOPs estimator: workload-aware | logical tokens/s: 152137.5 | valid tokens/s: 152137.5 | physical tokens/s: 152476.7 | loss tokens/s: 111392.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.913216E-01 | lm-global-supervised-tokens: 1.228394E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.151046E-02 | lm-token-buffer-mib: 5.809473E+01 | global_load_balancing_loss: 9.998813E-01 | loss scale: 1.0 | grad norm: 0.097 | params norm: 11437.479 | params rms: 0.061930 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 18:06:19.902947 | finish at 2026-10-11 16:30:00 + [2026-10-06 22:25:06] iteration 3131/ 6855 | consumed samples: 400768 | elapsed time per iteration (ms): 85596.9 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.78% | FLOPs estimator: workload-aware | logical tokens/s: 196002.4 | valid tokens/s: 196002.4 | physical tokens/s: 196532.8 | loss tokens/s: 145386.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.060271E-01 | lm-global-supervised-tokens: 1.244462E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.400436E-02 | lm-token-buffer-mib: 5.804771E+01 | global_load_balancing_loss: 9.996897E-01 | loss scale: 1.0 | grad norm: 0.215 | params norm: 11437.416 | params rms: 0.061930 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 16:32:42.697508 | finish at 2026-10-10 14:57:48 + [2026-10-06 22:26:37] iteration 3132/ 6855 | consumed samples: 400896 | elapsed time per iteration (ms): 90912.6 | throughput per GPU (TFLOP/s/GPU): 41.3 | MFU 13.25% | FLOPs estimator: workload-aware | logical tokens/s: 184541.6 | valid tokens/s: 184541.6 | physical tokens/s: 184911.9 | loss tokens/s: 130709.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.315140E-01 | lm-global-supervised-tokens: 1.188315E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.152321E-02 | lm-token-buffer-mib: 5.814320E+01 | global_load_balancing_loss: 1.000384E+00 | loss scale: 1.0 | grad norm: 0.155 | params norm: 11437.350 | params rms: 0.061930 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 22:01:07.465266 | finish at 2026-10-10 20:27:44 + [2026-10-06 22:28:08] iteration 3133/ 6855 | consumed samples: 401024 | elapsed time per iteration (ms): 91745.3 | throughput per GPU (TFLOP/s/GPU): 41.6 | MFU 13.33% | FLOPs estimator: workload-aware | logical tokens/s: 182866.4 | valid tokens/s: 182866.4 | physical tokens/s: 183409.8 | loss tokens/s: 137049.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.151366E-01 | lm-global-supervised-tokens: 1.257362E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.166780E-02 | lm-token-buffer-mib: 5.804716E+01 | global_load_balancing_loss: 9.988310E-01 | loss scale: 1.0 | grad norm: 0.129 | params norm: 11437.285 | params rms: 0.061929 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 22:51:16.131038 | finish at 2026-10-10 21:19:24 + [2026-10-06 22:29:34] iteration 3134/ 6855 | consumed samples: 401152 | elapsed time per iteration (ms): 85260.4 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.79% | FLOPs estimator: workload-aware | logical tokens/s: 196775.8 | valid tokens/s: 196775.8 | physical tokens/s: 197191.5 | loss tokens/s: 141469.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.643198E-01 | lm-global-supervised-tokens: 1.206178E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.777377E-02 | lm-token-buffer-mib: 5.806133E+01 | global_load_balancing_loss: 1.000775E+00 | loss scale: 1.0 | grad norm: 0.191 | params norm: 11437.221 | params rms: 0.061929 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 16:07:33.850137 | finish at 2026-10-10 14:37:07 + [2026-10-06 22:31:02] iteration 3135/ 6855 | consumed samples: 401280 | elapsed time per iteration (ms): 88530.6 | throughput per GPU (TFLOP/s/GPU): 41.7 | MFU 13.37% | FLOPs estimator: workload-aware | logical tokens/s: 189507.3 | valid tokens/s: 189507.3 | physical tokens/s: 189953.4 | loss tokens/s: 138869.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.485708E-01 | lm-global-supervised-tokens: 1.229417E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.321515E-02 | lm-token-buffer-mib: 5.813911E+01 | global_load_balancing_loss: 1.001150E+00 | loss scale: 1.0 | grad norm: 0.051 | params norm: 11437.156 | params rms: 0.061928 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 19:28:53.784370 | finish at 2026-10-10 17:59:56 + [2026-10-06 22:32:29] iteration 3136/ 6855 | consumed samples: 401408 | elapsed time per iteration (ms): 86862.9 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.52% | FLOPs estimator: workload-aware | logical tokens/s: 193145.4 | valid tokens/s: 193145.4 | physical tokens/s: 193599.0 | loss tokens/s: 143787.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.543782E-01 | lm-global-supervised-tokens: 1.248975E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.365280E-02 | lm-token-buffer-mib: 5.803145E+01 | global_load_balancing_loss: 1.000602E+00 | loss scale: 1.0 | grad norm: 0.083 | params norm: 11437.092 | params rms: 0.061928 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 17:44:03.003695 | finish at 2026-10-10 16:16:32 + [2026-10-06 22:33:57] iteration 3137/ 6855 | consumed samples: 401536 | elapsed time per iteration (ms): 88070.4 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.54% | FLOPs estimator: workload-aware | logical tokens/s: 190497.3 | valid tokens/s: 190497.3 | physical tokens/s: 190942.4 | loss tokens/s: 132201.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.217899E-01 | lm-global-supervised-tokens: 1.164300E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.184553E-02 | lm-token-buffer-mib: 5.835429E+01 | global_load_balancing_loss: 1.001176E+00 | loss scale: 1.0 | grad norm: 0.118 | params norm: 11437.029 | params rms: 0.061928 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 18:57:25.704649 | finish at 2026-10-10 17:31:23 + [2026-10-06 22:35:24] iteration 3138/ 6855 | consumed samples: 401664 | elapsed time per iteration (ms): 86975.7 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.76% | FLOPs estimator: workload-aware | logical tokens/s: 192895.4 | valid tokens/s: 192895.4 | physical tokens/s: 193413.8 | loss tokens/s: 143132.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.933473E-01 | lm-global-supervised-tokens: 1.244909E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.200898E-02 | lm-token-buffer-mib: 5.806791E+01 | global_load_balancing_loss: 1.000835E+00 | loss scale: 1.0 | grad norm: 0.078 | params norm: 11436.967 | params rms: 0.061927 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 17:48:08.694258 | finish at 2026-10-10 16:23:33 + [2026-10-06 22:36:54] iteration 3139/ 6855 | consumed samples: 401792 | elapsed time per iteration (ms): 89476.6 | throughput per GPU (TFLOP/s/GPU): 42.8 | MFU 13.70% | FLOPs estimator: workload-aware | logical tokens/s: 187503.6 | valid tokens/s: 187503.6 | physical tokens/s: 187810.9 | loss tokens/s: 134421.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.050350E-01 | lm-global-supervised-tokens: 1.202758E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.161874E-02 | lm-token-buffer-mib: 5.807153E+01 | global_load_balancing_loss: 9.990829E-01 | loss scale: 1.0 | grad norm: 0.066 | params norm: 11436.905 | params rms: 0.061927 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 20:21:34.910680 | finish at 2026-10-10 18:58:28 + [2026-10-06 22:38:40] iteration 3140/ 6855 | consumed samples: 401920 | elapsed time per iteration (ms): 106415.6 | throughput per GPU (TFLOP/s/GPU): 34.0 | MFU 10.91% | FLOPs estimator: workload-aware | logical tokens/s: 157657.4 | valid tokens/s: 157657.4 | physical tokens/s: 157997.6 | loss tokens/s: 119809.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.654043E-01 | lm-global-supervised-tokens: 1.274956E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.534858E-02 | lm-token-buffer-mib: 5.809188E+01 | global_load_balancing_loss: 9.991242E-01 | loss scale: 1.0 | grad norm: 0.098 | params norm: 11436.841 | params rms: 0.061927 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 13:48:54.008257 | finish at 2026-10-11 12:27:34 + [2026-10-06 22:40:03] iteration 3141/ 6855 | consumed samples: 402048 | elapsed time per iteration (ms): 82886.7 | throughput per GPU (TFLOP/s/GPU): 43.4 | MFU 13.90% | FLOPs estimator: workload-aware | logical tokens/s: 202410.7 | valid tokens/s: 202410.7 | physical tokens/s: 202919.1 | loss tokens/s: 150971.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.565917E-01 | lm-global-supervised-tokens: 1.251352E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.832529E-02 | lm-token-buffer-mib: 5.811310E+01 | global_load_balancing_loss: 9.996930E-01 | loss scale: 1.0 | grad norm: 0.067 | params norm: 11436.779 | params rms: 0.061926 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 13:30:41.176034 | finish at 2026-10-10 12:10:44 + [2026-10-06 22:41:27] iteration 3142/ 6855 | consumed samples: 402176 | elapsed time per iteration (ms): 84507.8 | throughput per GPU (TFLOP/s/GPU): 43.2 | MFU 13.84% | FLOPs estimator: workload-aware | logical tokens/s: 198528.0 | valid tokens/s: 198528.0 | physical tokens/s: 198964.7 | loss tokens/s: 144096.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.778067E-01 | lm-global-supervised-tokens: 1.217731E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.123515E-02 | lm-token-buffer-mib: 5.823141E+01 | global_load_balancing_loss: 1.000462E+00 | loss scale: 1.0 | grad norm: 0.120 | params norm: 11436.718 | params rms: 0.061926 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 15:09:37.409550 | finish at 2026-10-10 13:51:05 + [2026-10-06 22:42:53] iteration 3143/ 6855 | consumed samples: 402304 | elapsed time per iteration (ms): 85730.4 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.79% | FLOPs estimator: workload-aware | logical tokens/s: 195697.0 | valid tokens/s: 195697.0 | physical tokens/s: 196258.9 | loss tokens/s: 147616.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.829918E-01 | lm-global-supervised-tokens: 1.265521E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.430436E-02 | lm-token-buffer-mib: 5.822095E+01 | global_load_balancing_loss: 1.000107E+00 | loss scale: 1.0 | grad norm: 0.091 | params norm: 11436.658 | params rms: 0.061926 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 16:23:51.100861 | finish at 2026-10-10 15:06:44 + [2026-10-06 22:44:20] iteration 3144/ 6855 | consumed samples: 402432 | elapsed time per iteration (ms): 87065.5 | throughput per GPU (TFLOP/s/GPU): 42.8 | MFU 13.71% | FLOPs estimator: workload-aware | logical tokens/s: 192696.5 | valid tokens/s: 192696.5 | physical tokens/s: 193029.0 | loss tokens/s: 139669.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.354249E-01 | lm-global-supervised-tokens: 1.216037E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.433400E-02 | lm-token-buffer-mib: 5.805515E+01 | global_load_balancing_loss: 1.000077E+00 | loss scale: 1.0 | grad norm: 0.062 | params norm: 11436.599 | params rms: 0.061925 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 17:44:59.987409 | finish at 2026-10-10 16:29:20 + [2026-10-06 22:45:50] iteration 3145/ 6855 | consumed samples: 402560 | elapsed time per iteration (ms): 90089.7 | throughput per GPU (TFLOP/s/GPU): 41.7 | MFU 13.36% | FLOPs estimator: workload-aware | logical tokens/s: 186227.8 | valid tokens/s: 186227.8 | physical tokens/s: 186707.7 | loss tokens/s: 137183.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.012360E-01 | lm-global-supervised-tokens: 1.235883E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.032505E-02 | lm-token-buffer-mib: 5.820702E+01 | global_load_balancing_loss: 1.001888E+00 | loss scale: 1.0 | grad norm: 0.063 | params norm: 11436.539 | params rms: 0.061925 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 20:50:32.648067 | finish at 2026-10-10 19:36:23 + [2026-10-06 22:47:14] iteration 3146/ 6855 | consumed samples: 402688 | elapsed time per iteration (ms): 84159.1 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.56% | FLOPs estimator: workload-aware | logical tokens/s: 199350.6 | valid tokens/s: 199350.6 | physical tokens/s: 199804.5 | loss tokens/s: 149139.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.376516E-01 | lm-global-supervised-tokens: 1.255142E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.155319E-02 | lm-token-buffer-mib: 5.810366E+01 | global_load_balancing_loss: 1.000226E+00 | loss scale: 1.0 | grad norm: 0.049 | params norm: 11436.480 | params rms: 0.061925 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 14:42:26.011024 | finish at 2026-10-10 13:29:40 + [2026-10-06 22:48:43] iteration 3147/ 6855 | consumed samples: 402816 | elapsed time per iteration (ms): 88821.9 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.52% | FLOPs estimator: workload-aware | logical tokens/s: 188885.6 | valid tokens/s: 188885.6 | physical tokens/s: 189196.2 | loss tokens/s: 138772.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.210862E-01 | lm-global-supervised-tokens: 1.232602E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.516562E-02 | lm-token-buffer-mib: 5.812591E+01 | global_load_balancing_loss: 1.000834E+00 | loss scale: 1.0 | grad norm: 0.056 | params norm: 11436.421 | params rms: 0.061924 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 19:29:11.663143 | finish at 2026-10-10 18:17:55 + [2026-10-06 22:50:11] iteration 3148/ 6855 | consumed samples: 402944 | elapsed time per iteration (ms): 87772.0 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.56% | FLOPs estimator: workload-aware | logical tokens/s: 191145.1 | valid tokens/s: 191145.1 | physical tokens/s: 191655.7 | loss tokens/s: 145226.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.980243E-01 | lm-global-supervised-tokens: 1.274684E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.162988E-02 | lm-token-buffer-mib: 5.812837E+01 | global_load_balancing_loss: 1.000110E+00 | loss scale: 1.0 | grad norm: 0.063 | params norm: 11436.360 | params rms: 0.061924 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 18:22:50.720313 | finish at 2026-10-10 17:13:02 + [2026-10-06 22:51:38] iteration 3149/ 6855 | consumed samples: 403072 | elapsed time per iteration (ms): 86912.9 | throughput per GPU (TFLOP/s/GPU): 43.1 | MFU 13.83% | FLOPs estimator: workload-aware | logical tokens/s: 193034.7 | valid tokens/s: 193034.7 | physical tokens/s: 193411.4 | loss tokens/s: 139799.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.878424E-01 | lm-global-supervised-tokens: 1.215042E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.668751E-02 | lm-token-buffer-mib: 5.805715E+01 | global_load_balancing_loss: 9.998600E-01 | loss scale: 1.0 | grad norm: 0.040 | params norm: 11436.300 | params rms: 0.061924 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 17:28:19.345131 | finish at 2026-10-10 16:19:57 + [2026-10-06 22:53:30] iteration 3150/ 6855 | consumed samples: 403200 | elapsed time per iteration (ms): 112558.7 | throughput per GPU (TFLOP/s/GPU): 33.9 | MFU 10.85% | FLOPs estimator: workload-aware | logical tokens/s: 149052.8 | valid tokens/s: 149052.8 | physical tokens/s: 149331.3 | loss tokens/s: 104456.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.295884E-01 | lm-global-supervised-tokens: 1.175746E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.483917E-02 | lm-token-buffer-mib: 5.808744E+01 | global_load_balancing_loss: 9.999168E-01 | loss scale: 1.0 | grad norm: 0.114 | params norm: 11436.238 | params rms: 0.061924 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 19:50:30.138398 | finish at 2026-10-11 18:44:01 + [2026-10-06 22:54:58] iteration 3151/ 6855 | consumed samples: 403328 | elapsed time per iteration (ms): 87792.8 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.44% | FLOPs estimator: workload-aware | logical tokens/s: 191099.6 | valid tokens/s: 191099.6 | physical tokens/s: 191694.2 | loss tokens/s: 141090.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.580269E-01 | lm-global-supervised-tokens: 1.238670E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.136698E-02 | lm-token-buffer-mib: 5.809894E+01 | global_load_balancing_loss: 9.993033E-01 | loss scale: 1.0 | grad norm: 0.159 | params norm: 11436.178 | params rms: 0.061923 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 18:19:44.665245 | finish at 2026-10-10 17:14:43 + [2026-10-06 22:56:23] iteration 3152/ 6855 | consumed samples: 403456 | elapsed time per iteration (ms): 85050.8 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.63% | FLOPs estimator: workload-aware | logical tokens/s: 197260.5 | valid tokens/s: 197260.5 | physical tokens/s: 197924.2 | loss tokens/s: 149630.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.040538E-01 | lm-global-supervised-tokens: 1.272618E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.797921E-02 | lm-token-buffer-mib: 5.829890E+01 | global_load_balancing_loss: 9.987914E-01 | loss scale: 1.0 | grad norm: 0.117 | params norm: 11436.117 | params rms: 0.061923 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 15:29:02.959097 | finish at 2026-10-10 14:25:26 + [2026-10-06 22:57:48] iteration 3153/ 6855 | consumed samples: 403584 | elapsed time per iteration (ms): 85111.4 | throughput per GPU (TFLOP/s/GPU): 42.8 | MFU 13.71% | FLOPs estimator: workload-aware | logical tokens/s: 197120.4 | valid tokens/s: 197120.4 | physical tokens/s: 197588.9 | loss tokens/s: 155181.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.934015E-01 | lm-global-supervised-tokens: 1.320771E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.179921E-02 | lm-token-buffer-mib: 5.812749E+01 | global_load_balancing_loss: 9.988602E-01 | loss scale: 1.0 | grad norm: 0.051 | params norm: 11436.055 | params rms: 0.061923 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 15:31:22.414746 | finish at 2026-10-10 14:29:11 + [2026-10-06 22:59:15] iteration 3154/ 6855 | consumed samples: 403712 | elapsed time per iteration (ms): 86379.0 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.77% | FLOPs estimator: workload-aware | logical tokens/s: 194227.3 | valid tokens/s: 194227.3 | physical tokens/s: 194697.2 | loss tokens/s: 147421.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.969151E-01 | lm-global-supervised-tokens: 1.273412E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.390849E-02 | lm-token-buffer-mib: 5.815215E+01 | global_load_balancing_loss: 1.000141E+00 | loss scale: 1.0 | grad norm: 0.111 | params norm: 11435.997 | params rms: 0.061922 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 16:48:08.552628 | finish at 2026-10-10 15:47:23 + [2026-10-06 23:00:40] iteration 3155/ 6855 | consumed samples: 403840 | elapsed time per iteration (ms): 84782.3 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.55% | FLOPs estimator: workload-aware | logical tokens/s: 197885.5 | valid tokens/s: 197885.5 | physical tokens/s: 198351.7 | loss tokens/s: 148353.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.352506E-01 | lm-global-supervised-tokens: 1.257779E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.807383E-02 | lm-token-buffer-mib: 5.814687E+01 | global_load_balancing_loss: 1.001239E+00 | loss scale: 1.0 | grad norm: 0.052 | params norm: 11435.938 | params rms: 0.061922 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 15:08:14.562030 | finish at 2026-10-10 14:08:54 + [2026-10-06 23:02:09] iteration 3156/ 6855 | consumed samples: 403968 | elapsed time per iteration (ms): 89198.9 | throughput per GPU (TFLOP/s/GPU): 41.5 | MFU 13.31% | FLOPs estimator: workload-aware | logical tokens/s: 188087.1 | valid tokens/s: 188087.1 | physical tokens/s: 188569.9 | loss tokens/s: 143255.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.023691E-01 | lm-global-supervised-tokens: 1.277819E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.946504E-02 | lm-token-buffer-mib: 5.804557E+01 | global_load_balancing_loss: 1.001520E+00 | loss scale: 1.0 | grad norm: 0.123 | params norm: 11435.878 | params rms: 0.061922 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 19:39:06.608457 | finish at 2026-10-10 18:41:15 + [2026-10-06 23:03:35] iteration 3157/ 6855 | consumed samples: 404096 | elapsed time per iteration (ms): 86288.5 | throughput per GPU (TFLOP/s/GPU): 42.8 | MFU 13.73% | FLOPs estimator: workload-aware | logical tokens/s: 194431.2 | valid tokens/s: 194431.2 | physical tokens/s: 194747.8 | loss tokens/s: 148424.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.075470E-01 | lm-global-supervised-tokens: 1.280733E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.143957E-02 | lm-token-buffer-mib: 5.807453E+01 | global_load_balancing_loss: 1.002031E+00 | loss scale: 1.0 | grad norm: 0.042 | params norm: 11435.817 | params rms: 0.061921 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 16:38:14.806254 | finish at 2026-10-10 15:41:50 + [2026-10-06 23:04:59] iteration 3158/ 6855 | consumed samples: 404224 | elapsed time per iteration (ms): 84119.5 | throughput per GPU (TFLOP/s/GPU): 43.5 | MFU 13.94% | FLOPs estimator: workload-aware | logical tokens/s: 199444.2 | valid tokens/s: 199444.2 | physical tokens/s: 199963.0 | loss tokens/s: 148558.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.446713E-01 | lm-global-supervised-tokens: 1.249665E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.145075E-02 | lm-token-buffer-mib: 5.831287E+01 | global_load_balancing_loss: 1.001443E+00 | loss scale: 1.0 | grad norm: 0.293 | params norm: 11435.757 | params rms: 0.061921 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 14:23:09.765649 | finish at 2026-10-10 13:28:09 + [2026-10-06 23:06:28] iteration 3159/ 6855 | consumed samples: 404352 | elapsed time per iteration (ms): 89124.1 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.55% | FLOPs estimator: workload-aware | logical tokens/s: 188245.1 | valid tokens/s: 188245.1 | physical tokens/s: 188689.0 | loss tokens/s: 133062.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.530048E-01 | lm-global-supervised-tokens: 1.185910E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.356822E-02 | lm-token-buffer-mib: 5.815641E+01 | global_load_balancing_loss: 1.000282E+00 | loss scale: 1.0 | grad norm: 0.372 | params norm: 11435.696 | params rms: 0.061921 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 19:30:02.669964 | finish at 2026-10-10 18:36:31 + [2026-10-06 23:08:20] iteration 3160/ 6855 | consumed samples: 404480 | elapsed time per iteration (ms): 111814.0 | throughput per GPU (TFLOP/s/GPU): 33.0 | MFU 10.58% | FLOPs estimator: workload-aware | logical tokens/s: 150045.3 | valid tokens/s: 150045.3 | physical tokens/s: 150407.3 | loss tokens/s: 112500.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.338613E-01 | lm-global-supervised-tokens: 1.257915E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.491815E-02 | lm-token-buffer-mib: 5.810021E+01 | global_load_balancing_loss: 9.985389E-01 | loss scale: 1.0 | grad norm: 0.119 | params norm: 11435.635 | params rms: 0.061920 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 18:45:52.913718 | finish at 2026-10-11 17:54:13 +times across ranks (ms): + save-checkpoint: + rank 0: 23792.99 + rank 1: 23792.96 + rank 2: 23792.97 + rank 3: 23792.96 + rank 4: 23792.96 + rank 5: 23792.96 + rank 6: 23792.95 + rank 7: 23792.96 + rank 8: 23792.49 + rank 9: 23792.48 + rank 10: 23792.49 + rank 11: 23792.43 + rank 12: 23792.44 + rank 13: 23792.47 + rank 14: 23792.51 + rank 15: 23792.43 + rank 16: 23793.26 + rank 17: 23793.20 + rank 18: 23793.21 + rank 19: 23793.20 + rank 20: 23793.19 + rank 21: 23793.26 + rank 22: 23793.20 + rank 23: 23793.20 + rank 24: 23793.07 + rank 25: 23793.03 + rank 26: 23792.99 + rank 27: 23793.16 + rank 28: 23792.99 + rank 29: 23792.96 + rank 30: 23793.00 + rank 31: 23792.99 + rank 32: 23792.99 + rank 33: 23792.99 + rank 34: 23792.99 + rank 35: 23792.97 + rank 36: 23792.96 + rank 37: 23792.95 + rank 38: 23792.98 + rank 39: 23792.96 + rank 40: 23792.92 + rank 41: 23792.90 + rank 42: 23792.92 + rank 43: 23792.91 + rank 44: 23792.88 + rank 45: 23792.89 + rank 46: 23792.91 + rank 47: 23792.89 + rank 48: 23793.02 + rank 49: 23793.01 + rank 50: 23793.01 + rank 51: 23792.99 + rank 52: 23792.99 + rank 53: 23792.99 + rank 54: 23792.98 + rank 55: 23793.00 + rank 56: 23793.17 + rank 57: 23793.13 + rank 58: 23793.13 + rank 59: 23793.11 + rank 60: 23793.13 + rank 61: 23793.12 + rank 62: 23793.15 + rank 63: 23793.18 + [2026-10-06 23:10:26] iteration 3161/ 6855 | consumed samples: 404608 | elapsed time per iteration (ms): 101245.2 | throughput per GPU (TFLOP/s/GPU): 36.1 | MFU 11.58% | FLOPs estimator: workload-aware | logical tokens/s: 165708.4 | valid tokens/s: 165708.4 | physical tokens/s: 166239.6 | loss tokens/s: 124702.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.139725E-01 | lm-global-supervised-tokens: 1.262549E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 8.692589E-01 | lm-token-buffer-mib: 5.807427E+01 | global_load_balancing_loss: 9.987834E-01 | loss scale: 1.0 | grad norm: 0.185 | params norm: 11435.573 | params rms: 0.061920 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 7:53:19.823104 | finish at 2026-10-11 07:03:46 + [2026-10-06 23:11:58] iteration 3162/ 6855 | consumed samples: 404736 | elapsed time per iteration (ms): 92147.0 | throughput per GPU (TFLOP/s/GPU): 40.4 | MFU 12.95% | FLOPs estimator: workload-aware | logical tokens/s: 182069.5 | valid tokens/s: 182069.5 | physical tokens/s: 182414.4 | loss tokens/s: 133219.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.505454E-01 | lm-global-supervised-tokens: 1.227575E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.344735E-02 | lm-token-buffer-mib: 5.809662E+01 | global_load_balancing_loss: 9.998402E-01 | loss scale: 1.0 | grad norm: 0.070 | params norm: 11435.511 | params rms: 0.061920 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 22:31:39.035044 | finish at 2026-10-10 21:43:37 + [2026-10-06 23:13:23] iteration 3163/ 6855 | consumed samples: 404864 | elapsed time per iteration (ms): 84999.9 | throughput per GPU (TFLOP/s/GPU): 44.0 | MFU 14.10% | FLOPs estimator: workload-aware | logical tokens/s: 197378.2 | valid tokens/s: 197378.2 | physical tokens/s: 197888.3 | loss tokens/s: 150625.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.774895E-01 | lm-global-supervised-tokens: 1.280317E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.177543E-02 | lm-token-buffer-mib: 5.806710E+01 | global_load_balancing_loss: 9.989806E-01 | loss scale: 1.0 | grad norm: 0.046 | params norm: 11435.449 | params rms: 0.061919 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 15:10:19.542274 | finish at 2026-10-10 14:23:43 + [2026-10-06 23:14:47] iteration 3164/ 6855 | consumed samples: 404992 | elapsed time per iteration (ms): 83984.8 | throughput per GPU (TFLOP/s/GPU): 43.5 | MFU 13.95% | FLOPs estimator: workload-aware | logical tokens/s: 199764.6 | valid tokens/s: 199764.6 | physical tokens/s: 200208.5 | loss tokens/s: 153040.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.443877E-01 | lm-global-supervised-tokens: 1.285307E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.174346E-02 | lm-token-buffer-mib: 5.825816E+01 | global_load_balancing_loss: 1.000294E+00 | loss scale: 1.0 | grad norm: 0.163 | params norm: 11435.388 | params rms: 0.061919 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 14:06:28.001011 | finish at 2026-10-10 13:21:15 + [2026-10-06 23:16:16] iteration 3165/ 6855 | consumed samples: 405120 | elapsed time per iteration (ms): 88700.4 | throughput per GPU (TFLOP/s/GPU): 41.3 | MFU 13.23% | FLOPs estimator: workload-aware | logical tokens/s: 189144.6 | valid tokens/s: 189144.6 | physical tokens/s: 189621.6 | loss tokens/s: 134192.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.159310E-01 | lm-global-supervised-tokens: 1.190294E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.397185E-02 | lm-token-buffer-mib: 5.843928E+01 | global_load_balancing_loss: 1.001983E+00 | loss scale: 1.0 | grad norm: 0.052 | params norm: 11435.325 | params rms: 0.061919 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 18:55:04.604867 | finish at 2026-10-10 18:11:21 + [2026-10-06 23:17:42] iteration 3166/ 6855 | consumed samples: 405248 | elapsed time per iteration (ms): 86234.3 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.52% | FLOPs estimator: workload-aware | logical tokens/s: 194553.7 | valid tokens/s: 194553.7 | physical tokens/s: 194951.8 | loss tokens/s: 142961.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.509601E-01 | lm-global-supervised-tokens: 1.232816E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.501893E-02 | lm-token-buffer-mib: 5.805018E+01 | global_load_balancing_loss: 1.000982E+00 | loss scale: 1.0 | grad norm: 0.081 | params norm: 11435.263 | params rms: 0.061918 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 16:21:58.437867 | finish at 2026-10-10 15:39:41 + [2026-10-06 23:19:12] iteration 3167/ 6855 | consumed samples: 405376 | elapsed time per iteration (ms): 89305.3 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.49% | FLOPs estimator: workload-aware | logical tokens/s: 187863.7 | valid tokens/s: 187863.7 | physical tokens/s: 188261.4 | loss tokens/s: 132962.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.606348E-01 | lm-global-supervised-tokens: 1.187423E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.751318E-02 | lm-token-buffer-mib: 5.824528E+01 | global_load_balancing_loss: 1.000664E+00 | loss scale: 1.0 | grad norm: 0.130 | params norm: 11435.199 | params rms: 0.061918 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 19:29:17.775808 | finish at 2026-10-10 18:48:29 + [2026-10-06 23:20:38] iteration 3168/ 6855 | consumed samples: 405504 | elapsed time per iteration (ms): 86443.5 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.41% | FLOPs estimator: workload-aware | logical tokens/s: 194082.7 | valid tokens/s: 194082.7 | physical tokens/s: 194658.5 | loss tokens/s: 143872.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.464661E-01 | lm-global-supervised-tokens: 1.243680E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.907348E-02 | lm-token-buffer-mib: 5.812498E+01 | global_load_balancing_loss: 9.995595E-01 | loss scale: 1.0 | grad norm: 0.063 | params norm: 11435.134 | params rms: 0.061918 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 16:31:57.126637 | finish at 2026-10-10 15:52:35 + [2026-10-06 23:22:06] iteration 3169/ 6855 | consumed samples: 405632 | elapsed time per iteration (ms): 87898.6 | throughput per GPU (TFLOP/s/GPU): 41.4 | MFU 13.28% | FLOPs estimator: workload-aware | logical tokens/s: 190868.7 | valid tokens/s: 190868.7 | physical tokens/s: 191307.2 | loss tokens/s: 144120.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.599331E-01 | lm-global-supervised-tokens: 1.266799E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.181077E-02 | lm-token-buffer-mib: 5.803152E+01 | global_load_balancing_loss: 9.990336E-01 | loss scale: 1.0 | grad norm: 0.118 | params norm: 11435.068 | params rms: 0.061917 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 17:59:54.167912 | finish at 2026-10-10 17:22:00 + [2026-10-06 23:23:54] iteration 3170/ 6855 | consumed samples: 405760 | elapsed time per iteration (ms): 107873.1 | throughput per GPU (TFLOP/s/GPU): 33.6 | MFU 10.77% | FLOPs estimator: workload-aware | logical tokens/s: 155527.2 | valid tokens/s: 155527.2 | physical tokens/s: 155933.4 | loss tokens/s: 118013.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.722581E-01 | lm-global-supervised-tokens: 1.273049E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.113444E-02 | lm-token-buffer-mib: 5.813629E+01 | global_load_balancing_loss: 9.998627E-01 | loss scale: 1.0 | grad norm: 0.070 | params norm: 11435.001 | params rms: 0.061917 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 14:25:12.502806 | finish at 2026-10-11 13:49:06 + [2026-10-06 23:25:24] iteration 3171/ 6855 | consumed samples: 405888 | elapsed time per iteration (ms): 89642.2 | throughput per GPU (TFLOP/s/GPU): 40.9 | MFU 13.11% | FLOPs estimator: workload-aware | logical tokens/s: 187156.4 | valid tokens/s: 187156.3 | physical tokens/s: 187628.4 | loss tokens/s: 137735.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 5.989249E-01 | lm-global-supervised-tokens: 1.234694E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.147677E-02 | lm-token-buffer-mib: 5.808934E+01 | global_load_balancing_loss: 1.001701E+00 | loss scale: 1.0 | grad norm: 0.035 | params norm: 11434.936 | params rms: 0.061916 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 19:44:01.942946 | finish at 2026-10-10 19:09:25 + [2026-10-06 23:26:51] iteration 3172/ 6855 | consumed samples: 406016 | elapsed time per iteration (ms): 87683.0 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.66% | FLOPs estimator: workload-aware | logical tokens/s: 191338.9 | valid tokens/s: 191338.9 | physical tokens/s: 191645.6 | loss tokens/s: 137452.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.077548E-01 | lm-global-supervised-tokens: 1.205226E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.217680E-02 | lm-token-buffer-mib: 5.802773E+01 | global_load_balancing_loss: 1.000840E+00 | loss scale: 1.0 | grad norm: 0.058 | params norm: 11434.872 | params rms: 0.061916 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 17:42:16.476152 | finish at 2026-10-10 17:09:08 + [2026-10-06 23:28:18] iteration 3173/ 6855 | consumed samples: 406144 | elapsed time per iteration (ms): 87016.7 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.70% | FLOPs estimator: workload-aware | logical tokens/s: 192804.3 | valid tokens/s: 192804.3 | physical tokens/s: 193330.5 | loss tokens/s: 147201.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.983684E-01 | lm-global-supervised-tokens: 1.280901E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.312060E-02 | lm-token-buffer-mib: 5.815213E+01 | global_load_balancing_loss: 9.993837E-01 | loss scale: 1.0 | grad norm: 0.065 | params norm: 11434.805 | params rms: 0.061916 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 16:59:55.420157 | finish at 2026-10-10 16:28:14 + [2026-10-06 23:29:44] iteration 3174/ 6855 | consumed samples: 406272 | elapsed time per iteration (ms): 86088.4 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.69% | FLOPs estimator: workload-aware | logical tokens/s: 194883.0 | valid tokens/s: 194883.0 | physical tokens/s: 195359.5 | loss tokens/s: 152949.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.819370E-01 | lm-global-supervised-tokens: 1.316715E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.616737E-02 | lm-token-buffer-mib: 5.821629E+01 | global_load_balancing_loss: 9.997492E-01 | loss scale: 1.0 | grad norm: 0.047 | params norm: 11434.737 | params rms: 0.061915 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 16:01:31.226216 | finish at 2026-10-10 15:31:16 + [2026-10-06 23:31:12] iteration 3175/ 6855 | consumed samples: 406400 | elapsed time per iteration (ms): 87345.3 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.67% | FLOPs estimator: workload-aware | logical tokens/s: 192078.7 | valid tokens/s: 192078.7 | physical tokens/s: 192583.5 | loss tokens/s: 138188.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.069811E-01 | lm-global-supervised-tokens: 1.207008E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.308075E-02 | lm-token-buffer-mib: 5.815336E+01 | global_load_balancing_loss: 9.999233E-01 | loss scale: 1.0 | grad norm: 0.057 | params norm: 11434.670 | params rms: 0.061915 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 17:17:10.692444 | finish at 2026-10-10 16:48:22 + [2026-10-06 23:32:43] iteration 3176/ 6855 | consumed samples: 406528 | elapsed time per iteration (ms): 91539.0 | throughput per GPU (TFLOP/s/GPU): 41.0 | MFU 13.14% | FLOPs estimator: workload-aware | logical tokens/s: 183279.2 | valid tokens/s: 183279.2 | physical tokens/s: 183770.5 | loss tokens/s: 132862.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.756937E-01 | lm-global-supervised-tokens: 1.216212E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.338200E-02 | lm-token-buffer-mib: 5.820877E+01 | global_load_balancing_loss: 1.000460E+00 | loss scale: 1.0 | grad norm: 0.064 | params norm: 11434.603 | params rms: 0.061915 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 21:32:51.863589 | finish at 2026-10-10 21:05:35 + [2026-10-06 23:34:10] iteration 3177/ 6855 | consumed samples: 406656 | elapsed time per iteration (ms): 87055.9 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.57% | FLOPs estimator: workload-aware | logical tokens/s: 192717.5 | valid tokens/s: 192717.5 | physical tokens/s: 193300.0 | loss tokens/s: 141766.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.648518E-01 | lm-global-supervised-tokens: 1.234159E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.147015E-02 | lm-token-buffer-mib: 5.816331E+01 | global_load_balancing_loss: 9.999996E-01 | loss scale: 1.0 | grad norm: 0.039 | params norm: 11434.535 | params rms: 0.061914 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 16:56:31.531281 | finish at 2026-10-10 16:30:42 + [2026-10-06 23:35:35] iteration 3178/ 6855 | consumed samples: 406784 | elapsed time per iteration (ms): 84442.1 | throughput per GPU (TFLOP/s/GPU): 43.6 | MFU 13.98% | FLOPs estimator: workload-aware | logical tokens/s: 198682.5 | valid tokens/s: 198682.5 | physical tokens/s: 199203.1 | loss tokens/s: 152702.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.689207E-01 | lm-global-supervised-tokens: 1.289455E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.034091E-02 | lm-token-buffer-mib: 5.807051E+01 | global_load_balancing_loss: 9.998052E-01 | loss scale: 1.0 | grad norm: 0.051 | params norm: 11434.466 | params rms: 0.061914 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 14:14:53.452844 | finish at 2026-10-10 13:50:28 + [2026-10-06 23:37:00] iteration 3179/ 6855 | consumed samples: 406912 | elapsed time per iteration (ms): 85742.1 | throughput per GPU (TFLOP/s/GPU): 43.5 | MFU 13.93% | FLOPs estimator: workload-aware | logical tokens/s: 195670.2 | valid tokens/s: 195670.2 | physical tokens/s: 196049.9 | loss tokens/s: 142746.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.932226E-01 | lm-global-supervised-tokens: 1.223938E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.390356E-02 | lm-token-buffer-mib: 5.808433E+01 | global_load_balancing_loss: 1.000546E+00 | loss scale: 1.0 | grad norm: 0.050 | params norm: 11434.398 | params rms: 0.061914 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 15:33:07.925421 | finish at 2026-10-10 15:10:08 + [2026-10-06 23:38:53] iteration 3180/ 6855 | consumed samples: 407040 | elapsed time per iteration (ms): 113051.8 | throughput per GPU (TFLOP/s/GPU): 33.7 | MFU 10.81% | FLOPs estimator: workload-aware | logical tokens/s: 148402.2 | valid tokens/s: 148402.2 | physical tokens/s: 148728.5 | loss tokens/s: 105062.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.541107E-01 | lm-global-supervised-tokens: 1.187745E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.306023E-02 | lm-token-buffer-mib: 5.810827E+01 | global_load_balancing_loss: 1.001292E+00 | loss scale: 1.0 | grad norm: 0.067 | params norm: 11434.332 | params rms: 0.061913 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 19:24:25.220475 | finish at 2026-10-11 19:03:19 + [2026-10-06 23:40:20] iteration 3181/ 6855 | consumed samples: 407168 | elapsed time per iteration (ms): 86115.9 | throughput per GPU (TFLOP/s/GPU): 43.2 | MFU 13.84% | FLOPs estimator: workload-aware | logical tokens/s: 194820.6 | valid tokens/s: 194820.6 | physical tokens/s: 195340.6 | loss tokens/s: 139639.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.412818E-01 | lm-global-supervised-tokens: 1.202515E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.457274E-02 | lm-token-buffer-mib: 5.815613E+01 | global_load_balancing_loss: 1.000125E+00 | loss scale: 1.0 | grad norm: 0.032 | params norm: 11434.266 | params rms: 0.061913 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 15:53:09.970913 | finish at 2026-10-10 15:33:30 + [2026-10-06 23:41:49] iteration 3182/ 6855 | consumed samples: 407296 | elapsed time per iteration (ms): 89626.5 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.45% | FLOPs estimator: workload-aware | logical tokens/s: 187190.3 | valid tokens/s: 187190.3 | physical tokens/s: 187653.4 | loss tokens/s: 139166.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.278503E-01 | lm-global-supervised-tokens: 1.247302E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.136514E-02 | lm-token-buffer-mib: 5.809189E+01 | global_load_balancing_loss: 9.994597E-01 | loss scale: 1.0 | grad norm: 0.044 | params norm: 11434.200 | params rms: 0.061912 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 19:26:38.211163 | finish at 2026-10-10 19:08:27 + [2026-10-06 23:43:17] iteration 3183/ 6855 | consumed samples: 407424 | elapsed time per iteration (ms): 87697.0 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.63% | FLOPs estimator: workload-aware | logical tokens/s: 191308.6 | valid tokens/s: 191308.6 | physical tokens/s: 191590.3 | loss tokens/s: 142148.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.484686E-01 | lm-global-supervised-tokens: 1.246601E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.719716E-02 | lm-token-buffer-mib: 5.806182E+01 | global_load_balancing_loss: 1.000191E+00 | loss scale: 1.0 | grad norm: 0.085 | params norm: 11434.135 | params rms: 0.061912 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 17:27:03.286045 | finish at 2026-10-10 17:10:20 + [2026-10-06 23:44:45] iteration 3184/ 6855 | consumed samples: 407552 | elapsed time per iteration (ms): 88052.5 | throughput per GPU (TFLOP/s/GPU): 41.7 | MFU 13.36% | FLOPs estimator: workload-aware | logical tokens/s: 190535.8 | valid tokens/s: 190535.8 | physical tokens/s: 191080.1 | loss tokens/s: 141267.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.446208E-01 | lm-global-supervised-tokens: 1.243899E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.343184E-02 | lm-token-buffer-mib: 5.806935E+01 | global_load_balancing_loss: 1.000380E+00 | loss scale: 1.0 | grad norm: 0.068 | params norm: 11434.070 | params rms: 0.061912 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 17:47:20.548987 | finish at 2026-10-10 17:32:06 + [2026-10-06 23:46:14] iteration 3185/ 6855 | consumed samples: 407680 | elapsed time per iteration (ms): 89227.7 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.43% | FLOPs estimator: workload-aware | logical tokens/s: 188026.7 | valid tokens/s: 188026.6 | physical tokens/s: 188399.0 | loss tokens/s: 138931.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.000923E-01 | lm-global-supervised-tokens: 1.239652E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.153275E-02 | lm-token-buffer-mib: 5.809975E+01 | global_load_balancing_loss: 9.990584E-01 | loss scale: 1.0 | grad norm: 0.067 | params norm: 11434.005 | params rms: 0.061911 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 18:57:45.509357 | finish at 2026-10-10 18:44:00 + [2026-10-06 23:47:39] iteration 3186/ 6855 | consumed samples: 407808 | elapsed time per iteration (ms): 84336.5 | throughput per GPU (TFLOP/s/GPU): 43.1 | MFU 13.82% | FLOPs estimator: workload-aware | logical tokens/s: 198931.4 | valid tokens/s: 198931.4 | physical tokens/s: 199318.4 | loss tokens/s: 144655.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.891851E-01 | lm-global-supervised-tokens: 1.219975E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.340849E-02 | lm-token-buffer-mib: 5.812000E+01 | global_load_balancing_loss: 9.995310E-01 | loss scale: 1.0 | grad norm: 0.043 | params norm: 11433.942 | params rms: 0.061911 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 13:57:10.768699 | finish at 2026-10-10 13:44:49 + [2026-10-06 23:49:04] iteration 3187/ 6855 | consumed samples: 407936 | elapsed time per iteration (ms): 85832.9 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.75% | FLOPs estimator: workload-aware | logical tokens/s: 195463.8 | valid tokens/s: 195463.8 | physical tokens/s: 196080.1 | loss tokens/s: 144616.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.554043E-01 | lm-global-supervised-tokens: 1.241285E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.449504E-02 | lm-token-buffer-mib: 5.811318E+01 | global_load_balancing_loss: 9.998226E-01 | loss scale: 1.0 | grad norm: 0.042 | params norm: 11433.878 | params rms: 0.061911 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 15:27:14.898097 | finish at 2026-10-10 15:16:19 + [2026-10-06 23:50:34] iteration 3188/ 6855 | consumed samples: 408064 | elapsed time per iteration (ms): 89982.4 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.42% | FLOPs estimator: workload-aware | logical tokens/s: 186449.6 | valid tokens/s: 186449.6 | physical tokens/s: 186835.1 | loss tokens/s: 127939.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.304213E-01 | lm-global-supervised-tokens: 1.151226E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.159872E-02 | lm-token-buffer-mib: 5.811175E+01 | global_load_balancing_loss: 1.001005E+00 | loss scale: 1.0 | grad norm: 0.037 | params norm: 11433.814 | params rms: 0.061910 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 19:39:25.444845 | finish at 2026-10-10 19:30:00 + [2026-10-06 23:52:00] iteration 3189/ 6855 | consumed samples: 408192 | elapsed time per iteration (ms): 85881.3 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.52% | FLOPs estimator: workload-aware | logical tokens/s: 195352.7 | valid tokens/s: 195352.7 | physical tokens/s: 195860.6 | loss tokens/s: 147617.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.445738E-01 | lm-global-supervised-tokens: 1.267759E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.286694E-02 | lm-token-buffer-mib: 5.824239E+01 | global_load_balancing_loss: 1.000756E+00 | loss scale: 1.0 | grad norm: 0.063 | params norm: 11433.751 | params rms: 0.061910 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 15:27:20.974184 | finish at 2026-10-10 15:19:21 + [2026-10-06 23:53:51] iteration 3190/ 6855 | consumed samples: 408320 | elapsed time per iteration (ms): 110586.9 | throughput per GPU (TFLOP/s/GPU): 33.6 | MFU 10.77% | FLOPs estimator: workload-aware | logical tokens/s: 151710.2 | valid tokens/s: 151710.2 | physical tokens/s: 151985.2 | loss tokens/s: 111413.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.852716E-01 | lm-global-supervised-tokens: 1.232088E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.306967E-02 | lm-token-buffer-mib: 5.806564E+01 | global_load_balancing_loss: 1.000782E+00 | loss scale: 1.0 | grad norm: 0.037 | params norm: 11433.689 | params rms: 0.061910 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 16:35:01.006835 | finish at 2026-10-11 16:28:52 + [2026-10-06 23:55:19] iteration 3191/ 6855 | consumed samples: 408448 | elapsed time per iteration (ms): 88414.6 | throughput per GPU (TFLOP/s/GPU): 41.3 | MFU 13.25% | FLOPs estimator: workload-aware | logical tokens/s: 189755.1 | valid tokens/s: 189755.1 | physical tokens/s: 190339.1 | loss tokens/s: 147121.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.587458E-01 | lm-global-supervised-tokens: 1.300770E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.762199E-02 | lm-token-buffer-mib: 5.819617E+01 | global_load_balancing_loss: 1.000717E+00 | loss scale: 1.0 | grad norm: 0.068 | params norm: 11433.625 | params rms: 0.061909 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 17:59:10.992683 | finish at 2026-10-10 17:54:30 + [2026-10-06 23:56:50] iteration 3192/ 6855 | consumed samples: 408576 | elapsed time per iteration (ms): 90411.3 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.41% | FLOPs estimator: workload-aware | logical tokens/s: 185565.4 | valid tokens/s: 185565.4 | physical tokens/s: 186008.4 | loss tokens/s: 135686.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.334329E-01 | lm-global-supervised-tokens: 1.226759E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.210149E-02 | lm-token-buffer-mib: 5.817698E+01 | global_load_balancing_loss: 1.000721E+00 | loss scale: 1.0 | grad norm: 0.095 | params norm: 11433.561 | params rms: 0.061909 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 19:59:36.707847 | finish at 2026-10-10 19:56:26 + [2026-10-06 23:58:18] iteration 3193/ 6855 | consumed samples: 408704 | elapsed time per iteration (ms): 88080.8 | throughput per GPU (TFLOP/s/GPU): 43.3 | MFU 13.89% | FLOPs estimator: workload-aware | logical tokens/s: 190475.2 | valid tokens/s: 190475.2 | physical tokens/s: 190939.5 | loss tokens/s: 135267.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.546733E-01 | lm-global-supervised-tokens: 1.191448E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.312358E-02 | lm-token-buffer-mib: 5.807501E+01 | global_load_balancing_loss: 9.991527E-01 | loss scale: 1.0 | grad norm: 0.057 | params norm: 11433.497 | params rms: 0.061909 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 17:35:51.755351 | finish at 2026-10-10 17:34:09 + [2026-10-06 23:59:44] iteration 3194/ 6855 | consumed samples: 408832 | elapsed time per iteration (ms): 86139.0 | throughput per GPU (TFLOP/s/GPU): 43.2 | MFU 13.84% | FLOPs estimator: workload-aware | logical tokens/s: 194768.6 | valid tokens/s: 194768.6 | physical tokens/s: 195165.1 | loss tokens/s: 143574.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.744421E-01 | lm-global-supervised-tokens: 1.236737E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.765074E-02 | lm-token-buffer-mib: 5.806252E+01 | global_load_balancing_loss: 9.988258E-01 | loss scale: 1.0 | grad norm: 0.190 | params norm: 11433.434 | params rms: 0.061908 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 15:35:54.949477 | finish at 2026-10-10 15:35:39 + [2026-10-07 00:01:12] iteration 3195/ 6855 | consumed samples: 408960 | elapsed time per iteration (ms): 88138.1 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.57% | FLOPs estimator: workload-aware | logical tokens/s: 190351.1 | valid tokens/s: 190351.1 | physical tokens/s: 190731.6 | loss tokens/s: 140806.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.795779E-01 | lm-global-supervised-tokens: 1.241038E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.611343E-02 | lm-token-buffer-mib: 5.809147E+01 | global_load_balancing_loss: 9.999962E-01 | loss scale: 1.0 | grad norm: 0.232 | params norm: 11433.373 | params rms: 0.061908 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 17:36:25.453520 | finish at 2026-10-10 17:37:37 + [2026-10-07 00:02:39] iteration 3196/ 6855 | consumed samples: 409088 | elapsed time per iteration (ms): 86853.9 | throughput per GPU (TFLOP/s/GPU): 43.3 | MFU 13.88% | FLOPs estimator: workload-aware | logical tokens/s: 193165.8 | valid tokens/s: 193165.8 | physical tokens/s: 193635.5 | loss tokens/s: 132851.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.784876E-01 | lm-global-supervised-tokens: 1.153869E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.647422E-02 | lm-token-buffer-mib: 5.815952E+01 | global_load_balancing_loss: 1.000622E+00 | loss scale: 1.0 | grad norm: 0.109 | params norm: 11433.312 | params rms: 0.061908 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 16:16:38.508048 | finish at 2026-10-10 16:19:17 + [2026-10-07 00:04:03] iteration 3197/ 6855 | consumed samples: 409216 | elapsed time per iteration (ms): 84406.1 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.77% | FLOPs estimator: workload-aware | logical tokens/s: 198767.1 | valid tokens/s: 198767.1 | physical tokens/s: 199168.0 | loss tokens/s: 149419.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.638715E-01 | lm-global-supervised-tokens: 1.261193E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.148340E-02 | lm-token-buffer-mib: 5.820146E+01 | global_load_balancing_loss: 9.999623E-01 | loss scale: 1.0 | grad norm: 0.338 | params norm: 11433.250 | params rms: 0.061907 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 13:45:57.479042 | finish at 2026-10-10 13:50:01 + [2026-10-07 00:05:32] iteration 3198/ 6855 | consumed samples: 409344 | elapsed time per iteration (ms): 88734.5 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.54% | FLOPs estimator: workload-aware | logical tokens/s: 189071.3 | valid tokens/s: 189071.2 | physical tokens/s: 189566.6 | loss tokens/s: 137772.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.522877E-01 | lm-global-supervised-tokens: 1.222518E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.151434E-02 | lm-token-buffer-mib: 5.821123E+01 | global_load_balancing_loss: 1.002288E+00 | loss scale: 1.0 | grad norm: 0.074 | params norm: 11433.191 | params rms: 0.061907 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 18:08:21.916283 | finish at 2026-10-10 18:13:54 + [2026-10-07 00:06:57] iteration 3199/ 6855 | consumed samples: 409472 | elapsed time per iteration (ms): 85397.6 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.57% | FLOPs estimator: workload-aware | logical tokens/s: 196459.1 | valid tokens/s: 196459.1 | physical tokens/s: 196905.5 | loss tokens/s: 144422.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.620401E-01 | lm-global-supervised-tokens: 1.233336E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.443799E-02 | lm-token-buffer-mib: 5.812924E+01 | global_load_balancing_loss: 1.000763E+00 | loss scale: 1.0 | grad norm: 0.637 | params norm: 11433.129 | params rms: 0.061907 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 14:43:33.515535 | finish at 2026-10-10 14:50:31 + [2026-10-07 00:08:48] iteration 3200/ 6855 | consumed samples: 409600 | elapsed time per iteration (ms): 110568.4 | throughput per GPU (TFLOP/s/GPU): 33.8 | MFU 10.84% | FLOPs estimator: workload-aware | logical tokens/s: 151735.3 | valid tokens/s: 151735.3 | physical tokens/s: 152131.7 | loss tokens/s: 109090.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.960749E-01 | lm-global-supervised-tokens: 1.206196E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.351222E-02 | lm-token-buffer-mib: 5.820473E+01 | global_load_balancing_loss: 1.001118E+00 | loss scale: 1.0 | grad norm: 0.330 | params norm: 11433.065 | params rms: 0.061906 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 16:15:27.532157 | finish at 2026-10-11 16:24:15 +times across ranks (ms): + save-checkpoint: + rank 0: 24543.79 + rank 1: 24543.77 + rank 2: 24543.81 + rank 3: 24543.78 + rank 4: 24543.78 + rank 5: 24543.77 + rank 6: 24543.74 + rank 7: 24543.79 + rank 8: 24543.60 + rank 9: 24543.57 + rank 10: 24543.57 + rank 11: 24543.55 + rank 12: 24543.42 + rank 13: 24543.57 + rank 14: 24543.56 + rank 15: 24543.55 + rank 16: 24543.49 + rank 17: 24543.56 + rank 18: 24543.43 + rank 19: 24543.47 + rank 20: 24543.43 + rank 21: 24543.48 + rank 22: 24543.43 + rank 23: 24543.44 + rank 24: 24863.02 + rank 25: 24862.95 + rank 26: 24862.94 + rank 27: 24862.98 + rank 28: 24862.86 + rank 29: 24862.81 + rank 30: 24862.78 + rank 31: 24863.03 + rank 32: 24543.16 + rank 33: 24543.00 + rank 34: 24543.19 + rank 35: 24543.08 + rank 36: 24543.09 + rank 37: 24543.07 + rank 38: 24543.10 + rank 39: 24543.13 + rank 40: 24543.58 + rank 41: 24543.50 + rank 42: 24543.53 + rank 43: 24543.52 + rank 44: 24543.49 + rank 45: 24543.50 + rank 46: 24543.51 + rank 47: 24543.49 + rank 48: 24544.15 + rank 49: 24544.09 + rank 50: 24544.07 + rank 51: 24543.99 + rank 52: 24544.10 + rank 53: 24544.07 + rank 54: 24544.15 + rank 55: 24544.14 + rank 56: 24543.94 + rank 57: 24543.97 + rank 58: 24543.88 + rank 59: 24543.88 + rank 60: 24543.88 + rank 61: 24543.95 + rank 62: 24543.88 + rank 63: 24543.92 + [2026-10-07 00:10:44] iteration 3201/ 6855 | consumed samples: 409728 | elapsed time per iteration (ms): 90801.3 | throughput per GPU (TFLOP/s/GPU): 40.6 | MFU 13.02% | FLOPs estimator: workload-aware | logical tokens/s: 184767.0 | valid tokens/s: 184767.0 | physical tokens/s: 185206.4 | loss tokens/s: 135603.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.704366E-01 | lm-global-supervised-tokens: 1.231296E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.259861E+00 | lm-token-buffer-mib: 5.824481E+01 | global_load_balancing_loss: 9.994088E-01 | loss scale: 1.0 | grad norm: 0.339 | params norm: 11433.003 | params rms: 0.061906 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 20:09:48.063632 | finish at 2026-10-10 20:20:32 + [2026-10-07 00:12:17] iteration 3202/ 6855 | consumed samples: 409856 | elapsed time per iteration (ms): 93078.1 | throughput per GPU (TFLOP/s/GPU): 39.6 | MFU 12.68% | FLOPs estimator: workload-aware | logical tokens/s: 180248.7 | valid tokens/s: 180248.7 | physical tokens/s: 180652.3 | loss tokens/s: 131394.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.748762E-01 | lm-global-supervised-tokens: 1.222993E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.739343E-02 | lm-token-buffer-mib: 5.820204E+01 | global_load_balancing_loss: 9.992709E-01 | loss scale: 1.0 | grad norm: 0.146 | params norm: 11432.942 | params rms: 0.061906 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 22:26:54.156341 | finish at 2026-10-10 22:39:11 + [2026-10-07 00:13:43] iteration 3203/ 6855 | consumed samples: 409984 | elapsed time per iteration (ms): 86288.4 | throughput per GPU (TFLOP/s/GPU): 43.5 | MFU 13.94% | FLOPs estimator: workload-aware | logical tokens/s: 194431.6 | valid tokens/s: 194431.6 | physical tokens/s: 195039.9 | loss tokens/s: 146978.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.002981E-01 | lm-global-supervised-tokens: 1.268253E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.215530E-02 | lm-token-buffer-mib: 5.817803E+01 | global_load_balancing_loss: 9.982752E-01 | loss scale: 1.0 | grad norm: 0.105 | params norm: 11432.879 | params rms: 0.061905 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 15:32:05.170388 | finish at 2026-10-10 15:45:49 + [2026-10-07 00:15:12] iteration 3204/ 6855 | consumed samples: 410112 | elapsed time per iteration (ms): 88807.4 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.39% | FLOPs estimator: workload-aware | logical tokens/s: 188916.1 | valid tokens/s: 188916.1 | physical tokens/s: 189229.8 | loss tokens/s: 140247.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.147044E-01 | lm-global-supervised-tokens: 1.245501E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.377124E-02 | lm-token-buffer-mib: 5.813549E+01 | global_load_balancing_loss: 9.991114E-01 | loss scale: 1.0 | grad norm: 0.175 | params norm: 11432.815 | params rms: 0.061905 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 18:03:55.834766 | finish at 2026-10-10 18:19:08 + [2026-10-07 00:16:38] iteration 3205/ 6855 | consumed samples: 410240 | elapsed time per iteration (ms): 85526.9 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.55% | FLOPs estimator: workload-aware | logical tokens/s: 196162.3 | valid tokens/s: 196162.3 | physical tokens/s: 196761.1 | loss tokens/s: 146600.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.314144E-01 | lm-global-supervised-tokens: 1.253828E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.158932E-02 | lm-token-buffer-mib: 5.857890E+01 | global_load_balancing_loss: 1.000055E+00 | loss scale: 1.0 | grad norm: 0.218 | params norm: 11432.751 | params rms: 0.061905 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 14:42:53.187804 | finish at 2026-10-10 14:59:31 + [2026-10-07 00:18:05] iteration 3206/ 6855 | consumed samples: 410368 | elapsed time per iteration (ms): 86879.4 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.77% | FLOPs estimator: workload-aware | logical tokens/s: 193108.8 | valid tokens/s: 193108.8 | physical tokens/s: 193487.2 | loss tokens/s: 137545.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.754441E-01 | lm-global-supervised-tokens: 1.194989E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.437347E-02 | lm-token-buffer-mib: 5.810960E+01 | global_load_balancing_loss: 1.001812E+00 | loss scale: 1.0 | grad norm: 0.093 | params norm: 11432.689 | params rms: 0.061904 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 16:03:42.875845 | finish at 2026-10-10 16:21:48 + [2026-10-07 00:19:31] iteration 3207/ 6855 | consumed samples: 410496 | elapsed time per iteration (ms): 85856.6 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.74% | FLOPs estimator: workload-aware | logical tokens/s: 195408.4 | valid tokens/s: 195408.4 | physical tokens/s: 195906.3 | loss tokens/s: 145789.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.674535E-01 | lm-global-supervised-tokens: 1.251702E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.389142E-02 | lm-token-buffer-mib: 5.822511E+01 | global_load_balancing_loss: 1.001313E+00 | loss scale: 1.0 | grad norm: 0.142 | params norm: 11432.626 | params rms: 0.061904 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 15:00:05.018738 | finish at 2026-10-10 15:19:36 + [2026-10-07 00:20:57] iteration 3208/ 6855 | consumed samples: 410624 | elapsed time per iteration (ms): 86745.3 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.61% | FLOPs estimator: workload-aware | logical tokens/s: 193407.5 | valid tokens/s: 193407.5 | physical tokens/s: 193887.6 | loss tokens/s: 142112.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.440001E-01 | lm-global-supervised-tokens: 1.232763E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.365150E-02 | lm-token-buffer-mib: 5.815786E+01 | global_load_balancing_loss: 9.998912E-01 | loss scale: 1.0 | grad norm: 0.150 | params norm: 11432.564 | params rms: 0.061904 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 15:52:40.235378 | finish at 2026-10-10 16:13:37 + [2026-10-07 00:22:23] iteration 3209/ 6855 | consumed samples: 410752 | elapsed time per iteration (ms): 86134.4 | throughput per GPU (TFLOP/s/GPU): 41.7 | MFU 13.38% | FLOPs estimator: workload-aware | logical tokens/s: 194779.5 | valid tokens/s: 194779.5 | physical tokens/s: 195178.2 | loss tokens/s: 148153.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.399528E-01 | lm-global-supervised-tokens: 1.276108E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.666264E-02 | lm-token-buffer-mib: 5.805421E+01 | global_load_balancing_loss: 1.000054E+00 | loss scale: 1.0 | grad norm: 0.146 | params norm: 11432.502 | params rms: 0.061903 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 15:14:05.897696 | finish at 2026-10-10 15:36:29 + [2026-10-07 00:24:13] iteration 3210/ 6855 | consumed samples: 410880 | elapsed time per iteration (ms): 109836.3 | throughput per GPU (TFLOP/s/GPU): 33.7 | MFU 10.80% | FLOPs estimator: workload-aware | logical tokens/s: 152747.3 | valid tokens/s: 152747.3 | physical tokens/s: 153095.3 | loss tokens/s: 110745.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.559381E-01 | lm-global-supervised-tokens: 1.216383E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.175580E-02 | lm-token-buffer-mib: 5.819579E+01 | global_load_balancing_loss: 1.000398E+00 | loss scale: 1.0 | grad norm: 0.052 | params norm: 11432.438 | params rms: 0.061903 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 15:12:33.277491 | finish at 2026-10-11 15:36:46 + [2026-10-07 00:25:41] iteration 3211/ 6855 | consumed samples: 411008 | elapsed time per iteration (ms): 87966.3 | throughput per GPU (TFLOP/s/GPU): 42.8 | MFU 13.73% | FLOPs estimator: workload-aware | logical tokens/s: 190723.1 | valid tokens/s: 190723.1 | physical tokens/s: 191158.0 | loss tokens/s: 133891.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.877193E-01 | lm-global-supervised-tokens: 1.177794E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.355547E-02 | lm-token-buffer-mib: 5.803848E+01 | global_load_balancing_loss: 1.000599E+00 | loss scale: 1.0 | grad norm: 0.087 | params norm: 11432.373 | params rms: 0.061903 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 17:02:29.074739 | finish at 2026-10-10 17:28:10 + [2026-10-07 00:27:08] iteration 3212/ 6855 | consumed samples: 411136 | elapsed time per iteration (ms): 86359.7 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.75% | FLOPs estimator: workload-aware | logical tokens/s: 194270.0 | valid tokens/s: 194270.0 | physical tokens/s: 194811.2 | loss tokens/s: 133935.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.857705E-01 | lm-global-supervised-tokens: 1.156659E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.221341E-02 | lm-token-buffer-mib: 5.819079E+01 | global_load_balancing_loss: 1.000554E+00 | loss scale: 1.0 | grad norm: 0.151 | params norm: 11432.309 | params rms: 0.061902 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 15:23:28.208916 | finish at 2026-10-10 15:50:36 + [2026-10-07 00:28:37] iteration 3213/ 6855 | consumed samples: 411264 | elapsed time per iteration (ms): 89424.8 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.53% | FLOPs estimator: workload-aware | logical tokens/s: 187612.2 | valid tokens/s: 187612.2 | physical tokens/s: 188041.6 | loss tokens/s: 136852.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.879132E-01 | lm-global-supervised-tokens: 1.223798E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.115806E-02 | lm-token-buffer-mib: 5.815011E+01 | global_load_balancing_loss: 9.993785E-01 | loss scale: 1.0 | grad norm: 0.046 | params norm: 11432.246 | params rms: 0.061902 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 18:28:05.165590 | finish at 2026-10-10 18:56:42 + [2026-10-07 00:30:04] iteration 3214/ 6855 | consumed samples: 411392 | elapsed time per iteration (ms): 87459.0 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.47% | FLOPs estimator: workload-aware | logical tokens/s: 191829.2 | valid tokens/s: 191829.2 | physical tokens/s: 192287.9 | loss tokens/s: 142255.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.113874E-01 | lm-global-supervised-tokens: 1.244152E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.569752E-02 | lm-token-buffer-mib: 5.806046E+01 | global_load_balancing_loss: 9.994377E-01 | loss scale: 1.0 | grad norm: 0.095 | params norm: 11432.181 | params rms: 0.061902 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 16:27:18.081378 | finish at 2026-10-10 16:57:23 + [2026-10-07 00:31:36] iteration 3215/ 6855 | consumed samples: 411520 | elapsed time per iteration (ms): 91975.7 | throughput per GPU (TFLOP/s/GPU): 41.1 | MFU 13.18% | FLOPs estimator: workload-aware | logical tokens/s: 182409.0 | valid tokens/s: 182409.0 | physical tokens/s: 182826.6 | loss tokens/s: 134912.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.230191E-01 | lm-global-supervised-tokens: 1.240868E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.309325E-02 | lm-token-buffer-mib: 5.814472E+01 | global_load_balancing_loss: 9.998466E-01 | loss scale: 1.0 | grad norm: 0.147 | params norm: 11432.118 | params rms: 0.061901 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 20:59:51.492968 | finish at 2026-10-10 21:31:28 + [2026-10-07 00:33:05] iteration 3216/ 6855 | consumed samples: 411648 | elapsed time per iteration (ms): 88392.9 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.67% | FLOPs estimator: workload-aware | logical tokens/s: 189801.9 | valid tokens/s: 189801.9 | physical tokens/s: 190134.3 | loss tokens/s: 145023.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.714236E-01 | lm-global-supervised-tokens: 1.281907E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.163164E-02 | lm-token-buffer-mib: 5.825945E+01 | global_load_balancing_loss: 1.000153E+00 | loss scale: 1.0 | grad norm: 0.032 | params norm: 11432.056 | params rms: 0.061901 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 17:21:01.630320 | finish at 2026-10-10 17:54:06 + [2026-10-07 00:34:32] iteration 3217/ 6855 | consumed samples: 411776 | elapsed time per iteration (ms): 87060.5 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.62% | FLOPs estimator: workload-aware | logical tokens/s: 192707.4 | valid tokens/s: 192707.4 | physical tokens/s: 193086.6 | loss tokens/s: 137231.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.403080E-01 | lm-global-supervised-tokens: 1.194746E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.227664E-02 | lm-token-buffer-mib: 5.821016E+01 | global_load_balancing_loss: 1.000773E+00 | loss scale: 1.0 | grad norm: 0.101 | params norm: 11431.994 | params rms: 0.061901 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 15:58:46.253919 | finish at 2026-10-10 16:33:18 + [2026-10-07 00:35:54] iteration 3218/ 6855 | consumed samples: 411904 | elapsed time per iteration (ms): 82334.9 | throughput per GPU (TFLOP/s/GPU): 44.4 | MFU 14.22% | FLOPs estimator: workload-aware | logical tokens/s: 203767.6 | valid tokens/s: 203767.6 | physical tokens/s: 204206.5 | loss tokens/s: 151672.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.535178E-01 | lm-global-supervised-tokens: 1.248794E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.072369E-02 | lm-token-buffer-mib: 5.814127E+01 | global_load_balancing_loss: 1.000339E+00 | loss scale: 1.0 | grad norm: 0.084 | params norm: 11431.930 | params rms: 0.061900 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 11:10:51.900876 | finish at 2026-10-10 11:46:46 + [2026-10-07 00:37:19] iteration 3219/ 6855 | consumed samples: 412032 | elapsed time per iteration (ms): 85061.3 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.75% | FLOPs estimator: workload-aware | logical tokens/s: 197236.8 | valid tokens/s: 197236.8 | physical tokens/s: 197546.4 | loss tokens/s: 143894.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.306148E-01 | lm-global-supervised-tokens: 1.223986E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.333231E-02 | lm-token-buffer-mib: 5.804124E+01 | global_load_balancing_loss: 1.000514E+00 | loss scale: 1.0 | grad norm: 0.029 | params norm: 11431.866 | params rms: 0.061900 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 13:54:42.855735 | finish at 2026-10-10 14:32:02 + [2026-10-07 00:39:13] iteration 3220/ 6855 | consumed samples: 412160 | elapsed time per iteration (ms): 113977.1 | throughput per GPU (TFLOP/s/GPU): 33.4 | MFU 10.71% | FLOPs estimator: workload-aware | logical tokens/s: 147197.9 | valid tokens/s: 147197.9 | physical tokens/s: 147476.3 | loss tokens/s: 105729.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.134939E-01 | lm-global-supervised-tokens: 1.205072E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.222594E-02 | lm-token-buffer-mib: 5.812269E+01 | global_load_balancing_loss: 1.000258E+00 | loss scale: 1.0 | grad norm: 0.108 | params norm: 11431.803 | params rms: 0.061899 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 19:05:06.626389 | finish at 2026-10-11 19:44:20 + [2026-10-07 00:40:39] iteration 3221/ 6855 | consumed samples: 412288 | elapsed time per iteration (ms): 85987.3 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.70% | FLOPs estimator: workload-aware | logical tokens/s: 195112.4 | valid tokens/s: 195112.4 | physical tokens/s: 195710.3 | loss tokens/s: 143864.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.819357E-01 | lm-global-supervised-tokens: 1.237056E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.560685E-02 | lm-token-buffer-mib: 5.824516E+01 | global_load_balancing_loss: 9.998932E-01 | loss scale: 1.0 | grad norm: 0.072 | params norm: 11431.740 | params rms: 0.061899 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 14:47:57.994330 | finish at 2026-10-10 15:28:37 + [2026-10-07 00:42:06] iteration 3222/ 6855 | consumed samples: 412416 | elapsed time per iteration (ms): 86445.1 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.46% | FLOPs estimator: workload-aware | logical tokens/s: 194079.4 | valid tokens/s: 194079.4 | physical tokens/s: 194650.0 | loss tokens/s: 149546.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.031209E-01 | lm-global-supervised-tokens: 1.292752E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.363648E-02 | lm-token-buffer-mib: 5.817426E+01 | global_load_balancing_loss: 9.993636E-01 | loss scale: 1.0 | grad norm: 0.140 | params norm: 11431.678 | params rms: 0.061899 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 15:14:15.116979 | finish at 2026-10-10 15:56:21 + [2026-10-07 00:43:35] iteration 3223/ 6855 | consumed samples: 412544 | elapsed time per iteration (ms): 89500.7 | throughput per GPU (TFLOP/s/GPU): 41.7 | MFU 13.37% | FLOPs estimator: workload-aware | logical tokens/s: 187453.1 | valid tokens/s: 187453.1 | physical tokens/s: 187891.6 | loss tokens/s: 140950.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.217751E-01 | lm-global-supervised-tokens: 1.261519E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.155127E-02 | lm-token-buffer-mib: 5.816579E+01 | global_load_balancing_loss: 1.000190E+00 | loss scale: 1.0 | grad norm: 0.048 | params norm: 11431.614 | params rms: 0.061898 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 18:17:46.564903 | finish at 2026-10-10 19:01:22 + [2026-10-07 00:45:00] iteration 3224/ 6855 | consumed samples: 412672 | elapsed time per iteration (ms): 84494.3 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.62% | FLOPs estimator: workload-aware | logical tokens/s: 198559.7 | valid tokens/s: 198559.7 | physical tokens/s: 199088.2 | loss tokens/s: 145751.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.570064E-01 | lm-global-supervised-tokens: 1.231519E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.123195E-02 | lm-token-buffer-mib: 5.828395E+01 | global_load_balancing_loss: 1.000263E+00 | loss scale: 1.0 | grad norm: 0.060 | params norm: 11431.550 | params rms: 0.061898 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 13:13:18.897257 | finish at 2026-10-10 13:58:19 + [2026-10-07 00:46:30] iteration 3225/ 6855 | consumed samples: 412800 | elapsed time per iteration (ms): 90253.8 | throughput per GPU (TFLOP/s/GPU): 41.4 | MFU 13.26% | FLOPs estimator: workload-aware | logical tokens/s: 185888.8 | valid tokens/s: 185888.8 | physical tokens/s: 186413.0 | loss tokens/s: 134792.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.882040E-01 | lm-global-supervised-tokens: 1.216550E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.146360E-02 | lm-token-buffer-mib: 5.814700E+01 | global_load_balancing_loss: 9.999585E-01 | loss scale: 1.0 | grad norm: 0.074 | params norm: 11431.487 | params rms: 0.061898 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 19:00:21.242630 | finish at 2026-10-10 19:46:51 + [2026-10-07 00:47:57] iteration 3226/ 6855 | consumed samples: 412928 | elapsed time per iteration (ms): 87445.0 | throughput per GPU (TFLOP/s/GPU): 41.6 | MFU 13.34% | FLOPs estimator: workload-aware | logical tokens/s: 191859.7 | valid tokens/s: 191859.7 | physical tokens/s: 192221.8 | loss tokens/s: 146881.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.328377E-01 | lm-global-supervised-tokens: 1.284405E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.599467E-02 | lm-token-buffer-mib: 5.816059E+01 | global_load_balancing_loss: 1.000606E+00 | loss scale: 1.0 | grad norm: 0.068 | params norm: 11431.423 | params rms: 0.061897 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 16:08:58.078667 | finish at 2026-10-10 16:56:55 + [2026-10-07 00:49:24] iteration 3227/ 6855 | consumed samples: 413056 | elapsed time per iteration (ms): 86224.5 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.56% | FLOPs estimator: workload-aware | logical tokens/s: 194575.7 | valid tokens/s: 194575.7 | physical tokens/s: 195047.4 | loss tokens/s: 141230.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.056078E-01 | lm-global-supervised-tokens: 1.217750E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.232087E-02 | lm-token-buffer-mib: 5.811743E+01 | global_load_balancing_loss: 1.000427E+00 | loss scale: 1.0 | grad norm: 0.076 | params norm: 11431.360 | params rms: 0.061897 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 14:53:42.392367 | finish at 2026-10-10 15:43:06 + [2026-10-07 00:50:53] iteration 3228/ 6855 | consumed samples: 413184 | elapsed time per iteration (ms): 89145.9 | throughput per GPU (TFLOP/s/GPU): 41.5 | MFU 13.31% | FLOPs estimator: workload-aware | logical tokens/s: 188199.2 | valid tokens/s: 188199.2 | physical tokens/s: 188627.5 | loss tokens/s: 145030.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.301189E-01 | lm-global-supervised-tokens: 1.292886E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.671777E-02 | lm-token-buffer-mib: 5.809047E+01 | global_load_balancing_loss: 1.001451E+00 | loss scale: 1.0 | grad norm: 0.083 | params norm: 11431.295 | params rms: 0.061897 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 17:48:52.267544 | finish at 2026-10-10 18:39:45 + [2026-10-07 00:52:18] iteration 3229/ 6855 | consumed samples: 413312 | elapsed time per iteration (ms): 85539.7 | throughput per GPU (TFLOP/s/GPU): 43.8 | MFU 14.04% | FLOPs estimator: workload-aware | logical tokens/s: 196133.0 | valid tokens/s: 196133.0 | physical tokens/s: 196549.5 | loss tokens/s: 146197.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.956683E-01 | lm-global-supervised-tokens: 1.250568E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.276863E-02 | lm-token-buffer-mib: 5.817376E+01 | global_load_balancing_loss: 1.000227E+00 | loss scale: 1.0 | grad norm: 0.065 | params norm: 11431.232 | params rms: 0.061896 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 14:09:26.866727 | finish at 2026-10-10 15:01:45 + [2026-10-07 00:54:06] iteration 3230/ 6855 | consumed samples: 413440 | elapsed time per iteration (ms): 107555.0 | throughput per GPU (TFLOP/s/GPU): 33.6 | MFU 10.76% | FLOPs estimator: workload-aware | logical tokens/s: 155986.9 | valid tokens/s: 155986.9 | physical tokens/s: 156338.6 | loss tokens/s: 116508.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.754039E-01 | lm-global-supervised-tokens: 1.253111E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.192912E-02 | lm-token-buffer-mib: 5.818129E+01 | global_load_balancing_loss: 1.000666E+00 | loss scale: 1.0 | grad norm: 0.089 | params norm: 11431.167 | params rms: 0.061896 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 12:18:06.777580 | finish at 2026-10-11 13:12:13 + [2026-10-07 00:55:37] iteration 3231/ 6855 | consumed samples: 413568 | elapsed time per iteration (ms): 91098.9 | throughput per GPU (TFLOP/s/GPU): 40.4 | MFU 12.96% | FLOPs estimator: workload-aware | logical tokens/s: 184164.4 | valid tokens/s: 184164.4 | physical tokens/s: 184531.1 | loss tokens/s: 140646.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.319726E-01 | lm-global-supervised-tokens: 1.281277E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.150444E-02 | lm-token-buffer-mib: 5.803320E+01 | global_load_balancing_loss: 9.990002E-01 | loss scale: 1.0 | grad norm: 0.051 | params norm: 11431.103 | params rms: 0.061896 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 19:42:22.355999 | finish at 2026-10-10 20:37:59 + [2026-10-07 00:57:02] iteration 3232/ 6855 | consumed samples: 413696 | elapsed time per iteration (ms): 85401.4 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.75% | FLOPs estimator: workload-aware | logical tokens/s: 196450.4 | valid tokens/s: 196450.4 | physical tokens/s: 196876.6 | loss tokens/s: 143418.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.277409E-01 | lm-global-supervised-tokens: 1.224811E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.525767E-02 | lm-token-buffer-mib: 5.803220E+01 | global_load_balancing_loss: 9.997391E-01 | loss scale: 1.0 | grad norm: 0.059 | params norm: 11431.040 | params rms: 0.061895 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 13:56:49.213786 | finish at 2026-10-10 14:53:52 + [2026-10-07 00:58:30] iteration 3233/ 6855 | consumed samples: 413824 | elapsed time per iteration (ms): 87236.1 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.51% | FLOPs estimator: workload-aware | logical tokens/s: 192319.5 | valid tokens/s: 192319.5 | physical tokens/s: 192872.2 | loss tokens/s: 146519.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.016682E-01 | lm-global-supervised-tokens: 1.278178E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.127524E-02 | lm-token-buffer-mib: 5.814828E+01 | global_load_balancing_loss: 1.000315E+00 | loss scale: 1.0 | grad norm: 0.069 | params norm: 11430.977 | params rms: 0.061895 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 15:46:09.261993 | finish at 2026-10-10 16:44:39 + [2026-10-07 00:59:57] iteration 3234/ 6855 | consumed samples: 413952 | elapsed time per iteration (ms): 87476.6 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.80% | FLOPs estimator: workload-aware | logical tokens/s: 191790.2 | valid tokens/s: 191790.2 | physical tokens/s: 192261.6 | loss tokens/s: 145454.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.722412E-01 | lm-global-supervised-tokens: 1.272389E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 9.779523E-02 | lm-token-buffer-mib: 5.819541E+01 | global_load_balancing_loss: 1.000163E+00 | loss scale: 1.0 | grad norm: 0.043 | params norm: 11430.915 | params rms: 0.061895 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 15:59:12.645762 | finish at 2026-10-10 16:59:10 + [2026-10-07 01:01:24] iteration 3235/ 6855 | consumed samples: 414080 | elapsed time per iteration (ms): 86520.4 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.66% | FLOPs estimator: workload-aware | logical tokens/s: 193910.1 | valid tokens/s: 193910.0 | physical tokens/s: 194241.3 | loss tokens/s: 143828.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.827762E-01 | lm-global-supervised-tokens: 1.244414E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.875458E-02 | lm-token-buffer-mib: 5.806671E+01 | global_load_balancing_loss: 1.000069E+00 | loss scale: 1.0 | grad norm: 0.048 | params norm: 11430.851 | params rms: 0.061894 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 15:00:04.006114 | finish at 2026-10-10 16:01:28 + [2026-10-07 01:02:52] iteration 3236/ 6855 | consumed samples: 414208 | elapsed time per iteration (ms): 88546.2 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.51% | FLOPs estimator: workload-aware | logical tokens/s: 189473.6 | valid tokens/s: 189473.6 | physical tokens/s: 189953.1 | loss tokens/s: 139488.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.814990E-01 | lm-global-supervised-tokens: 1.235114E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.629125E-02 | lm-token-buffer-mib: 5.806123E+01 | global_load_balancing_loss: 1.000256E+00 | loss scale: 1.0 | grad norm: 0.097 | params norm: 11430.785 | params rms: 0.061894 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 17:00:48.813554 | finish at 2026-10-10 18:03:41 + [2026-10-07 01:04:20] iteration 3237/ 6855 | consumed samples: 414336 | elapsed time per iteration (ms): 87979.0 | throughput per GPU (TFLOP/s/GPU): 41.6 | MFU 13.32% | FLOPs estimator: workload-aware | logical tokens/s: 190695.2 | valid tokens/s: 190695.2 | physical tokens/s: 191160.9 | loss tokens/s: 147702.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.242498E-01 | lm-global-supervised-tokens: 1.299474E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.149403E-02 | lm-token-buffer-mib: 5.823256E+01 | global_load_balancing_loss: 9.999599E-01 | loss scale: 1.0 | grad norm: 0.123 | params norm: 11430.719 | params rms: 0.061894 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 16:25:07.993865 | finish at 2026-10-10 17:29:28 + [2026-10-07 01:05:48] iteration 3238/ 6855 | consumed samples: 414464 | elapsed time per iteration (ms): 88011.3 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.57% | FLOPs estimator: workload-aware | logical tokens/s: 190625.6 | valid tokens/s: 190625.6 | physical tokens/s: 191157.4 | loss tokens/s: 138988.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.982065E-01 | lm-global-supervised-tokens: 1.223256E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.188382E-02 | lm-token-buffer-mib: 5.806068E+01 | global_load_balancing_loss: 1.000402E+00 | loss scale: 1.0 | grad norm: 0.080 | params norm: 11430.653 | params rms: 0.061893 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 16:25:36.987971 | finish at 2026-10-10 17:31:25 + [2026-10-07 01:07:13] iteration 3239/ 6855 | consumed samples: 414592 | elapsed time per iteration (ms): 84765.5 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.66% | FLOPs estimator: workload-aware | logical tokens/s: 197924.6 | valid tokens/s: 197924.6 | physical tokens/s: 198387.7 | loss tokens/s: 147739.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.377581E-01 | lm-global-supervised-tokens: 1.252317E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.237486E-02 | lm-token-buffer-mib: 5.814621E+01 | global_load_balancing_loss: 1.000230E+00 | loss scale: 1.0 | grad norm: 0.044 | params norm: 11430.587 | params rms: 0.061893 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 13:08:32.032730 | finish at 2026-10-10 14:15:45 + [2026-10-07 01:09:04] iteration 3240/ 6855 | consumed samples: 414720 | elapsed time per iteration (ms): 110872.4 | throughput per GPU (TFLOP/s/GPU): 33.5 | MFU 10.73% | FLOPs estimator: workload-aware | logical tokens/s: 151319.7 | valid tokens/s: 151319.7 | physical tokens/s: 151764.6 | loss tokens/s: 109820.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.355961E-01 | lm-global-supervised-tokens: 1.217603E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.150472E-02 | lm-token-buffer-mib: 5.809202E+01 | global_load_balancing_loss: 1.000551E+00 | loss scale: 1.0 | grad norm: 0.184 | params norm: 11430.521 | params rms: 0.061893 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 15:20:03.604639 | finish at 2026-10-11 16:29:07 +times across ranks (ms): + save-checkpoint: + rank 0: 23708.47 + rank 1: 23708.48 + rank 2: 23708.58 + rank 3: 23708.47 + rank 4: 23708.48 + rank 5: 23708.47 + rank 6: 23708.46 + rank 7: 23708.47 + rank 8: 23707.36 + rank 9: 23707.35 + rank 10: 23707.35 + rank 11: 23707.36 + rank 12: 23707.37 + rank 13: 23707.32 + rank 14: 23707.37 + rank 15: 23707.34 + rank 16: 23707.74 + rank 17: 23707.74 + rank 18: 23707.74 + rank 19: 23707.78 + rank 20: 23707.79 + rank 21: 23707.74 + rank 22: 23707.73 + rank 23: 23707.76 + rank 24: 23707.21 + rank 25: 23707.17 + rank 26: 23707.18 + rank 27: 23707.27 + rank 28: 23707.19 + rank 29: 23707.18 + rank 30: 23707.18 + rank 31: 23707.29 + rank 32: 23705.82 + rank 33: 23705.79 + rank 34: 23705.85 + rank 35: 23705.78 + rank 36: 23705.82 + rank 37: 23705.80 + rank 38: 23705.80 + rank 39: 23705.78 + rank 40: 23707.51 + rank 41: 23707.49 + rank 42: 23707.50 + rank 43: 23707.48 + rank 44: 23707.48 + rank 45: 23707.49 + rank 46: 23707.48 + rank 47: 23707.46 + rank 48: 23708.79 + rank 49: 23708.86 + rank 50: 23708.72 + rank 51: 23708.71 + rank 52: 23708.76 + rank 53: 23708.69 + rank 54: 23708.72 + rank 55: 23708.71 + rank 56: 23708.04 + rank 57: 23708.01 + rank 58: 23707.99 + rank 59: 23707.97 + rank 60: 23707.98 + rank 61: 23707.99 + rank 62: 23707.97 + rank 63: 23707.99 + [2026-10-07 01:11:29] iteration 3241/ 6855 | consumed samples: 414848 | elapsed time per iteration (ms): 121634.9 | throughput per GPU (TFLOP/s/GPU): 30.2 | MFU 9.67% | FLOPs estimator: workload-aware | logical tokens/s: 137930.8 | valid tokens/s: 137930.8 | physical tokens/s: 138277.8 | loss tokens/s: 103106.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.491941E-01 | lm-global-supervised-tokens: 1.254130E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 3.985248E-01 | lm-token-buffer-mib: 5.816190E+01 | global_load_balancing_loss: 9.999495E-01 | loss scale: 1.0 | grad norm: 0.290 | params norm: 11430.453 | params rms: 0.061892 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 2:06:28.644397 | finish at 2026-10-12 03:17:58 + [2026-10-07 01:13:03] iteration 3242/ 6855 | consumed samples: 414976 | elapsed time per iteration (ms): 93794.9 | throughput per GPU (TFLOP/s/GPU): 39.2 | MFU 12.56% | FLOPs estimator: workload-aware | logical tokens/s: 178871.3 | valid tokens/s: 178871.3 | physical tokens/s: 179254.5 | loss tokens/s: 134828.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.259943E-01 | lm-global-supervised-tokens: 1.264626E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.253594E-02 | lm-token-buffer-mib: 5.812560E+01 | global_load_balancing_loss: 9.996474E-01 | loss scale: 1.0 | grad norm: 0.120 | params norm: 11430.387 | params rms: 0.061892 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 22:08:00.841690 | finish at 2026-10-10 23:21:04 + [2026-10-07 01:14:32] iteration 3243/ 6855 | consumed samples: 415104 | elapsed time per iteration (ms): 89269.9 | throughput per GPU (TFLOP/s/GPU): 41.6 | MFU 13.35% | FLOPs estimator: workload-aware | logical tokens/s: 187938.1 | valid tokens/s: 187938.1 | physical tokens/s: 188335.5 | loss tokens/s: 140466.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.739921E-01 | lm-global-supervised-tokens: 1.253942E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.155831E-02 | lm-token-buffer-mib: 5.802174E+01 | global_load_balancing_loss: 1.000661E+00 | loss scale: 1.0 | grad norm: 0.154 | params norm: 11430.320 | params rms: 0.061891 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 17:34:02.892019 | finish at 2026-10-10 18:48:35 + [2026-10-07 01:16:02] iteration 3244/ 6855 | consumed samples: 415232 | elapsed time per iteration (ms): 89796.4 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.55% | FLOPs estimator: workload-aware | logical tokens/s: 186835.5 | valid tokens/s: 186835.5 | physical tokens/s: 187241.0 | loss tokens/s: 142392.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.777697E-01 | lm-global-supervised-tokens: 1.278637E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.460580E-02 | lm-token-buffer-mib: 5.817245E+01 | global_load_balancing_loss: 1.000749E+00 | loss scale: 1.0 | grad norm: 0.150 | params norm: 11430.254 | params rms: 0.061891 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 18:04:14.786879 | finish at 2026-10-10 19:20:17 + [2026-10-07 01:17:28] iteration 3245/ 6855 | consumed samples: 415360 | elapsed time per iteration (ms): 85525.4 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.65% | FLOPs estimator: workload-aware | logical tokens/s: 196166.4 | valid tokens/s: 196166.4 | physical tokens/s: 196662.7 | loss tokens/s: 144255.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.429331E-01 | lm-global-supervised-tokens: 1.233748E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.784612E-02 | lm-token-buffer-mib: 5.813756E+01 | global_load_balancing_loss: 9.994264E-01 | loss scale: 1.0 | grad norm: 0.067 | params norm: 11430.188 | params rms: 0.061891 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 13:45:46.812499 | finish at 2026-10-10 15:03:15 + [2026-10-07 01:18:55] iteration 3246/ 6855 | consumed samples: 415488 | elapsed time per iteration (ms): 87356.5 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.56% | FLOPs estimator: workload-aware | logical tokens/s: 192054.7 | valid tokens/s: 192054.7 | physical tokens/s: 192583.6 | loss tokens/s: 146475.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.154496E-01 | lm-global-supervised-tokens: 1.279560E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.849983E-02 | lm-token-buffer-mib: 5.802935E+01 | global_load_balancing_loss: 9.989545E-01 | loss scale: 1.0 | grad norm: 0.124 | params norm: 11430.122 | params rms: 0.061890 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 15:34:29.449853 | finish at 2026-10-10 16:53:25 + [2026-10-07 01:20:21] iteration 3247/ 6855 | consumed samples: 415616 | elapsed time per iteration (ms): 85514.4 | throughput per GPU (TFLOP/s/GPU): 43.4 | MFU 13.92% | FLOPs estimator: workload-aware | logical tokens/s: 196190.8 | valid tokens/s: 196190.8 | physical tokens/s: 196750.8 | loss tokens/s: 140831.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.398531E-01 | lm-global-supervised-tokens: 1.204315E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.503036E-02 | lm-token-buffer-mib: 5.812593E+01 | global_load_balancing_loss: 1.000160E+00 | loss scale: 1.0 | grad norm: 0.066 | params norm: 11430.056 | params rms: 0.061890 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 13:42:15.980165 | finish at 2026-10-10 15:02:37 + [2026-10-07 01:21:50] iteration 3248/ 6855 | consumed samples: 415744 | elapsed time per iteration (ms): 89273.4 | throughput per GPU (TFLOP/s/GPU): 41.7 | MFU 13.36% | FLOPs estimator: workload-aware | logical tokens/s: 187930.1 | valid tokens/s: 187930.1 | physical tokens/s: 188414.3 | loss tokens/s: 135328.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.060367E-01 | lm-global-supervised-tokens: 1.208122E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.227080E-02 | lm-token-buffer-mib: 5.817853E+01 | global_load_balancing_loss: 1.000943E+00 | loss scale: 1.0 | grad norm: 0.077 | params norm: 11429.990 | params rms: 0.061890 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 17:26:49.057729 | finish at 2026-10-10 18:48:39 + [2026-10-07 01:23:16] iteration 3249/ 6855 | consumed samples: 415872 | elapsed time per iteration (ms): 85625.1 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.69% | FLOPs estimator: workload-aware | logical tokens/s: 195937.6 | valid tokens/s: 195937.6 | physical tokens/s: 196352.4 | loss tokens/s: 144997.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.631392E-01 | lm-global-supervised-tokens: 1.241547E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.157822E-02 | lm-token-buffer-mib: 5.808694E+01 | global_load_balancing_loss: 1.001443E+00 | loss scale: 1.0 | grad norm: 0.087 | params norm: 11429.923 | params rms: 0.061889 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 13:46:04.219417 | finish at 2026-10-10 15:09:20 + [2026-10-07 01:25:04] iteration 3250/ 6855 | consumed samples: 416000 | elapsed time per iteration (ms): 108579.3 | throughput per GPU (TFLOP/s/GPU): 34.0 | MFU 10.90% | FLOPs estimator: workload-aware | logical tokens/s: 154515.7 | valid tokens/s: 154515.7 | physical tokens/s: 154852.0 | loss tokens/s: 114897.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.571796E-01 | lm-global-supervised-tokens: 1.247547E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.207134E-02 | lm-token-buffer-mib: 5.815312E+01 | global_load_balancing_loss: 1.000523E+00 | loss scale: 1.0 | grad norm: 0.043 | params norm: 11429.859 | params rms: 0.061889 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 12:43:48.555012 | finish at 2026-10-11 14:08:53 + [2026-10-07 01:26:33] iteration 3251/ 6855 | consumed samples: 416128 | elapsed time per iteration (ms): 88671.3 | throughput per GPU (TFLOP/s/GPU): 40.8 | MFU 13.09% | FLOPs estimator: workload-aware | logical tokens/s: 189205.0 | valid tokens/s: 189205.0 | physical tokens/s: 189710.6 | loss tokens/s: 143143.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.961271E-01 | lm-global-supervised-tokens: 1.269272E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.436533E-02 | lm-token-buffer-mib: 5.813165E+01 | global_load_balancing_loss: 1.000617E+00 | loss scale: 1.0 | grad norm: 0.099 | params norm: 11429.794 | params rms: 0.061889 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 16:46:11.375275 | finish at 2026-10-10 18:12:44 + [2026-10-07 01:28:00] iteration 3252/ 6855 | consumed samples: 416256 | elapsed time per iteration (ms): 86746.9 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.68% | FLOPs estimator: workload-aware | logical tokens/s: 193403.5 | valid tokens/s: 193403.5 | physical tokens/s: 193877.6 | loss tokens/s: 137681.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.986632E-01 | lm-global-supervised-tokens: 1.194342E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.153800E-02 | lm-token-buffer-mib: 5.819930E+01 | global_load_balancing_loss: 9.991429E-01 | loss scale: 1.0 | grad norm: 0.031 | params norm: 11429.729 | params rms: 0.061888 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 14:49:09.154011 | finish at 2026-10-10 16:17:09 + [2026-10-07 01:29:30] iteration 3253/ 6855 | consumed samples: 416384 | elapsed time per iteration (ms): 90173.3 | throughput per GPU (TFLOP/s/GPU): 40.7 | MFU 13.05% | FLOPs estimator: workload-aware | logical tokens/s: 186054.9 | valid tokens/s: 186054.9 | physical tokens/s: 186563.3 | loss tokens/s: 139991.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.344550E-01 | lm-global-supervised-tokens: 1.262345E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.211374E-02 | lm-token-buffer-mib: 5.816830E+01 | global_load_balancing_loss: 1.000689E+00 | loss scale: 1.0 | grad norm: 0.097 | params norm: 11429.664 | params rms: 0.061888 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 18:13:24.118494 | finish at 2026-10-10 19:42:54 + [2026-10-07 01:30:54] iteration 3254/ 6855 | consumed samples: 416512 | elapsed time per iteration (ms): 83761.3 | throughput per GPU (TFLOP/s/GPU): 43.9 | MFU 14.06% | FLOPs estimator: workload-aware | logical tokens/s: 200298.0 | valid tokens/s: 200298.0 | physical tokens/s: 200791.1 | loss tokens/s: 138720.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.549754E-01 | lm-global-supervised-tokens: 1.161938E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.377179E-02 | lm-token-buffer-mib: 5.828400E+01 | global_load_balancing_loss: 1.000338E+00 | loss scale: 1.0 | grad norm: 0.086 | params norm: 11429.598 | params rms: 0.061888 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 11:47:04.267329 | finish at 2026-10-10 13:17:58 + [2026-10-07 01:32:20] iteration 3255/ 6855 | consumed samples: 416640 | elapsed time per iteration (ms): 86037.4 | throughput per GPU (TFLOP/s/GPU): 43.1 | MFU 13.80% | FLOPs estimator: workload-aware | logical tokens/s: 194999.0 | valid tokens/s: 194999.0 | physical tokens/s: 195391.2 | loss tokens/s: 142376.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.708714E-01 | lm-global-supervised-tokens: 1.224967E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.479090E-02 | lm-token-buffer-mib: 5.825651E+01 | global_load_balancing_loss: 1.000758E+00 | loss scale: 1.0 | grad norm: 0.077 | params norm: 11429.534 | params rms: 0.061887 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 14:02:14.773922 | finish at 2026-10-10 15:34:34 + [2026-10-07 01:33:46] iteration 3256/ 6855 | consumed samples: 416768 | elapsed time per iteration (ms): 86659.6 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.56% | FLOPs estimator: workload-aware | logical tokens/s: 193599.0 | valid tokens/s: 193598.9 | physical tokens/s: 194056.6 | loss tokens/s: 146504.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.390561E-01 | lm-global-supervised-tokens: 1.269605E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.446982E-02 | lm-token-buffer-mib: 5.817329E+01 | global_load_balancing_loss: 1.000332E+00 | loss scale: 1.0 | grad norm: 0.057 | params norm: 11429.470 | params rms: 0.061887 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 14:38:07.867005 | finish at 2026-10-10 16:11:54 + [2026-10-07 01:35:13] iteration 3257/ 6855 | consumed samples: 416896 | elapsed time per iteration (ms): 87144.1 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.76% | FLOPs estimator: workload-aware | logical tokens/s: 192521.9 | valid tokens/s: 192521.9 | physical tokens/s: 192974.9 | loss tokens/s: 136493.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.576667E-01 | lm-global-supervised-tokens: 1.189461E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.156598E-02 | lm-token-buffer-mib: 5.813578E+01 | global_load_balancing_loss: 9.997789E-01 | loss scale: 1.0 | grad norm: 0.091 | params norm: 11429.407 | params rms: 0.061887 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 15:05:44.494784 | finish at 2026-10-10 16:40:58 + [2026-10-07 01:36:39] iteration 3258/ 6855 | consumed samples: 417024 | elapsed time per iteration (ms): 85465.0 | throughput per GPU (TFLOP/s/GPU): 43.5 | MFU 13.95% | FLOPs estimator: workload-aware | logical tokens/s: 196304.8 | valid tokens/s: 196304.8 | physical tokens/s: 196879.4 | loss tokens/s: 143521.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.278197E-01 | lm-global-supervised-tokens: 1.226604E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.140671E-02 | lm-token-buffer-mib: 5.829047E+01 | global_load_balancing_loss: 1.000223E+00 | loss scale: 1.0 | grad norm: 0.037 | params norm: 11429.345 | params rms: 0.061886 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 13:23:37.716178 | finish at 2026-10-10 15:00:17 + [2026-10-07 01:38:04] iteration 3259/ 6855 | consumed samples: 417152 | elapsed time per iteration (ms): 85482.5 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.75% | FLOPs estimator: workload-aware | logical tokens/s: 196264.4 | valid tokens/s: 196264.4 | physical tokens/s: 196653.6 | loss tokens/s: 152354.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.544695E-01 | lm-global-supervised-tokens: 1.302367E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.514268E-02 | lm-token-buffer-mib: 5.808646E+01 | global_load_balancing_loss: 9.986532E-01 | loss scale: 1.0 | grad norm: 0.056 | params norm: 11429.284 | params rms: 0.061886 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 13:23:14.951960 | finish at 2026-10-10 15:01:19 + [2026-10-07 01:39:58] iteration 3260/ 6855 | consumed samples: 417280 | elapsed time per iteration (ms): 113649.6 | throughput per GPU (TFLOP/s/GPU): 33.3 | MFU 10.67% | FLOPs estimator: workload-aware | logical tokens/s: 147622.1 | valid tokens/s: 147622.1 | physical tokens/s: 148083.0 | loss tokens/s: 104864.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.478092E-01 | lm-global-supervised-tokens: 1.191783E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.163643E-02 | lm-token-buffer-mib: 5.818748E+01 | global_load_balancing_loss: 9.995289E-01 | loss scale: 1.0 | grad norm: 0.049 | params norm: 11429.223 | params rms: 0.061886 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 17:29:30.211822 | finish at 2026-10-11 19:09:28 + [2026-10-07 01:41:22] iteration 3261/ 6855 | consumed samples: 417408 | elapsed time per iteration (ms): 84462.4 | throughput per GPU (TFLOP/s/GPU): 43.9 | MFU 14.07% | FLOPs estimator: workload-aware | logical tokens/s: 198634.3 | valid tokens/s: 198634.3 | physical tokens/s: 199094.9 | loss tokens/s: 145784.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.384749E-01 | lm-global-supervised-tokens: 1.231334E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.606814E-02 | lm-token-buffer-mib: 5.806926E+01 | global_load_balancing_loss: 1.000984E+00 | loss scale: 1.0 | grad norm: 0.051 | params norm: 11429.160 | params rms: 0.061885 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 12:19:17.869739 | finish at 2026-10-10 14:00:40 + [2026-10-07 01:42:49] iteration 3262/ 6855 | consumed samples: 417536 | elapsed time per iteration (ms): 86442.5 | throughput per GPU (TFLOP/s/GPU): 43.2 | MFU 13.83% | FLOPs estimator: workload-aware | logical tokens/s: 194084.8 | valid tokens/s: 194084.8 | physical tokens/s: 194404.3 | loss tokens/s: 136650.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.683340E-01 | lm-global-supervised-tokens: 1.181242E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.303549E-02 | lm-token-buffer-mib: 5.808848E+01 | global_load_balancing_loss: 1.001269E+00 | loss scale: 1.0 | grad norm: 0.040 | params norm: 11429.096 | params rms: 0.061885 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 14:16:27.727300 | finish at 2026-10-10 15:59:17 + [2026-10-07 01:44:17] iteration 3263/ 6855 | consumed samples: 417664 | elapsed time per iteration (ms): 88061.1 | throughput per GPU (TFLOP/s/GPU): 41.3 | MFU 13.24% | FLOPs estimator: workload-aware | logical tokens/s: 190517.9 | valid tokens/s: 190517.9 | physical tokens/s: 190998.3 | loss tokens/s: 143491.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.926258E-01 | lm-global-supervised-tokens: 1.263602E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.160700E-02 | lm-token-buffer-mib: 5.833489E+01 | global_load_balancing_loss: 1.000491E+00 | loss scale: 1.0 | grad norm: 0.106 | params norm: 11429.033 | params rms: 0.061884 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 15:51:55.418125 | finish at 2026-10-10 17:36:12 + [2026-10-07 01:45:39] iteration 3264/ 6855 | consumed samples: 417792 | elapsed time per iteration (ms): 82254.8 | throughput per GPU (TFLOP/s/GPU): 43.8 | MFU 14.03% | FLOPs estimator: workload-aware | logical tokens/s: 203965.6 | valid tokens/s: 203965.6 | physical tokens/s: 204423.6 | loss tokens/s: 153703.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.291392E-01 | lm-global-supervised-tokens: 1.264281E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.273140E-02 | lm-token-buffer-mib: 5.812813E+01 | global_load_balancing_loss: 1.001447E+00 | loss scale: 1.0 | grad norm: 0.146 | params norm: 11428.970 | params rms: 0.061884 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 10:02:56.938291 | finish at 2026-10-10 11:48:36 + [2026-10-07 01:47:06] iteration 3265/ 6855 | consumed samples: 417920 | elapsed time per iteration (ms): 86854.5 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.68% | FLOPs estimator: workload-aware | logical tokens/s: 193163.8 | valid tokens/s: 193163.8 | physical tokens/s: 193616.1 | loss tokens/s: 143574.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.229124E-01 | lm-global-supervised-tokens: 1.247007E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.168168E-02 | lm-token-buffer-mib: 5.825867E+01 | global_load_balancing_loss: 1.001126E+00 | loss scale: 1.0 | grad norm: 0.080 | params norm: 11428.908 | params rms: 0.061884 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 14:36:47.686012 | finish at 2026-10-10 16:23:54 + [2026-10-07 01:48:33] iteration 3266/ 6855 | consumed samples: 418048 | elapsed time per iteration (ms): 87439.5 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.48% | FLOPs estimator: workload-aware | logical tokens/s: 191871.9 | valid tokens/s: 191871.9 | physical tokens/s: 192232.0 | loss tokens/s: 138484.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.295171E-01 | lm-global-supervised-tokens: 1.210899E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.158053E-02 | lm-token-buffer-mib: 5.811686E+01 | global_load_balancing_loss: 9.997693E-01 | loss scale: 1.0 | grad norm: 0.033 | params norm: 11428.845 | params rms: 0.061883 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 15:10:20.423166 | finish at 2026-10-10 16:58:54 + [2026-10-07 01:50:03] iteration 3267/ 6855 | consumed samples: 418176 | elapsed time per iteration (ms): 89656.4 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.69% | FLOPs estimator: workload-aware | logical tokens/s: 187127.4 | valid tokens/s: 187127.4 | physical tokens/s: 187412.9 | loss tokens/s: 134859.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.597635E-01 | lm-global-supervised-tokens: 1.209098E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.217378E-02 | lm-token-buffer-mib: 5.804379E+01 | global_load_balancing_loss: 9.986857E-01 | loss scale: 1.0 | grad norm: 0.046 | params norm: 11428.783 | params rms: 0.061883 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 17:21:27.234933 | finish at 2026-10-10 19:11:30 + [2026-10-07 01:51:30] iteration 3268/ 6855 | consumed samples: 418304 | elapsed time per iteration (ms): 86716.9 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.61% | FLOPs estimator: workload-aware | logical tokens/s: 193470.9 | valid tokens/s: 193470.9 | physical tokens/s: 194007.4 | loss tokens/s: 134496.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.152419E-01 | lm-global-supervised-tokens: 1.166308E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.152251E-02 | lm-token-buffer-mib: 5.809341E+01 | global_load_balancing_loss: 9.992546E-01 | loss scale: 1.0 | grad norm: 0.033 | params norm: 11428.720 | params rms: 0.061883 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 14:24:13.697723 | finish at 2026-10-10 16:15:44 + [2026-10-07 01:53:01] iteration 3269/ 6855 | consumed samples: 418432 | elapsed time per iteration (ms): 90888.0 | throughput per GPU (TFLOP/s/GPU): 41.0 | MFU 13.15% | FLOPs estimator: workload-aware | logical tokens/s: 184592.0 | valid tokens/s: 184592.0 | physical tokens/s: 185028.3 | loss tokens/s: 140265.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.633000E-01 | lm-global-supervised-tokens: 1.274844E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.174246E-02 | lm-token-buffer-mib: 5.813462E+01 | global_load_balancing_loss: 9.989031E-01 | loss scale: 1.0 | grad norm: 0.078 | params norm: 11428.657 | params rms: 0.061882 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 18:32:04.239796 | finish at 2026-10-10 20:25:05 + [2026-10-07 01:54:49] iteration 3270/ 6855 | consumed samples: 418560 | elapsed time per iteration (ms): 108567.2 | throughput per GPU (TFLOP/s/GPU): 34.6 | MFU 11.10% | FLOPs estimator: workload-aware | logical tokens/s: 154533.0 | valid tokens/s: 154533.0 | physical tokens/s: 154790.3 | loss tokens/s: 109545.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.884890E-01 | lm-global-supervised-tokens: 1.189309E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.732853E-02 | lm-token-buffer-mib: 5.806670E+01 | global_load_balancing_loss: 1.001150E+00 | loss scale: 1.0 | grad norm: 0.050 | params norm: 11428.594 | params rms: 0.061882 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 12:06:53.454541 | finish at 2026-10-11 14:01:43 + [2026-10-07 01:56:16] iteration 3271/ 6855 | consumed samples: 418688 | elapsed time per iteration (ms): 87146.1 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.64% | FLOPs estimator: workload-aware | logical tokens/s: 192518.2 | valid tokens/s: 192518.2 | physical tokens/s: 192875.0 | loss tokens/s: 140736.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.660486E-01 | lm-global-supervised-tokens: 1.226463E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.342761E-02 | lm-token-buffer-mib: 5.808582E+01 | global_load_balancing_loss: 1.000578E+00 | loss scale: 1.0 | grad norm: 0.066 | params norm: 11428.529 | params rms: 0.061882 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 14:45:31.668701 | finish at 2026-10-10 16:41:48 + [2026-10-07 01:57:43] iteration 3272/ 6855 | consumed samples: 418816 | elapsed time per iteration (ms): 86804.0 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.66% | FLOPs estimator: workload-aware | logical tokens/s: 193276.9 | valid tokens/s: 193276.9 | physical tokens/s: 193591.6 | loss tokens/s: 136439.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.499375E-01 | lm-global-supervised-tokens: 1.184353E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.368219E-02 | lm-token-buffer-mib: 5.809506E+01 | global_load_balancing_loss: 1.000821E+00 | loss scale: 1.0 | grad norm: 0.043 | params norm: 11428.463 | params rms: 0.061881 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 14:23:38.850386 | finish at 2026-10-10 16:21:22 + [2026-10-07 01:59:17] iteration 3273/ 6855 | consumed samples: 418944 | elapsed time per iteration (ms): 93364.4 | throughput per GPU (TFLOP/s/GPU): 40.9 | MFU 13.10% | FLOPs estimator: workload-aware | logical tokens/s: 179695.5 | valid tokens/s: 179695.5 | physical tokens/s: 180105.4 | loss tokens/s: 126613.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.572174E-01 | lm-global-supervised-tokens: 1.182120E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.214270E-02 | lm-token-buffer-mib: 5.816232E+01 | global_load_balancing_loss: 1.001008E+00 | loss scale: 1.0 | grad norm: 0.144 | params norm: 11428.399 | params rms: 0.061881 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 20:53:51.126755 | finish at 2026-10-10 22:53:08 + [2026-10-07 02:00:48] iteration 3274/ 6855 | consumed samples: 419072 | elapsed time per iteration (ms): 91640.2 | throughput per GPU (TFLOP/s/GPU): 41.2 | MFU 13.20% | FLOPs estimator: workload-aware | logical tokens/s: 183076.8 | valid tokens/s: 183076.8 | physical tokens/s: 183583.3 | loss tokens/s: 130267.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.217217E-01 | lm-global-supervised-tokens: 1.193775E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.291077E-02 | lm-token-buffer-mib: 5.812563E+01 | global_load_balancing_loss: 1.000431E+00 | loss scale: 1.0 | grad norm: 0.157 | params norm: 11428.335 | params rms: 0.061881 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 19:09:23.682200 | finish at 2026-10-10 21:10:12 + [2026-10-07 02:02:11] iteration 3275/ 6855 | consumed samples: 419200 | elapsed time per iteration (ms): 82534.1 | throughput per GPU (TFLOP/s/GPU): 43.5 | MFU 13.95% | FLOPs estimator: workload-aware | logical tokens/s: 203276.0 | valid tokens/s: 203276.0 | physical tokens/s: 203694.9 | loss tokens/s: 151886.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.651900E-01 | lm-global-supervised-tokens: 1.253582E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.156185E-02 | lm-token-buffer-mib: 5.817313E+01 | global_load_balancing_loss: 1.000193E+00 | loss scale: 1.0 | grad norm: 0.071 | params norm: 11428.270 | params rms: 0.061880 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 10:04:32.146482 | finish at 2026-10-10 12:06:43 + [2026-10-07 02:03:40] iteration 3276/ 6855 | consumed samples: 419328 | elapsed time per iteration (ms): 89068.0 | throughput per GPU (TFLOP/s/GPU): 41.7 | MFU 13.37% | FLOPs estimator: workload-aware | logical tokens/s: 188363.8 | valid tokens/s: 188363.8 | physical tokens/s: 188848.7 | loss tokens/s: 142271.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.726983E-01 | lm-global-supervised-tokens: 1.267187E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.390742E-02 | lm-token-buffer-mib: 5.820949E+01 | global_load_balancing_loss: 9.990440E-01 | loss scale: 1.0 | grad norm: 0.107 | params norm: 11428.205 | params rms: 0.061880 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 16:32:54.301456 | finish at 2026-10-10 18:36:34 + [2026-10-07 02:05:10] iteration 3277/ 6855 | consumed samples: 419456 | elapsed time per iteration (ms): 90356.2 | throughput per GPU (TFLOP/s/GPU): 41.3 | MFU 13.25% | FLOPs estimator: workload-aware | logical tokens/s: 185678.2 | valid tokens/s: 185678.2 | physical tokens/s: 186010.8 | loss tokens/s: 130112.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.559446E-01 | lm-global-supervised-tokens: 1.175642E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.065052E-02 | lm-token-buffer-mib: 5.808149E+01 | global_load_balancing_loss: 9.999641E-01 | loss scale: 1.0 | grad norm: 0.142 | params norm: 11428.141 | params rms: 0.061880 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 17:48:14.465613 | finish at 2026-10-10 19:53:25 + [2026-10-07 02:06:34] iteration 3278/ 6855 | consumed samples: 419584 | elapsed time per iteration (ms): 83741.3 | throughput per GPU (TFLOP/s/GPU): 43.3 | MFU 13.87% | FLOPs estimator: workload-aware | logical tokens/s: 200345.7 | valid tokens/s: 200345.7 | physical tokens/s: 201037.7 | loss tokens/s: 147406.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.553341E-01 | lm-global-supervised-tokens: 1.234399E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.113964E-02 | lm-token-buffer-mib: 5.813883E+01 | global_load_balancing_loss: 9.998703E-01 | loss scale: 1.0 | grad norm: 0.172 | params norm: 11428.077 | params rms: 0.061879 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 11:12:22.575899 | finish at 2026-10-10 13:18:57 + [2026-10-07 02:07:59] iteration 3279/ 6855 | consumed samples: 419712 | elapsed time per iteration (ms): 85389.5 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.64% | FLOPs estimator: workload-aware | logical tokens/s: 196478.5 | valid tokens/s: 196478.5 | physical tokens/s: 196913.4 | loss tokens/s: 146929.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.466217E-01 | lm-global-supervised-tokens: 1.254622E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.541384E-02 | lm-token-buffer-mib: 5.812918E+01 | global_load_balancing_loss: 9.990435E-01 | loss scale: 1.0 | grad norm: 0.093 | params norm: 11428.015 | params rms: 0.061879 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 12:49:12.879787 | finish at 2026-10-10 14:57:12 + [2026-10-07 02:09:51] iteration 3280/ 6855 | consumed samples: 419840 | elapsed time per iteration (ms): 111510.2 | throughput per GPU (TFLOP/s/GPU): 33.1 | MFU 10.60% | FLOPs estimator: workload-aware | logical tokens/s: 150453.5 | valid tokens/s: 150453.5 | physical tokens/s: 150923.2 | loss tokens/s: 113016.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.879665E-01 | lm-global-supervised-tokens: 1.260251E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.154136E-02 | lm-token-buffer-mib: 5.817007E+01 | global_load_balancing_loss: 1.000363E+00 | loss scale: 1.0 | grad norm: 0.073 | params norm: 11427.953 | params rms: 0.061879 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 14:44:08.844051 | finish at 2026-10-11 16:54:00 +times across ranks (ms): + save-checkpoint: + rank 0: 24454.69 + rank 1: 24454.69 + rank 2: 24454.73 + rank 3: 24454.70 + rank 4: 24454.70 + rank 5: 24454.69 + rank 6: 24454.68 + rank 7: 24454.69 + rank 8: 24453.38 + rank 9: 24453.37 + rank 10: 24453.38 + rank 11: 24453.36 + rank 12: 24453.37 + rank 13: 24453.37 + rank 14: 24453.37 + rank 15: 24453.37 + rank 16: 24454.41 + rank 17: 24454.33 + rank 18: 24454.34 + rank 19: 24454.32 + rank 20: 24454.37 + rank 21: 24454.35 + rank 22: 24454.31 + rank 23: 24454.37 + rank 24: 24453.67 + rank 25: 24453.67 + rank 26: 24453.79 + rank 27: 24453.68 + rank 28: 24453.67 + rank 29: 24453.66 + rank 30: 24453.63 + rank 31: 24453.67 + rank 32: 24454.05 + rank 33: 24454.06 + rank 34: 24454.09 + rank 35: 24454.04 + rank 36: 24454.00 + rank 37: 24454.01 + rank 38: 24454.01 + rank 39: 24453.99 + rank 40: 24454.57 + rank 41: 24454.56 + rank 42: 24454.56 + rank 43: 24454.58 + rank 44: 24454.57 + rank 45: 24454.58 + rank 46: 24454.61 + rank 47: 24454.56 + rank 48: 24453.79 + rank 49: 24453.74 + rank 50: 24453.69 + rank 51: 24453.71 + rank 52: 24453.77 + rank 53: 24453.74 + rank 54: 24453.73 + rank 55: 24453.71 + rank 56: 24453.63 + rank 57: 24453.60 + rank 58: 24453.60 + rank 59: 24453.63 + rank 60: 24453.59 + rank 61: 24453.61 + rank 62: 24453.60 + rank 63: 24453.68 + [2026-10-07 02:11:59] iteration 3281/ 6855 | consumed samples: 419968 | elapsed time per iteration (ms): 103914.2 | throughput per GPU (TFLOP/s/GPU): 36.8 | MFU 11.81% | FLOPs estimator: workload-aware | logical tokens/s: 161452.3 | valid tokens/s: 161452.3 | physical tokens/s: 161876.1 | loss tokens/s: 120146.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.480847E-01 | lm-global-supervised-tokens: 1.248495E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.535086E+00 | lm-token-buffer-mib: 5.811853E+01 | global_load_balancing_loss: 1.000859E+00 | loss scale: 1.0 | grad norm: 0.077 | params norm: 11427.891 | params rms: 0.061878 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 7:09:49.403020 | finish at 2026-10-11 09:21:49 + [2026-10-07 02:13:30] iteration 3282/ 6855 | consumed samples: 420096 | elapsed time per iteration (ms): 90463.0 | throughput per GPU (TFLOP/s/GPU): 40.5 | MFU 12.98% | FLOPs estimator: workload-aware | logical tokens/s: 185458.8 | valid tokens/s: 185458.8 | physical tokens/s: 185817.7 | loss tokens/s: 137752.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.682841E-01 | lm-global-supervised-tokens: 1.246154E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.743514E-02 | lm-token-buffer-mib: 5.810220E+01 | global_load_balancing_loss: 1.001109E+00 | loss scale: 1.0 | grad norm: 0.068 | params norm: 11427.830 | params rms: 0.061878 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 17:47:04.407399 | finish at 2026-10-10 20:00:34 + [2026-10-07 02:14:55] iteration 3283/ 6855 | consumed samples: 420224 | elapsed time per iteration (ms): 85254.8 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.79% | FLOPs estimator: workload-aware | logical tokens/s: 196788.0 | valid tokens/s: 196788.0 | physical tokens/s: 197222.3 | loss tokens/s: 149384.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.489856E-01 | lm-global-supervised-tokens: 1.273579E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.156435E-02 | lm-token-buffer-mib: 5.813566E+01 | global_load_balancing_loss: 1.000370E+00 | loss scale: 1.0 | grad norm: 0.049 | params norm: 11427.769 | params rms: 0.061878 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 12:35:30.311958 | finish at 2026-10-10 14:50:25 + [2026-10-07 02:16:26] iteration 3284/ 6855 | consumed samples: 420352 | elapsed time per iteration (ms): 91156.6 | throughput per GPU (TFLOP/s/GPU): 41.6 | MFU 13.32% | FLOPs estimator: workload-aware | logical tokens/s: 184047.8 | valid tokens/s: 184047.8 | physical tokens/s: 184433.1 | loss tokens/s: 140518.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.560813E-01 | lm-global-supervised-tokens: 1.280922E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.139556E-02 | lm-token-buffer-mib: 5.806130E+01 | global_load_balancing_loss: 1.000286E+00 | loss scale: 1.0 | grad norm: 0.062 | params norm: 11427.707 | params rms: 0.061877 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 18:25:20.362480 | finish at 2026-10-10 20:41:47 + [2026-10-07 02:17:55] iteration 3285/ 6855 | consumed samples: 420480 | elapsed time per iteration (ms): 88892.4 | throughput per GPU (TFLOP/s/GPU): 41.7 | MFU 13.37% | FLOPs estimator: workload-aware | logical tokens/s: 188735.7 | valid tokens/s: 188735.7 | physical tokens/s: 189274.7 | loss tokens/s: 143098.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.099993E-01 | lm-global-supervised-tokens: 1.272037E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.724595E-02 | lm-token-buffer-mib: 5.810938E+01 | global_load_balancing_loss: 9.993263E-01 | loss scale: 1.0 | grad norm: 0.072 | params norm: 11427.644 | params rms: 0.061877 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 16:09:05.924270 | finish at 2026-10-10 18:27:01 + [2026-10-07 02:19:20] iteration 3286/ 6855 | consumed samples: 420608 | elapsed time per iteration (ms): 85037.5 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.74% | FLOPs estimator: workload-aware | logical tokens/s: 197291.0 | valid tokens/s: 197291.0 | physical tokens/s: 198117.6 | loss tokens/s: 151600.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.798611E-01 | lm-global-supervised-tokens: 1.289170E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.330174E-02 | lm-token-buffer-mib: 5.828773E+01 | global_load_balancing_loss: 1.001056E+00 | loss scale: 1.0 | grad norm: 0.101 | params norm: 11427.580 | params rms: 0.061877 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 12:18:18.888215 | finish at 2026-10-10 14:37:39 + [2026-10-07 02:20:45] iteration 3287/ 6855 | consumed samples: 420736 | elapsed time per iteration (ms): 84853.8 | throughput per GPU (TFLOP/s/GPU): 43.2 | MFU 13.85% | FLOPs estimator: workload-aware | logical tokens/s: 197718.9 | valid tokens/s: 197718.9 | physical tokens/s: 198066.3 | loss tokens/s: 145423.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.147436E-01 | lm-global-supervised-tokens: 1.233973E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.420448E-02 | lm-token-buffer-mib: 5.809981E+01 | global_load_balancing_loss: 1.000884E+00 | loss scale: 1.0 | grad norm: 0.168 | params norm: 11427.518 | params rms: 0.061876 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 12:05:58.234409 | finish at 2026-10-10 14:26:43 + [2026-10-07 02:22:12] iteration 3288/ 6855 | consumed samples: 420864 | elapsed time per iteration (ms): 86513.5 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.66% | FLOPs estimator: workload-aware | logical tokens/s: 193926.0 | valid tokens/s: 193926.0 | physical tokens/s: 194553.1 | loss tokens/s: 147095.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.672540E-01 | lm-global-supervised-tokens: 1.272572E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.400058E-02 | lm-token-buffer-mib: 5.816018E+01 | global_load_balancing_loss: 1.000215E+00 | loss scale: 1.0 | grad norm: 0.064 | params norm: 11427.456 | params rms: 0.061876 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 13:43:13.632300 | finish at 2026-10-10 16:05:25 + [2026-10-07 02:23:38] iteration 3289/ 6855 | consumed samples: 420992 | elapsed time per iteration (ms): 86349.6 | throughput per GPU (TFLOP/s/GPU): 43.7 | MFU 14.02% | FLOPs estimator: workload-aware | logical tokens/s: 194293.5 | valid tokens/s: 194293.5 | physical tokens/s: 194734.3 | loss tokens/s: 139016.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.530187E-01 | lm-global-supervised-tokens: 1.200403E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.137343E-02 | lm-token-buffer-mib: 5.805730E+01 | global_load_balancing_loss: 9.995373E-01 | loss scale: 1.0 | grad norm: 0.142 | params norm: 11427.394 | params rms: 0.061876 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 13:32:02.675574 | finish at 2026-10-10 15:55:41 + [2026-10-07 02:25:25] iteration 3290/ 6855 | consumed samples: 421120 | elapsed time per iteration (ms): 107484.1 | throughput per GPU (TFLOP/s/GPU): 33.6 | MFU 10.78% | FLOPs estimator: workload-aware | logical tokens/s: 156090.1 | valid tokens/s: 156090.1 | physical tokens/s: 156435.5 | loss tokens/s: 111446.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.385794E-01 | lm-global-supervised-tokens: 1.197869E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.317891E-02 | lm-token-buffer-mib: 5.810852E+01 | global_load_balancing_loss: 1.000102E+00 | loss scale: 1.0 | grad norm: 0.061 | params norm: 11427.331 | params rms: 0.061875 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 10:26:20.963912 | finish at 2026-10-11 12:51:46 + [2026-10-07 02:26:52] iteration 3291/ 6855 | consumed samples: 421248 | elapsed time per iteration (ms): 86704.0 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.54% | FLOPs estimator: workload-aware | logical tokens/s: 193499.7 | valid tokens/s: 193499.7 | physical tokens/s: 193939.1 | loss tokens/s: 142702.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.675914E-01 | lm-global-supervised-tokens: 1.237286E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.259309E-02 | lm-token-buffer-mib: 5.807531E+01 | global_load_balancing_loss: 1.000003E+00 | loss scale: 1.0 | grad norm: 0.176 | params norm: 11427.269 | params rms: 0.061875 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 13:50:12.999905 | finish at 2026-10-10 16:17:05 + [2026-10-07 02:28:19] iteration 3292/ 6855 | consumed samples: 421376 | elapsed time per iteration (ms): 87345.3 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.58% | FLOPs estimator: workload-aware | logical tokens/s: 192078.9 | valid tokens/s: 192078.9 | physical tokens/s: 192529.5 | loss tokens/s: 143421.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.885419E-01 | lm-global-supervised-tokens: 1.252718E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.219983E-02 | lm-token-buffer-mib: 5.817704E+01 | global_load_balancing_loss: 9.992468E-01 | loss scale: 1.0 | grad norm: 0.126 | params norm: 11427.207 | params rms: 0.061875 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 14:26:51.254484 | finish at 2026-10-10 16:55:11 + [2026-10-07 02:29:49] iteration 3293/ 6855 | consumed samples: 421504 | elapsed time per iteration (ms): 89273.3 | throughput per GPU (TFLOP/s/GPU): 41.6 | MFU 13.32% | FLOPs estimator: workload-aware | logical tokens/s: 187930.8 | valid tokens/s: 187930.8 | physical tokens/s: 188282.7 | loss tokens/s: 138558.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.595943E-01 | lm-global-supervised-tokens: 1.236954E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.275926E-02 | lm-token-buffer-mib: 5.813119E+01 | global_load_balancing_loss: 1.000378E+00 | loss scale: 1.0 | grad norm: 0.200 | params norm: 11427.144 | params rms: 0.061874 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 16:19:51.349138 | finish at 2026-10-10 18:49:40 + [2026-10-07 02:31:16] iteration 3294/ 6855 | consumed samples: 421632 | elapsed time per iteration (ms): 87022.1 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.49% | FLOPs estimator: workload-aware | logical tokens/s: 192791.9 | valid tokens/s: 192791.9 | physical tokens/s: 193269.9 | loss tokens/s: 137205.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.403917E-01 | lm-global-supervised-tokens: 1.193991E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.254191E-02 | lm-token-buffer-mib: 5.812572E+01 | global_load_balancing_loss: 1.001220E+00 | loss scale: 1.0 | grad norm: 0.262 | params norm: 11427.082 | params rms: 0.061874 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 14:04:45.608005 | finish at 2026-10-10 16:36:01 + [2026-10-07 02:32:43] iteration 3295/ 6855 | consumed samples: 421760 | elapsed time per iteration (ms): 87418.9 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.66% | FLOPs estimator: workload-aware | logical tokens/s: 191917.3 | valid tokens/s: 191917.3 | physical tokens/s: 192257.2 | loss tokens/s: 139412.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.912714E-01 | lm-global-supervised-tokens: 1.218728E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.204537E-02 | lm-token-buffer-mib: 5.805009E+01 | global_load_balancing_loss: 9.990475E-01 | loss scale: 1.0 | grad norm: 0.093 | params norm: 11427.019 | params rms: 0.061874 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 14:26:51.327333 | finish at 2026-10-10 16:59:34 + [2026-10-07 02:34:12] iteration 3296/ 6855 | consumed samples: 421888 | elapsed time per iteration (ms): 88444.4 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.55% | FLOPs estimator: workload-aware | logical tokens/s: 189692.2 | valid tokens/s: 189692.2 | physical tokens/s: 190034.5 | loss tokens/s: 135238.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.688406E-01 | lm-global-supervised-tokens: 1.196108E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.136743E-02 | lm-token-buffer-mib: 5.814266E+01 | global_load_balancing_loss: 1.001049E+00 | loss scale: 1.0 | grad norm: 0.090 | params norm: 11426.958 | params rms: 0.061873 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 15:26:13.549428 | finish at 2026-10-10 18:00:25 + [2026-10-07 02:35:37] iteration 3297/ 6855 | consumed samples: 422016 | elapsed time per iteration (ms): 85192.5 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.76% | FLOPs estimator: workload-aware | logical tokens/s: 196932.0 | valid tokens/s: 196932.0 | physical tokens/s: 197354.2 | loss tokens/s: 150209.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.879745E-01 | lm-global-supervised-tokens: 1.279673E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.851489E-02 | lm-token-buffer-mib: 5.814710E+01 | global_load_balancing_loss: 1.000300E+00 | loss scale: 1.0 | grad norm: 0.088 | params norm: 11426.897 | params rms: 0.061873 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 12:11:55.006175 | finish at 2026-10-10 14:47:32 + [2026-10-07 02:37:07] iteration 3298/ 6855 | consumed samples: 422144 | elapsed time per iteration (ms): 90058.6 | throughput per GPU (TFLOP/s/GPU): 41.6 | MFU 13.33% | FLOPs estimator: workload-aware | logical tokens/s: 186291.6 | valid tokens/s: 186291.6 | physical tokens/s: 186907.7 | loss tokens/s: 138422.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.783084E-01 | lm-global-supervised-tokens: 1.246616E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.249048E-02 | lm-token-buffer-mib: 5.810419E+01 | global_load_balancing_loss: 9.996914E-01 | loss scale: 1.0 | grad norm: 0.108 | params norm: 11426.836 | params rms: 0.061873 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 16:58:58.613869 | finish at 2026-10-10 19:36:05 + [2026-10-07 02:38:34] iteration 3299/ 6855 | consumed samples: 422272 | elapsed time per iteration (ms): 87381.7 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.66% | FLOPs estimator: workload-aware | logical tokens/s: 191998.8 | valid tokens/s: 191998.8 | physical tokens/s: 192499.1 | loss tokens/s: 149827.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.011278E-01 | lm-global-supervised-tokens: 1.309216E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.276639E-02 | lm-token-buffer-mib: 5.816125E+01 | global_load_balancing_loss: 1.000210E+00 | loss scale: 1.0 | grad norm: 0.096 | params norm: 11426.776 | params rms: 0.061872 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 14:18:49.225296 | finish at 2026-10-10 16:57:23 + [2026-10-07 02:40:25] iteration 3300/ 6855 | consumed samples: 422400 | elapsed time per iteration (ms): 111014.5 | throughput per GPU (TFLOP/s/GPU): 33.4 | MFU 10.70% | FLOPs estimator: workload-aware | logical tokens/s: 151126.1 | valid tokens/s: 151126.1 | physical tokens/s: 151392.0 | loss tokens/s: 107719.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.400083E-01 | lm-global-supervised-tokens: 1.195839E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.149088E-02 | lm-token-buffer-mib: 5.814626E+01 | global_load_balancing_loss: 9.998220E-01 | loss scale: 1.0 | grad norm: 0.090 | params norm: 11426.715 | params rms: 0.061872 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 13:37:36.402218 | finish at 2026-10-11 16:18:02 + [2026-10-07 02:41:52] iteration 3301/ 6855 | consumed samples: 422528 | elapsed time per iteration (ms): 87236.3 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.69% | FLOPs estimator: workload-aware | logical tokens/s: 192318.1 | valid tokens/s: 192318.1 | physical tokens/s: 192861.5 | loss tokens/s: 140724.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.895281E-01 | lm-global-supervised-tokens: 1.227626E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.172920E-02 | lm-token-buffer-mib: 5.819913E+01 | global_load_balancing_loss: 1.000475E+00 | loss scale: 1.0 | grad norm: 0.082 | params norm: 11426.655 | params rms: 0.061872 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 14:07:17.644092 | finish at 2026-10-10 16:49:10 + [2026-10-07 02:43:17] iteration 3302/ 6855 | consumed samples: 422656 | elapsed time per iteration (ms): 84220.8 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.77% | FLOPs estimator: workload-aware | logical tokens/s: 199204.4 | valid tokens/s: 199204.4 | physical tokens/s: 199583.8 | loss tokens/s: 151069.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.267725E-01 | lm-global-supervised-tokens: 1.272318E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.354423E-02 | lm-token-buffer-mib: 5.812145E+01 | global_load_balancing_loss: 1.000157E+00 | loss scale: 1.0 | grad norm: 0.069 | params norm: 11426.592 | params rms: 0.061871 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 11:07:16.409792 | finish at 2026-10-10 13:50:33 + [2026-10-07 02:44:44] iteration 3303/ 6855 | consumed samples: 422784 | elapsed time per iteration (ms): 87033.7 | throughput per GPU (TFLOP/s/GPU): 41.5 | MFU 13.31% | FLOPs estimator: workload-aware | logical tokens/s: 192766.6 | valid tokens/s: 192766.6 | physical tokens/s: 193156.2 | loss tokens/s: 142547.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.339913E-01 | lm-global-supervised-tokens: 1.240648E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.155529E-02 | lm-token-buffer-mib: 5.813227E+01 | global_load_balancing_loss: 1.000991E+00 | loss scale: 1.0 | grad norm: 0.079 | params norm: 11426.529 | params rms: 0.061871 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 13:52:23.734543 | finish at 2026-10-10 16:37:07 + [2026-10-07 02:46:15] iteration 3304/ 6855 | consumed samples: 422912 | elapsed time per iteration (ms): 90845.8 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.50% | FLOPs estimator: workload-aware | logical tokens/s: 184677.8 | valid tokens/s: 184677.8 | physical tokens/s: 185080.3 | loss tokens/s: 131945.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.427117E-01 | lm-global-supervised-tokens: 1.198673E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.623904E-02 | lm-token-buffer-mib: 5.814397E+01 | global_load_balancing_loss: 1.000626E+00 | loss scale: 1.0 | grad norm: 0.071 | params norm: 11426.466 | params rms: 0.061871 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 17:36:33.605698 | finish at 2026-10-10 20:22:48 + [2026-10-07 02:47:44] iteration 3305/ 6855 | consumed samples: 423040 | elapsed time per iteration (ms): 89641.1 | throughput per GPU (TFLOP/s/GPU): 41.4 | MFU 13.27% | FLOPs estimator: workload-aware | logical tokens/s: 187159.9 | valid tokens/s: 187159.9 | physical tokens/s: 187565.1 | loss tokens/s: 134446.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.684942E-01 | lm-global-supervised-tokens: 1.205197E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.545222E-02 | lm-token-buffer-mib: 5.805319E+01 | global_load_balancing_loss: 1.000297E+00 | loss scale: 1.0 | grad norm: 0.064 | params norm: 11426.403 | params rms: 0.061870 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 16:23:45.859046 | finish at 2026-10-10 19:11:30 + [2026-10-07 02:49:13] iteration 3306/ 6855 | consumed samples: 423168 | elapsed time per iteration (ms): 89196.9 | throughput per GPU (TFLOP/s/GPU): 41.7 | MFU 13.35% | FLOPs estimator: workload-aware | logical tokens/s: 188091.8 | valid tokens/s: 188091.8 | physical tokens/s: 188513.7 | loss tokens/s: 139085.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.606760E-01 | lm-global-supervised-tokens: 1.240598E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.579176E-02 | lm-token-buffer-mib: 5.818906E+01 | global_load_balancing_loss: 9.999345E-01 | loss scale: 1.0 | grad norm: 0.038 | params norm: 11426.339 | params rms: 0.061870 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 15:55:59.925481 | finish at 2026-10-10 18:45:13 + [2026-10-07 02:50:36] iteration 3307/ 6855 | consumed samples: 423296 | elapsed time per iteration (ms): 82234.0 | throughput per GPU (TFLOP/s/GPU): 44.5 | MFU 14.26% | FLOPs estimator: workload-aware | logical tokens/s: 204017.9 | valid tokens/s: 204017.9 | physical tokens/s: 204415.2 | loss tokens/s: 158528.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.704639E-01 | lm-global-supervised-tokens: 1.303641E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.162402E-02 | lm-token-buffer-mib: 5.808494E+01 | global_load_balancing_loss: 1.000764E+00 | loss scale: 1.0 | grad norm: 0.048 | params norm: 11426.277 | params rms: 0.061870 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 9:02:46.353696 | finish at 2026-10-10 11:53:22 + [2026-10-07 02:52:04] iteration 3308/ 6855 | consumed samples: 423424 | elapsed time per iteration (ms): 88479.7 | throughput per GPU (TFLOP/s/GPU): 41.6 | MFU 13.32% | FLOPs estimator: workload-aware | logical tokens/s: 189615.9 | valid tokens/s: 189615.9 | physical tokens/s: 190174.3 | loss tokens/s: 142505.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.682983E-01 | lm-global-supervised-tokens: 1.260883E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.154737E-02 | lm-token-buffer-mib: 5.811127E+01 | global_load_balancing_loss: 9.999522E-01 | loss scale: 1.0 | grad norm: 0.060 | params norm: 11426.214 | params rms: 0.061869 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 15:10:37.379511 | finish at 2026-10-10 18:02:42 + [2026-10-07 02:53:30] iteration 3309/ 6855 | consumed samples: 423552 | elapsed time per iteration (ms): 85917.5 | throughput per GPU (TFLOP/s/GPU): 43.1 | MFU 13.80% | FLOPs estimator: workload-aware | logical tokens/s: 195270.5 | valid tokens/s: 195270.5 | physical tokens/s: 195681.1 | loss tokens/s: 143067.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.051148E-01 | lm-global-supervised-tokens: 1.229198E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.313446E-02 | lm-token-buffer-mib: 5.827847E+01 | global_load_balancing_loss: 9.999210E-01 | loss scale: 1.0 | grad norm: 0.062 | params norm: 11426.151 | params rms: 0.061869 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 12:37:43.548911 | finish at 2026-10-10 15:31:14 + [2026-10-07 02:55:19] iteration 3310/ 6855 | consumed samples: 423680 | elapsed time per iteration (ms): 109285.2 | throughput per GPU (TFLOP/s/GPU): 33.9 | MFU 10.87% | FLOPs estimator: workload-aware | logical tokens/s: 153517.2 | valid tokens/s: 153517.2 | physical tokens/s: 153841.7 | loss tokens/s: 114505.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.369323E-01 | lm-global-supervised-tokens: 1.251372E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.597775E-02 | lm-token-buffer-mib: 5.804089E+01 | global_load_balancing_loss: 1.000111E+00 | loss scale: 1.0 | grad norm: 0.051 | params norm: 11426.087 | params rms: 0.061869 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 11:36:55.995543 | finish at 2026-10-11 14:32:15 + [2026-10-07 02:56:44] iteration 3311/ 6855 | consumed samples: 423808 | elapsed time per iteration (ms): 84474.3 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.65% | FLOPs estimator: workload-aware | logical tokens/s: 198607.2 | valid tokens/s: 198607.2 | physical tokens/s: 199434.7 | loss tokens/s: 144830.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 5.972018E-01 | lm-global-supervised-tokens: 1.223444E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.943947E-02 | lm-token-buffer-mib: 5.825484E+01 | global_load_balancing_loss: 1.000827E+00 | loss scale: 1.0 | grad norm: 0.094 | params norm: 11426.025 | params rms: 0.061868 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 11:09:36.902819 | finish at 2026-10-10 14:06:21 + [2026-10-07 02:58:15] iteration 3312/ 6855 | consumed samples: 423936 | elapsed time per iteration (ms): 90874.0 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.44% | FLOPs estimator: workload-aware | logical tokens/s: 184620.3 | valid tokens/s: 184620.3 | physical tokens/s: 185011.4 | loss tokens/s: 127748.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.498540E-01 | lm-global-supervised-tokens: 1.160898E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.180952E-02 | lm-token-buffer-mib: 5.804872E+01 | global_load_balancing_loss: 1.000816E+00 | loss scale: 1.0 | grad norm: 0.283 | params norm: 11425.960 | params rms: 0.061868 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 17:26:06.495251 | finish at 2026-10-10 20:24:21 + [2026-10-07 02:59:40] iteration 3313/ 6855 | consumed samples: 424064 | elapsed time per iteration (ms): 85288.2 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.59% | FLOPs estimator: workload-aware | logical tokens/s: 196712.0 | valid tokens/s: 196712.0 | physical tokens/s: 197328.1 | loss tokens/s: 148058.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.415350E-01 | lm-global-supervised-tokens: 1.262767E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.088062E-02 | lm-token-buffer-mib: 5.823515E+01 | global_load_balancing_loss: 1.000407E+00 | loss scale: 1.0 | grad norm: 0.399 | params norm: 11425.895 | params rms: 0.061868 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 11:54:50.779562 | finish at 2026-10-10 14:54:31 + [2026-10-07 03:01:09] iteration 3314/ 6855 | consumed samples: 424192 | elapsed time per iteration (ms): 89340.7 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.52% | FLOPs estimator: workload-aware | logical tokens/s: 187789.1 | valid tokens/s: 187789.1 | physical tokens/s: 188402.6 | loss tokens/s: 138390.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.598669E-01 | lm-global-supervised-tokens: 1.236386E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.577615E-02 | lm-token-buffer-mib: 5.824066E+01 | global_load_balancing_loss: 9.994771E-01 | loss scale: 1.0 | grad norm: 0.091 | params norm: 11425.829 | params rms: 0.061867 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 15:52:35.406566 | finish at 2026-10-10 18:53:45 + [2026-10-07 03:02:34] iteration 3315/ 6855 | consumed samples: 424320 | elapsed time per iteration (ms): 85100.6 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.63% | FLOPs estimator: workload-aware | logical tokens/s: 197145.4 | valid tokens/s: 197145.4 | physical tokens/s: 197559.5 | loss tokens/s: 150270.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.821303E-01 | lm-global-supervised-tokens: 1.278811E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.273667E-02 | lm-token-buffer-mib: 5.803653E+01 | global_load_balancing_loss: 9.987828E-01 | loss scale: 1.0 | grad norm: 0.348 | params norm: 11425.761 | params rms: 0.061867 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 11:40:56.180563 | finish at 2026-10-10 14:43:31 + [2026-10-07 03:03:59] iteration 3316/ 6855 | consumed samples: 424448 | elapsed time per iteration (ms): 84544.0 | throughput per GPU (TFLOP/s/GPU): 44.2 | MFU 14.17% | FLOPs estimator: workload-aware | logical tokens/s: 198442.7 | valid tokens/s: 198442.7 | physical tokens/s: 199119.7 | loss tokens/s: 144537.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.896222E-01 | lm-global-supervised-tokens: 1.221977E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.391161E-02 | lm-token-buffer-mib: 5.818986E+01 | global_load_balancing_loss: 9.988857E-01 | loss scale: 1.0 | grad norm: 0.048 | params norm: 11425.695 | params rms: 0.061866 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 11:06:41.386967 | finish at 2026-10-10 14:10:40 + [2026-10-07 03:05:26] iteration 3317/ 6855 | consumed samples: 424576 | elapsed time per iteration (ms): 87491.9 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.65% | FLOPs estimator: workload-aware | logical tokens/s: 191756.5 | valid tokens/s: 191756.5 | physical tokens/s: 192192.2 | loss tokens/s: 138667.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.605313E-01 | lm-global-supervised-tokens: 1.213224E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.366116E-02 | lm-token-buffer-mib: 5.829742E+01 | global_load_balancing_loss: 1.000644E+00 | loss scale: 1.0 | grad norm: 0.273 | params norm: 11425.627 | params rms: 0.061866 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 13:59:06.328588 | finish at 2026-10-10 17:04:33 + [2026-10-07 03:06:54] iteration 3318/ 6855 | consumed samples: 424704 | elapsed time per iteration (ms): 87047.6 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.67% | FLOPs estimator: workload-aware | logical tokens/s: 192735.3 | valid tokens/s: 192735.2 | physical tokens/s: 193259.8 | loss tokens/s: 146322.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.254636E-01 | lm-global-supervised-tokens: 1.273702E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.267406E-02 | lm-token-buffer-mib: 5.808083E+01 | global_load_balancing_loss: 1.000604E+00 | loss scale: 1.0 | grad norm: 0.158 | params norm: 11425.559 | params rms: 0.061866 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 13:31:27.345287 | finish at 2026-10-10 16:38:21 + [2026-10-07 03:08:18] iteration 3319/ 6855 | consumed samples: 424832 | elapsed time per iteration (ms): 84298.3 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.67% | FLOPs estimator: workload-aware | logical tokens/s: 199022.1 | valid tokens/s: 199022.1 | physical tokens/s: 199688.7 | loss tokens/s: 145130.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 5.939967E-01 | lm-global-supervised-tokens: 1.223427E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.159916E-02 | lm-token-buffer-mib: 5.827246E+01 | global_load_balancing_loss: 1.002156E+00 | loss scale: 1.0 | grad norm: 0.188 | params norm: 11425.493 | params rms: 0.061865 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 10:47:58.683517 | finish at 2026-10-10 13:56:16 + [2026-10-07 03:10:10] iteration 3320/ 6855 | consumed samples: 424960 | elapsed time per iteration (ms): 112203.8 | throughput per GPU (TFLOP/s/GPU): 33.3 | MFU 10.69% | FLOPs estimator: workload-aware | logical tokens/s: 149524.4 | valid tokens/s: 149524.4 | physical tokens/s: 149950.9 | loss tokens/s: 112533.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.749774E-01 | lm-global-supervised-tokens: 1.262669E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.630928E-02 | lm-token-buffer-mib: 5.821434E+01 | global_load_balancing_loss: 9.994956E-01 | loss scale: 1.0 | grad norm: 0.184 | params norm: 11425.427 | params rms: 0.061865 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 14:10:40.363759 | finish at 2026-10-11 17:20:50 +times across ranks (ms): + save-checkpoint: + rank 0: 27808.54 + rank 1: 27808.42 + rank 2: 27808.48 + rank 3: 27808.51 + rank 4: 27808.43 + rank 5: 27808.43 + rank 6: 27808.45 + rank 7: 27808.44 + rank 8: 27810.12 + rank 9: 27810.08 + rank 10: 27810.12 + rank 11: 27810.08 + rank 12: 27810.08 + rank 13: 27810.08 + rank 14: 27810.14 + rank 15: 27810.08 + rank 16: 27808.40 + rank 17: 27808.38 + rank 18: 27808.37 + rank 19: 27808.39 + rank 20: 27808.41 + rank 21: 27808.38 + rank 22: 27808.37 + rank 23: 27808.34 + rank 24: 27808.46 + rank 25: 27808.74 + rank 26: 27808.88 + rank 27: 27808.79 + rank 28: 27808.72 + rank 29: 27809.73 + rank 30: 27809.72 + rank 31: 27808.74 + rank 32: 27808.48 + rank 33: 27808.50 + rank 34: 27808.52 + rank 35: 27808.45 + rank 36: 27808.41 + rank 37: 27808.43 + rank 38: 27808.41 + rank 39: 27808.38 + rank 40: 27808.26 + rank 41: 27808.28 + rank 42: 27808.25 + rank 43: 27808.27 + rank 44: 27808.23 + rank 45: 27808.24 + rank 46: 27808.24 + rank 47: 27808.22 + rank 48: 27808.51 + rank 49: 27808.47 + rank 50: 27808.47 + rank 51: 27808.49 + rank 52: 27808.53 + rank 53: 27808.42 + rank 54: 27808.55 + rank 55: 27808.51 + rank 56: 27808.28 + rank 57: 27808.20 + rank 58: 27808.19 + rank 59: 27808.18 + rank 60: 27808.21 + rank 61: 27808.20 + rank 62: 27808.18 + rank 63: 27808.22 + [2026-10-07 03:12:19] iteration 3321/ 6855 | consumed samples: 425088 | elapsed time per iteration (ms): 101267.3 | throughput per GPU (TFLOP/s/GPU): 36.1 | MFU 11.57% | FLOPs estimator: workload-aware | logical tokens/s: 165672.6 | valid tokens/s: 165672.6 | physical tokens/s: 166043.9 | loss tokens/s: 127698.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.785147E-01 | lm-global-supervised-tokens: 1.293166E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.690161E+00 | lm-token-buffer-mib: 5.804618E+01 | global_load_balancing_loss: 1.000816E+00 | loss scale: 1.0 | grad norm: 0.113 | params norm: 11425.361 | params rms: 0.061865 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 3:24:38.540917 | finish at 2026-10-11 06:36:58 + [2026-10-07 03:13:50] iteration 3322/ 6855 | consumed samples: 425216 | elapsed time per iteration (ms): 90568.8 | throughput per GPU (TFLOP/s/GPU): 40.4 | MFU 12.94% | FLOPs estimator: workload-aware | logical tokens/s: 185242.8 | valid tokens/s: 185242.8 | physical tokens/s: 185736.7 | loss tokens/s: 138847.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.628680E-01 | lm-global-supervised-tokens: 1.257520E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.455672E-02 | lm-token-buffer-mib: 5.837426E+01 | global_load_balancing_loss: 1.000576E+00 | loss scale: 1.0 | grad norm: 0.096 | params norm: 11425.295 | params rms: 0.061864 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 16:52:59.569461 | finish at 2026-10-10 20:06:49 + [2026-10-07 03:15:18] iteration 3323/ 6855 | consumed samples: 425344 | elapsed time per iteration (ms): 88010.6 | throughput per GPU (TFLOP/s/GPU): 43.2 | MFU 13.84% | FLOPs estimator: workload-aware | logical tokens/s: 190626.7 | valid tokens/s: 190626.7 | physical tokens/s: 190985.1 | loss tokens/s: 135750.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.664755E-01 | lm-global-supervised-tokens: 1.194753E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.337208E-02 | lm-token-buffer-mib: 5.816735E+01 | global_load_balancing_loss: 1.000576E+00 | loss scale: 1.0 | grad norm: 0.104 | params norm: 11425.230 | params rms: 0.061864 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 14:20:53.556569 | finish at 2026-10-10 17:36:11 + [2026-10-07 03:16:42] iteration 3324/ 6855 | consumed samples: 425472 | elapsed time per iteration (ms): 84455.2 | throughput per GPU (TFLOP/s/GPU): 43.4 | MFU 13.91% | FLOPs estimator: workload-aware | logical tokens/s: 198652.1 | valid tokens/s: 198652.1 | physical tokens/s: 199158.9 | loss tokens/s: 145019.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.283614E-01 | lm-global-supervised-tokens: 1.224767E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.286300E-02 | lm-token-buffer-mib: 5.822282E+01 | global_load_balancing_loss: 9.995134E-01 | loss scale: 1.0 | grad norm: 0.066 | params norm: 11425.166 | params rms: 0.061864 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 10:50:11.168774 | finish at 2026-10-10 14:06:53 + [2026-10-07 03:18:10] iteration 3325/ 6855 | consumed samples: 425600 | elapsed time per iteration (ms): 87989.0 | throughput per GPU (TFLOP/s/GPU): 41.6 | MFU 13.34% | FLOPs estimator: workload-aware | logical tokens/s: 190673.1 | valid tokens/s: 190673.1 | physical tokens/s: 191137.3 | loss tokens/s: 143239.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.585974E-01 | lm-global-supervised-tokens: 1.260346E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.120851E-02 | lm-token-buffer-mib: 5.808199E+01 | global_load_balancing_loss: 1.000088E+00 | loss scale: 1.0 | grad norm: 0.076 | params norm: 11425.099 | params rms: 0.061863 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 14:16:41.021323 | finish at 2026-10-10 17:34:51 + [2026-10-07 03:19:37] iteration 3326/ 6855 | consumed samples: 425728 | elapsed time per iteration (ms): 86881.1 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.67% | FLOPs estimator: workload-aware | logical tokens/s: 193105.2 | valid tokens/s: 193105.2 | physical tokens/s: 193579.8 | loss tokens/s: 141397.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.209122E-01 | lm-global-supervised-tokens: 1.228477E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.236036E-02 | lm-token-buffer-mib: 5.813979E+01 | global_load_balancing_loss: 1.000010E+00 | loss scale: 1.0 | grad norm: 0.148 | params norm: 11425.032 | params rms: 0.061863 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 13:10:03.246872 | finish at 2026-10-10 16:29:40 + [2026-10-07 03:21:06] iteration 3327/ 6855 | consumed samples: 425856 | elapsed time per iteration (ms): 88722.4 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.63% | FLOPs estimator: workload-aware | logical tokens/s: 189097.8 | valid tokens/s: 189097.8 | physical tokens/s: 189447.9 | loss tokens/s: 133877.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.951513E-01 | lm-global-supervised-tokens: 1.187792E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.776749E-02 | lm-token-buffer-mib: 5.806979E+01 | global_load_balancing_loss: 9.990379E-01 | loss scale: 1.0 | grad norm: 0.049 | params norm: 11424.967 | params rms: 0.061862 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 14:56:52.645529 | finish at 2026-10-10 18:17:58 + [2026-10-07 03:22:33] iteration 3328/ 6855 | consumed samples: 425984 | elapsed time per iteration (ms): 87501.0 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.65% | FLOPs estimator: workload-aware | logical tokens/s: 191736.8 | valid tokens/s: 191736.8 | physical tokens/s: 192099.5 | loss tokens/s: 142102.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.575326E-01 | lm-global-supervised-tokens: 1.243406E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.318143E-02 | lm-token-buffer-mib: 5.816789E+01 | global_load_balancing_loss: 9.997431E-01 | loss scale: 1.0 | grad norm: 0.109 | params norm: 11424.900 | params rms: 0.061862 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 13:43:36.007404 | finish at 2026-10-10 17:06:09 + [2026-10-07 03:23:59] iteration 3329/ 6855 | consumed samples: 426112 | elapsed time per iteration (ms): 85379.5 | throughput per GPU (TFLOP/s/GPU): 43.5 | MFU 13.95% | FLOPs estimator: workload-aware | logical tokens/s: 196501.2 | valid tokens/s: 196501.1 | physical tokens/s: 196938.3 | loss tokens/s: 145624.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.533262E-01 | lm-global-supervised-tokens: 1.243335E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.178234E-02 | lm-token-buffer-mib: 5.805981E+01 | global_load_balancing_loss: 1.001441E+00 | loss scale: 1.0 | grad norm: 0.041 | params norm: 11424.834 | params rms: 0.061862 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 11:37:28.146955 | finish at 2026-10-10 15:01:27 + [2026-10-07 03:25:50] iteration 3330/ 6855 | consumed samples: 426240 | elapsed time per iteration (ms): 111355.6 | throughput per GPU (TFLOP/s/GPU): 33.2 | MFU 10.65% | FLOPs estimator: workload-aware | logical tokens/s: 150663.3 | valid tokens/s: 150663.3 | physical tokens/s: 151048.4 | loss tokens/s: 112980.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.957329E-01 | lm-global-supervised-tokens: 1.258096E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.158741E-02 | lm-token-buffer-mib: 5.810212E+01 | global_load_balancing_loss: 9.997922E-01 | loss scale: 1.0 | grad norm: 0.072 | params norm: 11424.767 | params rms: 0.061861 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 13:02:08.329057 | finish at 2026-10-11 16:27:58 + [2026-10-07 03:27:20] iteration 3331/ 6855 | consumed samples: 426368 | elapsed time per iteration (ms): 90330.6 | throughput per GPU (TFLOP/s/GPU): 41.5 | MFU 13.29% | FLOPs estimator: workload-aware | logical tokens/s: 185731.2 | valid tokens/s: 185731.1 | physical tokens/s: 186250.0 | loss tokens/s: 139484.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.528167E-01 | lm-global-supervised-tokens: 1.259970E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.630268E-02 | lm-token-buffer-mib: 5.823593E+01 | global_load_balancing_loss: 1.000369E+00 | loss scale: 1.0 | grad norm: 0.052 | params norm: 11424.700 | params rms: 0.061861 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 16:25:24.886609 | finish at 2026-10-10 19:52:45 + [2026-10-07 03:28:47] iteration 3332/ 6855 | consumed samples: 426496 | elapsed time per iteration (ms): 86462.7 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.68% | FLOPs estimator: workload-aware | logical tokens/s: 194039.3 | valid tokens/s: 194039.3 | physical tokens/s: 194401.1 | loss tokens/s: 137285.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.182572E-01 | lm-global-supervised-tokens: 1.187007E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.246396E-02 | lm-token-buffer-mib: 5.822367E+01 | global_load_balancing_loss: 1.001642E+00 | loss scale: 1.0 | grad norm: 0.096 | params norm: 11424.635 | params rms: 0.061861 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 12:36:48.168148 | finish at 2026-10-10 16:05:35 + [2026-10-07 03:30:15] iteration 3333/ 6855 | consumed samples: 426624 | elapsed time per iteration (ms): 87957.8 | throughput per GPU (TFLOP/s/GPU): 41.7 | MFU 13.37% | FLOPs estimator: workload-aware | logical tokens/s: 190741.3 | valid tokens/s: 190741.3 | physical tokens/s: 191199.0 | loss tokens/s: 140344.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.947893E-01 | lm-global-supervised-tokens: 1.234436E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.650428E-02 | lm-token-buffer-mib: 5.811433E+01 | global_load_balancing_loss: 9.996281E-01 | loss scale: 1.0 | grad norm: 0.045 | params norm: 11424.570 | params rms: 0.061860 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 14:03:07.259607 | finish at 2026-10-10 17:33:22 + [2026-10-07 03:31:44] iteration 3334/ 6855 | consumed samples: 426752 | elapsed time per iteration (ms): 88964.4 | throughput per GPU (TFLOP/s/GPU): 40.4 | MFU 12.96% | FLOPs estimator: workload-aware | logical tokens/s: 188583.1 | valid tokens/s: 188583.1 | physical tokens/s: 189167.3 | loss tokens/s: 138473.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.262639E-01 | lm-global-supervised-tokens: 1.231923E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.412495E-02 | lm-token-buffer-mib: 5.806763E+01 | global_load_balancing_loss: 9.992982E-01 | loss scale: 1.0 | grad norm: 0.132 | params norm: 11424.505 | params rms: 0.061860 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 15:00:43.640834 | finish at 2026-10-10 18:32:27 + [2026-10-07 03:33:12] iteration 3335/ 6855 | consumed samples: 426880 | elapsed time per iteration (ms): 88558.7 | throughput per GPU (TFLOP/s/GPU): 41.4 | MFU 13.27% | FLOPs estimator: workload-aware | logical tokens/s: 189447.0 | valid tokens/s: 189447.0 | physical tokens/s: 189833.2 | loss tokens/s: 143369.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.595105E-01 | lm-global-supervised-tokens: 1.269663E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.187693E-02 | lm-token-buffer-mib: 5.820755E+01 | global_load_balancing_loss: 1.000342E+00 | loss scale: 1.0 | grad norm: 0.124 | params norm: 11424.441 | params rms: 0.061860 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 14:35:26.761093 | finish at 2026-10-10 18:08:39 + [2026-10-07 03:34:40] iteration 3336/ 6855 | consumed samples: 427008 | elapsed time per iteration (ms): 87523.8 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.60% | FLOPs estimator: workload-aware | logical tokens/s: 191687.1 | valid tokens/s: 191687.1 | physical tokens/s: 192061.9 | loss tokens/s: 145432.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.363636E-01 | lm-global-supervised-tokens: 1.272883E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.428193E-02 | lm-token-buffer-mib: 5.813506E+01 | global_load_balancing_loss: 9.997508E-01 | loss scale: 1.0 | grad norm: 0.068 | params norm: 11424.377 | params rms: 0.061859 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 13:33:16.166257 | finish at 2026-10-10 17:07:56 + [2026-10-07 03:36:02] iteration 3337/ 6855 | consumed samples: 427136 | elapsed time per iteration (ms): 82395.0 | throughput per GPU (TFLOP/s/GPU): 43.6 | MFU 13.98% | FLOPs estimator: workload-aware | logical tokens/s: 203619.2 | valid tokens/s: 203619.2 | physical tokens/s: 204101.8 | loss tokens/s: 159942.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.503350E-01 | lm-global-supervised-tokens: 1.317846E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.777103E-02 | lm-token-buffer-mib: 5.812732E+01 | global_load_balancing_loss: 9.996158E-01 | loss scale: 1.0 | grad norm: 0.034 | params norm: 11424.313 | params rms: 0.061859 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 8:31:05.752522 | finish at 2026-10-10 12:07:08 + [2026-10-07 03:37:26] iteration 3338/ 6855 | consumed samples: 427264 | elapsed time per iteration (ms): 83666.1 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.78% | FLOPs estimator: workload-aware | logical tokens/s: 200525.2 | valid tokens/s: 200525.1 | physical tokens/s: 201132.9 | loss tokens/s: 147436.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.357306E-01 | lm-global-supervised-tokens: 1.233540E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.444991E-02 | lm-token-buffer-mib: 5.804469E+01 | global_load_balancing_loss: 9.999306E-01 | loss scale: 1.0 | grad norm: 0.135 | params norm: 11424.250 | params rms: 0.061859 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 9:44:13.517824 | finish at 2026-10-10 13:21:39 + [2026-10-07 03:38:52] iteration 3339/ 6855 | consumed samples: 427392 | elapsed time per iteration (ms): 86293.3 | throughput per GPU (TFLOP/s/GPU): 43.2 | MFU 13.86% | FLOPs estimator: workload-aware | logical tokens/s: 194420.0 | valid tokens/s: 194420.0 | physical tokens/s: 194792.8 | loss tokens/s: 139700.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.685800E-01 | lm-global-supervised-tokens: 1.205521E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.268568E-02 | lm-token-buffer-mib: 5.802508E+01 | global_load_balancing_loss: 1.001270E+00 | loss scale: 1.0 | grad norm: 0.084 | params norm: 11424.188 | params rms: 0.061858 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 12:16:47.151123 | finish at 2026-10-10 15:55:39 + [2026-10-07 03:40:43] iteration 3340/ 6855 | consumed samples: 427520 | elapsed time per iteration (ms): 111165.1 | throughput per GPU (TFLOP/s/GPU): 34.8 | MFU 11.16% | FLOPs estimator: workload-aware | logical tokens/s: 150921.5 | valid tokens/s: 150921.5 | physical tokens/s: 151147.4 | loss tokens/s: 107969.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.674553E-01 | lm-global-supervised-tokens: 1.200243E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.604564E-02 | lm-token-buffer-mib: 5.803908E+01 | global_load_balancing_loss: 1.000268E+00 | loss scale: 1.0 | grad norm: 0.046 | params norm: 11424.129 | params rms: 0.061858 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 12:32:25.451711 | finish at 2026-10-11 16:13:09 + [2026-10-07 03:42:08] iteration 3341/ 6855 | consumed samples: 427648 | elapsed time per iteration (ms): 85005.4 | throughput per GPU (TFLOP/s/GPU): 43.3 | MFU 13.88% | FLOPs estimator: workload-aware | logical tokens/s: 197365.7 | valid tokens/s: 197365.7 | physical tokens/s: 197884.4 | loss tokens/s: 144568.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.494597E-01 | lm-global-supervised-tokens: 1.228910E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.150649E-02 | lm-token-buffer-mib: 5.812389E+01 | global_load_balancing_loss: 1.000427E+00 | loss scale: 1.0 | grad norm: 0.062 | params norm: 11424.067 | params rms: 0.061858 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 10:58:29.106933 | finish at 2026-10-10 14:40:37 + [2026-10-07 03:43:36] iteration 3342/ 6855 | consumed samples: 427776 | elapsed time per iteration (ms): 87714.0 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.47% | FLOPs estimator: workload-aware | logical tokens/s: 191271.7 | valid tokens/s: 191271.7 | physical tokens/s: 191656.3 | loss tokens/s: 137058.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.672828E-01 | lm-global-supervised-tokens: 1.202198E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.051682E-02 | lm-token-buffer-mib: 5.807192E+01 | global_load_balancing_loss: 1.000278E+00 | loss scale: 1.0 | grad norm: 0.043 | params norm: 11424.005 | params rms: 0.061857 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 13:35:39.372410 | finish at 2026-10-10 17:19:15 + [2026-10-07 03:45:02] iteration 3343/ 6855 | consumed samples: 427904 | elapsed time per iteration (ms): 85485.7 | throughput per GPU (TFLOP/s/GPU): 43.1 | MFU 13.80% | FLOPs estimator: workload-aware | logical tokens/s: 196257.1 | valid tokens/s: 196257.1 | physical tokens/s: 196794.2 | loss tokens/s: 137007.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.261897E-01 | lm-global-supervised-tokens: 1.171219E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.266445E-02 | lm-token-buffer-mib: 5.802811E+01 | global_load_balancing_loss: 1.000698E+00 | loss scale: 1.0 | grad norm: 0.059 | params norm: 11423.943 | params rms: 0.061857 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 11:23:45.808165 | finish at 2026-10-10 15:08:47 + [2026-10-07 03:46:27] iteration 3344/ 6855 | consumed samples: 428032 | elapsed time per iteration (ms): 85607.6 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.79% | FLOPs estimator: workload-aware | logical tokens/s: 195977.3 | valid tokens/s: 195977.3 | physical tokens/s: 196551.2 | loss tokens/s: 143177.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.819035E-01 | lm-global-supervised-tokens: 1.225709E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.156495E-02 | lm-token-buffer-mib: 5.817744E+01 | global_load_balancing_loss: 9.998868E-01 | loss scale: 1.0 | grad norm: 0.062 | params norm: 11423.882 | params rms: 0.061857 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 11:29:28.155433 | finish at 2026-10-10 15:15:55 + [2026-10-07 03:47:54] iteration 3345/ 6855 | consumed samples: 428160 | elapsed time per iteration (ms): 87188.4 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.52% | FLOPs estimator: workload-aware | logical tokens/s: 192424.4 | valid tokens/s: 192424.4 | physical tokens/s: 192908.4 | loss tokens/s: 138155.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 5.686310E-01 | lm-global-supervised-tokens: 1.204552E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.274503E-02 | lm-token-buffer-mib: 5.818764E+01 | global_load_balancing_loss: 1.000135E+00 | loss scale: 1.0 | grad norm: 0.047 | params norm: 11423.820 | params rms: 0.061856 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 13:00:31.232221 | finish at 2026-10-10 16:48:26 + [2026-10-07 03:49:19] iteration 3346/ 6855 | consumed samples: 428288 | elapsed time per iteration (ms): 84896.3 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.75% | FLOPs estimator: workload-aware | logical tokens/s: 197619.7 | valid tokens/s: 197619.7 | physical tokens/s: 198062.9 | loss tokens/s: 148558.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.748119E-01 | lm-global-supervised-tokens: 1.261206E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.216614E-02 | lm-token-buffer-mib: 5.827091E+01 | global_load_balancing_loss: 9.993095E-01 | loss scale: 1.0 | grad norm: 0.042 | params norm: 11423.759 | params rms: 0.061856 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 10:45:01.227404 | finish at 2026-10-10 14:34:20 + [2026-10-07 03:50:47] iteration 3347/ 6855 | consumed samples: 428416 | elapsed time per iteration (ms): 87423.3 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.60% | FLOPs estimator: workload-aware | logical tokens/s: 191907.7 | valid tokens/s: 191907.7 | physical tokens/s: 192308.0 | loss tokens/s: 141236.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.558691E-01 | lm-global-supervised-tokens: 1.234735E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.445659E-02 | lm-token-buffer-mib: 5.811308E+01 | global_load_balancing_loss: 9.997424E-01 | loss scale: 1.0 | grad norm: 0.049 | params norm: 11423.699 | params rms: 0.061856 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 13:11:21.085372 | finish at 2026-10-10 17:02:08 + [2026-10-07 03:52:16] iteration 3348/ 6855 | consumed samples: 428544 | elapsed time per iteration (ms): 89307.6 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.52% | FLOPs estimator: workload-aware | logical tokens/s: 187857.8 | valid tokens/s: 187857.8 | physical tokens/s: 188292.9 | loss tokens/s: 141848.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.677247E-01 | lm-global-supervised-tokens: 1.266816E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.343991E-02 | lm-token-buffer-mib: 5.813314E+01 | global_load_balancing_loss: 9.996579E-01 | loss scale: 1.0 | grad norm: 0.061 | params norm: 11423.640 | params rms: 0.061855 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 15:00:01.647086 | finish at 2026-10-10 18:52:18 + [2026-10-07 03:53:43] iteration 3349/ 6855 | consumed samples: 428672 | elapsed time per iteration (ms): 86683.3 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.53% | FLOPs estimator: workload-aware | logical tokens/s: 193545.6 | valid tokens/s: 193545.6 | physical tokens/s: 194013.8 | loss tokens/s: 143548.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.208071E-01 | lm-global-supervised-tokens: 1.244328E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.342838E-02 | lm-token-buffer-mib: 5.809633E+01 | global_load_balancing_loss: 1.000930E+00 | loss scale: 1.0 | grad norm: 0.067 | params norm: 11423.581 | params rms: 0.061855 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 12:25:11.735961 | finish at 2026-10-10 16:18:54 + [2026-10-07 03:55:31] iteration 3350/ 6855 | consumed samples: 428800 | elapsed time per iteration (ms): 108032.4 | throughput per GPU (TFLOP/s/GPU): 34.1 | MFU 10.92% | FLOPs estimator: workload-aware | logical tokens/s: 155297.8 | valid tokens/s: 155297.8 | physical tokens/s: 155641.6 | loss tokens/s: 115035.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.552012E-01 | lm-global-supervised-tokens: 1.242753E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.535793E-02 | lm-token-buffer-mib: 5.809104E+01 | global_load_balancing_loss: 1.000873E+00 | loss scale: 1.0 | grad norm: 0.091 | params norm: 11423.521 | params rms: 0.061855 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 9:10:53.564131 | finish at 2026-10-11 13:06:24 + [2026-10-07 03:56:57] iteration 3351/ 6855 | consumed samples: 428928 | elapsed time per iteration (ms): 86416.3 | throughput per GPU (TFLOP/s/GPU): 41.7 | MFU 13.36% | FLOPs estimator: workload-aware | logical tokens/s: 194143.7 | valid tokens/s: 194143.7 | physical tokens/s: 194637.2 | loss tokens/s: 142581.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.543754E-01 | lm-global-supervised-tokens: 1.232140E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.444077E-02 | lm-token-buffer-mib: 5.819888E+01 | global_load_balancing_loss: 1.001297E+00 | loss scale: 1.0 | grad norm: 0.057 | params norm: 11423.461 | params rms: 0.061854 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 12:06:42.685329 | finish at 2026-10-10 16:03:40 + [2026-10-07 03:58:25] iteration 3352/ 6855 | consumed samples: 429056 | elapsed time per iteration (ms): 87914.8 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.56% | FLOPs estimator: workload-aware | logical tokens/s: 190834.8 | valid tokens/s: 190834.8 | physical tokens/s: 191336.1 | loss tokens/s: 135775.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.893735E-01 | lm-global-supervised-tokens: 1.193666E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.450676E-02 | lm-token-buffer-mib: 5.816715E+01 | global_load_balancing_loss: 1.000528E+00 | loss scale: 1.0 | grad norm: 0.050 | params norm: 11423.399 | params rms: 0.061854 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 13:32:45.613470 | finish at 2026-10-10 17:31:11 + [2026-10-07 03:59:51] iteration 3353/ 6855 | consumed samples: 429184 | elapsed time per iteration (ms): 85515.4 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.76% | FLOPs estimator: workload-aware | logical tokens/s: 196189.0 | valid tokens/s: 196189.0 | physical tokens/s: 196951.3 | loss tokens/s: 145388.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.539089E-01 | lm-global-supervised-tokens: 1.243298E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.317906E-02 | lm-token-buffer-mib: 5.814142E+01 | global_load_balancing_loss: 9.994372E-01 | loss scale: 1.0 | grad norm: 0.109 | params norm: 11423.336 | params rms: 0.061854 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 11:11:14.973147 | finish at 2026-10-10 15:11:06 + [2026-10-07 04:01:18] iteration 3354/ 6855 | consumed samples: 429312 | elapsed time per iteration (ms): 87553.5 | throughput per GPU (TFLOP/s/GPU): 43.3 | MFU 13.88% | FLOPs estimator: workload-aware | logical tokens/s: 191621.1 | valid tokens/s: 191621.1 | physical tokens/s: 191971.0 | loss tokens/s: 149381.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.747409E-01 | lm-global-supervised-tokens: 1.307884E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.404929E-02 | lm-token-buffer-mib: 5.808496E+01 | global_load_balancing_loss: 9.987500E-01 | loss scale: 1.0 | grad norm: 0.083 | params norm: 11423.273 | params rms: 0.061853 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 13:08:44.939336 | finish at 2026-10-10 17:10:03 + [2026-10-07 04:02:43] iteration 3355/ 6855 | consumed samples: 429440 | elapsed time per iteration (ms): 84466.5 | throughput per GPU (TFLOP/s/GPU): 43.6 | MFU 13.96% | FLOPs estimator: workload-aware | logical tokens/s: 198625.1 | valid tokens/s: 198625.1 | physical tokens/s: 199063.3 | loss tokens/s: 144129.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.209821E-01 | lm-global-supervised-tokens: 1.217408E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.225622E-02 | lm-token-buffer-mib: 5.818012E+01 | global_load_balancing_loss: 9.992179E-01 | loss scale: 1.0 | grad norm: 0.072 | params norm: 11423.210 | params rms: 0.061853 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 10:07:12.681727 | finish at 2026-10-10 14:09:55 + [2026-10-07 04:04:08] iteration 3356/ 6855 | consumed samples: 429568 | elapsed time per iteration (ms): 85743.6 | throughput per GPU (TFLOP/s/GPU): 43.1 | MFU 13.81% | FLOPs estimator: workload-aware | logical tokens/s: 195667.2 | valid tokens/s: 195667.2 | physical tokens/s: 196087.2 | loss tokens/s: 141951.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.819765E-01 | lm-global-supervised-tokens: 1.217139E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.756701E-02 | lm-token-buffer-mib: 5.812889E+01 | global_load_balancing_loss: 1.001096E+00 | loss scale: 1.0 | grad norm: 0.095 | params norm: 11423.146 | params rms: 0.061853 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 11:20:16.959465 | finish at 2026-10-10 15:24:25 + [2026-10-07 04:05:34] iteration 3357/ 6855 | consumed samples: 429696 | elapsed time per iteration (ms): 85214.1 | throughput per GPU (TFLOP/s/GPU): 42.8 | MFU 13.71% | FLOPs estimator: workload-aware | logical tokens/s: 196882.4 | valid tokens/s: 196882.4 | physical tokens/s: 197446.0 | loss tokens/s: 144962.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.204190E-01 | lm-global-supervised-tokens: 1.235281E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.459496E-02 | lm-token-buffer-mib: 5.822611E+01 | global_load_balancing_loss: 1.000707E+00 | loss scale: 1.0 | grad norm: 0.090 | params norm: 11423.081 | params rms: 0.061852 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 10:47:58.985611 | finish at 2026-10-10 14:53:33 + [2026-10-07 04:07:00] iteration 3358/ 6855 | consumed samples: 429824 | elapsed time per iteration (ms): 86575.3 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.59% | FLOPs estimator: workload-aware | logical tokens/s: 193787.3 | valid tokens/s: 193787.3 | physical tokens/s: 194381.9 | loss tokens/s: 144698.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.678416E-01 | lm-global-supervised-tokens: 1.252731E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.153662E-02 | lm-token-buffer-mib: 5.808639E+01 | global_load_balancing_loss: 1.000249E+00 | loss scale: 1.0 | grad norm: 0.129 | params norm: 11423.016 | params rms: 0.061852 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 12:05:53.861543 | finish at 2026-10-10 16:12:54 + [2026-10-07 04:08:29] iteration 3359/ 6855 | consumed samples: 429952 | elapsed time per iteration (ms): 88677.0 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.63% | FLOPs estimator: workload-aware | logical tokens/s: 189193.7 | valid tokens/s: 189193.7 | physical tokens/s: 189705.4 | loss tokens/s: 131472.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.334872E-01 | lm-global-supervised-tokens: 1.165854E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.144902E-02 | lm-token-buffer-mib: 5.816525E+01 | global_load_balancing_loss: 1.000197E+00 | loss scale: 1.0 | grad norm: 0.060 | params norm: 11422.950 | params rms: 0.061852 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 14:06:54.692972 | finish at 2026-10-10 18:15:23 + [2026-10-07 04:10:15] iteration 3360/ 6855 | consumed samples: 430080 | elapsed time per iteration (ms): 106229.3 | throughput per GPU (TFLOP/s/GPU): 34.8 | MFU 11.14% | FLOPs estimator: workload-aware | logical tokens/s: 157933.8 | valid tokens/s: 157933.8 | physical tokens/s: 158338.1 | loss tokens/s: 112744.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.701401E-01 | lm-global-supervised-tokens: 1.197679E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.127097E-02 | lm-token-buffer-mib: 5.807126E+01 | global_load_balancing_loss: 1.001045E+00 | loss scale: 1.0 | grad norm: 0.154 | params norm: 11422.885 | params rms: 0.061851 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 7:07:51.315250 | finish at 2026-10-11 11:18:06 +times across ranks (ms): + save-checkpoint: + rank 0: 24856.82 + rank 1: 24856.79 + rank 2: 24856.83 + rank 3: 24856.82 + rank 4: 24856.82 + rank 5: 24856.80 + rank 6: 24856.77 + rank 7: 24856.83 + rank 8: 24856.63 + rank 9: 24856.58 + rank 10: 24856.67 + rank 11: 24856.58 + rank 12: 24856.62 + rank 13: 24856.59 + rank 14: 24856.62 + rank 15: 24856.59 + rank 16: 24856.56 + rank 17: 24856.55 + rank 18: 24856.53 + rank 19: 24856.56 + rank 20: 24856.56 + rank 21: 24856.62 + rank 22: 24856.52 + rank 23: 24856.53 + rank 24: 24856.52 + rank 25: 24856.56 + rank 26: 24856.52 + rank 27: 24856.49 + rank 28: 24856.44 + rank 29: 24856.49 + rank 30: 24856.47 + rank 31: 24856.46 + rank 32: 24856.82 + rank 33: 24856.58 + rank 34: 24856.71 + rank 35: 24856.78 + rank 36: 24856.70 + rank 37: 24856.81 + rank 38: 24856.79 + rank 39: 24856.74 + rank 40: 24856.47 + rank 41: 24856.44 + rank 42: 24856.44 + rank 43: 24856.47 + rank 44: 24856.44 + rank 45: 24856.45 + rank 46: 24856.44 + rank 47: 24856.45 + rank 48: 24856.96 + rank 49: 24856.87 + rank 50: 24856.78 + rank 51: 24856.91 + rank 52: 24856.99 + rank 53: 24856.87 + rank 54: 24857.25 + rank 55: 24856.89 + rank 56: 24856.80 + rank 57: 24856.79 + rank 58: 24856.73 + rank 59: 24856.75 + rank 60: 24856.79 + rank 61: 24856.77 + rank 62: 24856.74 + rank 63: 24856.76 + [2026-10-07 04:12:10] iteration 3361/ 6855 | consumed samples: 430208 | elapsed time per iteration (ms): 90367.2 | throughput per GPU (TFLOP/s/GPU): 40.3 | MFU 12.92% | FLOPs estimator: workload-aware | logical tokens/s: 185655.6 | valid tokens/s: 185655.6 | physical tokens/s: 186089.7 | loss tokens/s: 139762.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.374676E-01 | lm-global-supervised-tokens: 1.262991E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.427338E-01 | lm-token-buffer-mib: 5.819049E+01 | global_load_balancing_loss: 9.998263E-01 | loss scale: 1.0 | grad norm: 0.035 | params norm: 11422.820 | params rms: 0.061851 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 15:42:22.843997 | finish at 2026-10-10 19:54:33 + [2026-10-07 04:13:43] iteration 3362/ 6855 | consumed samples: 430336 | elapsed time per iteration (ms): 92593.4 | throughput per GPU (TFLOP/s/GPU): 40.0 | MFU 12.82% | FLOPs estimator: workload-aware | logical tokens/s: 181191.3 | valid tokens/s: 181191.3 | physical tokens/s: 181478.7 | loss tokens/s: 128708.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.657413E-01 | lm-global-supervised-tokens: 1.191758E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.653370E-02 | lm-token-buffer-mib: 5.811865E+01 | global_load_balancing_loss: 9.998333E-01 | loss scale: 1.0 | grad norm: 0.115 | params norm: 11422.756 | params rms: 0.061851 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 17:50:28.679582 | finish at 2026-10-10 22:04:12 + [2026-10-07 04:15:10] iteration 3363/ 6855 | consumed samples: 430464 | elapsed time per iteration (ms): 87324.6 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.67% | FLOPs estimator: workload-aware | logical tokens/s: 192124.7 | valid tokens/s: 192124.7 | physical tokens/s: 192525.8 | loss tokens/s: 141413.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.626301E-01 | lm-global-supervised-tokens: 1.234889E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.210575E-02 | lm-token-buffer-mib: 5.807462E+01 | global_load_balancing_loss: 1.000312E+00 | loss scale: 1.0 | grad norm: 0.034 | params norm: 11422.694 | params rms: 0.061850 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 12:42:17.563079 | finish at 2026-10-10 16:57:28 + [2026-10-07 04:16:35] iteration 3364/ 6855 | consumed samples: 430592 | elapsed time per iteration (ms): 84615.4 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.78% | FLOPs estimator: workload-aware | logical tokens/s: 198275.5 | valid tokens/s: 198275.5 | physical tokens/s: 198814.4 | loss tokens/s: 147387.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.609461E-01 | lm-global-supervised-tokens: 1.247124E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.502138E-02 | lm-token-buffer-mib: 5.819982E+01 | global_load_balancing_loss: 1.000692E+00 | loss scale: 1.0 | grad norm: 0.128 | params norm: 11422.630 | params rms: 0.061850 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 10:03:12.492339 | finish at 2026-10-10 14:19:47 + [2026-10-07 04:18:02] iteration 3365/ 6855 | consumed samples: 430720 | elapsed time per iteration (ms): 86879.1 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.57% | FLOPs estimator: workload-aware | logical tokens/s: 193109.5 | valid tokens/s: 193109.5 | physical tokens/s: 193497.7 | loss tokens/s: 141803.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.919196E-01 | lm-global-supervised-tokens: 1.231971E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.336398E-02 | lm-token-buffer-mib: 5.808025E+01 | global_load_balancing_loss: 9.995944E-01 | loss scale: 1.0 | grad norm: 0.062 | params norm: 11422.567 | params rms: 0.061849 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 12:13:27.894542 | finish at 2026-10-10 16:31:30 + [2026-10-07 04:19:28] iteration 3366/ 6855 | consumed samples: 430848 | elapsed time per iteration (ms): 86197.4 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.59% | FLOPs estimator: workload-aware | logical tokens/s: 194636.2 | valid tokens/s: 194636.2 | physical tokens/s: 195051.3 | loss tokens/s: 142255.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.506652E-01 | lm-global-supervised-tokens: 1.226203E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.369287E-02 | lm-token-buffer-mib: 5.807294E+01 | global_load_balancing_loss: 9.997606E-01 | loss scale: 1.0 | grad norm: 0.088 | params norm: 11422.504 | params rms: 0.061849 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 11:32:22.752216 | finish at 2026-10-10 15:51:51 + [2026-10-07 04:20:55] iteration 3367/ 6855 | consumed samples: 430976 | elapsed time per iteration (ms): 87343.8 | throughput per GPU (TFLOP/s/GPU): 41.7 | MFU 13.37% | FLOPs estimator: workload-aware | logical tokens/s: 192082.3 | valid tokens/s: 192082.3 | physical tokens/s: 192489.7 | loss tokens/s: 144393.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.603804E-01 | lm-global-supervised-tokens: 1.261185E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.130606E-02 | lm-token-buffer-mib: 5.818804E+01 | global_load_balancing_loss: 1.000021E+00 | loss scale: 1.0 | grad norm: 0.036 | params norm: 11422.440 | params rms: 0.061849 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 12:37:35.268608 | finish at 2026-10-10 16:58:31 + [2026-10-07 04:22:23] iteration 3368/ 6855 | consumed samples: 431104 | elapsed time per iteration (ms): 87876.3 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.65% | FLOPs estimator: workload-aware | logical tokens/s: 190918.3 | valid tokens/s: 190918.3 | physical tokens/s: 191222.8 | loss tokens/s: 143584.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.490993E-01 | lm-global-supervised-tokens: 1.261769E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.265578E-02 | lm-token-buffer-mib: 5.812381E+01 | global_load_balancing_loss: 1.000949E+00 | loss scale: 1.0 | grad norm: 0.058 | params norm: 11422.378 | params rms: 0.061848 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 13:07:04.655942 | finish at 2026-10-10 17:29:28 + [2026-10-07 04:23:52] iteration 3369/ 6855 | consumed samples: 431232 | elapsed time per iteration (ms): 88413.5 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.64% | FLOPs estimator: workload-aware | logical tokens/s: 189758.3 | valid tokens/s: 189758.3 | physical tokens/s: 190095.1 | loss tokens/s: 136701.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.673041E-01 | lm-global-supervised-tokens: 1.208625E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.523940E-02 | lm-token-buffer-mib: 5.809924E+01 | global_load_balancing_loss: 9.999958E-01 | loss scale: 1.0 | grad norm: 0.063 | params norm: 11422.317 | params rms: 0.061848 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 13:36:49.514438 | finish at 2026-10-10 18:00:41 + [2026-10-07 04:25:41] iteration 3370/ 6855 | consumed samples: 431360 | elapsed time per iteration (ms): 109722.2 | throughput per GPU (TFLOP/s/GPU): 34.1 | MFU 10.92% | FLOPs estimator: workload-aware | logical tokens/s: 152906.1 | valid tokens/s: 152906.1 | physical tokens/s: 153289.2 | loss tokens/s: 115426.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.130972E-01 | lm-global-supervised-tokens: 1.266486E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.262622E-02 | lm-token-buffer-mib: 5.816315E+01 | global_load_balancing_loss: 9.998633E-01 | loss scale: 1.0 | grad norm: 0.121 | params norm: 11422.255 | params rms: 0.061848 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 10:13:01.844276 | finish at 2026-10-11 14:38:43 + [2026-10-07 04:27:13] iteration 3371/ 6855 | consumed samples: 431488 | elapsed time per iteration (ms): 91809.7 | throughput per GPU (TFLOP/s/GPU): 40.8 | MFU 13.08% | FLOPs estimator: workload-aware | logical tokens/s: 182738.6 | valid tokens/s: 182738.6 | physical tokens/s: 183090.7 | loss tokens/s: 129843.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.031845E-01 | lm-global-supervised-tokens: 1.192088E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.789441E-02 | lm-token-buffer-mib: 5.810007E+01 | global_load_balancing_loss: 1.000269E+00 | loss scale: 1.0 | grad norm: 0.055 | params norm: 11422.192 | params rms: 0.061847 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 16:51:05.038036 | finish at 2026-10-10 21:18:18 + [2026-10-07 04:28:39] iteration 3372/ 6855 | consumed samples: 431616 | elapsed time per iteration (ms): 85583.4 | throughput per GPU (TFLOP/s/GPU): 43.2 | MFU 13.86% | FLOPs estimator: workload-aware | logical tokens/s: 196032.7 | valid tokens/s: 196032.7 | physical tokens/s: 196447.8 | loss tokens/s: 152748.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.101191E-01 | lm-global-supervised-tokens: 1.307276E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.330889E-02 | lm-token-buffer-mib: 5.810735E+01 | global_load_balancing_loss: 1.000147E+00 | loss scale: 1.0 | grad norm: 0.056 | params norm: 11422.132 | params rms: 0.061847 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 10:48:07.024590 | finish at 2026-10-10 15:16:46 + [2026-10-07 04:30:04] iteration 3373/ 6855 | consumed samples: 431744 | elapsed time per iteration (ms): 85175.5 | throughput per GPU (TFLOP/s/GPU): 43.7 | MFU 14.00% | FLOPs estimator: workload-aware | logical tokens/s: 196970.9 | valid tokens/s: 196970.9 | physical tokens/s: 197382.5 | loss tokens/s: 145824.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.878587E-01 | lm-global-supervised-tokens: 1.242071E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.177623E-02 | lm-token-buffer-mib: 5.819475E+01 | global_load_balancing_loss: 1.000893E+00 | loss scale: 1.0 | grad norm: 0.060 | params norm: 11422.073 | params rms: 0.061847 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 10:23:01.138028 | finish at 2026-10-10 14:53:05 + [2026-10-07 04:31:31] iteration 3374/ 6855 | consumed samples: 431872 | elapsed time per iteration (ms): 86941.3 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.75% | FLOPs estimator: workload-aware | logical tokens/s: 192971.4 | valid tokens/s: 192971.4 | physical tokens/s: 193445.6 | loss tokens/s: 139548.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.188029E-01 | lm-global-supervised-tokens: 1.213254E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.343330E-02 | lm-token-buffer-mib: 5.818651E+01 | global_load_balancing_loss: 9.996496E-01 | loss scale: 1.0 | grad norm: 0.056 | params norm: 11422.016 | params rms: 0.061846 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 12:04:02.685754 | finish at 2026-10-10 16:35:34 + [2026-10-07 04:33:00] iteration 3375/ 6855 | consumed samples: 432000 | elapsed time per iteration (ms): 89124.3 | throughput per GPU (TFLOP/s/GPU): 41.7 | MFU 13.38% | FLOPs estimator: workload-aware | logical tokens/s: 188244.4 | valid tokens/s: 188244.4 | physical tokens/s: 188633.9 | loss tokens/s: 135548.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.390712E-01 | lm-global-supervised-tokens: 1.208064E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.167678E-02 | lm-token-buffer-mib: 5.806950E+01 | global_load_balancing_loss: 1.000330E+00 | loss scale: 1.0 | grad norm: 0.053 | params norm: 11421.954 | params rms: 0.061846 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 14:09:12.715845 | finish at 2026-10-10 18:42:13 + [2026-10-07 04:34:27] iteration 3376/ 6855 | consumed samples: 432128 | elapsed time per iteration (ms): 87231.4 | throughput per GPU (TFLOP/s/GPU): 43.2 | MFU 13.84% | FLOPs estimator: workload-aware | logical tokens/s: 192329.7 | valid tokens/s: 192329.7 | physical tokens/s: 192663.0 | loss tokens/s: 135461.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.809423E-01 | lm-global-supervised-tokens: 1.181652E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.379427E-02 | lm-token-buffer-mib: 5.804868E+01 | global_load_balancing_loss: 1.000337E+00 | loss scale: 1.0 | grad norm: 0.042 | params norm: 11421.894 | params rms: 0.061846 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 12:17:57.868947 | finish at 2026-10-10 16:52:25 + [2026-10-07 04:35:55] iteration 3377/ 6855 | consumed samples: 432256 | elapsed time per iteration (ms): 88172.2 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.63% | FLOPs estimator: workload-aware | logical tokens/s: 190277.7 | valid tokens/s: 190277.7 | physical tokens/s: 190516.0 | loss tokens/s: 139254.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.583964E-01 | lm-global-supervised-tokens: 1.227837E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.168026E-02 | lm-token-buffer-mib: 5.805515E+01 | global_load_balancing_loss: 9.999835E-01 | loss scale: 1.0 | grad norm: 0.043 | params norm: 11421.835 | params rms: 0.061846 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 13:11:02.861723 | finish at 2026-10-10 17:46:58 + [2026-10-07 04:37:25] iteration 3378/ 6855 | consumed samples: 432384 | elapsed time per iteration (ms): 89423.4 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.55% | FLOPs estimator: workload-aware | logical tokens/s: 187614.7 | valid tokens/s: 187614.7 | physical tokens/s: 188187.2 | loss tokens/s: 139817.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.413122E-01 | lm-global-supervised-tokens: 1.250292E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.611034E-02 | lm-token-buffer-mib: 5.817175E+01 | global_load_balancing_loss: 1.000588E+00 | loss scale: 1.0 | grad norm: 0.064 | params norm: 11421.776 | params rms: 0.061845 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 14:22:05.037193 | finish at 2026-10-10 18:59:30 + [2026-10-07 04:38:50] iteration 3379/ 6855 | consumed samples: 432512 | elapsed time per iteration (ms): 84792.9 | throughput per GPU (TFLOP/s/GPU): 43.3 | MFU 13.88% | FLOPs estimator: workload-aware | logical tokens/s: 197860.4 | valid tokens/s: 197860.4 | physical tokens/s: 198252.9 | loss tokens/s: 144561.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.185707E-01 | lm-global-supervised-tokens: 1.225779E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.494380E-02 | lm-token-buffer-mib: 5.808951E+01 | global_load_balancing_loss: 9.998042E-01 | loss scale: 1.0 | grad norm: 0.046 | params norm: 11421.717 | params rms: 0.061845 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 9:52:20.080917 | finish at 2026-10-10 14:31:10 + [2026-10-07 04:40:39] iteration 3380/ 6855 | consumed samples: 432640 | elapsed time per iteration (ms): 109221.9 | throughput per GPU (TFLOP/s/GPU): 34.3 | MFU 10.98% | FLOPs estimator: workload-aware | logical tokens/s: 153606.4 | valid tokens/s: 153606.4 | physical tokens/s: 153992.0 | loss tokens/s: 111558.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.622310E-01 | lm-global-supervised-tokens: 1.218466E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.178263E-02 | lm-token-buffer-mib: 5.818406E+01 | global_load_balancing_loss: 1.001034E+00 | loss scale: 1.0 | grad norm: 0.076 | params norm: 11421.661 | params rms: 0.061845 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 9:25:46.100795 | finish at 2026-10-11 14:06:25 + [2026-10-07 04:42:04] iteration 3381/ 6855 | consumed samples: 432768 | elapsed time per iteration (ms): 85332.9 | throughput per GPU (TFLOP/s/GPU): 43.2 | MFU 13.86% | FLOPs estimator: workload-aware | logical tokens/s: 196608.6 | valid tokens/s: 196608.6 | physical tokens/s: 197169.9 | loss tokens/s: 141368.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.031306E-01 | lm-global-supervised-tokens: 1.206339E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.358029E-02 | lm-token-buffer-mib: 5.853140E+01 | global_load_balancing_loss: 1.000535E+00 | loss scale: 1.0 | grad norm: 0.086 | params norm: 11421.603 | params rms: 0.061844 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 10:20:46.430988 | finish at 2026-10-10 15:02:51 + [2026-10-07 04:43:30] iteration 3382/ 6855 | consumed samples: 432896 | elapsed time per iteration (ms): 85685.5 | throughput per GPU (TFLOP/s/GPU): 43.7 | MFU 14.00% | FLOPs estimator: workload-aware | logical tokens/s: 195799.9 | valid tokens/s: 195799.9 | physical tokens/s: 196095.0 | loss tokens/s: 144800.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.812003E-01 | lm-global-supervised-tokens: 1.240727E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.202290E-02 | lm-token-buffer-mib: 5.810003E+01 | global_load_balancing_loss: 9.992984E-01 | loss scale: 1.0 | grad norm: 0.137 | params norm: 11421.545 | params rms: 0.061844 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 10:39:45.716335 | finish at 2026-10-10 15:23:16 + [2026-10-07 04:44:56] iteration 3383/ 6855 | consumed samples: 433024 | elapsed time per iteration (ms): 86583.6 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.63% | FLOPs estimator: workload-aware | logical tokens/s: 193768.9 | valid tokens/s: 193768.9 | physical tokens/s: 194152.5 | loss tokens/s: 136559.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.488046E-01 | lm-global-supervised-tokens: 1.182385E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.607293E-02 | lm-token-buffer-mib: 5.814831E+01 | global_load_balancing_loss: 1.000789E+00 | loss scale: 1.0 | grad norm: 0.156 | params norm: 11421.486 | params rms: 0.061844 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 11:30:18.425739 | finish at 2026-10-10 16:15:15 + [2026-10-07 04:46:23] iteration 3384/ 6855 | consumed samples: 433152 | elapsed time per iteration (ms): 86912.8 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.78% | FLOPs estimator: workload-aware | logical tokens/s: 193035.1 | valid tokens/s: 193035.1 | physical tokens/s: 193433.0 | loss tokens/s: 141714.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.591530E-01 | lm-global-supervised-tokens: 1.231678E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.120873E-02 | lm-token-buffer-mib: 5.812732E+01 | global_load_balancing_loss: 9.999729E-01 | loss scale: 1.0 | grad norm: 0.077 | params norm: 11421.428 | params rms: 0.061843 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 11:47:54.168511 | finish at 2026-10-10 16:34:18 + [2026-10-07 04:47:51] iteration 3385/ 6855 | consumed samples: 433280 | elapsed time per iteration (ms): 87814.1 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.48% | FLOPs estimator: workload-aware | logical tokens/s: 191052.9 | valid tokens/s: 191052.9 | physical tokens/s: 191442.2 | loss tokens/s: 147344.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.734844E-01 | lm-global-supervised-tokens: 1.293894E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.335264E-02 | lm-token-buffer-mib: 5.807575E+01 | global_load_balancing_loss: 9.990565E-01 | loss scale: 1.0 | grad norm: 0.051 | params norm: 11421.370 | params rms: 0.061843 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 12:38:34.982524 | finish at 2026-10-10 17:26:26 + [2026-10-07 04:49:21] iteration 3386/ 6855 | consumed samples: 433408 | elapsed time per iteration (ms): 90031.0 | throughput per GPU (TFLOP/s/GPU): 41.2 | MFU 13.22% | FLOPs estimator: workload-aware | logical tokens/s: 186349.3 | valid tokens/s: 186349.3 | physical tokens/s: 186937.4 | loss tokens/s: 136558.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.258860E-01 | lm-global-supervised-tokens: 1.229447E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.138585E-02 | lm-token-buffer-mib: 5.819727E+01 | global_load_balancing_loss: 1.000108E+00 | loss scale: 1.0 | grad norm: 0.096 | params norm: 11421.312 | params rms: 0.061843 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 14:45:17.498123 | finish at 2026-10-10 19:34:39 + [2026-10-07 04:50:49] iteration 3387/ 6855 | consumed samples: 433536 | elapsed time per iteration (ms): 87659.8 | throughput per GPU (TFLOP/s/GPU): 43.2 | MFU 13.85% | FLOPs estimator: workload-aware | logical tokens/s: 191389.4 | valid tokens/s: 191389.4 | physical tokens/s: 191757.1 | loss tokens/s: 138980.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.112683E-01 | lm-global-supervised-tokens: 1.218295E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.130175E-02 | lm-token-buffer-mib: 5.814322E+01 | global_load_balancing_loss: 1.000584E+00 | loss scale: 1.0 | grad norm: 0.149 | params norm: 11421.254 | params rms: 0.061842 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 12:26:44.024523 | finish at 2026-10-10 17:17:33 + [2026-10-07 04:52:17] iteration 3388/ 6855 | consumed samples: 433664 | elapsed time per iteration (ms): 87873.8 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.43% | FLOPs estimator: workload-aware | logical tokens/s: 190923.2 | valid tokens/s: 190923.2 | physical tokens/s: 191365.7 | loss tokens/s: 146832.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.568927E-01 | lm-global-supervised-tokens: 1.290277E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.750804E-02 | lm-token-buffer-mib: 5.812823E+01 | global_load_balancing_loss: 1.000379E+00 | loss scale: 1.0 | grad norm: 0.129 | params norm: 11421.196 | params rms: 0.061842 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 12:37:38.411007 | finish at 2026-10-10 17:29:55 + [2026-10-07 04:53:42] iteration 3389/ 6855 | consumed samples: 433792 | elapsed time per iteration (ms): 85552.7 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.62% | FLOPs estimator: workload-aware | logical tokens/s: 196103.7 | valid tokens/s: 196103.7 | physical tokens/s: 196618.9 | loss tokens/s: 146024.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.082405E-01 | lm-global-supervised-tokens: 1.249276E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.976192E-02 | lm-token-buffer-mib: 5.815217E+01 | global_load_balancing_loss: 1.000652E+00 | loss scale: 1.0 | grad norm: 0.034 | params norm: 11421.136 | params rms: 0.061842 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 10:22:05.652564 | finish at 2026-10-10 15:15:48 + [2026-10-07 04:55:32] iteration 3390/ 6855 | consumed samples: 433920 | elapsed time per iteration (ms): 109956.7 | throughput per GPU (TFLOP/s/GPU): 33.7 | MFU 10.81% | FLOPs estimator: workload-aware | logical tokens/s: 152580.0 | valid tokens/s: 152579.9 | physical tokens/s: 152865.9 | loss tokens/s: 108579.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.849396E-01 | lm-global-supervised-tokens: 1.193899E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.496351E-02 | lm-token-buffer-mib: 5.807801E+01 | global_load_balancing_loss: 1.000222E+00 | loss scale: 1.0 | grad norm: 0.093 | params norm: 11421.075 | params rms: 0.061841 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 9:49:59.827838 | finish at 2026-10-11 14:45:32 + [2026-10-07 04:56:59] iteration 3391/ 6855 | consumed samples: 434048 | elapsed time per iteration (ms): 86995.3 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.52% | FLOPs estimator: workload-aware | logical tokens/s: 192850.9 | valid tokens/s: 192850.9 | physical tokens/s: 193405.5 | loss tokens/s: 144468.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.641151E-01 | lm-global-supervised-tokens: 1.256804E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.253571E-02 | lm-token-buffer-mib: 5.814612E+01 | global_load_balancing_loss: 9.993600E-01 | loss scale: 1.0 | grad norm: 0.056 | params norm: 11421.012 | params rms: 0.061841 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 11:42:31.881262 | finish at 2026-10-10 16:39:31 + [2026-10-07 04:58:26] iteration 3392/ 6855 | consumed samples: 434176 | elapsed time per iteration (ms): 86309.4 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.46% | FLOPs estimator: workload-aware | logical tokens/s: 194384.6 | valid tokens/s: 194384.6 | physical tokens/s: 194822.1 | loss tokens/s: 148544.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.619490E-01 | lm-global-supervised-tokens: 1.282080E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.159306E-02 | lm-token-buffer-mib: 5.826851E+01 | global_load_balancing_loss: 1.000286E+00 | loss scale: 1.0 | grad norm: 0.072 | params norm: 11420.950 | params rms: 0.061841 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 11:01:29.383954 | finish at 2026-10-10 15:59:55 + [2026-10-07 04:59:52] iteration 3393/ 6855 | consumed samples: 434304 | elapsed time per iteration (ms): 86478.9 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.49% | FLOPs estimator: workload-aware | logical tokens/s: 194003.2 | valid tokens/s: 194003.2 | physical tokens/s: 194708.2 | loss tokens/s: 140114.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.622702E-01 | lm-global-supervised-tokens: 1.211696E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.158966E-02 | lm-token-buffer-mib: 5.828729E+01 | global_load_balancing_loss: 1.001985E+00 | loss scale: 1.0 | grad norm: 0.042 | params norm: 11420.889 | params rms: 0.061840 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 11:09:49.892218 | finish at 2026-10-10 16:09:42 + [2026-10-07 05:01:23] iteration 3394/ 6855 | consumed samples: 434432 | elapsed time per iteration (ms): 91174.4 | throughput per GPU (TFLOP/s/GPU): 40.8 | MFU 13.08% | FLOPs estimator: workload-aware | logical tokens/s: 184011.7 | valid tokens/s: 184011.7 | physical tokens/s: 184401.1 | loss tokens/s: 139609.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.623623E-01 | lm-global-supervised-tokens: 1.272885E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.162402E-02 | lm-token-buffer-mib: 5.808925E+01 | global_load_balancing_loss: 1.000477E+00 | loss scale: 1.0 | grad norm: 0.077 | params norm: 11420.828 | params rms: 0.061840 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 15:39:14.758798 | finish at 2026-10-10 20:40:38 + [2026-10-07 05:02:51] iteration 3395/ 6855 | consumed samples: 434560 | elapsed time per iteration (ms): 87569.6 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.68% | FLOPs estimator: workload-aware | logical tokens/s: 191587.0 | valid tokens/s: 191587.0 | physical tokens/s: 192195.2 | loss tokens/s: 143771.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.086343E-01 | lm-global-supervised-tokens: 1.258999E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.188550E-02 | lm-token-buffer-mib: 5.820805E+01 | global_load_balancing_loss: 1.000013E+00 | loss scale: 1.0 | grad norm: 0.034 | params norm: 11420.764 | params rms: 0.061840 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 12:09:50.961552 | finish at 2026-10-10 17:12:42 + [2026-10-07 05:04:15] iteration 3396/ 6855 | consumed samples: 434688 | elapsed time per iteration (ms): 83924.4 | throughput per GPU (TFLOP/s/GPU): 43.2 | MFU 13.84% | FLOPs estimator: workload-aware | logical tokens/s: 199908.3 | valid tokens/s: 199908.3 | physical tokens/s: 200346.1 | loss tokens/s: 147266.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.104749E-01 | lm-global-supervised-tokens: 1.235930E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.901740E-02 | lm-token-buffer-mib: 5.805299E+01 | global_load_balancing_loss: 9.993968E-01 | loss scale: 1.0 | grad norm: 0.091 | params norm: 11420.702 | params rms: 0.061839 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 8:38:14.656606 | finish at 2026-10-10 13:42:29 + [2026-10-07 05:05:40] iteration 3397/ 6855 | consumed samples: 434816 | elapsed time per iteration (ms): 85731.4 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.67% | FLOPs estimator: workload-aware | logical tokens/s: 195695.0 | valid tokens/s: 195695.0 | physical tokens/s: 196115.7 | loss tokens/s: 143200.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.622623E-01 | lm-global-supervised-tokens: 1.227676E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.715493E-02 | lm-token-buffer-mib: 5.809597E+01 | global_load_balancing_loss: 1.000373E+00 | loss scale: 1.0 | grad norm: 0.050 | params norm: 11420.640 | params rms: 0.061839 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 10:20:59.295844 | finish at 2026-10-10 15:26:40 + [2026-10-07 05:07:06] iteration 3398/ 6855 | consumed samples: 434944 | elapsed time per iteration (ms): 85801.8 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.63% | FLOPs estimator: workload-aware | logical tokens/s: 195534.0 | valid tokens/s: 195534.0 | physical tokens/s: 195943.7 | loss tokens/s: 143220.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.323590E-01 | lm-global-supervised-tokens: 1.228860E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.777407E-02 | lm-token-buffer-mib: 5.813543E+01 | global_load_balancing_loss: 9.992919E-01 | loss scale: 1.0 | grad norm: 0.070 | params norm: 11420.578 | params rms: 0.061839 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 10:23:36.991607 | finish at 2026-10-10 15:30:43 + [2026-10-07 05:08:37] iteration 3399/ 6855 | consumed samples: 435072 | elapsed time per iteration (ms): 90503.3 | throughput per GPU (TFLOP/s/GPU): 40.8 | MFU 13.08% | FLOPs estimator: workload-aware | logical tokens/s: 185376.6 | valid tokens/s: 185376.6 | physical tokens/s: 185696.0 | loss tokens/s: 128707.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 5.894809E-01 | lm-global-supervised-tokens: 1.164847E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.018418E-02 | lm-token-buffer-mib: 5.805283E+01 | global_load_balancing_loss: 1.000096E+00 | loss scale: 1.0 | grad norm: 0.038 | params norm: 11420.517 | params rms: 0.061838 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 14:52:59.252197 | finish at 2026-10-10 20:01:36 + [2026-10-07 05:10:27] iteration 3400/ 6855 | consumed samples: 435200 | elapsed time per iteration (ms): 110423.7 | throughput per GPU (TFLOP/s/GPU): 33.7 | MFU 10.80% | FLOPs estimator: workload-aware | logical tokens/s: 151934.7 | valid tokens/s: 151934.7 | physical tokens/s: 152255.0 | loss tokens/s: 112143.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.564558E-01 | lm-global-supervised-tokens: 1.238327E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.170306E-02 | lm-token-buffer-mib: 5.808091E+01 | global_load_balancing_loss: 1.000203E+00 | loss scale: 1.0 | grad norm: 0.125 | params norm: 11420.458 | params rms: 0.061838 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 9:58:33.833578 | finish at 2026-10-11 15:09:01 +times across ranks (ms): + save-checkpoint: + rank 0: 25079.46 + rank 1: 25079.44 + rank 2: 25079.50 + rank 3: 25079.47 + rank 4: 25079.46 + rank 5: 25079.44 + rank 6: 25079.43 + rank 7: 25079.45 + rank 8: 25080.20 + rank 9: 25080.16 + rank 10: 25080.15 + rank 11: 25080.15 + rank 12: 25080.16 + rank 13: 25080.17 + rank 14: 25080.16 + rank 15: 25080.15 + rank 16: 25080.27 + rank 17: 25080.26 + rank 18: 25080.25 + rank 19: 25080.27 + rank 20: 25080.25 + rank 21: 25080.28 + rank 22: 25080.29 + rank 23: 25080.25 + rank 24: 25080.38 + rank 25: 25080.37 + rank 26: 25080.39 + rank 27: 25080.38 + rank 28: 25080.41 + rank 29: 25080.48 + rank 30: 25080.36 + rank 31: 25080.37 + rank 32: 25079.27 + rank 33: 25079.32 + rank 34: 25079.25 + rank 35: 25079.24 + rank 36: 25079.25 + rank 37: 25079.29 + rank 38: 25079.33 + rank 39: 25079.23 + rank 40: 25079.68 + rank 41: 25079.65 + rank 42: 25079.65 + rank 43: 25079.65 + rank 44: 25079.65 + rank 45: 25079.64 + rank 46: 25079.64 + rank 47: 25079.63 + rank 48: 25081.28 + rank 49: 25081.27 + rank 50: 25081.26 + rank 51: 25081.25 + rank 52: 25081.25 + rank 53: 25081.26 + rank 54: 25081.27 + rank 55: 25081.27 + rank 56: 25079.86 + rank 57: 25079.86 + rank 58: 25079.82 + rank 59: 25079.82 + rank 60: 25079.82 + rank 61: 25079.83 + rank 62: 25079.82 + rank 63: 25079.82 + [2026-10-07 05:12:44] iteration 3401/ 6855 | consumed samples: 435328 | elapsed time per iteration (ms): 109764.7 | throughput per GPU (TFLOP/s/GPU): 33.6 | MFU 10.77% | FLOPs estimator: workload-aware | logical tokens/s: 152847.0 | valid tokens/s: 152847.0 | physical tokens/s: 153205.6 | loss tokens/s: 110725.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.262954E-01 | lm-global-supervised-tokens: 1.215378E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 7.995265E-01 | lm-token-buffer-mib: 5.820462E+01 | global_load_balancing_loss: 1.000256E+00 | loss scale: 1.0 | grad norm: 0.130 | params norm: 11420.398 | params rms: 0.061838 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 9:18:47.209346 | finish at 2026-10-11 14:31:31 + [2026-10-07 05:14:19] iteration 3402/ 6855 | consumed samples: 435456 | elapsed time per iteration (ms): 94924.6 | throughput per GPU (TFLOP/s/GPU): 38.8 | MFU 12.43% | FLOPs estimator: workload-aware | logical tokens/s: 176742.4 | valid tokens/s: 176742.4 | physical tokens/s: 177033.7 | loss tokens/s: 133314.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.609052E-01 | lm-global-supervised-tokens: 1.265484E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.912309E-02 | lm-token-buffer-mib: 5.807344E+01 | global_load_balancing_loss: 1.001155E+00 | loss scale: 1.0 | grad norm: 0.036 | params norm: 11420.336 | params rms: 0.061837 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 19:02:54.512508 | finish at 2026-10-11 00:17:13 + [2026-10-07 05:15:46] iteration 3403/ 6855 | consumed samples: 435584 | elapsed time per iteration (ms): 87590.8 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.44% | FLOPs estimator: workload-aware | logical tokens/s: 191539.6 | valid tokens/s: 191539.6 | physical tokens/s: 191919.0 | loss tokens/s: 138122.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.101978E-01 | lm-global-supervised-tokens: 1.209826E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.755182E-02 | lm-token-buffer-mib: 5.814340E+01 | global_load_balancing_loss: 1.001642E+00 | loss scale: 1.0 | grad norm: 0.139 | params norm: 11420.273 | params rms: 0.061837 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 11:59:23.442585 | finish at 2026-10-10 17:15:10 + [2026-10-07 05:17:13] iteration 3404/ 6855 | consumed samples: 435712 | elapsed time per iteration (ms): 86798.6 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.62% | FLOPs estimator: workload-aware | logical tokens/s: 193288.6 | valid tokens/s: 193288.6 | physical tokens/s: 193683.3 | loss tokens/s: 142759.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.573962E-01 | lm-global-supervised-tokens: 1.239134E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.419488E-02 | lm-token-buffer-mib: 5.814324E+01 | global_load_balancing_loss: 1.000056E+00 | loss scale: 1.0 | grad norm: 0.050 | params norm: 11420.211 | params rms: 0.061837 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 11:12:22.010418 | finish at 2026-10-10 16:29:35 + [2026-10-07 05:18:40] iteration 3405/ 6855 | consumed samples: 435840 | elapsed time per iteration (ms): 86558.8 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.69% | FLOPs estimator: workload-aware | logical tokens/s: 193824.5 | valid tokens/s: 193824.5 | physical tokens/s: 194335.6 | loss tokens/s: 150583.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.677997E-01 | lm-global-supervised-tokens: 1.303434E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.395764E-02 | lm-token-buffer-mib: 5.819130E+01 | global_load_balancing_loss: 9.995500E-01 | loss scale: 1.0 | grad norm: 0.094 | params norm: 11420.149 | params rms: 0.061836 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 10:57:07.793312 | finish at 2026-10-10 16:15:47 + [2026-10-07 05:20:06] iteration 3406/ 6855 | consumed samples: 435968 | elapsed time per iteration (ms): 85800.4 | throughput per GPU (TFLOP/s/GPU): 43.2 | MFU 13.86% | FLOPs estimator: workload-aware | logical tokens/s: 195537.6 | valid tokens/s: 195537.6 | physical tokens/s: 195877.8 | loss tokens/s: 141271.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 5.851920E-01 | lm-global-supervised-tokens: 1.212115E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.118574E-02 | lm-token-buffer-mib: 5.816431E+01 | global_load_balancing_loss: 9.995422E-01 | loss scale: 1.0 | grad norm: 0.069 | params norm: 11420.087 | params rms: 0.061836 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 10:12:05.706300 | finish at 2026-10-10 15:32:11 + [2026-10-07 05:21:33] iteration 3407/ 6855 | consumed samples: 436096 | elapsed time per iteration (ms): 87642.1 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.43% | FLOPs estimator: workload-aware | logical tokens/s: 191428.6 | valid tokens/s: 191428.6 | physical tokens/s: 191826.2 | loss tokens/s: 139749.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.046707E-01 | lm-global-supervised-tokens: 1.224794E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.026713E-02 | lm-token-buffer-mib: 5.808506E+01 | global_load_balancing_loss: 9.991674E-01 | loss scale: 1.0 | grad norm: 0.052 | params norm: 11420.023 | params rms: 0.061836 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 11:56:30.126366 | finish at 2026-10-10 17:18:03 + [2026-10-07 05:23:01] iteration 3408/ 6855 | consumed samples: 436224 | elapsed time per iteration (ms): 87442.9 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.44% | FLOPs estimator: workload-aware | logical tokens/s: 191864.6 | valid tokens/s: 191864.6 | physical tokens/s: 192321.1 | loss tokens/s: 142452.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.872702E-01 | lm-global-supervised-tokens: 1.245647E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.125441E-02 | lm-token-buffer-mib: 5.809885E+01 | global_load_balancing_loss: 1.000117E+00 | loss scale: 1.0 | grad norm: 0.052 | params norm: 11419.960 | params rms: 0.061835 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 11:43:35.519954 | finish at 2026-10-10 17:06:36 + [2026-10-07 05:24:28] iteration 3409/ 6855 | consumed samples: 436352 | elapsed time per iteration (ms): 87811.5 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.44% | FLOPs estimator: workload-aware | logical tokens/s: 191058.4 | valid tokens/s: 191058.4 | physical tokens/s: 191557.0 | loss tokens/s: 140060.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.484156E-01 | lm-global-supervised-tokens: 1.229891E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.401098E-02 | lm-token-buffer-mib: 5.812436E+01 | global_load_balancing_loss: 1.001361E+00 | loss scale: 1.0 | grad norm: 0.044 | params norm: 11419.898 | params rms: 0.061835 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 12:03:18.437466 | finish at 2026-10-10 17:27:47 + [2026-10-07 05:26:16] iteration 3410/ 6855 | consumed samples: 436480 | elapsed time per iteration (ms): 107836.9 | throughput per GPU (TFLOP/s/GPU): 34.2 | MFU 10.98% | FLOPs estimator: workload-aware | logical tokens/s: 155579.0 | valid tokens/s: 155579.0 | physical tokens/s: 156057.2 | loss tokens/s: 112008.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.857318E-01 | lm-global-supervised-tokens: 1.207867E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.829869E-02 | lm-token-buffer-mib: 5.833304E+01 | global_load_balancing_loss: 1.001638E+00 | loss scale: 1.0 | grad norm: 0.046 | params norm: 11419.836 | params rms: 0.061835 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 7:11:38.266686 | finish at 2026-10-11 12:37:55 + [2026-10-07 05:27:40] iteration 3411/ 6855 | consumed samples: 436608 | elapsed time per iteration (ms): 83356.8 | throughput per GPU (TFLOP/s/GPU): 43.9 | MFU 14.06% | FLOPs estimator: workload-aware | logical tokens/s: 201269.8 | valid tokens/s: 201269.8 | physical tokens/s: 201584.8 | loss tokens/s: 146849.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.746117E-01 | lm-global-supervised-tokens: 1.224090E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.474274E-02 | lm-token-buffer-mib: 5.814787E+01 | global_load_balancing_loss: 9.997552E-01 | loss scale: 1.0 | grad norm: 0.042 | params norm: 11419.775 | params rms: 0.061834 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 7:44:40.939356 | finish at 2026-10-10 13:12:21 + [2026-10-07 05:29:11] iteration 3412/ 6855 | consumed samples: 436736 | elapsed time per iteration (ms): 91397.9 | throughput per GPU (TFLOP/s/GPU): 40.5 | MFU 12.99% | FLOPs estimator: workload-aware | logical tokens/s: 183562.4 | valid tokens/s: 183562.4 | physical tokens/s: 183947.2 | loss tokens/s: 134324.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.489384E-01 | lm-global-supervised-tokens: 1.227693E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.234722E-02 | lm-token-buffer-mib: 5.815755E+01 | global_load_balancing_loss: 9.992690E-01 | loss scale: 1.0 | grad norm: 0.036 | params norm: 11419.716 | params rms: 0.061834 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 15:24:42.973343 | finish at 2026-10-10 20:53:54 + [2026-10-07 05:30:43] iteration 3413/ 6855 | consumed samples: 436864 | elapsed time per iteration (ms): 91903.2 | throughput per GPU (TFLOP/s/GPU): 41.1 | MFU 13.17% | FLOPs estimator: workload-aware | logical tokens/s: 182552.9 | valid tokens/s: 182552.9 | physical tokens/s: 183016.1 | loss tokens/s: 129461.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.385175E-01 | lm-global-supervised-tokens: 1.189796E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.133905E-02 | lm-token-buffer-mib: 5.812707E+01 | global_load_balancing_loss: 9.994876E-01 | loss scale: 1.0 | grad norm: 0.064 | params norm: 11419.655 | params rms: 0.061834 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 15:52:10.967553 | finish at 2026-10-10 21:22:54 + [2026-10-07 05:32:09] iteration 3414/ 6855 | consumed samples: 436992 | elapsed time per iteration (ms): 86065.8 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.69% | FLOPs estimator: workload-aware | logical tokens/s: 194934.8 | valid tokens/s: 194934.8 | physical tokens/s: 195512.0 | loss tokens/s: 143914.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.683480E-01 | lm-global-supervised-tokens: 1.238614E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.445403E-02 | lm-token-buffer-mib: 5.816277E+01 | global_load_balancing_loss: 9.997184E-01 | loss scale: 1.0 | grad norm: 0.092 | params norm: 11419.593 | params rms: 0.061833 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 10:15:52.373332 | finish at 2026-10-10 15:48:01 + [2026-10-07 05:33:35] iteration 3415/ 6855 | consumed samples: 437120 | elapsed time per iteration (ms): 86268.8 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.58% | FLOPs estimator: workload-aware | logical tokens/s: 194475.4 | valid tokens/s: 194475.4 | physical tokens/s: 195188.5 | loss tokens/s: 143069.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.810898E-01 | lm-global-supervised-tokens: 1.234244E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.437880E-02 | lm-token-buffer-mib: 5.813197E+01 | global_load_balancing_loss: 1.000326E+00 | loss scale: 1.0 | grad norm: 0.145 | params norm: 11419.532 | params rms: 0.061833 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 10:26:04.594975 | finish at 2026-10-10 15:59:40 + [2026-10-07 05:35:02] iteration 3416/ 6855 | consumed samples: 437248 | elapsed time per iteration (ms): 86932.1 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.57% | FLOPs estimator: workload-aware | logical tokens/s: 192991.5 | valid tokens/s: 192991.5 | physical tokens/s: 193534.7 | loss tokens/s: 139474.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.524143E-01 | lm-global-supervised-tokens: 1.212481E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.208637E-02 | lm-token-buffer-mib: 5.821802E+01 | global_load_balancing_loss: 1.000355E+00 | loss scale: 1.0 | grad norm: 0.213 | params norm: 11419.473 | params rms: 0.061833 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 11:02:39.622466 | finish at 2026-10-10 16:37:42 + [2026-10-07 05:36:27] iteration 3417/ 6855 | consumed samples: 437376 | elapsed time per iteration (ms): 84658.1 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.75% | FLOPs estimator: workload-aware | logical tokens/s: 198175.7 | valid tokens/s: 198175.7 | physical tokens/s: 198630.5 | loss tokens/s: 147058.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.689571E-01 | lm-global-supervised-tokens: 1.244967E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.510006E-02 | lm-token-buffer-mib: 5.809873E+01 | global_load_balancing_loss: 1.000594E+00 | loss scale: 1.0 | grad norm: 0.193 | params norm: 11419.414 | params rms: 0.061832 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 8:50:54.558897 | finish at 2026-10-10 14:27:21 + [2026-10-07 05:37:55] iteration 3418/ 6855 | consumed samples: 437504 | elapsed time per iteration (ms): 88416.2 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.48% | FLOPs estimator: workload-aware | logical tokens/s: 189751.5 | valid tokens/s: 189751.5 | physical tokens/s: 190323.4 | loss tokens/s: 143252.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.855382E-01 | lm-global-supervised-tokens: 1.266588E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.169321E-02 | lm-token-buffer-mib: 5.847009E+01 | global_load_balancing_loss: 1.000059E+00 | loss scale: 1.0 | grad norm: 0.094 | params norm: 11419.353 | params rms: 0.061832 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 12:24:46.638926 | finish at 2026-10-10 18:02:42 + [2026-10-07 05:39:23] iteration 3419/ 6855 | consumed samples: 437632 | elapsed time per iteration (ms): 87789.0 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.47% | FLOPs estimator: workload-aware | logical tokens/s: 191107.8 | valid tokens/s: 191107.8 | physical tokens/s: 191538.4 | loss tokens/s: 140899.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.526940E-01 | lm-global-supervised-tokens: 1.236939E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.102442E-02 | lm-token-buffer-mib: 5.807024E+01 | global_load_balancing_loss: 1.000069E+00 | loss scale: 1.0 | grad norm: 0.059 | params norm: 11419.291 | params rms: 0.061832 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 11:47:22.911548 | finish at 2026-10-10 17:26:46 + [2026-10-07 05:41:13] iteration 3420/ 6855 | consumed samples: 437760 | elapsed time per iteration (ms): 110450.3 | throughput per GPU (TFLOP/s/GPU): 34.3 | MFU 10.98% | FLOPs estimator: workload-aware | logical tokens/s: 151897.8 | valid tokens/s: 151897.8 | physical tokens/s: 152122.9 | loss tokens/s: 109495.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.686357E-01 | lm-global-supervised-tokens: 1.209378E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.415507E-02 | lm-token-buffer-mib: 5.806003E+01 | global_load_balancing_loss: 1.001837E+00 | loss scale: 1.0 | grad norm: 0.044 | params norm: 11419.231 | params rms: 0.061831 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 9:23:16.939305 | finish at 2026-10-11 15:04:30 + [2026-10-07 05:42:43] iteration 3421/ 6855 | consumed samples: 437888 | elapsed time per iteration (ms): 89362.6 | throughput per GPU (TFLOP/s/GPU): 41.3 | MFU 13.22% | FLOPs estimator: workload-aware | logical tokens/s: 187742.8 | valid tokens/s: 187742.8 | physical tokens/s: 188174.1 | loss tokens/s: 138789.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.887578E-01 | lm-global-supervised-tokens: 1.240256E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.736857E-02 | lm-token-buffer-mib: 5.817519E+01 | global_load_balancing_loss: 1.000312E+00 | loss scale: 1.0 | grad norm: 0.056 | params norm: 11419.169 | params rms: 0.061831 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 13:14:31.149053 | finish at 2026-10-10 18:57:14 + [2026-10-07 05:44:08] iteration 3422/ 6855 | consumed samples: 438016 | elapsed time per iteration (ms): 84926.4 | throughput per GPU (TFLOP/s/GPU): 43.5 | MFU 13.95% | FLOPs estimator: workload-aware | logical tokens/s: 197550.0 | valid tokens/s: 197550.0 | physical tokens/s: 197998.2 | loss tokens/s: 145939.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.458614E-01 | lm-global-supervised-tokens: 1.239412E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.478725E-02 | lm-token-buffer-mib: 5.809304E+01 | global_load_balancing_loss: 9.993567E-01 | loss scale: 1.0 | grad norm: 0.035 | params norm: 11419.107 | params rms: 0.061831 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 8:59:12.434964 | finish at 2026-10-10 14:43:20 + [2026-10-07 05:45:38] iteration 3423/ 6855 | consumed samples: 438144 | elapsed time per iteration (ms): 90447.4 | throughput per GPU (TFLOP/s/GPU): 41.0 | MFU 13.13% | FLOPs estimator: workload-aware | logical tokens/s: 185491.3 | valid tokens/s: 185491.3 | physical tokens/s: 185920.3 | loss tokens/s: 142230.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.744351E-01 | lm-global-supervised-tokens: 1.286437E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.139538E-02 | lm-token-buffer-mib: 5.807392E+01 | global_load_balancing_loss: 9.999490E-01 | loss scale: 1.0 | grad norm: 0.055 | params norm: 11419.044 | params rms: 0.061830 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 14:13:35.578583 | finish at 2026-10-10 19:59:14 + [2026-10-07 05:47:02] iteration 3424/ 6855 | consumed samples: 438272 | elapsed time per iteration (ms): 83352.3 | throughput per GPU (TFLOP/s/GPU): 43.5 | MFU 13.94% | FLOPs estimator: workload-aware | logical tokens/s: 201279.9 | valid tokens/s: 201279.9 | physical tokens/s: 201794.3 | loss tokens/s: 153959.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.334006E-01 | lm-global-supervised-tokens: 1.283288E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.347238E-02 | lm-token-buffer-mib: 5.811731E+01 | global_load_balancing_loss: 9.989960E-01 | loss scale: 1.0 | grad norm: 0.052 | params norm: 11418.983 | params rms: 0.061830 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 7:26:21.898114 | finish at 2026-10-10 13:13:23 + [2026-10-07 05:48:29] iteration 3425/ 6855 | consumed samples: 438400 | elapsed time per iteration (ms): 87378.9 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.43% | FLOPs estimator: workload-aware | logical tokens/s: 192005.4 | valid tokens/s: 192005.3 | physical tokens/s: 192391.2 | loss tokens/s: 141845.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.299520E-01 | lm-global-supervised-tokens: 1.239428E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.161152E-02 | lm-token-buffer-mib: 5.816272E+01 | global_load_balancing_loss: 9.996277E-01 | loss scale: 1.0 | grad norm: 0.053 | params norm: 11418.923 | params rms: 0.061830 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 11:15:09.581380 | finish at 2026-10-10 17:03:39 + [2026-10-07 05:50:01] iteration 3426/ 6855 | consumed samples: 438528 | elapsed time per iteration (ms): 91668.4 | throughput per GPU (TFLOP/s/GPU): 41.5 | MFU 13.29% | FLOPs estimator: workload-aware | logical tokens/s: 183020.7 | valid tokens/s: 183020.7 | physical tokens/s: 183341.8 | loss tokens/s: 133090.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.711382E-01 | lm-global-supervised-tokens: 1.220019E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.141756E-02 | lm-token-buffer-mib: 5.805886E+01 | global_load_balancing_loss: 9.998600E-01 | loss scale: 1.0 | grad norm: 0.071 | params norm: 11418.864 | params rms: 0.061829 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 15:18:51.019800 | finish at 2026-10-10 21:08:52 + [2026-10-07 05:51:30] iteration 3427/ 6855 | consumed samples: 438656 | elapsed time per iteration (ms): 89817.9 | throughput per GPU (TFLOP/s/GPU): 41.3 | MFU 13.23% | FLOPs estimator: workload-aware | logical tokens/s: 186790.8 | valid tokens/s: 186790.8 | physical tokens/s: 187358.5 | loss tokens/s: 137147.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.840826E-01 | lm-global-supervised-tokens: 1.231827E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.165050E-02 | lm-token-buffer-mib: 5.823370E+01 | global_load_balancing_loss: 1.001177E+00 | loss scale: 1.0 | grad norm: 0.039 | params norm: 11418.802 | params rms: 0.061829 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 13:31:35.754464 | finish at 2026-10-10 19:23:06 + [2026-10-07 05:52:58] iteration 3428/ 6855 | consumed samples: 438784 | elapsed time per iteration (ms): 87411.7 | throughput per GPU (TFLOP/s/GPU): 41.3 | MFU 13.24% | FLOPs estimator: workload-aware | logical tokens/s: 191932.8 | valid tokens/s: 191932.8 | physical tokens/s: 192563.0 | loss tokens/s: 146979.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.033347E-01 | lm-global-supervised-tokens: 1.284774E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.419122E-02 | lm-token-buffer-mib: 5.809492E+01 | global_load_balancing_loss: 1.001153E+00 | loss scale: 1.0 | grad norm: 0.082 | params norm: 11418.740 | params rms: 0.061829 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 11:12:40.047091 | finish at 2026-10-10 17:05:38 + [2026-10-07 05:54:24] iteration 3429/ 6855 | consumed samples: 438912 | elapsed time per iteration (ms): 85961.6 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.67% | FLOPs estimator: workload-aware | logical tokens/s: 195170.7 | valid tokens/s: 195170.7 | physical tokens/s: 195728.7 | loss tokens/s: 146709.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.058456E-01 | lm-global-supervised-tokens: 1.261141E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.956235E-02 | lm-token-buffer-mib: 5.818853E+01 | global_load_balancing_loss: 1.001673E+00 | loss scale: 1.0 | grad norm: 0.128 | params norm: 11418.677 | params rms: 0.061828 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 9:48:24.420586 | finish at 2026-10-10 15:42:48 + [2026-10-07 05:56:10] iteration 3430/ 6855 | consumed samples: 439040 | elapsed time per iteration (ms): 106090.5 | throughput per GPU (TFLOP/s/GPU): 33.9 | MFU 10.87% | FLOPs estimator: workload-aware | logical tokens/s: 158140.0 | valid tokens/s: 158140.0 | physical tokens/s: 158528.7 | loss tokens/s: 119032.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.333926E-01 | lm-global-supervised-tokens: 1.262822E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.158902E-02 | lm-token-buffer-mib: 5.806065E+01 | global_load_balancing_loss: 1.001133E+00 | loss scale: 1.0 | grad norm: 0.075 | params norm: 11418.615 | params rms: 0.061828 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 4:55:59.940034 | finish at 2026-10-11 10:52:10 + [2026-10-07 05:57:38] iteration 3431/ 6855 | consumed samples: 439168 | elapsed time per iteration (ms): 88507.8 | throughput per GPU (TFLOP/s/GPU): 41.5 | MFU 13.31% | FLOPs estimator: workload-aware | logical tokens/s: 189555.5 | valid tokens/s: 189555.5 | physical tokens/s: 190110.9 | loss tokens/s: 145151.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.850781E-01 | lm-global-supervised-tokens: 1.284706E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.140119E-02 | lm-token-buffer-mib: 5.810347E+01 | global_load_balancing_loss: 9.988412E-01 | loss scale: 1.0 | grad norm: 0.187 | params norm: 11418.553 | params rms: 0.061828 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 12:10:50.587547 | finish at 2026-10-10 18:08:29 + [2026-10-07 05:59:08] iteration 3432/ 6855 | consumed samples: 439296 | elapsed time per iteration (ms): 89190.0 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.59% | FLOPs estimator: workload-aware | logical tokens/s: 188106.0 | valid tokens/s: 188106.0 | physical tokens/s: 188506.4 | loss tokens/s: 138591.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.682839E-01 | lm-global-supervised-tokens: 1.236099E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.152218E-02 | lm-token-buffer-mib: 5.814250E+01 | global_load_balancing_loss: 9.997261E-01 | loss scale: 1.0 | grad norm: 0.294 | params norm: 11418.491 | params rms: 0.061827 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 12:48:17.517911 | finish at 2026-10-10 18:47:25 + [2026-10-07 06:00:36] iteration 3433/ 6855 | consumed samples: 439424 | elapsed time per iteration (ms): 88399.1 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.47% | FLOPs estimator: workload-aware | logical tokens/s: 189789.4 | valid tokens/s: 189789.3 | physical tokens/s: 190196.6 | loss tokens/s: 143442.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.373897E-01 | lm-global-supervised-tokens: 1.268022E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.150560E-02 | lm-token-buffer-mib: 5.813027E+01 | global_load_balancing_loss: 9.991028E-01 | loss scale: 1.0 | grad norm: 0.095 | params norm: 11418.428 | params rms: 0.061827 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 12:01:41.645363 | finish at 2026-10-10 18:02:18 + [2026-10-07 06:02:02] iteration 3434/ 6855 | consumed samples: 439552 | elapsed time per iteration (ms): 85829.8 | throughput per GPU (TFLOP/s/GPU): 44.4 | MFU 14.23% | FLOPs estimator: workload-aware | logical tokens/s: 195470.5 | valid tokens/s: 195470.5 | physical tokens/s: 195760.3 | loss tokens/s: 138762.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.719682E-01 | lm-global-supervised-tokens: 1.190999E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.186223E-02 | lm-token-buffer-mib: 5.803953E+01 | global_load_balancing_loss: 9.999866E-01 | loss scale: 1.0 | grad norm: 0.149 | params norm: 11418.365 | params rms: 0.061827 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 9:33:43.755213 | finish at 2026-10-10 15:35:46 + [2026-10-07 06:03:27] iteration 3435/ 6855 | consumed samples: 439680 | elapsed time per iteration (ms): 85607.4 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.76% | FLOPs estimator: workload-aware | logical tokens/s: 195977.7 | valid tokens/s: 195977.7 | physical tokens/s: 196372.1 | loss tokens/s: 147303.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.671413E-01 | lm-global-supervised-tokens: 1.261028E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.296502E-02 | lm-token-buffer-mib: 5.806377E+01 | global_load_balancing_loss: 1.000046E+00 | loss scale: 1.0 | grad norm: 0.124 | params norm: 11418.302 | params rms: 0.061826 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 9:19:37.428474 | finish at 2026-10-10 15:23:05 + [2026-10-07 06:04:55] iteration 3436/ 6855 | consumed samples: 439808 | elapsed time per iteration (ms): 87179.9 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.41% | FLOPs estimator: workload-aware | logical tokens/s: 192443.4 | valid tokens/s: 192443.4 | physical tokens/s: 192887.7 | loss tokens/s: 143375.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.055820E-01 | lm-global-supervised-tokens: 1.249947E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.370948E-02 | lm-token-buffer-mib: 5.808094E+01 | global_load_balancing_loss: 1.001009E+00 | loss scale: 1.0 | grad norm: 0.097 | params norm: 11418.238 | params rms: 0.061826 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 10:47:48.122697 | finish at 2026-10-10 16:52:43 + [2026-10-07 06:06:24] iteration 3437/ 6855 | consumed samples: 439936 | elapsed time per iteration (ms): 89128.0 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.62% | FLOPs estimator: workload-aware | logical tokens/s: 188236.5 | valid tokens/s: 188236.5 | physical tokens/s: 188573.0 | loss tokens/s: 138966.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.515364E-01 | lm-global-supervised-tokens: 1.238579E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.174564E-02 | lm-token-buffer-mib: 5.809601E+01 | global_load_balancing_loss: 1.001066E+00 | loss scale: 1.0 | grad norm: 0.059 | params norm: 11418.177 | params rms: 0.061826 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 12:37:19.471475 | finish at 2026-10-10 18:43:43 + [2026-10-07 06:07:50] iteration 3438/ 6855 | consumed samples: 440064 | elapsed time per iteration (ms): 86461.1 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.70% | FLOPs estimator: workload-aware | logical tokens/s: 194043.6 | valid tokens/s: 194043.6 | physical tokens/s: 194448.5 | loss tokens/s: 143245.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.585973E-01 | lm-global-supervised-tokens: 1.238513E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.313295E-02 | lm-token-buffer-mib: 5.803865E+01 | global_load_balancing_loss: 9.997624E-01 | loss scale: 1.0 | grad norm: 0.171 | params norm: 11418.116 | params rms: 0.061825 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 10:03:57.447069 | finish at 2026-10-10 16:11:48 + [2026-10-07 06:09:17] iteration 3439/ 6855 | consumed samples: 440192 | elapsed time per iteration (ms): 86316.6 | throughput per GPU (TFLOP/s/GPU): 43.2 | MFU 13.84% | FLOPs estimator: workload-aware | logical tokens/s: 194368.3 | valid tokens/s: 194368.2 | physical tokens/s: 194850.0 | loss tokens/s: 137546.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.488826E-01 | lm-global-supervised-tokens: 1.187250E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.374784E-02 | lm-token-buffer-mib: 5.822028E+01 | global_load_balancing_loss: 1.000475E+00 | loss scale: 1.0 | grad norm: 0.097 | params norm: 11418.057 | params rms: 0.061825 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 9:54:17.351959 | finish at 2026-10-10 16:03:34 + [2026-10-07 06:11:08] iteration 3440/ 6855 | consumed samples: 440320 | elapsed time per iteration (ms): 111183.0 | throughput per GPU (TFLOP/s/GPU): 33.7 | MFU 10.80% | FLOPs estimator: workload-aware | logical tokens/s: 150896.6 | valid tokens/s: 150896.6 | physical tokens/s: 151308.4 | loss tokens/s: 108421.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.693207E-01 | lm-global-supervised-tokens: 1.205459E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.161662E-02 | lm-token-buffer-mib: 5.817144E+01 | global_load_balancing_loss: 9.987584E-01 | loss scale: 1.0 | grad norm: 0.191 | params norm: 11417.995 | params rms: 0.061825 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 9:28:10.018578 | finish at 2026-10-11 15:39:18 +times across ranks (ms): + save-checkpoint: + rank 0: 24769.07 + rank 1: 24769.08 + rank 2: 24769.09 + rank 3: 24769.06 + rank 4: 24769.07 + rank 5: 24769.07 + rank 6: 24769.07 + rank 7: 24769.09 + rank 8: 24769.94 + rank 9: 24769.92 + rank 10: 24769.91 + rank 11: 24769.91 + rank 12: 24769.93 + rank 13: 24769.93 + rank 14: 24769.91 + rank 15: 24769.89 + rank 16: 24769.23 + rank 17: 24769.24 + rank 18: 24769.23 + rank 19: 24769.23 + rank 20: 24769.21 + rank 21: 24769.23 + rank 22: 24769.21 + rank 23: 24769.21 + rank 24: 24769.70 + rank 25: 24769.68 + rank 26: 24769.69 + rank 27: 24769.71 + rank 28: 24769.68 + rank 29: 24769.69 + rank 30: 24769.67 + rank 31: 24769.67 + rank 32: 24769.62 + rank 33: 24769.61 + rank 34: 24769.60 + rank 35: 24769.60 + rank 36: 24769.58 + rank 37: 24769.59 + rank 38: 24769.59 + rank 39: 24769.57 + rank 40: 24768.97 + rank 41: 24768.96 + rank 42: 24768.94 + rank 43: 24768.97 + rank 44: 24768.94 + rank 45: 24768.96 + rank 46: 24769.02 + rank 47: 24768.99 + rank 48: 24768.75 + rank 49: 24768.76 + rank 50: 24768.71 + rank 51: 24768.74 + rank 52: 24768.76 + rank 53: 24768.71 + rank 54: 24768.71 + rank 55: 24768.73 + rank 56: 24769.43 + rank 57: 24769.44 + rank 58: 24769.45 + rank 59: 24769.43 + rank 60: 24769.43 + rank 61: 24769.44 + rank 62: 24769.42 + rank 63: 24769.44 + [2026-10-07 06:13:13] iteration 3441/ 6855 | consumed samples: 440448 | elapsed time per iteration (ms): 99362.1 | throughput per GPU (TFLOP/s/GPU): 36.8 | MFU 11.78% | FLOPs estimator: workload-aware | logical tokens/s: 168849.2 | valid tokens/s: 168849.2 | physical tokens/s: 169142.0 | loss tokens/s: 125759.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.799057E-01 | lm-global-supervised-tokens: 1.249568E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 4.794252E+00 | lm-token-buffer-mib: 5.809458E+01 | global_load_balancing_loss: 1.001427E+00 | loss scale: 1.0 | grad norm: 0.054 | params norm: 11417.932 | params rms: 0.061824 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 22:13:42.216336 | finish at 2026-10-11 04:26:55 + [2026-10-07 06:14:43] iteration 3442/ 6855 | consumed samples: 440576 | elapsed time per iteration (ms): 90357.1 | throughput per GPU (TFLOP/s/GPU): 41.4 | MFU 13.26% | FLOPs estimator: workload-aware | logical tokens/s: 185676.5 | valid tokens/s: 185676.5 | physical tokens/s: 186094.7 | loss tokens/s: 141038.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.978342E-01 | lm-global-supervised-tokens: 1.274382E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.418421E-02 | lm-token-buffer-mib: 5.815627E+01 | global_load_balancing_loss: 9.995171E-01 | loss scale: 1.0 | grad norm: 0.255 | params norm: 11417.870 | params rms: 0.061824 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 13:39:48.778265 | finish at 2026-10-10 19:54:32 + [2026-10-07 06:16:10] iteration 3443/ 6855 | consumed samples: 440704 | elapsed time per iteration (ms): 86456.0 | throughput per GPU (TFLOP/s/GPU): 43.3 | MFU 13.88% | FLOPs estimator: workload-aware | logical tokens/s: 194054.9 | valid tokens/s: 194054.9 | physical tokens/s: 194631.9 | loss tokens/s: 144022.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.680859E-01 | lm-global-supervised-tokens: 1.245164E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.710115E-02 | lm-token-buffer-mib: 5.834589E+01 | global_load_balancing_loss: 9.999634E-01 | loss scale: 1.0 | grad norm: 0.079 | params norm: 11417.809 | params rms: 0.061824 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 9:56:27.962603 | finish at 2026-10-10 16:12:38 + [2026-10-07 06:17:34] iteration 3444/ 6855 | consumed samples: 440832 | elapsed time per iteration (ms): 83926.6 | throughput per GPU (TFLOP/s/GPU): 43.1 | MFU 13.82% | FLOPs estimator: workload-aware | logical tokens/s: 199903.4 | valid tokens/s: 199903.4 | physical tokens/s: 200238.2 | loss tokens/s: 149119.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.256559E-01 | lm-global-supervised-tokens: 1.251511E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.344115E-02 | lm-token-buffer-mib: 5.814817E+01 | global_load_balancing_loss: 1.001301E+00 | loss scale: 1.0 | grad norm: 0.200 | params norm: 11417.748 | params rms: 0.061823 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 7:31:13.587801 | finish at 2026-10-10 13:48:47 + [2026-10-07 06:19:01] iteration 3445/ 6855 | consumed samples: 440960 | elapsed time per iteration (ms): 87208.1 | throughput per GPU (TFLOP/s/GPU): 41.7 | MFU 13.38% | FLOPs estimator: workload-aware | logical tokens/s: 192381.5 | valid tokens/s: 192381.5 | physical tokens/s: 192729.0 | loss tokens/s: 141486.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.266127E-01 | lm-global-supervised-tokens: 1.233872E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.364111E-02 | lm-token-buffer-mib: 5.815071E+01 | global_load_balancing_loss: 1.000127E+00 | loss scale: 1.0 | grad norm: 0.129 | params norm: 11417.688 | params rms: 0.061823 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 10:36:19.464364 | finish at 2026-10-10 16:55:20 + [2026-10-07 06:20:28] iteration 3446/ 6855 | consumed samples: 441088 | elapsed time per iteration (ms): 87471.9 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.48% | FLOPs estimator: workload-aware | logical tokens/s: 191800.5 | valid tokens/s: 191800.5 | physical tokens/s: 192252.1 | loss tokens/s: 138842.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.205236E-01 | lm-global-supervised-tokens: 1.214482E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.117332E-02 | lm-token-buffer-mib: 5.818412E+01 | global_load_balancing_loss: 1.000826E+00 | loss scale: 1.0 | grad norm: 0.056 | params norm: 11417.628 | params rms: 0.061823 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 10:49:51.694862 | finish at 2026-10-10 17:10:20 + [2026-10-07 06:21:56] iteration 3447/ 6855 | consumed samples: 441216 | elapsed time per iteration (ms): 87838.1 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.42% | FLOPs estimator: workload-aware | logical tokens/s: 191000.8 | valid tokens/s: 191000.8 | physical tokens/s: 191423.4 | loss tokens/s: 139271.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.664034E-01 | lm-global-supervised-tokens: 1.223337E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.145481E-02 | lm-token-buffer-mib: 5.810783E+01 | global_load_balancing_loss: 1.000699E+00 | loss scale: 1.0 | grad norm: 0.084 | params norm: 11417.566 | params rms: 0.061822 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 11:09:12.257652 | finish at 2026-10-10 17:31:09 + [2026-10-07 06:23:26] iteration 3448/ 6855 | consumed samples: 441344 | elapsed time per iteration (ms): 89728.0 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.41% | FLOPs estimator: workload-aware | logical tokens/s: 186977.6 | valid tokens/s: 186977.5 | physical tokens/s: 187325.0 | loss tokens/s: 137539.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.049984E-01 | lm-global-supervised-tokens: 1.234114E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.106669E-02 | lm-token-buffer-mib: 5.802812E+01 | global_load_balancing_loss: 9.995799E-01 | loss scale: 1.0 | grad norm: 0.119 | params norm: 11417.504 | params rms: 0.061822 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 12:55:03.410280 | finish at 2026-10-10 19:18:29 + [2026-10-07 06:24:54] iteration 3449/ 6855 | consumed samples: 441472 | elapsed time per iteration (ms): 87955.0 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.43% | FLOPs estimator: workload-aware | logical tokens/s: 190747.4 | valid tokens/s: 190747.4 | physical tokens/s: 191245.3 | loss tokens/s: 137690.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.514446E-01 | lm-global-supervised-tokens: 1.211060E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.960999E-02 | lm-token-buffer-mib: 5.812762E+01 | global_load_balancing_loss: 9.993774E-01 | loss scale: 1.0 | grad norm: 0.064 | params norm: 11417.441 | params rms: 0.061822 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 11:12:54.655843 | finish at 2026-10-10 17:37:49 + [2026-10-07 06:26:44] iteration 3450/ 6855 | consumed samples: 441600 | elapsed time per iteration (ms): 109615.4 | throughput per GPU (TFLOP/s/GPU): 33.7 | MFU 10.81% | FLOPs estimator: workload-aware | logical tokens/s: 153055.0 | valid tokens/s: 153055.0 | physical tokens/s: 153358.2 | loss tokens/s: 111224.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.921983E-01 | lm-global-supervised-tokens: 1.219188E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.655228E-02 | lm-token-buffer-mib: 5.811555E+01 | global_load_balancing_loss: 9.993582E-01 | loss scale: 1.0 | grad norm: 0.044 | params norm: 11417.378 | params rms: 0.061821 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 7:40:40.317922 | finish at 2026-10-11 14:07:24 + [2026-10-07 06:28:12] iteration 3451/ 6855 | consumed samples: 441728 | elapsed time per iteration (ms): 88125.4 | throughput per GPU (TFLOP/s/GPU): 41.6 | MFU 13.32% | FLOPs estimator: workload-aware | logical tokens/s: 190378.6 | valid tokens/s: 190378.6 | physical tokens/s: 190721.2 | loss tokens/s: 143380.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.514968E-01 | lm-global-supervised-tokens: 1.263542E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.286012E-02 | lm-token-buffer-mib: 5.809177E+01 | global_load_balancing_loss: 9.995280E-01 | loss scale: 1.0 | grad norm: 0.048 | params norm: 11417.315 | params rms: 0.061821 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 11:19:38.732785 | finish at 2026-10-10 17:47:50 + [2026-10-07 06:29:43] iteration 3452/ 6855 | consumed samples: 441856 | elapsed time per iteration (ms): 91273.9 | throughput per GPU (TFLOP/s/GPU): 41.1 | MFU 13.16% | FLOPs estimator: workload-aware | logical tokens/s: 183811.2 | valid tokens/s: 183811.2 | physical tokens/s: 184185.6 | loss tokens/s: 129536.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.698426E-01 | lm-global-supervised-tokens: 1.182335E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.510701E-02 | lm-token-buffer-mib: 5.808723E+01 | global_load_balancing_loss: 1.001413E+00 | loss scale: 1.0 | grad norm: 0.078 | params norm: 11417.253 | params rms: 0.061821 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 14:16:45.240831 | finish at 2026-10-10 20:46:28 + [2026-10-07 06:31:14] iteration 3453/ 6855 | consumed samples: 441984 | elapsed time per iteration (ms): 91064.5 | throughput per GPU (TFLOP/s/GPU): 40.2 | MFU 12.88% | FLOPs estimator: workload-aware | logical tokens/s: 184234.1 | valid tokens/s: 184234.1 | physical tokens/s: 184726.6 | loss tokens/s: 138101.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.758128E-01 | lm-global-supervised-tokens: 1.257616E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.545042E-02 | lm-token-buffer-mib: 5.827770E+01 | global_load_balancing_loss: 9.997917E-01 | loss scale: 1.0 | grad norm: 0.058 | params norm: 11417.190 | params rms: 0.061820 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 14:03:21.426074 | finish at 2026-10-10 20:34:35 + [2026-10-07 06:32:41] iteration 3454/ 6855 | consumed samples: 442112 | elapsed time per iteration (ms): 86815.6 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.67% | FLOPs estimator: workload-aware | logical tokens/s: 193250.5 | valid tokens/s: 193250.5 | physical tokens/s: 193671.5 | loss tokens/s: 143420.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.586990E-01 | lm-global-supervised-tokens: 1.245111E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.813458E-02 | lm-token-buffer-mib: 5.823640E+01 | global_load_balancing_loss: 1.000719E+00 | loss scale: 1.0 | grad norm: 0.035 | params norm: 11417.129 | params rms: 0.061820 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 10:00:59.692339 | finish at 2026-10-10 16:33:41 + [2026-10-07 06:34:11] iteration 3455/ 6855 | consumed samples: 442240 | elapsed time per iteration (ms): 90276.9 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.53% | FLOPs estimator: workload-aware | logical tokens/s: 185841.5 | valid tokens/s: 185841.5 | physical tokens/s: 186179.3 | loss tokens/s: 137108.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.901969E-01 | lm-global-supervised-tokens: 1.237773E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.275476E-02 | lm-token-buffer-mib: 5.813600E+01 | global_load_balancing_loss: 1.000434E+00 | loss scale: 1.0 | grad norm: 0.039 | params norm: 11417.067 | params rms: 0.061820 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 13:15:41.612577 | finish at 2026-10-10 19:49:53 + [2026-10-07 06:35:34] iteration 3456/ 6855 | consumed samples: 442368 | elapsed time per iteration (ms): 82806.6 | throughput per GPU (TFLOP/s/GPU): 43.2 | MFU 13.84% | FLOPs estimator: workload-aware | logical tokens/s: 202606.8 | valid tokens/s: 202606.8 | physical tokens/s: 202995.8 | loss tokens/s: 154059.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.258678E-01 | lm-global-supervised-tokens: 1.275712E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.139278E-02 | lm-token-buffer-mib: 5.802662E+01 | global_load_balancing_loss: 1.000319E+00 | loss scale: 1.0 | grad norm: 0.088 | params norm: 11417.004 | params rms: 0.061819 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 6:10:59.588337 | finish at 2026-10-10 12:46:34 + [2026-10-07 06:37:03] iteration 3457/ 6855 | consumed samples: 442496 | elapsed time per iteration (ms): 88605.4 | throughput per GPU (TFLOP/s/GPU): 41.7 | MFU 13.37% | FLOPs estimator: workload-aware | logical tokens/s: 189347.6 | valid tokens/s: 189347.6 | physical tokens/s: 190019.7 | loss tokens/s: 139501.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.522097E-01 | lm-global-supervised-tokens: 1.236060E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.158054E-02 | lm-token-buffer-mib: 5.820221E+01 | global_load_balancing_loss: 1.000249E+00 | loss scale: 1.0 | grad norm: 0.135 | params norm: 11416.943 | params rms: 0.061819 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 11:38:01.049032 | finish at 2026-10-10 18:15:04 + [2026-10-07 06:38:30] iteration 3458/ 6855 | consumed samples: 442624 | elapsed time per iteration (ms): 87426.1 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.51% | FLOPs estimator: workload-aware | logical tokens/s: 191901.7 | valid tokens/s: 191901.7 | physical tokens/s: 192290.6 | loss tokens/s: 140780.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.326428E-01 | lm-global-supervised-tokens: 1.230785E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.535811E-02 | lm-token-buffer-mib: 5.813847E+01 | global_load_balancing_loss: 9.997426E-01 | loss scale: 1.0 | grad norm: 0.221 | params norm: 11416.880 | params rms: 0.061819 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 10:29:46.495769 | finish at 2026-10-10 17:08:16 + [2026-10-07 06:39:57] iteration 3459/ 6855 | consumed samples: 442752 | elapsed time per iteration (ms): 87533.3 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.70% | FLOPs estimator: workload-aware | logical tokens/s: 191666.3 | valid tokens/s: 191666.3 | physical tokens/s: 192148.6 | loss tokens/s: 137817.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.334651E-01 | lm-global-supervised-tokens: 1.206363E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.137840E-02 | lm-token-buffer-mib: 5.825010E+01 | global_load_balancing_loss: 1.000948E+00 | loss scale: 1.0 | grad norm: 0.277 | params norm: 11416.817 | params rms: 0.061818 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 10:34:22.998284 | finish at 2026-10-10 17:14:20 + [2026-10-07 06:41:43] iteration 3460/ 6855 | consumed samples: 442880 | elapsed time per iteration (ms): 105828.8 | throughput per GPU (TFLOP/s/GPU): 33.4 | MFU 10.72% | FLOPs estimator: workload-aware | logical tokens/s: 158531.1 | valid tokens/s: 158531.1 | physical tokens/s: 159046.2 | loss tokens/s: 120059.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.219420E-01 | lm-global-supervised-tokens: 1.270578E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.430626E-02 | lm-token-buffer-mib: 5.816373E+01 | global_load_balancing_loss: 1.000525E+00 | loss scale: 1.0 | grad norm: 0.274 | params norm: 11416.753 | params rms: 0.061818 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 3:48:08.787206 | finish at 2026-10-11 10:29:52 + [2026-10-07 06:43:13] iteration 3461/ 6855 | consumed samples: 443008 | elapsed time per iteration (ms): 90009.4 | throughput per GPU (TFLOP/s/GPU): 41.6 | MFU 13.33% | FLOPs estimator: workload-aware | logical tokens/s: 186393.6 | valid tokens/s: 186393.6 | physical tokens/s: 186915.8 | loss tokens/s: 132043.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.480466E-01 | lm-global-supervised-tokens: 1.188514E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.551469E-02 | lm-token-buffer-mib: 5.813642E+01 | global_load_balancing_loss: 1.000493E+00 | loss scale: 1.0 | grad norm: 0.187 | params norm: 11416.688 | params rms: 0.061818 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 12:51:32.063450 | finish at 2026-10-10 19:34:45 + [2026-10-07 06:44:43] iteration 3462/ 6855 | consumed samples: 443136 | elapsed time per iteration (ms): 89553.1 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.38% | FLOPs estimator: workload-aware | logical tokens/s: 187342.8 | valid tokens/s: 187342.8 | physical tokens/s: 187649.3 | loss tokens/s: 138118.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.747497E-01 | lm-global-supervised-tokens: 1.236894E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.718605E-02 | lm-token-buffer-mib: 5.806664E+01 | global_load_balancing_loss: 9.997734E-01 | loss scale: 1.0 | grad norm: 0.080 | params norm: 11416.623 | params rms: 0.061817 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 12:24:13.610425 | finish at 2026-10-10 19:08:56 + [2026-10-07 06:46:09] iteration 3463/ 6855 | consumed samples: 443264 | elapsed time per iteration (ms): 85884.4 | throughput per GPU (TFLOP/s/GPU): 43.3 | MFU 13.88% | FLOPs estimator: workload-aware | logical tokens/s: 195346.1 | valid tokens/s: 195346.1 | physical tokens/s: 195707.3 | loss tokens/s: 150301.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.016912E-01 | lm-global-supervised-tokens: 1.290852E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.453978E-02 | lm-token-buffer-mib: 5.803429E+01 | global_load_balancing_loss: 9.986849E-01 | loss scale: 1.0 | grad norm: 0.065 | params norm: 11416.559 | params rms: 0.061817 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 8:55:19.922440 | finish at 2026-10-10 15:41:29 + [2026-10-07 06:47:33] iteration 3464/ 6855 | consumed samples: 443392 | elapsed time per iteration (ms): 84235.5 | throughput per GPU (TFLOP/s/GPU): 43.1 | MFU 13.80% | FLOPs estimator: workload-aware | logical tokens/s: 199170.3 | valid tokens/s: 199170.3 | physical tokens/s: 199676.4 | loss tokens/s: 148982.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.552580E-01 | lm-global-supervised-tokens: 1.254962E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.169474E-02 | lm-token-buffer-mib: 5.816273E+01 | global_load_balancing_loss: 1.000021E+00 | loss scale: 1.0 | grad norm: 0.237 | params norm: 11416.494 | params rms: 0.061817 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 7:20:42.619637 | finish at 2026-10-10 14:08:16 + [2026-10-07 06:49:01] iteration 3465/ 6855 | consumed samples: 443520 | elapsed time per iteration (ms): 88026.8 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.55% | FLOPs estimator: workload-aware | logical tokens/s: 190591.8 | valid tokens/s: 190591.8 | physical tokens/s: 190969.7 | loss tokens/s: 139224.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.602077E-01 | lm-global-supervised-tokens: 1.225552E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.152509E-02 | lm-token-buffer-mib: 5.806715E+01 | global_load_balancing_loss: 1.000712E+00 | loss scale: 1.0 | grad norm: 0.172 | params norm: 11416.430 | params rms: 0.061816 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 10:53:30.784636 | finish at 2026-10-10 17:42:32 + [2026-10-07 06:50:27] iteration 3466/ 6855 | consumed samples: 443648 | elapsed time per iteration (ms): 85517.8 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.53% | FLOPs estimator: workload-aware | logical tokens/s: 196183.3 | valid tokens/s: 196183.3 | physical tokens/s: 196684.5 | loss tokens/s: 146010.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.531709E-01 | lm-global-supervised-tokens: 1.248654E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.473182E-02 | lm-token-buffer-mib: 5.815082E+01 | global_load_balancing_loss: 1.000486E+00 | loss scale: 1.0 | grad norm: 0.072 | params norm: 11416.367 | params rms: 0.061816 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 8:30:19.939250 | finish at 2026-10-10 15:20:46 + [2026-10-07 06:51:54] iteration 3467/ 6855 | consumed samples: 443776 | elapsed time per iteration (ms): 87157.6 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.67% | FLOPs estimator: workload-aware | logical tokens/s: 192492.8 | valid tokens/s: 192492.8 | physical tokens/s: 193070.3 | loss tokens/s: 141965.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.939486E-01 | lm-global-supervised-tokens: 1.237337E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.468752E-02 | lm-token-buffer-mib: 5.808041E+01 | global_load_balancing_loss: 9.999390E-01 | loss scale: 1.0 | grad norm: 0.080 | params norm: 11416.303 | params rms: 0.061816 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 10:01:29.843540 | finish at 2026-10-10 16:53:24 + [2026-10-07 06:53:20] iteration 3468/ 6855 | consumed samples: 443904 | elapsed time per iteration (ms): 86456.6 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.69% | FLOPs estimator: workload-aware | logical tokens/s: 194052.3 | valid tokens/s: 194052.3 | physical tokens/s: 194511.7 | loss tokens/s: 137918.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.297366E-01 | lm-global-supervised-tokens: 1.192396E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.506622E-02 | lm-token-buffer-mib: 5.821694E+01 | global_load_balancing_loss: 1.001476E+00 | loss scale: 1.0 | grad norm: 0.045 | params norm: 11416.241 | params rms: 0.061815 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 9:20:28.411976 | finish at 2026-10-10 16:13:49 + [2026-10-07 06:54:47] iteration 3469/ 6855 | consumed samples: 444032 | elapsed time per iteration (ms): 86731.7 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.58% | FLOPs estimator: workload-aware | logical tokens/s: 193437.6 | valid tokens/s: 193437.6 | physical tokens/s: 193889.4 | loss tokens/s: 146806.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.614027E-01 | lm-global-supervised-tokens: 1.273275E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.506382E-02 | lm-token-buffer-mib: 5.810465E+01 | global_load_balancing_loss: 1.001057E+00 | loss scale: 1.0 | grad norm: 0.185 | params norm: 11416.177 | params rms: 0.061815 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 9:34:33.429447 | finish at 2026-10-10 16:29:20 + [2026-10-07 06:56:37] iteration 3470/ 6855 | consumed samples: 444160 | elapsed time per iteration (ms): 109823.5 | throughput per GPU (TFLOP/s/GPU): 33.8 | MFU 10.83% | FLOPs estimator: workload-aware | logical tokens/s: 152764.8 | valid tokens/s: 152764.8 | physical tokens/s: 153135.8 | loss tokens/s: 112520.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.020456E-01 | lm-global-supervised-tokens: 1.235738E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.373611E-02 | lm-token-buffer-mib: 5.823319E+01 | global_load_balancing_loss: 9.999017E-01 | loss scale: 1.0 | grad norm: 0.149 | params norm: 11416.114 | params rms: 0.061815 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 7:15:52.543187 | finish at 2026-10-11 14:12:29 + [2026-10-07 06:58:06] iteration 3471/ 6855 | consumed samples: 444288 | elapsed time per iteration (ms): 89463.5 | throughput per GPU (TFLOP/s/GPU): 41.5 | MFU 13.30% | FLOPs estimator: workload-aware | logical tokens/s: 187530.3 | valid tokens/s: 187530.3 | physical tokens/s: 187995.6 | loss tokens/s: 133675.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.285846E-01 | lm-global-supervised-tokens: 1.195912E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.967112E-02 | lm-token-buffer-mib: 5.816496E+01 | global_load_balancing_loss: 1.000249E+00 | loss scale: 1.0 | grad norm: 0.117 | params norm: 11416.051 | params rms: 0.061814 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 12:05:44.584929 | finish at 2026-10-10 19:03:51 + [2026-10-07 06:59:30] iteration 3472/ 6855 | consumed samples: 444416 | elapsed time per iteration (ms): 84322.1 | throughput per GPU (TFLOP/s/GPU): 43.3 | MFU 13.89% | FLOPs estimator: workload-aware | logical tokens/s: 198965.4 | valid tokens/s: 198965.4 | physical tokens/s: 199532.0 | loss tokens/s: 149451.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.736212E-01 | lm-global-supervised-tokens: 1.260203E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.149478E-02 | lm-token-buffer-mib: 5.807543E+01 | global_load_balancing_loss: 9.987854E-01 | loss scale: 1.0 | grad norm: 0.094 | params norm: 11415.989 | params rms: 0.061814 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 7:14:21.669689 | finish at 2026-10-10 14:13:52 + [2026-10-07 07:00:59] iteration 3473/ 6855 | consumed samples: 444544 | elapsed time per iteration (ms): 88008.6 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.78% | FLOPs estimator: workload-aware | logical tokens/s: 190630.9 | valid tokens/s: 190630.9 | physical tokens/s: 191269.7 | loss tokens/s: 130495.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.845052E-01 | lm-global-supervised-tokens: 1.148470E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.114579E-02 | lm-token-buffer-mib: 5.816456E+01 | global_load_balancing_loss: 9.986100E-01 | loss scale: 1.0 | grad norm: 0.110 | params norm: 11415.929 | params rms: 0.061814 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 10:40:45.118248 | finish at 2026-10-10 17:41:44 + [2026-10-07 07:02:24] iteration 3474/ 6855 | consumed samples: 444672 | elapsed time per iteration (ms): 85176.1 | throughput per GPU (TFLOP/s/GPU): 43.7 | MFU 13.99% | FLOPs estimator: workload-aware | logical tokens/s: 196970.9 | valid tokens/s: 196970.9 | physical tokens/s: 197344.0 | loss tokens/s: 146995.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.188377E-01 | lm-global-supervised-tokens: 1.252046E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.128924E-02 | lm-token-buffer-mib: 5.820892E+01 | global_load_balancing_loss: 1.000581E+00 | loss scale: 1.0 | grad norm: 0.035 | params norm: 11415.868 | params rms: 0.061813 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 7:59:40.338532 | finish at 2026-10-10 15:02:04 + [2026-10-07 07:03:48] iteration 3475/ 6855 | consumed samples: 444800 | elapsed time per iteration (ms): 84022.6 | throughput per GPU (TFLOP/s/GPU): 43.2 | MFU 13.86% | FLOPs estimator: workload-aware | logical tokens/s: 199674.4 | valid tokens/s: 199674.4 | physical tokens/s: 200215.8 | loss tokens/s: 144386.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.275550E-01 | lm-global-supervised-tokens: 1.213177E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.251970E-02 | lm-token-buffer-mib: 5.809524E+01 | global_load_balancing_loss: 1.000503E+00 | loss scale: 1.0 | grad norm: 0.085 | params norm: 11415.807 | params rms: 0.061813 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 6:53:16.494961 | finish at 2026-10-10 13:57:04 + [2026-10-07 07:05:15] iteration 3476/ 6855 | consumed samples: 444928 | elapsed time per iteration (ms): 87308.1 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.69% | FLOPs estimator: workload-aware | logical tokens/s: 192160.9 | valid tokens/s: 192160.9 | physical tokens/s: 192756.3 | loss tokens/s: 140976.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.398505E-01 | lm-global-supervised-tokens: 1.230839E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.603126E-02 | lm-token-buffer-mib: 5.814783E+01 | global_load_balancing_loss: 1.001475E+00 | loss scale: 1.0 | grad norm: 0.045 | params norm: 11415.748 | params rms: 0.061813 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 9:56:54.019902 | finish at 2026-10-10 17:02:09 + [2026-10-07 07:06:46] iteration 3477/ 6855 | consumed samples: 445056 | elapsed time per iteration (ms): 91404.2 | throughput per GPU (TFLOP/s/GPU): 41.1 | MFU 13.18% | FLOPs estimator: workload-aware | logical tokens/s: 183549.6 | valid tokens/s: 183549.6 | physical tokens/s: 183994.8 | loss tokens/s: 130763.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.218329E-01 | lm-global-supervised-tokens: 1.195234E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.232804E-02 | lm-token-buffer-mib: 5.811351E+01 | global_load_balancing_loss: 1.001046E+00 | loss scale: 1.0 | grad norm: 0.109 | params norm: 11415.689 | params rms: 0.061812 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 13:46:03.500613 | finish at 2026-10-10 20:52:50 + [2026-10-07 07:08:12] iteration 3478/ 6855 | consumed samples: 445184 | elapsed time per iteration (ms): 85903.3 | throughput per GPU (TFLOP/s/GPU): 42.8 | MFU 13.71% | FLOPs estimator: workload-aware | logical tokens/s: 195302.7 | valid tokens/s: 195302.7 | physical tokens/s: 195717.8 | loss tokens/s: 149323.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.564771E-01 | lm-global-supervised-tokens: 1.282741E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.555856E-02 | lm-token-buffer-mib: 5.815300E+01 | global_load_balancing_loss: 9.998230E-01 | loss scale: 1.0 | grad norm: 0.084 | params norm: 11415.629 | params rms: 0.061812 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 8:34:55.308190 | finish at 2026-10-10 15:43:08 + [2026-10-07 07:09:40] iteration 3479/ 6855 | consumed samples: 445312 | elapsed time per iteration (ms): 87905.4 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.48% | FLOPs estimator: workload-aware | logical tokens/s: 190855.2 | valid tokens/s: 190855.2 | physical tokens/s: 191215.8 | loss tokens/s: 137560.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.127869E-01 | lm-global-supervised-tokens: 1.209231E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.144578E-02 | lm-token-buffer-mib: 5.809863E+01 | global_load_balancing_loss: 1.000331E+00 | loss scale: 1.0 | grad norm: 0.069 | params norm: 11415.569 | params rms: 0.061812 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 10:26:08.568550 | finish at 2026-10-10 17:35:49 + [2026-10-07 07:11:29] iteration 3480/ 6855 | consumed samples: 445440 | elapsed time per iteration (ms): 109085.2 | throughput per GPU (TFLOP/s/GPU): 33.4 | MFU 10.69% | FLOPs estimator: workload-aware | logical tokens/s: 153798.9 | valid tokens/s: 153798.9 | physical tokens/s: 154094.8 | loss tokens/s: 115922.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.900622E-01 | lm-global-supervised-tokens: 1.264538E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.179714E-02 | lm-token-buffer-mib: 5.812358E+01 | global_load_balancing_loss: 9.987739E-01 | loss scale: 1.0 | grad norm: 0.050 | params norm: 11415.510 | params rms: 0.061811 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 6:16:02.484258 | finish at 2026-10-11 13:27:32 +times across ranks (ms): + save-checkpoint: + rank 0: 23975.63 + rank 1: 23975.62 + rank 2: 23975.66 + rank 3: 23975.65 + rank 4: 23975.66 + rank 5: 23975.62 + rank 6: 23975.60 + rank 7: 23975.66 + rank 8: 23975.35 + rank 9: 23975.29 + rank 10: 23975.29 + rank 11: 23975.30 + rank 12: 23975.28 + rank 13: 23975.30 + rank 14: 23975.29 + rank 15: 23975.30 + rank 16: 23975.93 + rank 17: 23975.96 + rank 18: 23975.92 + rank 19: 23975.97 + rank 20: 23975.95 + rank 21: 23975.95 + rank 22: 23975.92 + rank 23: 23975.92 + rank 24: 23975.24 + rank 25: 23975.21 + rank 26: 23975.21 + rank 27: 23975.22 + rank 28: 23975.18 + rank 29: 23975.18 + rank 30: 23975.19 + rank 31: 23975.25 + rank 32: 23975.44 + rank 33: 23975.46 + rank 34: 23975.46 + rank 35: 23975.41 + rank 36: 23975.40 + rank 37: 23975.38 + rank 38: 23975.40 + rank 39: 23975.38 + rank 40: 23975.36 + rank 41: 23975.36 + rank 42: 23975.35 + rank 43: 23975.40 + rank 44: 23975.34 + rank 45: 23975.35 + rank 46: 23975.38 + rank 47: 23975.33 + rank 48: 23975.56 + rank 49: 23975.57 + rank 50: 23975.54 + rank 51: 23975.60 + rank 52: 23975.54 + rank 53: 23975.53 + rank 54: 23975.54 + rank 55: 23975.54 + rank 56: 23975.58 + rank 57: 23975.54 + rank 58: 23975.54 + rank 59: 23975.55 + rank 60: 23975.54 + rank 61: 23975.55 + rank 62: 23975.55 + rank 63: 23975.53 + [2026-10-07 07:13:38] iteration 3481/ 6855 | consumed samples: 445568 | elapsed time per iteration (ms): 104318.5 | throughput per GPU (TFLOP/s/GPU): 35.4 | MFU 11.34% | FLOPs estimator: workload-aware | logical tokens/s: 160826.8 | valid tokens/s: 160826.8 | physical tokens/s: 161289.7 | loss tokens/s: 118325.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.695755E-01 | lm-global-supervised-tokens: 1.234349E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 4.402375E-01 | lm-token-buffer-mib: 5.825083E+01 | global_load_balancing_loss: 9.995071E-01 | loss scale: 1.0 | grad norm: 0.222 | params norm: 11415.452 | params rms: 0.061811 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 1:46:10.673038 | finish at 2026-10-11 08:59:48 + [2026-10-07 07:15:11] iteration 3482/ 6855 | consumed samples: 445696 | elapsed time per iteration (ms): 93584.2 | throughput per GPU (TFLOP/s/GPU): 39.5 | MFU 12.67% | FLOPs estimator: workload-aware | logical tokens/s: 179273.6 | valid tokens/s: 179273.6 | physical tokens/s: 179700.6 | loss tokens/s: 132823.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.214868E-01 | lm-global-supervised-tokens: 1.243022E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.271813E-02 | lm-token-buffer-mib: 5.815604E+01 | global_load_balancing_loss: 1.000045E+00 | loss scale: 1.0 | grad norm: 0.326 | params norm: 11415.391 | params rms: 0.061811 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 15:40:59.460442 | finish at 2026-10-10 22:56:11 + [2026-10-07 07:16:37] iteration 3483/ 6855 | consumed samples: 445824 | elapsed time per iteration (ms): 86040.2 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.46% | FLOPs estimator: workload-aware | logical tokens/s: 194992.3 | valid tokens/s: 194992.3 | physical tokens/s: 195279.4 | loss tokens/s: 138882.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.193413E-01 | lm-global-supervised-tokens: 1.194948E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.322686E-02 | lm-token-buffer-mib: 5.805620E+01 | global_load_balancing_loss: 1.001591E+00 | loss scale: 1.0 | grad norm: 0.269 | params norm: 11415.332 | params rms: 0.061810 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 8:35:27.407261 | finish at 2026-10-10 15:52:05 + [2026-10-07 07:18:06] iteration 3484/ 6855 | consumed samples: 445952 | elapsed time per iteration (ms): 88901.6 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.61% | FLOPs estimator: workload-aware | logical tokens/s: 188716.6 | valid tokens/s: 188716.6 | physical tokens/s: 189129.6 | loss tokens/s: 136121.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.388538E-01 | lm-global-supervised-tokens: 1.210141E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.336367E-02 | lm-token-buffer-mib: 5.817803E+01 | global_load_balancing_loss: 1.001771E+00 | loss scale: 1.0 | grad norm: 0.123 | params norm: 11415.271 | params rms: 0.061810 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 11:14:47.430643 | finish at 2026-10-10 18:32:54 + [2026-10-07 07:19:33] iteration 3485/ 6855 | consumed samples: 446080 | elapsed time per iteration (ms): 86474.7 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.48% | FLOPs estimator: workload-aware | logical tokens/s: 194012.7 | valid tokens/s: 194012.7 | physical tokens/s: 194583.1 | loss tokens/s: 147746.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.818905E-01 | lm-global-supervised-tokens: 1.277633E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.176875E-02 | lm-token-buffer-mib: 5.813622E+01 | global_load_balancing_loss: 9.999591E-01 | loss scale: 1.0 | grad norm: 0.340 | params norm: 11415.210 | params rms: 0.061810 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 8:56:59.630444 | finish at 2026-10-10 16:16:32 + [2026-10-07 07:20:55] iteration 3486/ 6855 | consumed samples: 446208 | elapsed time per iteration (ms): 82692.6 | throughput per GPU (TFLOP/s/GPU): 44.1 | MFU 14.12% | FLOPs estimator: workload-aware | logical tokens/s: 202886.5 | valid tokens/s: 202886.5 | physical tokens/s: 203453.7 | loss tokens/s: 152933.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.715850E-01 | lm-global-supervised-tokens: 1.264647E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.247436E-02 | lm-token-buffer-mib: 5.838435E+01 | global_load_balancing_loss: 9.979373E-01 | loss scale: 1.0 | grad norm: 0.036 | params norm: 11415.148 | params rms: 0.061809 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 5:23:11.333294 | finish at 2026-10-10 12:44:07 + [2026-10-07 07:22:21] iteration 3487/ 6855 | consumed samples: 446336 | elapsed time per iteration (ms): 85746.1 | throughput per GPU (TFLOP/s/GPU): 41.6 | MFU 13.32% | FLOPs estimator: workload-aware | logical tokens/s: 195660.8 | valid tokens/s: 195660.8 | physical tokens/s: 196374.5 | loss tokens/s: 144851.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 5.962728E-01 | lm-global-supervised-tokens: 1.242047E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.530652E-02 | lm-token-buffer-mib: 5.835921E+01 | global_load_balancing_loss: 9.990422E-01 | loss scale: 1.0 | grad norm: 0.306 | params norm: 11415.087 | params rms: 0.061809 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 8:13:13.005152 | finish at 2026-10-10 15:35:34 + [2026-10-07 07:23:50] iteration 3488/ 6855 | consumed samples: 446464 | elapsed time per iteration (ms): 89060.0 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.51% | FLOPs estimator: workload-aware | logical tokens/s: 188381.0 | valid tokens/s: 188381.0 | physical tokens/s: 188785.2 | loss tokens/s: 139186.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.756845E-01 | lm-global-supervised-tokens: 1.239593E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.820442E-02 | lm-token-buffer-mib: 5.803821E+01 | global_load_balancing_loss: 9.988857E-01 | loss scale: 1.0 | grad norm: 0.037 | params norm: 11415.027 | params rms: 0.061809 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 11:17:45.074395 | finish at 2026-10-10 18:41:35 + [2026-10-07 07:25:22] iteration 3489/ 6855 | consumed samples: 446592 | elapsed time per iteration (ms): 91507.3 | throughput per GPU (TFLOP/s/GPU): 41.1 | MFU 13.16% | FLOPs estimator: workload-aware | logical tokens/s: 183342.6 | valid tokens/s: 183342.6 | physical tokens/s: 183728.9 | loss tokens/s: 134273.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.207828E-01 | lm-global-supervised-tokens: 1.228703E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.158200E-02 | lm-token-buffer-mib: 5.808379E+01 | global_load_balancing_loss: 1.001495E+00 | loss scale: 1.0 | grad norm: 0.126 | params norm: 11414.965 | params rms: 0.061808 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 13:33:33.421393 | finish at 2026-10-10 20:58:55 + [2026-10-07 07:27:15] iteration 3490/ 6855 | consumed samples: 446720 | elapsed time per iteration (ms): 113298.0 | throughput per GPU (TFLOP/s/GPU): 32.6 | MFU 10.44% | FLOPs estimator: workload-aware | logical tokens/s: 148079.7 | valid tokens/s: 148079.7 | physical tokens/s: 148542.0 | loss tokens/s: 106513.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.342472E-01 | lm-global-supervised-tokens: 1.206778E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.149230E-02 | lm-token-buffer-mib: 5.845392E+01 | global_load_balancing_loss: 1.001488E+00 | loss scale: 1.0 | grad norm: 0.202 | params norm: 11414.902 | params rms: 0.061808 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 9:54:07.601849 | finish at 2026-10-11 17:21:23 + [2026-10-07 07:28:41] iteration 3491/ 6855 | consumed samples: 446848 | elapsed time per iteration (ms): 86449.4 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.51% | FLOPs estimator: workload-aware | logical tokens/s: 194069.1 | valid tokens/s: 194069.1 | physical tokens/s: 194626.0 | loss tokens/s: 145253.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.518332E-01 | lm-global-supervised-tokens: 1.255708E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.306847E-02 | lm-token-buffer-mib: 5.805644E+01 | global_load_balancing_loss: 1.000715E+00 | loss scale: 1.0 | grad norm: 0.060 | params norm: 11414.841 | params rms: 0.061808 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 8:46:55.929001 | finish at 2026-10-10 16:15:37 + [2026-10-07 07:30:09] iteration 3492/ 6855 | consumed samples: 446976 | elapsed time per iteration (ms): 87291.2 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.67% | FLOPs estimator: workload-aware | logical tokens/s: 192198.1 | valid tokens/s: 192198.1 | physical tokens/s: 192532.7 | loss tokens/s: 142809.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.835838E-01 | lm-global-supervised-tokens: 1.246601E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.447044E-02 | lm-token-buffer-mib: 5.806059E+01 | global_load_balancing_loss: 1.000327E+00 | loss scale: 1.0 | grad norm: 0.148 | params norm: 11414.780 | params rms: 0.061807 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 9:32:40.425610 | finish at 2026-10-10 17:02:49 + [2026-10-07 07:31:35] iteration 3493/ 6855 | consumed samples: 447104 | elapsed time per iteration (ms): 86203.6 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.69% | FLOPs estimator: workload-aware | logical tokens/s: 194622.4 | valid tokens/s: 194622.4 | physical tokens/s: 195133.0 | loss tokens/s: 146470.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.800858E-01 | lm-global-supervised-tokens: 1.262632E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.218805E-02 | lm-token-buffer-mib: 5.818386E+01 | global_load_balancing_loss: 9.996766E-01 | loss scale: 1.0 | grad norm: 0.207 | params norm: 11414.717 | params rms: 0.061807 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 8:30:16.532622 | finish at 2026-10-10 16:01:51 + [2026-10-07 07:33:02] iteration 3494/ 6855 | consumed samples: 447232 | elapsed time per iteration (ms): 87345.6 | throughput per GPU (TFLOP/s/GPU): 41.3 | MFU 13.23% | FLOPs estimator: workload-aware | logical tokens/s: 192077.8 | valid tokens/s: 192077.8 | physical tokens/s: 192487.0 | loss tokens/s: 142944.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.413560E-01 | lm-global-supervised-tokens: 1.248560E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.125386E-02 | lm-token-buffer-mib: 5.815992E+01 | global_load_balancing_loss: 9.990774E-01 | loss scale: 1.0 | grad norm: 0.104 | params norm: 11414.653 | params rms: 0.061807 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 9:32:48.582064 | finish at 2026-10-10 17:05:51 + [2026-10-07 07:34:28] iteration 3495/ 6855 | consumed samples: 447360 | elapsed time per iteration (ms): 85235.6 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.57% | FLOPs estimator: workload-aware | logical tokens/s: 196832.9 | valid tokens/s: 196832.9 | physical tokens/s: 197322.6 | loss tokens/s: 144933.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.694590E-01 | lm-global-supervised-tokens: 1.235347E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.416890E-02 | lm-token-buffer-mib: 5.801945E+01 | global_load_balancing_loss: 1.000156E+00 | loss scale: 1.0 | grad norm: 0.388 | params norm: 11414.590 | params rms: 0.061806 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 7:33:11.711311 | finish at 2026-10-10 15:07:39 + [2026-10-07 07:35:54] iteration 3496/ 6855 | consumed samples: 447488 | elapsed time per iteration (ms): 86419.9 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.46% | FLOPs estimator: workload-aware | logical tokens/s: 194135.2 | valid tokens/s: 194135.2 | physical tokens/s: 194661.3 | loss tokens/s: 150262.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.660117E-01 | lm-global-supervised-tokens: 1.298562E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.320870E-02 | lm-token-buffer-mib: 5.816732E+01 | global_load_balancing_loss: 9.995561E-01 | loss scale: 1.0 | grad norm: 0.058 | params norm: 11414.526 | params rms: 0.061806 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 8:38:04.465523 | finish at 2026-10-10 16:13:58 + [2026-10-07 07:37:24] iteration 3497/ 6855 | consumed samples: 447616 | elapsed time per iteration (ms): 89932.7 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.39% | FLOPs estimator: workload-aware | logical tokens/s: 186552.4 | valid tokens/s: 186552.4 | physical tokens/s: 187048.4 | loss tokens/s: 135813.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.479092E-01 | lm-global-supervised-tokens: 1.221408E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.156955E-02 | lm-token-buffer-mib: 5.808778E+01 | global_load_balancing_loss: 1.000639E+00 | loss scale: 1.0 | grad norm: 0.400 | params norm: 11414.462 | params rms: 0.061806 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 11:53:13.859816 | finish at 2026-10-10 19:30:38 + [2026-10-07 07:38:51] iteration 3498/ 6855 | consumed samples: 447744 | elapsed time per iteration (ms): 87199.1 | throughput per GPU (TFLOP/s/GPU): 41.7 | MFU 13.36% | FLOPs estimator: workload-aware | logical tokens/s: 192401.3 | valid tokens/s: 192401.3 | physical tokens/s: 192769.1 | loss tokens/s: 139389.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.184672E-01 | lm-global-supervised-tokens: 1.215461E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.387829E-02 | lm-token-buffer-mib: 5.810229E+01 | global_load_balancing_loss: 1.001157E+00 | loss scale: 1.0 | grad norm: 0.072 | params norm: 11414.397 | params rms: 0.061805 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 9:18:47.225088 | finish at 2026-10-10 16:57:38 + [2026-10-07 07:40:20] iteration 3499/ 6855 | consumed samples: 447872 | elapsed time per iteration (ms): 88813.6 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.54% | FLOPs estimator: workload-aware | logical tokens/s: 188902.7 | valid tokens/s: 188902.7 | physical tokens/s: 189338.5 | loss tokens/s: 136487.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.614454E-01 | lm-global-supervised-tokens: 1.212198E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.285650E-02 | lm-token-buffer-mib: 5.826125E+01 | global_load_balancing_loss: 1.000785E+00 | loss scale: 1.0 | grad norm: 0.142 | params norm: 11414.332 | params rms: 0.061805 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 10:47:38.521826 | finish at 2026-10-10 18:27:58 + [2026-10-07 07:42:05] iteration 3500/ 6855 | consumed samples: 448000 | elapsed time per iteration (ms): 105229.4 | throughput per GPU (TFLOP/s/GPU): 34.6 | MFU 11.10% | FLOPs estimator: workload-aware | logical tokens/s: 159434.5 | valid tokens/s: 159434.5 | physical tokens/s: 159856.8 | loss tokens/s: 115747.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.195422E-01 | lm-global-supervised-tokens: 1.218004E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.194386E-02 | lm-token-buffer-mib: 5.824097E+01 | global_load_balancing_loss: 9.993721E-01 | loss scale: 1.0 | grad norm: 0.123 | params norm: 11414.270 | params rms: 0.061805 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 2:04:04.559140 | finish at 2026-10-11 09:46:10 + [2026-10-07 07:43:26] iteration 3501/ 6855 | consumed samples: 448128 | elapsed time per iteration (ms): 80687.8 | throughput per GPU (TFLOP/s/GPU): 43.2 | MFU 13.86% | FLOPs estimator: workload-aware | logical tokens/s: 207926.4 | valid tokens/s: 207926.4 | physical tokens/s: 208792.6 | loss tokens/s: 161059.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.655133E-01 | lm-global-supervised-tokens: 1.299555E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.207968E-02 | lm-token-buffer-mib: 5.815285E+01 | global_load_balancing_loss: 1.000149E+00 | loss scale: 1.0 | grad norm: 0.200 | params norm: 11414.208 | params rms: 0.061804 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 3:10:26.956135 | finish at 2026-10-10 10:53:53 + [2026-10-07 07:44:52] iteration 3502/ 6855 | consumed samples: 448256 | elapsed time per iteration (ms): 85728.1 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.52% | FLOPs estimator: workload-aware | logical tokens/s: 195701.9 | valid tokens/s: 195701.9 | physical tokens/s: 196173.9 | loss tokens/s: 145581.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.844305E-01 | lm-global-supervised-tokens: 1.248039E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.481581E-02 | lm-token-buffer-mib: 5.812918E+01 | global_load_balancing_loss: 9.991184E-01 | loss scale: 1.0 | grad norm: 0.064 | params norm: 11414.146 | params rms: 0.061804 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 7:50:46.358677 | finish at 2026-10-10 15:35:38 + [2026-10-07 07:46:18] iteration 3503/ 6855 | consumed samples: 448384 | elapsed time per iteration (ms): 86221.6 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.76% | FLOPs estimator: workload-aware | logical tokens/s: 194581.7 | valid tokens/s: 194581.7 | physical tokens/s: 194936.5 | loss tokens/s: 151149.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.350316E-01 | lm-global-supervised-tokens: 1.303236E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.303193E-02 | lm-token-buffer-mib: 5.819082E+01 | global_load_balancing_loss: 9.991592E-01 | loss scale: 1.0 | grad norm: 0.133 | params norm: 11414.086 | params rms: 0.061804 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 8:16:54.865723 | finish at 2026-10-10 16:03:13 + [2026-10-07 07:47:46] iteration 3504/ 6855 | consumed samples: 448512 | elapsed time per iteration (ms): 88714.0 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.44% | FLOPs estimator: workload-aware | logical tokens/s: 189115.5 | valid tokens/s: 189115.5 | physical tokens/s: 189613.4 | loss tokens/s: 137486.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.702942E-01 | lm-global-supervised-tokens: 1.219697E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.635493E-02 | lm-token-buffer-mib: 5.823311E+01 | global_load_balancing_loss: 1.000857E+00 | loss scale: 1.0 | grad norm: 0.303 | params norm: 11414.025 | params rms: 0.061803 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 10:34:40.562024 | finish at 2026-10-10 18:22:27 + [2026-10-07 07:49:14] iteration 3505/ 6855 | consumed samples: 448640 | elapsed time per iteration (ms): 87066.8 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.52% | FLOPs estimator: workload-aware | logical tokens/s: 192693.1 | valid tokens/s: 192693.1 | physical tokens/s: 192991.5 | loss tokens/s: 144435.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.451091E-01 | lm-global-supervised-tokens: 1.257551E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.544435E-02 | lm-token-buffer-mib: 5.813954E+01 | global_load_balancing_loss: 1.000472E+00 | loss scale: 1.0 | grad norm: 0.058 | params norm: 11413.963 | params rms: 0.061803 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 9:01:13.797166 | finish at 2026-10-10 16:50:27 + [2026-10-07 07:50:40] iteration 3506/ 6855 | consumed samples: 448768 | elapsed time per iteration (ms): 86942.0 | throughput per GPU (TFLOP/s/GPU): 41.6 | MFU 13.32% | FLOPs estimator: workload-aware | logical tokens/s: 192969.8 | valid tokens/s: 192969.8 | physical tokens/s: 193283.2 | loss tokens/s: 150485.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.155329E-01 | lm-global-supervised-tokens: 1.308348E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.380421E-02 | lm-token-buffer-mib: 5.809113E+01 | global_load_balancing_loss: 1.000924E+00 | loss scale: 1.0 | grad norm: 0.203 | params norm: 11413.902 | params rms: 0.061803 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 8:52:48.806412 | finish at 2026-10-10 16:43:29 + [2026-10-07 07:52:10] iteration 3507/ 6855 | consumed samples: 448896 | elapsed time per iteration (ms): 89825.0 | throughput per GPU (TFLOP/s/GPU): 41.1 | MFU 13.16% | FLOPs estimator: workload-aware | logical tokens/s: 186776.6 | valid tokens/s: 186776.6 | physical tokens/s: 187222.9 | loss tokens/s: 133958.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.049730E-01 | lm-global-supervised-tokens: 1.203278E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.318698E-02 | lm-token-buffer-mib: 5.820661E+01 | global_load_balancing_loss: 1.001282E+00 | loss scale: 1.0 | grad norm: 0.246 | params norm: 11413.839 | params rms: 0.061802 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 11:32:14.065038 | finish at 2026-10-10 19:24:24 + [2026-10-07 07:53:41] iteration 3508/ 6855 | consumed samples: 449024 | elapsed time per iteration (ms): 90242.7 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.39% | FLOPs estimator: workload-aware | logical tokens/s: 185912.1 | valid tokens/s: 185912.1 | physical tokens/s: 186350.1 | loss tokens/s: 137418.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.798427E-01 | lm-global-supervised-tokens: 1.240103E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.307605E-02 | lm-token-buffer-mib: 5.812675E+01 | global_load_balancing_loss: 1.000057E+00 | loss scale: 1.0 | grad norm: 0.099 | params norm: 11413.776 | params rms: 0.061802 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 11:54:02.345164 | finish at 2026-10-10 19:47:43 + [2026-10-07 07:55:05] iteration 3509/ 6855 | consumed samples: 449152 | elapsed time per iteration (ms): 83957.1 | throughput per GPU (TFLOP/s/GPU): 43.7 | MFU 14.02% | FLOPs estimator: workload-aware | logical tokens/s: 199829.9 | valid tokens/s: 199829.9 | physical tokens/s: 200407.8 | loss tokens/s: 149915.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.935502E-01 | lm-global-supervised-tokens: 1.258647E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.128925E-02 | lm-token-buffer-mib: 5.820066E+01 | global_load_balancing_loss: 9.995869E-01 | loss scale: 1.0 | grad norm: 0.201 | params norm: 11413.715 | params rms: 0.061802 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 6:02:00.522527 | finish at 2026-10-10 13:57:05 + [2026-10-07 07:56:53] iteration 3510/ 6855 | consumed samples: 449280 | elapsed time per iteration (ms): 108391.0 | throughput per GPU (TFLOP/s/GPU): 33.7 | MFU 10.80% | FLOPs estimator: workload-aware | logical tokens/s: 154783.3 | valid tokens/s: 154783.3 | physical tokens/s: 155256.6 | loss tokens/s: 121529.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.468303E-01 | lm-global-supervised-tokens: 1.317268E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.274644E-02 | lm-token-buffer-mib: 5.805384E+01 | global_load_balancing_loss: 9.991091E-01 | loss scale: 1.0 | grad norm: 0.144 | params norm: 11413.651 | params rms: 0.061801 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 4:42:47.876766 | finish at 2026-10-11 12:39:41 + [2026-10-07 07:58:17] iteration 3511/ 6855 | consumed samples: 449408 | elapsed time per iteration (ms): 84365.3 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.58% | FLOPs estimator: workload-aware | logical tokens/s: 198863.8 | valid tokens/s: 198863.8 | physical tokens/s: 199376.3 | loss tokens/s: 149555.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.249871E-01 | lm-global-supervised-tokens: 1.261730E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.408141E-02 | lm-token-buffer-mib: 5.813739E+01 | global_load_balancing_loss: 9.997203E-01 | loss scale: 1.0 | grad norm: 0.125 | params norm: 11413.585 | params rms: 0.061801 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 6:21:57.431450 | finish at 2026-10-10 14:20:15 + [2026-10-07 07:59:45] iteration 3512/ 6855 | consumed samples: 449536 | elapsed time per iteration (ms): 87785.6 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.54% | FLOPs estimator: workload-aware | logical tokens/s: 191115.2 | valid tokens/s: 191115.2 | physical tokens/s: 191473.2 | loss tokens/s: 133672.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.663947E-01 | lm-global-supervised-tokens: 1.173453E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.625479E-02 | lm-token-buffer-mib: 5.813048E+01 | global_load_balancing_loss: 1.001225E+00 | loss scale: 1.0 | grad norm: 0.060 | params norm: 11413.522 | params rms: 0.061801 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 9:31:07.358658 | finish at 2026-10-10 17:30:52 + [2026-10-07 08:01:16] iteration 3513/ 6855 | consumed samples: 449664 | elapsed time per iteration (ms): 90571.4 | throughput per GPU (TFLOP/s/GPU): 41.5 | MFU 13.31% | FLOPs estimator: workload-aware | logical tokens/s: 185237.1 | valid tokens/s: 185237.1 | physical tokens/s: 185735.5 | loss tokens/s: 128787.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.644391E-01 | lm-global-supervised-tokens: 1.166447E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.244474E-02 | lm-token-buffer-mib: 5.816437E+01 | global_load_balancing_loss: 1.001570E+00 | loss scale: 1.0 | grad norm: 0.122 | params norm: 11413.459 | params rms: 0.061800 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 12:04:49.759589 | finish at 2026-10-10 20:06:05 + [2026-10-07 08:02:46] iteration 3514/ 6855 | consumed samples: 449792 | elapsed time per iteration (ms): 90346.3 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.62% | FLOPs estimator: workload-aware | logical tokens/s: 185698.6 | valid tokens/s: 185698.6 | physical tokens/s: 186026.1 | loss tokens/s: 132869.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.781595E-01 | lm-global-supervised-tokens: 1.200424E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.341881E-02 | lm-token-buffer-mib: 5.809813E+01 | global_load_balancing_loss: 1.000452E+00 | loss scale: 1.0 | grad norm: 0.068 | params norm: 11413.396 | params rms: 0.061800 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 11:50:46.903486 | finish at 2026-10-10 19:53:33 + [2026-10-07 08:04:11] iteration 3515/ 6855 | consumed samples: 449920 | elapsed time per iteration (ms): 85239.7 | throughput per GPU (TFLOP/s/GPU): 43.3 | MFU 13.89% | FLOPs estimator: workload-aware | logical tokens/s: 196823.8 | valid tokens/s: 196823.8 | physical tokens/s: 197264.8 | loss tokens/s: 148383.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.908291E-01 | lm-global-supervised-tokens: 1.264818E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.293651E-02 | lm-token-buffer-mib: 5.810703E+01 | global_load_balancing_loss: 9.993750E-01 | loss scale: 1.0 | grad norm: 0.105 | params norm: 11413.333 | params rms: 0.061799 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 7:05:00.485187 | finish at 2026-10-10 15:09:12 + [2026-10-07 08:05:38] iteration 3516/ 6855 | consumed samples: 450048 | elapsed time per iteration (ms): 86410.6 | throughput per GPU (TFLOP/s/GPU): 43.5 | MFU 13.94% | FLOPs estimator: workload-aware | logical tokens/s: 194156.7 | valid tokens/s: 194156.7 | physical tokens/s: 194656.5 | loss tokens/s: 142189.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.513891E-01 | lm-global-supervised-tokens: 1.228665E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.224144E-02 | lm-token-buffer-mib: 5.813986E+01 | global_load_balancing_loss: 9.996222E-01 | loss scale: 1.0 | grad norm: 0.085 | params norm: 11413.269 | params rms: 0.061799 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 8:08:44.990039 | finish at 2026-10-10 16:14:23 + [2026-10-07 08:07:03] iteration 3517/ 6855 | consumed samples: 450176 | elapsed time per iteration (ms): 85192.7 | throughput per GPU (TFLOP/s/GPU): 43.2 | MFU 13.85% | FLOPs estimator: workload-aware | logical tokens/s: 196932.5 | valid tokens/s: 196932.5 | physical tokens/s: 197401.7 | loss tokens/s: 146596.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.715745E-01 | lm-global-supervised-tokens: 1.248897E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.099555E-02 | lm-token-buffer-mib: 5.815298E+01 | global_load_balancing_loss: 9.997750E-01 | loss scale: 1.0 | grad norm: 0.086 | params norm: 11413.207 | params rms: 0.061799 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 6:59:33.271293 | finish at 2026-10-10 15:06:36 + [2026-10-07 08:08:31] iteration 3518/ 6855 | consumed samples: 450304 | elapsed time per iteration (ms): 88061.6 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.43% | FLOPs estimator: workload-aware | logical tokens/s: 190516.3 | valid tokens/s: 190516.3 | physical tokens/s: 191037.8 | loss tokens/s: 145807.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.682612E-01 | lm-global-supervised-tokens: 1.284000E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.562318E-02 | lm-token-buffer-mib: 5.806889E+01 | global_load_balancing_loss: 9.995173E-01 | loss scale: 1.0 | grad norm: 0.115 | params norm: 11413.143 | params rms: 0.061798 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 9:37:41.654572 | finish at 2026-10-10 17:46:13 + [2026-10-07 08:10:00] iteration 3519/ 6855 | consumed samples: 450432 | elapsed time per iteration (ms): 89395.0 | throughput per GPU (TFLOP/s/GPU): 41.1 | MFU 13.16% | FLOPs estimator: workload-aware | logical tokens/s: 187675.1 | valid tokens/s: 187675.1 | physical tokens/s: 188001.4 | loss tokens/s: 137926.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.957118E-01 | lm-global-supervised-tokens: 1.232998E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.198285E-02 | lm-token-buffer-mib: 5.819895E+01 | global_load_balancing_loss: 1.001250E+00 | loss scale: 1.0 | grad norm: 0.044 | params norm: 11413.081 | params rms: 0.061798 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 10:50:21.647558 | finish at 2026-10-10 19:00:22 + [2026-10-07 08:11:56] iteration 3520/ 6855 | consumed samples: 450560 | elapsed time per iteration (ms): 115884.9 | throughput per GPU (TFLOP/s/GPU): 32.9 | MFU 10.53% | FLOPs estimator: workload-aware | logical tokens/s: 144774.6 | valid tokens/s: 144774.6 | physical tokens/s: 145095.5 | loss tokens/s: 103713.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.116247E-01 | lm-global-supervised-tokens: 1.201882E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.149690E-02 | lm-token-buffer-mib: 5.819862E+01 | global_load_balancing_loss: 1.000684E+00 | loss scale: 1.0 | grad norm: 0.071 | params norm: 11413.021 | params rms: 0.061798 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 11:21:16.030493 | finish at 2026-10-11 19:33:12 +times across ranks (ms): + save-checkpoint: + rank 0: 23934.50 + rank 1: 23934.50 + rank 2: 23934.53 + rank 3: 23934.51 + rank 4: 23934.50 + rank 5: 23934.48 + rank 6: 23934.47 + rank 7: 23934.50 + rank 8: 23934.92 + rank 9: 23934.88 + rank 10: 23934.88 + rank 11: 23934.87 + rank 12: 23934.88 + rank 13: 23934.88 + rank 14: 23934.88 + rank 15: 23934.87 + rank 16: 23934.12 + rank 17: 23934.14 + rank 18: 23934.09 + rank 19: 23934.12 + rank 20: 23934.11 + rank 21: 23934.12 + rank 22: 23934.10 + rank 23: 23934.10 + rank 24: 23935.29 + rank 25: 23935.25 + rank 26: 23935.30 + rank 27: 23935.30 + rank 28: 23935.27 + rank 29: 23935.27 + rank 30: 23935.27 + rank 31: 23935.28 + rank 32: 23934.43 + rank 33: 23934.43 + rank 34: 23934.42 + rank 35: 23934.42 + rank 36: 23934.40 + rank 37: 23934.38 + rank 38: 23934.38 + rank 39: 23934.39 + rank 40: 23934.57 + rank 41: 23934.57 + rank 42: 23934.56 + rank 43: 23934.57 + rank 44: 23934.57 + rank 45: 23934.55 + rank 46: 23934.60 + rank 47: 23934.57 + rank 48: 23934.76 + rank 49: 23934.75 + rank 50: 23934.74 + rank 51: 23934.73 + rank 52: 23934.74 + rank 53: 23934.74 + rank 54: 23934.73 + rank 55: 23934.74 + rank 56: 23934.82 + rank 57: 23934.81 + rank 58: 23934.81 + rank 59: 23934.81 + rank 60: 23934.82 + rank 61: 23934.81 + rank 62: 23934.82 + rank 63: 23934.80 + [2026-10-07 08:13:59] iteration 3521/ 6855 | consumed samples: 450688 | elapsed time per iteration (ms): 98911.3 | throughput per GPU (TFLOP/s/GPU): 37.1 | MFU 11.91% | FLOPs estimator: workload-aware | logical tokens/s: 169618.8 | valid tokens/s: 169618.7 | physical tokens/s: 169935.7 | loss tokens/s: 125981.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.686477E-01 | lm-global-supervised-tokens: 1.246099E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 5.318313E-01 | lm-token-buffer-mib: 5.806622E+01 | global_load_balancing_loss: 1.000739E+00 | loss scale: 1.0 | grad norm: 0.107 | params norm: 11412.960 | params rms: 0.061797 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 19:36:10.297860 | finish at 2026-10-11 03:50:10 + [2026-10-07 08:15:30] iteration 3522/ 6855 | consumed samples: 450816 | elapsed time per iteration (ms): 90730.3 | throughput per GPU (TFLOP/s/GPU): 39.8 | MFU 12.76% | FLOPs estimator: workload-aware | logical tokens/s: 184912.3 | valid tokens/s: 184912.3 | physical tokens/s: 185511.1 | loss tokens/s: 136733.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.557826E-01 | lm-global-supervised-tokens: 1.240588E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.549732E-02 | lm-token-buffer-mib: 5.817606E+01 | global_load_balancing_loss: 1.000489E+00 | loss scale: 1.0 | grad norm: 0.054 | params norm: 11412.899 | params rms: 0.061797 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 12:00:03.966562 | finish at 2026-10-10 20:15:34 + [2026-10-07 08:16:59] iteration 3523/ 6855 | consumed samples: 450944 | elapsed time per iteration (ms): 88764.2 | throughput per GPU (TFLOP/s/GPU): 41.3 | MFU 13.24% | FLOPs estimator: workload-aware | logical tokens/s: 189008.0 | valid tokens/s: 189008.0 | physical tokens/s: 189428.3 | loss tokens/s: 147518.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.068104E-01 | lm-global-supervised-tokens: 1.309437E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.011130E-02 | lm-token-buffer-mib: 5.811919E+01 | global_load_balancing_loss: 9.991074E-01 | loss scale: 1.0 | grad norm: 0.107 | params norm: 11412.840 | params rms: 0.061797 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 10:09:22.176080 | finish at 2026-10-10 18:26:21 + [2026-10-07 08:18:24] iteration 3524/ 6855 | consumed samples: 451072 | elapsed time per iteration (ms): 85281.4 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.52% | FLOPs estimator: workload-aware | logical tokens/s: 196727.6 | valid tokens/s: 196727.6 | physical tokens/s: 197374.6 | loss tokens/s: 151100.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.614670E-01 | lm-global-supervised-tokens: 1.288610E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.358831E-02 | lm-token-buffer-mib: 5.817538E+01 | global_load_balancing_loss: 1.000574E+00 | loss scale: 1.0 | grad norm: 0.062 | params norm: 11412.782 | params rms: 0.061797 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 6:54:32.363139 | finish at 2026-10-10 15:12:56 + [2026-10-07 08:19:52] iteration 3525/ 6855 | consumed samples: 451200 | elapsed time per iteration (ms): 87800.3 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.61% | FLOPs estimator: workload-aware | logical tokens/s: 191083.3 | valid tokens/s: 191083.3 | physical tokens/s: 191489.0 | loss tokens/s: 136385.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.376039E-01 | lm-global-supervised-tokens: 1.197469E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.168710E-02 | lm-token-buffer-mib: 5.811326E+01 | global_load_balancing_loss: 9.997457E-01 | loss scale: 1.0 | grad norm: 0.042 | params norm: 11412.724 | params rms: 0.061796 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 9:12:54.867611 | finish at 2026-10-10 17:32:47 + [2026-10-07 08:21:19] iteration 3526/ 6855 | consumed samples: 451328 | elapsed time per iteration (ms): 86719.9 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.65% | FLOPs estimator: workload-aware | logical tokens/s: 193464.4 | valid tokens/s: 193464.4 | physical tokens/s: 193879.3 | loss tokens/s: 138911.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.456276E-01 | lm-global-supervised-tokens: 1.204635E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.154539E-02 | lm-token-buffer-mib: 5.812085E+01 | global_load_balancing_loss: 1.000007E+00 | loss scale: 1.0 | grad norm: 0.042 | params norm: 11412.663 | params rms: 0.061796 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 8:11:30.599127 | finish at 2026-10-10 16:32:49 + [2026-10-07 08:22:47] iteration 3527/ 6855 | consumed samples: 451456 | elapsed time per iteration (ms): 87972.5 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.46% | FLOPs estimator: workload-aware | logical tokens/s: 190709.5 | valid tokens/s: 190709.5 | physical tokens/s: 191286.0 | loss tokens/s: 140352.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.442916E-01 | lm-global-supervised-tokens: 1.234716E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.158240E-02 | lm-token-buffer-mib: 5.814371E+01 | global_load_balancing_loss: 1.000932E+00 | loss scale: 1.0 | grad norm: 0.046 | params norm: 11412.603 | params rms: 0.061796 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 9:19:32.385071 | finish at 2026-10-10 17:42:19 + [2026-10-07 08:24:14] iteration 3528/ 6855 | consumed samples: 451584 | elapsed time per iteration (ms): 87940.9 | throughput per GPU (TFLOP/s/GPU): 41.4 | MFU 13.27% | FLOPs estimator: workload-aware | logical tokens/s: 190777.5 | valid tokens/s: 190777.5 | physical tokens/s: 191190.7 | loss tokens/s: 141406.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.711841E-01 | lm-global-supervised-tokens: 1.243540E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.145632E-02 | lm-token-buffer-mib: 5.811813E+01 | global_load_balancing_loss: 1.000347E+00 | loss scale: 1.0 | grad norm: 0.077 | params norm: 11412.545 | params rms: 0.061795 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 9:16:19.471363 | finish at 2026-10-10 17:40:34 + [2026-10-07 08:25:41] iteration 3529/ 6855 | consumed samples: 451712 | elapsed time per iteration (ms): 86329.0 | throughput per GPU (TFLOP/s/GPU): 41.7 | MFU 13.36% | FLOPs estimator: workload-aware | logical tokens/s: 194340.2 | valid tokens/s: 194340.2 | physical tokens/s: 194984.0 | loss tokens/s: 148226.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.745298E-01 | lm-global-supervised-tokens: 1.279625E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.371995E-02 | lm-token-buffer-mib: 5.818826E+01 | global_load_balancing_loss: 9.996193E-01 | loss scale: 1.0 | grad norm: 0.089 | params norm: 11412.487 | params rms: 0.061795 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 7:45:30.095391 | finish at 2026-10-10 16:11:11 + [2026-10-07 08:27:34] iteration 3530/ 6855 | consumed samples: 451840 | elapsed time per iteration (ms): 112772.8 | throughput per GPU (TFLOP/s/GPU): 33.2 | MFU 10.64% | FLOPs estimator: workload-aware | logical tokens/s: 148769.6 | valid tokens/s: 148769.6 | physical tokens/s: 149177.3 | loss tokens/s: 107660.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.173665E-01 | lm-global-supervised-tokens: 1.214119E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.929291E-02 | lm-token-buffer-mib: 5.820095E+01 | global_load_balancing_loss: 9.999592E-01 | loss scale: 1.0 | grad norm: 0.032 | params norm: 11412.426 | params rms: 0.061795 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 8:09:29.532150 | finish at 2026-10-11 16:37:03 + [2026-10-07 08:29:03] iteration 3531/ 6855 | consumed samples: 451968 | elapsed time per iteration (ms): 89588.5 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.39% | FLOPs estimator: workload-aware | logical tokens/s: 187269.3 | valid tokens/s: 187269.3 | physical tokens/s: 187790.7 | loss tokens/s: 134905.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.921148E-01 | lm-global-supervised-tokens: 1.208601E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.370208E-02 | lm-token-buffer-mib: 5.807497E+01 | global_load_balancing_loss: 1.000038E+00 | loss scale: 1.0 | grad norm: 0.057 | params norm: 11412.366 | params rms: 0.061794 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 10:43:12.166151 | finish at 2026-10-10 19:12:15 + [2026-10-07 08:30:32] iteration 3532/ 6855 | consumed samples: 452096 | elapsed time per iteration (ms): 88806.2 | throughput per GPU (TFLOP/s/GPU): 40.9 | MFU 13.10% | FLOPs estimator: workload-aware | logical tokens/s: 188919.2 | valid tokens/s: 188919.1 | physical tokens/s: 189372.7 | loss tokens/s: 143050.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.430268E-01 | lm-global-supervised-tokens: 1.270378E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.166970E-02 | lm-token-buffer-mib: 5.807544E+01 | global_load_balancing_loss: 9.993953E-01 | loss scale: 1.0 | grad norm: 0.037 | params norm: 11412.305 | params rms: 0.061794 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 9:58:22.978131 | finish at 2026-10-10 18:28:55 + [2026-10-07 08:31:57] iteration 3533/ 6855 | consumed samples: 452224 | elapsed time per iteration (ms): 85352.9 | throughput per GPU (TFLOP/s/GPU): 43.1 | MFU 13.82% | FLOPs estimator: workload-aware | logical tokens/s: 196562.2 | valid tokens/s: 196562.2 | physical tokens/s: 197086.0 | loss tokens/s: 142591.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.662018E-01 | lm-global-supervised-tokens: 1.217058E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.170337E-02 | lm-token-buffer-mib: 5.821833E+01 | global_load_balancing_loss: 1.000933E+00 | loss scale: 1.0 | grad norm: 0.052 | params norm: 11412.243 | params rms: 0.061794 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 6:45:42.455074 | finish at 2026-10-10 15:17:40 + [2026-10-07 08:33:23] iteration 3534/ 6855 | consumed samples: 452352 | elapsed time per iteration (ms): 85516.0 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.74% | FLOPs estimator: workload-aware | logical tokens/s: 196187.3 | valid tokens/s: 196187.3 | physical tokens/s: 196638.8 | loss tokens/s: 145496.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.733118E-01 | lm-global-supervised-tokens: 1.244230E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.527670E-02 | lm-token-buffer-mib: 5.835250E+01 | global_load_balancing_loss: 1.000650E+00 | loss scale: 1.0 | grad norm: 0.037 | params norm: 11412.183 | params rms: 0.061793 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 6:53:18.623439 | finish at 2026-10-10 15:26:41 + [2026-10-07 08:34:48] iteration 3535/ 6855 | consumed samples: 452480 | elapsed time per iteration (ms): 85403.6 | throughput per GPU (TFLOP/s/GPU): 43.5 | MFU 13.95% | FLOPs estimator: workload-aware | logical tokens/s: 196445.7 | valid tokens/s: 196445.7 | physical tokens/s: 196844.2 | loss tokens/s: 139781.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.776318E-01 | lm-global-supervised-tokens: 1.193781E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.245146E-02 | lm-token-buffer-mib: 5.807761E+01 | global_load_balancing_loss: 9.997996E-01 | loss scale: 1.0 | grad norm: 0.038 | params norm: 11412.120 | params rms: 0.061793 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 6:45:39.941635 | finish at 2026-10-10 15:20:28 + [2026-10-07 08:36:17] iteration 3536/ 6855 | consumed samples: 452608 | elapsed time per iteration (ms): 88364.1 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.48% | FLOPs estimator: workload-aware | logical tokens/s: 189864.7 | valid tokens/s: 189864.7 | physical tokens/s: 190373.0 | loss tokens/s: 139687.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.544676E-01 | lm-global-supervised-tokens: 1.234339E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.490504E-02 | lm-token-buffer-mib: 5.813037E+01 | global_load_balancing_loss: 9.994309E-01 | loss scale: 1.0 | grad norm: 0.053 | params norm: 11412.058 | params rms: 0.061793 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 9:28:00.313309 | finish at 2026-10-10 18:04:17 + [2026-10-07 08:37:45] iteration 3537/ 6855 | consumed samples: 452736 | elapsed time per iteration (ms): 88278.0 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.48% | FLOPs estimator: workload-aware | logical tokens/s: 190049.5 | valid tokens/s: 190049.5 | physical tokens/s: 190531.2 | loss tokens/s: 142499.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.390375E-01 | lm-global-supervised-tokens: 1.257956E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.545403E-02 | lm-token-buffer-mib: 5.837201E+01 | global_load_balancing_loss: 1.000748E+00 | loss scale: 1.0 | grad norm: 0.068 | params norm: 11411.996 | params rms: 0.061792 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 9:21:46.426536 | finish at 2026-10-10 17:59:31 + [2026-10-07 08:39:08] iteration 3538/ 6855 | consumed samples: 452864 | elapsed time per iteration (ms): 82948.5 | throughput per GPU (TFLOP/s/GPU): 43.1 | MFU 13.83% | FLOPs estimator: workload-aware | logical tokens/s: 202260.2 | valid tokens/s: 202260.2 | physical tokens/s: 202922.6 | loss tokens/s: 152577.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.411946E-01 | lm-global-supervised-tokens: 1.265611E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.474016E-02 | lm-token-buffer-mib: 5.823158E+01 | global_load_balancing_loss: 1.000650E+00 | loss scale: 1.0 | grad norm: 0.046 | params norm: 11411.933 | params rms: 0.061792 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 4:25:40.163947 | finish at 2026-10-10 13:04:48 + [2026-10-07 08:40:35] iteration 3539/ 6855 | consumed samples: 452992 | elapsed time per iteration (ms): 87167.1 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.57% | FLOPs estimator: workload-aware | logical tokens/s: 192471.3 | valid tokens/s: 192471.3 | physical tokens/s: 193043.9 | loss tokens/s: 144478.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.852741E-01 | lm-global-supervised-tokens: 1.259379E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.136917E-02 | lm-token-buffer-mib: 5.821512E+01 | global_load_balancing_loss: 1.000811E+00 | loss scale: 1.0 | grad norm: 0.139 | params norm: 11411.870 | params rms: 0.061792 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 8:17:25.974576 | finish at 2026-10-10 16:58:01 + [2026-10-07 08:42:24] iteration 3540/ 6855 | consumed samples: 453120 | elapsed time per iteration (ms): 109010.7 | throughput per GPU (TFLOP/s/GPU): 33.5 | MFU 10.75% | FLOPs estimator: workload-aware | logical tokens/s: 153904.3 | valid tokens/s: 153904.3 | physical tokens/s: 154346.8 | loss tokens/s: 114790.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.395881E-01 | lm-global-supervised-tokens: 1.251339E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.303412E-02 | lm-token-buffer-mib: 5.813371E+01 | global_load_balancing_loss: 1.000783E+00 | loss scale: 1.0 | grad norm: 0.235 | params norm: 11411.808 | params rms: 0.061791 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 4:22:50.439634 | finish at 2026-10-11 13:05:14 + [2026-10-07 08:43:50] iteration 3541/ 6855 | consumed samples: 453248 | elapsed time per iteration (ms): 86459.3 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.70% | FLOPs estimator: workload-aware | logical tokens/s: 194047.6 | valid tokens/s: 194047.6 | physical tokens/s: 194711.3 | loss tokens/s: 148437.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.687876E-01 | lm-global-supervised-tokens: 1.283380E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.526813E-02 | lm-token-buffer-mib: 5.812746E+01 | global_load_balancing_loss: 9.994915E-01 | loss scale: 1.0 | grad norm: 0.219 | params norm: 11411.746 | params rms: 0.061791 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 7:35:25.972584 | finish at 2026-10-10 16:19:16 + [2026-10-07 08:45:20] iteration 3542/ 6855 | consumed samples: 453376 | elapsed time per iteration (ms): 89949.6 | throughput per GPU (TFLOP/s/GPU): 40.8 | MFU 13.07% | FLOPs estimator: workload-aware | logical tokens/s: 186517.0 | valid tokens/s: 186517.0 | physical tokens/s: 186963.5 | loss tokens/s: 138706.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.739081E-01 | lm-global-supervised-tokens: 1.247657E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.124503E-02 | lm-token-buffer-mib: 5.808141E+01 | global_load_balancing_loss: 1.000730E+00 | loss scale: 1.0 | grad norm: 0.069 | params norm: 11411.684 | params rms: 0.061791 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 10:46:42.889668 | finish at 2026-10-10 19:32:03 + [2026-10-07 08:46:47] iteration 3543/ 6855 | consumed samples: 453504 | elapsed time per iteration (ms): 87024.1 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.40% | FLOPs estimator: workload-aware | logical tokens/s: 192787.4 | valid tokens/s: 192787.4 | physical tokens/s: 193264.5 | loss tokens/s: 142049.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.555405E-01 | lm-global-supervised-tokens: 1.236177E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.417346E-02 | lm-token-buffer-mib: 5.812164E+01 | global_load_balancing_loss: 1.000349E+00 | loss scale: 1.0 | grad norm: 0.230 | params norm: 11411.620 | params rms: 0.061790 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 8:03:43.857319 | finish at 2026-10-10 16:50:31 + [2026-10-07 08:48:13] iteration 3544/ 6855 | consumed samples: 453632 | elapsed time per iteration (ms): 85506.0 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.77% | FLOPs estimator: workload-aware | logical tokens/s: 196210.8 | valid tokens/s: 196210.7 | physical tokens/s: 196624.6 | loss tokens/s: 148208.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.955355E-01 | lm-global-supervised-tokens: 1.267269E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.568437E-02 | lm-token-buffer-mib: 5.812639E+01 | global_load_balancing_loss: 9.997634E-01 | loss scale: 1.0 | grad norm: 0.036 | params norm: 11411.557 | params rms: 0.061790 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 6:38:30.236677 | finish at 2026-10-10 15:26:43 + [2026-10-07 08:49:43] iteration 3545/ 6855 | consumed samples: 453760 | elapsed time per iteration (ms): 90524.2 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.42% | FLOPs estimator: workload-aware | logical tokens/s: 185334.1 | valid tokens/s: 185334.1 | physical tokens/s: 185738.9 | loss tokens/s: 133685.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.604345E-01 | lm-global-supervised-tokens: 1.210181E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.175608E-02 | lm-token-buffer-mib: 5.812119E+01 | global_load_balancing_loss: 1.000338E+00 | loss scale: 1.0 | grad norm: 0.149 | params norm: 11411.495 | params rms: 0.061790 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 11:13:55.011911 | finish at 2026-10-10 20:03:38 + [2026-10-07 08:51:07] iteration 3546/ 6855 | consumed samples: 453888 | elapsed time per iteration (ms): 83447.6 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.78% | FLOPs estimator: workload-aware | logical tokens/s: 201050.2 | valid tokens/s: 201050.2 | physical tokens/s: 201702.4 | loss tokens/s: 151110.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.192121E-01 | lm-global-supervised-tokens: 1.260984E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.115864E-02 | lm-token-buffer-mib: 5.824819E+01 | global_load_balancing_loss: 1.000527E+00 | loss scale: 1.0 | grad norm: 0.121 | params norm: 11411.433 | params rms: 0.061789 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 4:42:08.203489 | finish at 2026-10-10 13:33:15 + [2026-10-07 08:52:31] iteration 3547/ 6855 | consumed samples: 454016 | elapsed time per iteration (ms): 83886.4 | throughput per GPU (TFLOP/s/GPU): 42.8 | MFU 13.73% | FLOPs estimator: workload-aware | logical tokens/s: 199999.3 | valid tokens/s: 199999.3 | physical tokens/s: 200443.5 | loss tokens/s: 151718.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.443658E-01 | lm-global-supervised-tokens: 1.272710E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.158055E-02 | lm-token-buffer-mib: 5.820361E+01 | global_load_balancing_loss: 1.000171E+00 | loss scale: 1.0 | grad norm: 0.105 | params norm: 11411.372 | params rms: 0.061789 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 5:04:56.112562 | finish at 2026-10-10 13:57:27 + [2026-10-07 08:53:55] iteration 3548/ 6855 | consumed samples: 454144 | elapsed time per iteration (ms): 84479.9 | throughput per GPU (TFLOP/s/GPU): 43.2 | MFU 13.85% | FLOPs estimator: workload-aware | logical tokens/s: 198594.0 | valid tokens/s: 198594.0 | physical tokens/s: 198912.8 | loss tokens/s: 145568.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.429765E-01 | lm-global-supervised-tokens: 1.229761E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.885900E-02 | lm-token-buffer-mib: 5.810433E+01 | global_load_balancing_loss: 1.000065E+00 | loss scale: 1.0 | grad norm: 0.117 | params norm: 11411.313 | params rms: 0.061789 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 5:36:15.166104 | finish at 2026-10-10 14:30:10 + [2026-10-07 08:55:25] iteration 3549/ 6855 | consumed samples: 454272 | elapsed time per iteration (ms): 89329.5 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.44% | FLOPs estimator: workload-aware | logical tokens/s: 187811.9 | valid tokens/s: 187811.9 | physical tokens/s: 188316.2 | loss tokens/s: 139834.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.782131E-01 | lm-global-supervised-tokens: 1.249137E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.818480E-02 | lm-token-buffer-mib: 5.816452E+01 | global_load_balancing_loss: 1.000526E+00 | loss scale: 1.0 | grad norm: 0.068 | params norm: 11411.255 | params rms: 0.061788 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 10:02:03.395442 | finish at 2026-10-10 18:57:28 + [2026-10-07 08:57:16] iteration 3550/ 6855 | consumed samples: 454400 | elapsed time per iteration (ms): 110970.9 | throughput per GPU (TFLOP/s/GPU): 34.1 | MFU 10.94% | FLOPs estimator: workload-aware | logical tokens/s: 151185.4 | valid tokens/s: 151185.4 | physical tokens/s: 151542.3 | loss tokens/s: 109326.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.499414E-01 | lm-global-supervised-tokens: 1.213204E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.558410E-02 | lm-token-buffer-mib: 5.812708E+01 | global_load_balancing_loss: 1.000348E+00 | loss scale: 1.0 | grad norm: 0.078 | params norm: 11411.195 | params rms: 0.061788 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 5:52:38.886944 | finish at 2026-10-11 14:49:54 + [2026-10-07 08:58:44] iteration 3551/ 6855 | consumed samples: 454528 | elapsed time per iteration (ms): 88812.1 | throughput per GPU (TFLOP/s/GPU): 41.5 | MFU 13.31% | FLOPs estimator: workload-aware | logical tokens/s: 188906.5 | valid tokens/s: 188906.5 | physical tokens/s: 189440.8 | loss tokens/s: 143068.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.682737E-01 | lm-global-supervised-tokens: 1.270621E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.324694E-02 | lm-token-buffer-mib: 5.815987E+01 | global_load_balancing_loss: 9.993237E-01 | loss scale: 1.0 | grad norm: 0.123 | params norm: 11411.135 | params rms: 0.061788 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 9:30:35.260893 | finish at 2026-10-10 18:29:20 + [2026-10-07 09:00:10] iteration 3552/ 6855 | consumed samples: 454656 | elapsed time per iteration (ms): 85530.9 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.67% | FLOPs estimator: workload-aware | logical tokens/s: 196153.7 | valid tokens/s: 196153.7 | physical tokens/s: 196694.3 | loss tokens/s: 144352.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.230565E-01 | lm-global-supervised-tokens: 1.234663E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.301114E-02 | lm-token-buffer-mib: 5.811511E+01 | global_load_balancing_loss: 1.000197E+00 | loss scale: 1.0 | grad norm: 0.065 | params norm: 11411.075 | params rms: 0.061787 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 6:28:28.493405 | finish at 2026-10-10 15:28:38 + [2026-10-07 09:01:37] iteration 3553/ 6855 | consumed samples: 454784 | elapsed time per iteration (ms): 87214.1 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.45% | FLOPs estimator: workload-aware | logical tokens/s: 192368.0 | valid tokens/s: 192368.0 | physical tokens/s: 192783.4 | loss tokens/s: 141183.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.327498E-01 | lm-global-supervised-tokens: 1.231315E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.241993E-02 | lm-token-buffer-mib: 5.805187E+01 | global_load_balancing_loss: 1.000976E+00 | loss scale: 1.0 | grad norm: 0.069 | params norm: 11411.014 | params rms: 0.061787 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 7:59:40.897985 | finish at 2026-10-10 17:01:18 + [2026-10-07 09:03:08] iteration 3554/ 6855 | consumed samples: 454912 | elapsed time per iteration (ms): 90647.8 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.66% | FLOPs estimator: workload-aware | logical tokens/s: 185080.9 | valid tokens/s: 185080.9 | physical tokens/s: 185463.5 | loss tokens/s: 133764.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.712224E-01 | lm-global-supervised-tokens: 1.212547E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.124777E-02 | lm-token-buffer-mib: 5.819936E+01 | global_load_balancing_loss: 1.000689E+00 | loss scale: 1.0 | grad norm: 0.049 | params norm: 11410.953 | params rms: 0.061787 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 11:07:08.293254 | finish at 2026-10-10 20:10:16 + [2026-10-07 09:04:34] iteration 3555/ 6855 | consumed samples: 455040 | elapsed time per iteration (ms): 86496.3 | throughput per GPU (TFLOP/s/GPU): 41.5 | MFU 13.29% | FLOPs estimator: workload-aware | logical tokens/s: 193964.1 | valid tokens/s: 193964.1 | physical tokens/s: 194369.6 | loss tokens/s: 149387.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.417091E-01 | lm-global-supervised-tokens: 1.292143E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.827807E-02 | lm-token-buffer-mib: 5.811735E+01 | global_load_balancing_loss: 1.000047E+00 | loss scale: 1.0 | grad norm: 0.054 | params norm: 11410.894 | params rms: 0.061786 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 7:17:17.808824 | finish at 2026-10-10 16:21:52 + [2026-10-07 09:05:58] iteration 3556/ 6855 | consumed samples: 455168 | elapsed time per iteration (ms): 83743.5 | throughput per GPU (TFLOP/s/GPU): 43.3 | MFU 13.89% | FLOPs estimator: workload-aware | logical tokens/s: 200339.2 | valid tokens/s: 200339.1 | physical tokens/s: 200842.3 | loss tokens/s: 150780.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.725413E-01 | lm-global-supervised-tokens: 1.262691E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.245017E-02 | lm-token-buffer-mib: 5.819641E+01 | global_load_balancing_loss: 1.000506E+00 | loss scale: 1.0 | grad norm: 0.069 | params norm: 11410.834 | params rms: 0.061786 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 4:44:29.719175 | finish at 2026-10-10 13:50:28 + [2026-10-07 09:07:25] iteration 3557/ 6855 | consumed samples: 455296 | elapsed time per iteration (ms): 87171.5 | throughput per GPU (TFLOP/s/GPU): 41.3 | MFU 13.24% | FLOPs estimator: workload-aware | logical tokens/s: 192462.0 | valid tokens/s: 192462.0 | physical tokens/s: 193015.1 | loss tokens/s: 143139.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.700391E-01 | lm-global-supervised-tokens: 1.247767E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.410269E-02 | lm-token-buffer-mib: 5.840788E+01 | global_load_balancing_loss: 1.000215E+00 | loss scale: 1.0 | grad norm: 0.072 | params norm: 11410.774 | params rms: 0.061786 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 7:51:31.466931 | finish at 2026-10-10 16:58:57 + [2026-10-07 09:08:54] iteration 3558/ 6855 | consumed samples: 455424 | elapsed time per iteration (ms): 88826.4 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.61% | FLOPs estimator: workload-aware | logical tokens/s: 188876.5 | valid tokens/s: 188876.5 | physical tokens/s: 189250.7 | loss tokens/s: 136539.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.106320E-01 | lm-global-supervised-tokens: 1.212833E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.844953E-02 | lm-token-buffer-mib: 5.803361E+01 | global_load_balancing_loss: 9.999701E-01 | loss scale: 1.0 | grad norm: 0.052 | params norm: 11410.712 | params rms: 0.061785 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 9:21:00.512808 | finish at 2026-10-10 18:29:55 + [2026-10-07 09:10:20] iteration 3559/ 6855 | consumed samples: 455552 | elapsed time per iteration (ms): 86313.4 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.67% | FLOPs estimator: workload-aware | logical tokens/s: 194375.2 | valid tokens/s: 194375.2 | physical tokens/s: 194687.1 | loss tokens/s: 141846.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.233261E-01 | lm-global-supervised-tokens: 1.224324E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.345804E-02 | lm-token-buffer-mib: 5.810084E+01 | global_load_balancing_loss: 1.000058E+00 | loss scale: 1.0 | grad norm: 0.060 | params norm: 11410.650 | params rms: 0.061785 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 7:01:28.858841 | finish at 2026-10-10 16:11:49 + [2026-10-07 09:12:12] iteration 3560/ 6855 | consumed samples: 455680 | elapsed time per iteration (ms): 111504.5 | throughput per GPU (TFLOP/s/GPU): 33.4 | MFU 10.71% | FLOPs estimator: workload-aware | logical tokens/s: 150462.2 | valid tokens/s: 150462.2 | physical tokens/s: 150935.5 | loss tokens/s: 108360.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.819516E-01 | lm-global-supervised-tokens: 1.208268E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.182543E-02 | lm-token-buffer-mib: 5.804327E+01 | global_load_balancing_loss: 1.000913E+00 | loss scale: 1.0 | grad norm: 0.045 | params norm: 11410.589 | params rms: 0.061785 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 6:03:27.294216 | finish at 2026-10-11 15:15:39 +times across ranks (ms): + save-checkpoint: + rank 0: 25761.05 + rank 1: 25761.03 + rank 2: 25761.08 + rank 3: 25761.07 + rank 4: 25761.04 + rank 5: 25761.03 + rank 6: 25761.03 + rank 7: 25761.04 + rank 8: 25762.45 + rank 9: 25762.45 + rank 10: 25762.44 + rank 11: 25762.44 + rank 12: 25762.49 + rank 13: 25762.43 + rank 14: 25762.42 + rank 15: 25762.43 + rank 16: 25761.57 + rank 17: 25761.55 + rank 18: 25761.58 + rank 19: 25761.57 + rank 20: 25761.54 + rank 21: 25761.57 + rank 22: 25761.54 + rank 23: 25761.54 + rank 24: 25761.64 + rank 25: 25761.53 + rank 26: 25761.61 + rank 27: 25761.60 + rank 28: 25761.77 + rank 29: 25761.84 + rank 30: 25761.54 + rank 31: 25761.73 + rank 32: 25760.79 + rank 33: 25760.80 + rank 34: 25761.00 + rank 35: 25760.95 + rank 36: 25760.89 + rank 37: 25760.84 + rank 38: 25760.70 + rank 39: 25760.88 + rank 40: 25761.04 + rank 41: 25760.98 + rank 42: 25761.01 + rank 43: 25761.05 + rank 44: 25760.99 + rank 45: 25760.98 + rank 46: 25761.02 + rank 47: 25761.00 + rank 48: 25760.84 + rank 49: 25761.03 + rank 50: 25760.93 + rank 51: 25760.89 + rank 52: 25760.97 + rank 53: 25760.87 + rank 54: 25760.98 + rank 55: 25760.97 + rank 56: 25761.19 + rank 57: 25761.16 + rank 58: 25761.16 + rank 59: 25761.12 + rank 60: 25761.12 + rank 61: 25761.17 + rank 62: 25761.14 + rank 63: 25761.16 + [2026-10-07 09:14:08] iteration 3561/ 6855 | consumed samples: 455808 | elapsed time per iteration (ms): 89921.4 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.53% | FLOPs estimator: workload-aware | logical tokens/s: 186575.8 | valid tokens/s: 186575.8 | physical tokens/s: 187028.6 | loss tokens/s: 135226.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.492028E-01 | lm-global-supervised-tokens: 1.215978E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.621550E+00 | lm-token-buffer-mib: 5.812719E+01 | global_load_balancing_loss: 9.991072E-01 | loss scale: 1.0 | grad norm: 0.045 | params norm: 11410.526 | params rms: 0.061784 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 10:16:41.054920 | finish at 2026-10-10 19:30:49 + [2026-10-07 09:15:43] iteration 3562/ 6855 | consumed samples: 455936 | elapsed time per iteration (ms): 95616.8 | throughput per GPU (TFLOP/s/GPU): 40.3 | MFU 12.93% | FLOPs estimator: workload-aware | logical tokens/s: 175462.8 | valid tokens/s: 175462.8 | physical tokens/s: 175670.1 | loss tokens/s: 126252.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.282886E-01 | lm-global-supervised-tokens: 1.207188E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.540827E-02 | lm-token-buffer-mib: 5.804437E+01 | global_load_balancing_loss: 9.990082E-01 | loss scale: 1.0 | grad norm: 0.036 | params norm: 11410.464 | params rms: 0.061784 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 15:27:46.000152 | finish at 2026-10-11 00:43:29 + [2026-10-07 09:17:12] iteration 3563/ 6855 | consumed samples: 456064 | elapsed time per iteration (ms): 88313.1 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.53% | FLOPs estimator: workload-aware | logical tokens/s: 189973.7 | valid tokens/s: 189973.7 | physical tokens/s: 190605.0 | loss tokens/s: 142956.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.722497E-01 | lm-global-supervised-tokens: 1.262494E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.123368E-02 | lm-token-buffer-mib: 5.815849E+01 | global_load_balancing_loss: 1.000301E+00 | loss scale: 1.0 | grad norm: 0.138 | params norm: 11410.402 | params rms: 0.061784 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 8:45:26.574832 | finish at 2026-10-10 18:02:38 + [2026-10-07 09:18:40] iteration 3564/ 6855 | consumed samples: 456192 | elapsed time per iteration (ms): 88502.8 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.55% | FLOPs estimator: workload-aware | logical tokens/s: 189566.7 | valid tokens/s: 189566.7 | physical tokens/s: 190255.0 | loss tokens/s: 140261.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.143390E-01 | lm-global-supervised-tokens: 1.241356E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.163187E-02 | lm-token-buffer-mib: 5.824319E+01 | global_load_balancing_loss: 9.998231E-01 | loss scale: 1.0 | grad norm: 0.169 | params norm: 11410.338 | params rms: 0.061783 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 8:54:22.874041 | finish at 2026-10-10 18:13:03 + [2026-10-07 09:20:09] iteration 3565/ 6855 | consumed samples: 456320 | elapsed time per iteration (ms): 89211.4 | throughput per GPU (TFLOP/s/GPU): 41.6 | MFU 13.33% | FLOPs estimator: workload-aware | logical tokens/s: 188060.8 | valid tokens/s: 188060.8 | physical tokens/s: 188447.8 | loss tokens/s: 139975.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.932506E-01 | lm-global-supervised-tokens: 1.248744E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.788041E-02 | lm-token-buffer-mib: 5.806511E+01 | global_load_balancing_loss: 1.001536E+00 | loss scale: 1.0 | grad norm: 0.187 | params norm: 11410.276 | params rms: 0.061783 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 9:31:45.604939 | finish at 2026-10-10 18:51:55 + [2026-10-07 09:21:31] iteration 3566/ 6855 | consumed samples: 456448 | elapsed time per iteration (ms): 81876.3 | throughput per GPU (TFLOP/s/GPU): 43.6 | MFU 13.98% | FLOPs estimator: workload-aware | logical tokens/s: 204909.3 | valid tokens/s: 204909.3 | physical tokens/s: 205487.8 | loss tokens/s: 158325.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.628295E-01 | lm-global-supervised-tokens: 1.296309E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.469342E-02 | lm-token-buffer-mib: 5.813849E+01 | global_load_balancing_loss: 1.000518E+00 | loss scale: 1.0 | grad norm: 0.159 | params norm: 11410.216 | params rms: 0.061783 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 2:48:11.075738 | finish at 2026-10-10 12:09:42 + [2026-10-07 09:22:58] iteration 3567/ 6855 | consumed samples: 456576 | elapsed time per iteration (ms): 86799.3 | throughput per GPU (TFLOP/s/GPU): 41.6 | MFU 13.32% | FLOPs estimator: workload-aware | logical tokens/s: 193287.1 | valid tokens/s: 193287.0 | physical tokens/s: 193725.7 | loss tokens/s: 148140.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.491662E-01 | lm-global-supervised-tokens: 1.285851E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.210135E-02 | lm-token-buffer-mib: 5.810761E+01 | global_load_balancing_loss: 1.000872E+00 | loss scale: 1.0 | grad norm: 0.037 | params norm: 11410.156 | params rms: 0.061782 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 7:16:36.014374 | finish at 2026-10-10 16:39:34 + [2026-10-07 09:24:21] iteration 3568/ 6855 | consumed samples: 456704 | elapsed time per iteration (ms): 83404.5 | throughput per GPU (TFLOP/s/GPU): 43.7 | MFU 13.99% | FLOPs estimator: workload-aware | logical tokens/s: 201154.1 | valid tokens/s: 201154.1 | physical tokens/s: 201605.7 | loss tokens/s: 147186.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.384189E-01 | lm-global-supervised-tokens: 1.227600E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.667449E-02 | lm-token-buffer-mib: 5.829150E+01 | global_load_balancing_loss: 1.000546E+00 | loss scale: 1.0 | grad norm: 0.078 | params norm: 11410.096 | params rms: 0.061782 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 4:09:10.464569 | finish at 2026-10-10 13:33:32 + [2026-10-07 09:25:50] iteration 3569/ 6855 | consumed samples: 456832 | elapsed time per iteration (ms): 88339.9 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.48% | FLOPs estimator: workload-aware | logical tokens/s: 189916.2 | valid tokens/s: 189916.2 | physical tokens/s: 190267.7 | loss tokens/s: 133638.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.847404E-01 | lm-global-supervised-tokens: 1.180566E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.448024E-02 | lm-token-buffer-mib: 5.811613E+01 | global_load_balancing_loss: 1.001274E+00 | loss scale: 1.0 | grad norm: 0.054 | params norm: 11410.036 | params rms: 0.061782 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 8:38:05.037590 | finish at 2026-10-10 18:03:55 + [2026-10-07 09:27:40] iteration 3570/ 6855 | consumed samples: 456960 | elapsed time per iteration (ms): 110118.4 | throughput per GPU (TFLOP/s/GPU): 32.9 | MFU 10.55% | FLOPs estimator: workload-aware | logical tokens/s: 152354.9 | valid tokens/s: 152354.9 | physical tokens/s: 152760.2 | loss tokens/s: 121100.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.631330E-01 | lm-global-supervised-tokens: 1.333536E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.169124E-02 | lm-token-buffer-mib: 5.823808E+01 | global_load_balancing_loss: 1.000275E+00 | loss scale: 1.0 | grad norm: 0.052 | params norm: 11409.975 | params rms: 0.061781 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 4:28:58.937269 | finish at 2026-10-11 13:56:39 + [2026-10-07 09:29:08] iteration 3571/ 6855 | consumed samples: 457088 | elapsed time per iteration (ms): 87707.2 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.47% | FLOPs estimator: workload-aware | logical tokens/s: 191285.6 | valid tokens/s: 191285.6 | physical tokens/s: 191829.0 | loss tokens/s: 138905.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.259016E-01 | lm-global-supervised-tokens: 1.218297E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.150638E-02 | lm-token-buffer-mib: 5.813644E+01 | global_load_balancing_loss: 1.000138E+00 | loss scale: 1.0 | grad norm: 0.124 | params norm: 11409.915 | params rms: 0.061781 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 8:00:30.427740 | finish at 2026-10-10 17:29:38 + [2026-10-07 09:30:33] iteration 3572/ 6855 | consumed samples: 457216 | elapsed time per iteration (ms): 85632.9 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.59% | FLOPs estimator: workload-aware | logical tokens/s: 195920.1 | valid tokens/s: 195920.1 | physical tokens/s: 196557.8 | loss tokens/s: 147347.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.673492E-01 | lm-global-supervised-tokens: 1.261782E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.196758E-02 | lm-token-buffer-mib: 5.833797E+01 | global_load_balancing_loss: 9.991555E-01 | loss scale: 1.0 | grad norm: 0.047 | params norm: 11409.855 | params rms: 0.061781 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 6:05:32.957852 | finish at 2026-10-10 15:36:06 + [2026-10-07 09:32:03] iteration 3573/ 6855 | consumed samples: 457344 | elapsed time per iteration (ms): 89809.8 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.40% | FLOPs estimator: workload-aware | logical tokens/s: 186807.7 | valid tokens/s: 186807.6 | physical tokens/s: 187181.3 | loss tokens/s: 136965.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.952518E-01 | lm-global-supervised-tokens: 1.230080E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.364456E-02 | lm-token-buffer-mib: 5.811037E+01 | global_load_balancing_loss: 9.993330E-01 | loss scale: 1.0 | grad norm: 0.076 | params norm: 11409.793 | params rms: 0.061780 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 9:52:35.850160 | finish at 2026-10-10 19:24:39 + [2026-10-07 09:33:29] iteration 3574/ 6855 | consumed samples: 457472 | elapsed time per iteration (ms): 85940.9 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.50% | FLOPs estimator: workload-aware | logical tokens/s: 195217.8 | valid tokens/s: 195217.8 | physical tokens/s: 195880.2 | loss tokens/s: 149476.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.817670E-01 | lm-global-supervised-tokens: 1.284613E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.372108E-02 | lm-token-buffer-mib: 5.830717E+01 | global_load_balancing_loss: 9.988775E-01 | loss scale: 1.0 | grad norm: 0.078 | params norm: 11409.733 | params rms: 0.061780 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 6:19:32.004010 | finish at 2026-10-10 15:53:01 + [2026-10-07 09:34:56] iteration 3575/ 6855 | consumed samples: 457600 | elapsed time per iteration (ms): 86579.0 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.52% | FLOPs estimator: workload-aware | logical tokens/s: 193779.2 | valid tokens/s: 193779.2 | physical tokens/s: 194299.7 | loss tokens/s: 144848.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.662048E-01 | lm-global-supervised-tokens: 1.254085E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.569052E-02 | lm-token-buffer-mib: 5.814807E+01 | global_load_balancing_loss: 9.998628E-01 | loss scale: 1.0 | grad norm: 0.154 | params norm: 11409.673 | params rms: 0.061780 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 6:52:59.115295 | finish at 2026-10-10 16:27:55 + [2026-10-07 09:36:22] iteration 3576/ 6855 | consumed samples: 457728 | elapsed time per iteration (ms): 86774.8 | throughput per GPU (TFLOP/s/GPU): 41.7 | MFU 13.35% | FLOPs estimator: workload-aware | logical tokens/s: 193341.9 | valid tokens/s: 193341.9 | physical tokens/s: 193977.2 | loss tokens/s: 146884.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.565294E-01 | lm-global-supervised-tokens: 1.274588E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.484388E-02 | lm-token-buffer-mib: 5.813330E+01 | global_load_balancing_loss: 1.001230E+00 | loss scale: 1.0 | grad norm: 0.109 | params norm: 11409.615 | params rms: 0.061779 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 7:02:14.585039 | finish at 2026-10-10 16:38:37 + [2026-10-07 09:37:49] iteration 3577/ 6855 | consumed samples: 457856 | elapsed time per iteration (ms): 86659.6 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.53% | FLOPs estimator: workload-aware | logical tokens/s: 193598.6 | valid tokens/s: 193598.6 | physical tokens/s: 194008.2 | loss tokens/s: 144394.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.484135E-01 | lm-global-supervised-tokens: 1.251314E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.144528E-02 | lm-token-buffer-mib: 5.804398E+01 | global_load_balancing_loss: 1.001787E+00 | loss scale: 1.0 | grad norm: 0.041 | params norm: 11409.555 | params rms: 0.061779 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 6:54:30.132132 | finish at 2026-10-10 16:32:19 + [2026-10-07 09:39:20] iteration 3578/ 6855 | consumed samples: 457984 | elapsed time per iteration (ms): 91150.3 | throughput per GPU (TFLOP/s/GPU): 41.1 | MFU 13.16% | FLOPs estimator: workload-aware | logical tokens/s: 184060.0 | valid tokens/s: 184060.0 | physical tokens/s: 184482.5 | loss tokens/s: 139655.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.749707E-01 | lm-global-supervised-tokens: 1.272967E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.162159E-02 | lm-token-buffer-mib: 5.811858E+01 | global_load_balancing_loss: 1.000125E+00 | loss scale: 1.0 | grad norm: 0.065 | params norm: 11409.493 | params rms: 0.061779 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 10:58:19.684757 | finish at 2026-10-10 20:37:40 + [2026-10-07 09:40:50] iteration 3579/ 6855 | consumed samples: 458112 | elapsed time per iteration (ms): 89946.1 | throughput per GPU (TFLOP/s/GPU): 40.4 | MFU 12.95% | FLOPs estimator: workload-aware | logical tokens/s: 186524.4 | valid tokens/s: 186524.4 | physical tokens/s: 186996.2 | loss tokens/s: 138609.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.195057E-01 | lm-global-supervised-tokens: 1.246737E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.231666E-02 | lm-token-buffer-mib: 5.817078E+01 | global_load_balancing_loss: 1.000039E+00 | loss scale: 1.0 | grad norm: 0.070 | params norm: 11409.431 | params rms: 0.061778 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 9:51:03.497008 | finish at 2026-10-10 19:31:54 + [2026-10-07 09:42:38] iteration 3580/ 6855 | consumed samples: 458240 | elapsed time per iteration (ms): 107669.1 | throughput per GPU (TFLOP/s/GPU): 35.4 | MFU 11.34% | FLOPs estimator: workload-aware | logical tokens/s: 155821.8 | valid tokens/s: 155821.8 | physical tokens/s: 156143.0 | loss tokens/s: 114452.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.092887E-01 | lm-global-supervised-tokens: 1.232296E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.707431E-02 | lm-token-buffer-mib: 5.813786E+01 | global_load_balancing_loss: 9.997244E-01 | loss scale: 1.0 | grad norm: 0.040 | params norm: 11409.370 | params rms: 0.061778 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 1:56:56.207391 | finish at 2026-10-11 11:39:34 + [2026-10-07 09:44:05] iteration 3581/ 6855 | consumed samples: 458368 | elapsed time per iteration (ms): 87173.3 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.62% | FLOPs estimator: workload-aware | logical tokens/s: 192458.1 | valid tokens/s: 192458.1 | physical tokens/s: 192960.0 | loss tokens/s: 146304.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.281486E-01 | lm-global-supervised-tokens: 1.275388E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.646471E-02 | lm-token-buffer-mib: 5.807709E+01 | global_load_balancing_loss: 1.000406E+00 | loss scale: 1.0 | grad norm: 0.073 | params norm: 11409.307 | params rms: 0.061778 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 7:16:45.446738 | finish at 2026-10-10 17:00:50 + [2026-10-07 09:45:30] iteration 3582/ 6855 | consumed samples: 458496 | elapsed time per iteration (ms): 84803.1 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.65% | FLOPs estimator: workload-aware | logical tokens/s: 197837.1 | valid tokens/s: 197837.1 | physical tokens/s: 198343.8 | loss tokens/s: 150519.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.133627E-01 | lm-global-supervised-tokens: 1.276449E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.288267E-02 | lm-token-buffer-mib: 5.808087E+01 | global_load_balancing_loss: 1.001438E+00 | loss scale: 1.0 | grad norm: 0.060 | params norm: 11409.246 | params rms: 0.061777 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 5:06:00.612986 | finish at 2026-10-10 14:51:30 + [2026-10-07 09:46:56] iteration 3583/ 6855 | consumed samples: 458624 | elapsed time per iteration (ms): 86406.7 | throughput per GPU (TFLOP/s/GPU): 43.2 | MFU 13.86% | FLOPs estimator: workload-aware | logical tokens/s: 194165.7 | valid tokens/s: 194165.7 | physical tokens/s: 194447.4 | loss tokens/s: 138657.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.736062E-01 | lm-global-supervised-tokens: 1.198094E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.212109E-02 | lm-token-buffer-mib: 5.803416E+01 | global_load_balancing_loss: 1.000367E+00 | loss scale: 1.0 | grad norm: 0.043 | params norm: 11409.185 | params rms: 0.061777 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 6:32:02.564478 | finish at 2026-10-10 16:18:59 + [2026-10-07 09:48:23] iteration 3584/ 6855 | consumed samples: 458752 | elapsed time per iteration (ms): 86358.3 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.56% | FLOPs estimator: workload-aware | logical tokens/s: 194273.5 | valid tokens/s: 194273.5 | physical tokens/s: 194795.4 | loss tokens/s: 146330.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.288607E-01 | lm-global-supervised-tokens: 1.263687E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.132319E-02 | lm-token-buffer-mib: 5.805197E+01 | global_load_balancing_loss: 1.000822E+00 | loss scale: 1.0 | grad norm: 0.060 | params norm: 11409.125 | params rms: 0.061777 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 6:27:58.110725 | finish at 2026-10-10 16:16:21 + [2026-10-07 09:49:51] iteration 3585/ 6855 | consumed samples: 458880 | elapsed time per iteration (ms): 88119.0 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.52% | FLOPs estimator: workload-aware | logical tokens/s: 190392.5 | valid tokens/s: 190392.5 | physical tokens/s: 190756.1 | loss tokens/s: 140192.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.603680E-01 | lm-global-supervised-tokens: 1.235360E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.278086E-02 | lm-token-buffer-mib: 5.808707E+01 | global_load_balancing_loss: 9.989266E-01 | loss scale: 1.0 | grad norm: 0.038 | params norm: 11409.065 | params rms: 0.061776 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 8:02:29.070611 | finish at 2026-10-10 17:52:20 + [2026-10-07 09:51:21] iteration 3586/ 6855 | consumed samples: 459008 | elapsed time per iteration (ms): 89916.1 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.41% | FLOPs estimator: workload-aware | logical tokens/s: 186587.1 | valid tokens/s: 186587.1 | physical tokens/s: 186916.5 | loss tokens/s: 131600.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.252342E-01 | lm-global-supervised-tokens: 1.183296E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.269690E-02 | lm-token-buffer-mib: 5.814059E+01 | global_load_balancing_loss: 9.991637E-01 | loss scale: 1.0 | grad norm: 0.075 | params norm: 11409.006 | params rms: 0.061776 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 9:38:55.658499 | finish at 2026-10-10 19:30:16 + [2026-10-07 09:52:48] iteration 3587/ 6855 | consumed samples: 459136 | elapsed time per iteration (ms): 87191.8 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.43% | FLOPs estimator: workload-aware | logical tokens/s: 192416.6 | valid tokens/s: 192416.6 | physical tokens/s: 192795.2 | loss tokens/s: 139961.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.146584E-01 | lm-global-supervised-tokens: 1.220348E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.537041E-02 | lm-token-buffer-mib: 5.812917E+01 | global_load_balancing_loss: 9.995633E-01 | loss scale: 1.0 | grad norm: 0.145 | params norm: 11408.948 | params rms: 0.061776 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 7:09:02.761489 | finish at 2026-10-10 17:01:51 + [2026-10-07 09:54:15] iteration 3588/ 6855 | consumed samples: 459264 | elapsed time per iteration (ms): 87075.7 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.40% | FLOPs estimator: workload-aware | logical tokens/s: 192673.8 | valid tokens/s: 192673.8 | physical tokens/s: 193034.2 | loss tokens/s: 146762.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.557342E-01 | lm-global-supervised-tokens: 1.277944E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.518489E-02 | lm-token-buffer-mib: 5.810586E+01 | global_load_balancing_loss: 9.994983E-01 | loss scale: 1.0 | grad norm: 0.053 | params norm: 11408.890 | params rms: 0.061775 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 7:01:16.379032 | finish at 2026-10-10 16:55:31 + [2026-10-07 09:55:43] iteration 3589/ 6855 | consumed samples: 459392 | elapsed time per iteration (ms): 87856.4 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.61% | FLOPs estimator: workload-aware | logical tokens/s: 190961.5 | valid tokens/s: 190961.5 | physical tokens/s: 191384.2 | loss tokens/s: 136750.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.269382E-01 | lm-global-supervised-tokens: 1.201440E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.352345E-02 | lm-token-buffer-mib: 5.824400E+01 | global_load_balancing_loss: 1.001267E+00 | loss scale: 1.0 | grad norm: 0.155 | params norm: 11408.832 | params rms: 0.061775 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 7:42:19.104449 | finish at 2026-10-10 17:38:02 + [2026-10-07 09:57:34] iteration 3590/ 6855 | consumed samples: 459520 | elapsed time per iteration (ms): 110975.3 | throughput per GPU (TFLOP/s/GPU): 33.6 | MFU 10.78% | FLOPs estimator: workload-aware | logical tokens/s: 151179.7 | valid tokens/s: 151179.7 | physical tokens/s: 151449.6 | loss tokens/s: 111574.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.527174E-01 | lm-global-supervised-tokens: 1.238197E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.156418E-02 | lm-token-buffer-mib: 5.812010E+01 | global_load_balancing_loss: 1.000723E+00 | loss scale: 1.0 | grad norm: 0.157 | params norm: 11408.773 | params rms: 0.061775 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 4:38:54.440368 | finish at 2026-10-11 14:36:28 + [2026-10-07 09:59:02] iteration 3591/ 6855 | consumed samples: 459648 | elapsed time per iteration (ms): 88268.0 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.66% | FLOPs estimator: workload-aware | logical tokens/s: 190070.8 | valid tokens/s: 190070.8 | physical tokens/s: 190514.1 | loss tokens/s: 135158.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.977482E-01 | lm-global-supervised-tokens: 1.193013E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.310705E-02 | lm-token-buffer-mib: 5.844201E+01 | global_load_balancing_loss: 1.000187E+00 | loss scale: 1.0 | grad norm: 0.071 | params norm: 11408.712 | params rms: 0.061774 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 8:01:46.809219 | finish at 2026-10-10 18:00:49 + [2026-10-07 10:00:30] iteration 3592/ 6855 | consumed samples: 459776 | elapsed time per iteration (ms): 88324.9 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.53% | FLOPs estimator: workload-aware | logical tokens/s: 189948.3 | valid tokens/s: 189948.3 | physical tokens/s: 190369.2 | loss tokens/s: 133643.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.444100E-01 | lm-global-supervised-tokens: 1.180403E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.139925E-02 | lm-token-buffer-mib: 5.820913E+01 | global_load_balancing_loss: 1.000993E+00 | loss scale: 1.0 | grad norm: 0.187 | params norm: 11408.651 | params rms: 0.061774 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 8:03:24.085398 | finish at 2026-10-10 18:03:54 + [2026-10-07 10:01:54] iteration 3593/ 6855 | consumed samples: 459904 | elapsed time per iteration (ms): 83267.1 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.49% | FLOPs estimator: workload-aware | logical tokens/s: 201485.7 | valid tokens/s: 201485.6 | physical tokens/s: 202212.1 | loss tokens/s: 154872.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.636479E-01 | lm-global-supervised-tokens: 1.289579E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.436844E-02 | lm-token-buffer-mib: 5.815061E+01 | global_load_balancing_loss: 1.000557E+00 | loss scale: 1.0 | grad norm: 0.084 | params norm: 11408.592 | params rms: 0.061774 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 3:26:57.233071 | finish at 2026-10-10 13:28:51 + [2026-10-07 10:03:21] iteration 3594/ 6855 | consumed samples: 460032 | elapsed time per iteration (ms): 87435.2 | throughput per GPU (TFLOP/s/GPU): 43.5 | MFU 13.93% | FLOPs estimator: workload-aware | logical tokens/s: 191881.7 | valid tokens/s: 191881.7 | physical tokens/s: 192152.2 | loss tokens/s: 140847.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.010743E-01 | lm-global-supervised-tokens: 1.231500E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.339309E-02 | lm-token-buffer-mib: 5.808660E+01 | global_load_balancing_loss: 1.000309E+00 | loss scale: 1.0 | grad norm: 0.065 | params norm: 11408.534 | params rms: 0.061774 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 7:12:06.080606 | finish at 2026-10-10 17:15:27 + [2026-10-07 10:04:48] iteration 3595/ 6855 | consumed samples: 460160 | elapsed time per iteration (ms): 87155.5 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.68% | FLOPs estimator: workload-aware | logical tokens/s: 192497.4 | valid tokens/s: 192497.4 | physical tokens/s: 193104.9 | loss tokens/s: 139326.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.757396E-01 | lm-global-supervised-tokens: 1.214310E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.801313E-02 | lm-token-buffer-mib: 5.803732E+01 | global_load_balancing_loss: 1.000223E+00 | loss scale: 1.0 | grad norm: 0.051 | params norm: 11408.476 | params rms: 0.061773 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 6:55:27.083683 | finish at 2026-10-10 17:00:15 + [2026-10-07 10:06:15] iteration 3596/ 6855 | consumed samples: 460288 | elapsed time per iteration (ms): 86793.2 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.39% | FLOPs estimator: workload-aware | logical tokens/s: 193301.0 | valid tokens/s: 193301.0 | physical tokens/s: 194273.7 | loss tokens/s: 145594.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.830728E-01 | lm-global-supervised-tokens: 1.263663E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.439650E-02 | lm-token-buffer-mib: 5.810270E+01 | global_load_balancing_loss: 9.999306E-01 | loss scale: 1.0 | grad norm: 0.095 | params norm: 11408.417 | params rms: 0.061773 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 6:34:18.936287 | finish at 2026-10-10 16:40:34 + [2026-10-07 10:07:44] iteration 3597/ 6855 | consumed samples: 460416 | elapsed time per iteration (ms): 88973.7 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.68% | FLOPs estimator: workload-aware | logical tokens/s: 188563.5 | valid tokens/s: 188563.5 | physical tokens/s: 188943.9 | loss tokens/s: 142320.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.989741E-01 | lm-global-supervised-tokens: 1.266281E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.167542E-02 | lm-token-buffer-mib: 5.802904E+01 | global_load_balancing_loss: 9.987556E-01 | loss scale: 1.0 | grad norm: 0.081 | params norm: 11408.359 | params rms: 0.061773 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 8:31:16.302323 | finish at 2026-10-10 18:39:00 + [2026-10-07 10:09:11] iteration 3598/ 6855 | consumed samples: 460544 | elapsed time per iteration (ms): 86857.6 | throughput per GPU (TFLOP/s/GPU): 43.2 | MFU 13.83% | FLOPs estimator: workload-aware | logical tokens/s: 193157.3 | valid tokens/s: 193157.3 | physical tokens/s: 193654.4 | loss tokens/s: 151161.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.936141E-01 | lm-global-supervised-tokens: 1.312954E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.130084E-02 | lm-token-buffer-mib: 5.810720E+01 | global_load_balancing_loss: 1.000480E+00 | loss scale: 1.0 | grad norm: 0.154 | params norm: 11408.300 | params rms: 0.061772 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 6:34:55.337454 | finish at 2026-10-10 16:44:06 + [2026-10-07 10:10:41] iteration 3599/ 6855 | consumed samples: 460672 | elapsed time per iteration (ms): 90433.4 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.46% | FLOPs estimator: workload-aware | logical tokens/s: 185519.9 | valid tokens/s: 185519.9 | physical tokens/s: 185827.2 | loss tokens/s: 131394.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.598994E-01 | lm-global-supervised-tokens: 1.188244E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.222204E-02 | lm-token-buffer-mib: 5.803864E+01 | global_load_balancing_loss: 1.001327E+00 | loss scale: 1.0 | grad norm: 0.137 | params norm: 11408.241 | params rms: 0.061772 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 9:47:31.008064 | finish at 2026-10-10 19:58:12 + [2026-10-07 10:12:32] iteration 3600/ 6855 | consumed samples: 460800 | elapsed time per iteration (ms): 110489.9 | throughput per GPU (TFLOP/s/GPU): 33.8 | MFU 10.82% | FLOPs estimator: workload-aware | logical tokens/s: 151843.6 | valid tokens/s: 151843.6 | physical tokens/s: 152131.5 | loss tokens/s: 111844.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.093789E-01 | lm-global-supervised-tokens: 1.235766E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.203430E-02 | lm-token-buffer-mib: 5.804140E+01 | global_load_balancing_loss: 1.000200E+00 | loss scale: 1.0 | grad norm: 0.106 | params norm: 11408.183 | params rms: 0.061772 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 3:54:04.544932 | finish at 2026-10-11 14:06:36 +times across ranks (ms): + save-checkpoint: + rank 0: 27874.55 + rank 1: 27874.54 + rank 2: 27874.58 + rank 3: 27874.58 + rank 4: 27874.63 + rank 5: 27874.54 + rank 6: 27874.59 + rank 7: 27874.58 + rank 8: 27874.69 + rank 9: 27874.66 + rank 10: 27874.66 + rank 11: 27874.66 + rank 12: 27874.68 + rank 13: 27874.67 + rank 14: 27874.68 + rank 15: 27874.66 + rank 16: 27874.64 + rank 17: 27874.66 + rank 18: 27874.55 + rank 19: 27874.62 + rank 20: 27874.68 + rank 21: 27874.58 + rank 22: 27874.55 + rank 23: 27874.55 + rank 24: 27878.21 + rank 25: 27878.79 + rank 26: 27877.67 + rank 27: 27878.26 + rank 28: 27878.14 + rank 29: 27878.15 + rank 30: 27878.13 + rank 31: 27878.18 + rank 32: 27875.10 + rank 33: 27875.05 + rank 34: 27875.09 + rank 35: 27875.02 + rank 36: 27875.05 + rank 37: 27874.99 + rank 38: 27875.06 + rank 39: 27875.00 + rank 40: 27874.33 + rank 41: 27874.25 + rank 42: 27874.29 + rank 43: 27874.29 + rank 44: 27874.26 + rank 45: 27874.25 + rank 46: 27874.26 + rank 47: 27874.26 + rank 48: 27874.94 + rank 49: 27874.91 + rank 50: 27874.89 + rank 51: 27874.90 + rank 52: 27874.92 + rank 53: 27874.86 + rank 54: 27874.91 + rank 55: 27874.88 + rank 56: 27875.27 + rank 57: 27875.25 + rank 58: 27875.19 + rank 59: 27875.21 + rank 60: 27875.20 + rank 61: 27875.27 + rank 62: 27875.29 + rank 63: 27875.24 + [2026-10-07 10:14:43] iteration 3601/ 6855 | consumed samples: 460928 | elapsed time per iteration (ms): 102766.3 | throughput per GPU (TFLOP/s/GPU): 36.4 | MFU 11.65% | FLOPs estimator: workload-aware | logical tokens/s: 163255.8 | valid tokens/s: 163255.8 | physical tokens/s: 163587.8 | loss tokens/s: 119016.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.620237E-01 | lm-global-supervised-tokens: 1.223091E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 5.098855E-01 | lm-token-buffer-mib: 5.818884E+01 | global_load_balancing_loss: 9.997866E-01 | loss scale: 1.0 | grad norm: 0.120 | params norm: 11408.125 | params rms: 0.061771 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 20:53:21.518357 | finish at 2026-10-11 07:08:04 + [2026-10-07 10:16:17] iteration 3602/ 6855 | consumed samples: 461056 | elapsed time per iteration (ms): 93619.9 | throughput per GPU (TFLOP/s/GPU): 39.3 | MFU 12.59% | FLOPs estimator: workload-aware | logical tokens/s: 179205.1 | valid tokens/s: 179205.1 | physical tokens/s: 179580.2 | loss tokens/s: 136612.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.761540E-01 | lm-global-supervised-tokens: 1.278961E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.929111E-02 | lm-token-buffer-mib: 5.824717E+01 | global_load_balancing_loss: 1.000395E+00 | loss scale: 1.0 | grad norm: 0.061 | params norm: 11408.068 | params rms: 0.061771 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 12:35:45.630833 | finish at 2026-10-10 22:52:02 + [2026-10-07 10:17:41] iteration 3603/ 6855 | consumed samples: 461184 | elapsed time per iteration (ms): 84951.2 | throughput per GPU (TFLOP/s/GPU): 43.5 | MFU 13.94% | FLOPs estimator: workload-aware | logical tokens/s: 197491.9 | valid tokens/s: 197491.9 | physical tokens/s: 198015.0 | loss tokens/s: 143726.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.447309E-01 | lm-global-supervised-tokens: 1.220972E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.152309E-02 | lm-token-buffer-mib: 5.810945E+01 | global_load_balancing_loss: 9.999008E-01 | loss scale: 1.0 | grad norm: 0.058 | params norm: 11408.013 | params rms: 0.061771 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 4:44:21.309405 | finish at 2026-10-10 15:02:03 + [2026-10-07 10:19:08] iteration 3604/ 6855 | consumed samples: 461312 | elapsed time per iteration (ms): 86845.4 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.61% | FLOPs estimator: workload-aware | logical tokens/s: 193183.6 | valid tokens/s: 193183.6 | physical tokens/s: 193768.0 | loss tokens/s: 144578.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.385481E-01 | lm-global-supervised-tokens: 1.255593E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.457483E-02 | lm-token-buffer-mib: 5.820854E+01 | global_load_balancing_loss: 1.000358E+00 | loss scale: 1.0 | grad norm: 0.136 | params norm: 11407.958 | params rms: 0.061770 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 6:25:34.305022 | finish at 2026-10-10 16:44:43 + [2026-10-07 10:20:38] iteration 3605/ 6855 | consumed samples: 461440 | elapsed time per iteration (ms): 90042.4 | throughput per GPU (TFLOP/s/GPU): 41.4 | MFU 13.25% | FLOPs estimator: workload-aware | logical tokens/s: 186324.8 | valid tokens/s: 186324.8 | physical tokens/s: 186829.3 | loss tokens/s: 139680.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.783649E-01 | lm-global-supervised-tokens: 1.257714E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.337253E-02 | lm-token-buffer-mib: 5.821115E+01 | global_load_balancing_loss: 1.000160E+00 | loss scale: 1.0 | grad norm: 0.140 | params norm: 11407.902 | params rms: 0.061770 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 9:17:17.765527 | finish at 2026-10-10 19:37:56 + [2026-10-07 10:22:08] iteration 3606/ 6855 | consumed samples: 461568 | elapsed time per iteration (ms): 89943.5 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.46% | FLOPs estimator: workload-aware | logical tokens/s: 186530.5 | valid tokens/s: 186530.5 | physical tokens/s: 187001.7 | loss tokens/s: 135502.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.984547E-01 | lm-global-supervised-tokens: 1.218759E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.102106E-02 | lm-token-buffer-mib: 5.831587E+01 | global_load_balancing_loss: 1.001303E+00 | loss scale: 1.0 | grad norm: 0.074 | params norm: 11407.845 | params rms: 0.061770 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 9:10:26.315443 | finish at 2026-10-10 19:32:35 + [2026-10-07 10:23:34] iteration 3607/ 6855 | consumed samples: 461696 | elapsed time per iteration (ms): 85590.6 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.67% | FLOPs estimator: workload-aware | logical tokens/s: 196016.6 | valid tokens/s: 196016.6 | physical tokens/s: 196460.8 | loss tokens/s: 149193.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.433597E-01 | lm-global-supervised-tokens: 1.276958E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.458297E-02 | lm-token-buffer-mib: 5.817295E+01 | global_load_balancing_loss: 1.000934E+00 | loss scale: 1.0 | grad norm: 0.183 | params norm: 11407.786 | params rms: 0.061769 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 5:13:18.301369 | finish at 2026-10-10 15:36:52 + [2026-10-07 10:25:02] iteration 3608/ 6855 | consumed samples: 461824 | elapsed time per iteration (ms): 88076.3 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.68% | FLOPs estimator: workload-aware | logical tokens/s: 190484.0 | valid tokens/s: 190484.0 | physical tokens/s: 190952.7 | loss tokens/s: 148792.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.769898E-01 | lm-global-supervised-tokens: 1.310506E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.379682E-02 | lm-token-buffer-mib: 5.804013E+01 | global_load_balancing_loss: 9.986958E-01 | loss scale: 1.0 | grad norm: 0.055 | params norm: 11407.729 | params rms: 0.061769 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 7:26:23.695475 | finish at 2026-10-10 17:51:26 + [2026-10-07 10:26:28] iteration 3609/ 6855 | consumed samples: 461952 | elapsed time per iteration (ms): 86033.6 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.55% | FLOPs estimator: workload-aware | logical tokens/s: 195007.2 | valid tokens/s: 195007.2 | physical tokens/s: 195498.0 | loss tokens/s: 148885.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.817038E-01 | lm-global-supervised-tokens: 1.280915E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.559710E-02 | lm-token-buffer-mib: 5.812276E+01 | global_load_balancing_loss: 9.993206E-01 | loss scale: 1.0 | grad norm: 0.135 | params norm: 11407.671 | params rms: 0.061769 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 5:34:25.207523 | finish at 2026-10-10 16:00:53 + [2026-10-07 10:28:19] iteration 3610/ 6855 | consumed samples: 462080 | elapsed time per iteration (ms): 111034.8 | throughput per GPU (TFLOP/s/GPU): 34.7 | MFU 11.11% | FLOPs estimator: workload-aware | logical tokens/s: 151098.5 | valid tokens/s: 151098.5 | physical tokens/s: 151458.8 | loss tokens/s: 108570.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.682280E-01 | lm-global-supervised-tokens: 1.205508E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.232850E-02 | lm-token-buffer-mib: 5.820782E+01 | global_load_balancing_loss: 1.000231E+00 | loss scale: 1.0 | grad norm: 0.046 | params norm: 11407.613 | params rms: 0.061769 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 4:05:08.055373 | finish at 2026-10-11 14:33:27 + [2026-10-07 10:29:44] iteration 3611/ 6855 | consumed samples: 462208 | elapsed time per iteration (ms): 85443.7 | throughput per GPU (TFLOP/s/GPU): 42.8 | MFU 13.71% | FLOPs estimator: workload-aware | logical tokens/s: 196353.6 | valid tokens/s: 196353.6 | physical tokens/s: 196866.8 | loss tokens/s: 141441.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.943717E-01 | lm-global-supervised-tokens: 1.208525E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.206436E-02 | lm-token-buffer-mib: 5.810954E+01 | global_load_balancing_loss: 1.000566E+00 | loss scale: 1.0 | grad norm: 0.090 | params norm: 11407.554 | params rms: 0.061768 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 4:59:39.302716 | finish at 2026-10-10 15:29:24 + [2026-10-07 10:31:10] iteration 3612/ 6855 | consumed samples: 462336 | elapsed time per iteration (ms): 85600.0 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.46% | FLOPs estimator: workload-aware | logical tokens/s: 195994.2 | valid tokens/s: 195994.2 | physical tokens/s: 196933.3 | loss tokens/s: 145470.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.309864E-01 | lm-global-supervised-tokens: 1.245224E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.829681E-02 | lm-token-buffer-mib: 5.882694E+01 | global_load_balancing_loss: 1.000996E+00 | loss scale: 1.0 | grad norm: 0.107 | params norm: 11407.493 | params rms: 0.061768 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 5:06:40.780361 | finish at 2026-10-10 15:37:51 + [2026-10-07 10:32:36] iteration 3613/ 6855 | consumed samples: 462464 | elapsed time per iteration (ms): 86245.6 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.67% | FLOPs estimator: workload-aware | logical tokens/s: 194528.2 | valid tokens/s: 194528.2 | physical tokens/s: 194839.7 | loss tokens/s: 145781.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.308631E-01 | lm-global-supervised-tokens: 1.257304E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.246772E-02 | lm-token-buffer-mib: 5.812907E+01 | global_load_balancing_loss: 1.000153E+00 | loss scale: 1.0 | grad norm: 0.037 | params norm: 11407.433 | params rms: 0.061768 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 5:40:08.166359 | finish at 2026-10-10 16:12:45 + [2026-10-07 10:34:05] iteration 3614/ 6855 | consumed samples: 462592 | elapsed time per iteration (ms): 89135.1 | throughput per GPU (TFLOP/s/GPU): 41.4 | MFU 13.28% | FLOPs estimator: workload-aware | logical tokens/s: 188222.0 | valid tokens/s: 188222.0 | physical tokens/s: 188716.6 | loss tokens/s: 135714.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.436623E-01 | lm-global-supervised-tokens: 1.209692E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.606641E-02 | lm-token-buffer-mib: 5.822584E+01 | global_load_balancing_loss: 9.997298E-01 | loss scale: 1.0 | grad norm: 0.164 | params norm: 11407.374 | params rms: 0.061767 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 8:14:46.763693 | finish at 2026-10-10 18:48:52 + [2026-10-07 10:35:34] iteration 3615/ 6855 | consumed samples: 462720 | elapsed time per iteration (ms): 88361.0 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.67% | FLOPs estimator: workload-aware | logical tokens/s: 189870.5 | valid tokens/s: 189870.5 | physical tokens/s: 190235.3 | loss tokens/s: 141169.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.748658E-01 | lm-global-supervised-tokens: 1.247384E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.536071E-02 | lm-token-buffer-mib: 5.815153E+01 | global_load_balancing_loss: 9.997044E-01 | loss scale: 1.0 | grad norm: 0.030 | params norm: 11407.313 | params rms: 0.061767 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 7:31:29.789286 | finish at 2026-10-10 18:07:04 + [2026-10-07 10:36:58] iteration 3616/ 6855 | consumed samples: 462848 | elapsed time per iteration (ms): 83783.4 | throughput per GPU (TFLOP/s/GPU): 43.6 | MFU 13.97% | FLOPs estimator: workload-aware | logical tokens/s: 200245.1 | valid tokens/s: 200245.1 | physical tokens/s: 200801.2 | loss tokens/s: 148822.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.904287E-01 | lm-global-supervised-tokens: 1.246882E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.140166E-02 | lm-token-buffer-mib: 5.824113E+01 | global_load_balancing_loss: 1.000319E+00 | loss scale: 1.0 | grad norm: 0.132 | params norm: 11407.252 | params rms: 0.061767 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 3:22:54.461363 | finish at 2026-10-10 13:59:52 + [2026-10-07 10:38:24] iteration 3617/ 6855 | consumed samples: 462976 | elapsed time per iteration (ms): 86626.7 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.38% | FLOPs estimator: workload-aware | logical tokens/s: 193672.1 | valid tokens/s: 193672.1 | physical tokens/s: 194255.0 | loss tokens/s: 149887.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.289499E-01 | lm-global-supervised-tokens: 1.298429E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.654204E-02 | lm-token-buffer-mib: 5.818101E+01 | global_load_balancing_loss: 1.000427E+00 | loss scale: 1.0 | grad norm: 0.109 | params norm: 11407.193 | params rms: 0.061766 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 5:54:57.394859 | finish at 2026-10-10 16:33:22 + [2026-10-07 10:39:52] iteration 3618/ 6855 | consumed samples: 463104 | elapsed time per iteration (ms): 87275.1 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.49% | FLOPs estimator: workload-aware | logical tokens/s: 192233.7 | valid tokens/s: 192233.7 | physical tokens/s: 192797.3 | loss tokens/s: 140348.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.559254E-01 | lm-global-supervised-tokens: 1.224893E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.175771E-02 | lm-token-buffer-mib: 5.827824E+01 | global_load_balancing_loss: 1.000673E+00 | loss scale: 1.0 | grad norm: 0.108 | params norm: 11407.135 | params rms: 0.061766 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 6:28:29.413783 | finish at 2026-10-10 17:08:21 + [2026-10-07 10:41:16] iteration 3619/ 6855 | consumed samples: 463232 | elapsed time per iteration (ms): 84774.7 | throughput per GPU (TFLOP/s/GPU): 43.4 | MFU 13.91% | FLOPs estimator: workload-aware | logical tokens/s: 197903.6 | valid tokens/s: 197903.6 | physical tokens/s: 198317.1 | loss tokens/s: 145080.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.124904E-01 | lm-global-supervised-tokens: 1.229917E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.485854E-02 | lm-token-buffer-mib: 5.804062E+01 | global_load_balancing_loss: 1.000120E+00 | loss scale: 1.0 | grad norm: 0.156 | params norm: 11407.077 | params rms: 0.061766 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 4:12:10.861839 | finish at 2026-10-10 14:53:27 + [2026-10-07 10:43:04] iteration 3620/ 6855 | consumed samples: 463360 | elapsed time per iteration (ms): 107706.7 | throughput per GPU (TFLOP/s/GPU): 34.7 | MFU 11.13% | FLOPs estimator: workload-aware | logical tokens/s: 155767.4 | valid tokens/s: 155767.4 | physical tokens/s: 156182.1 | loss tokens/s: 111626.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.407355E-01 | lm-global-supervised-tokens: 1.202294E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.293326E-02 | lm-token-buffer-mib: 5.812585E+01 | global_load_balancing_loss: 1.000680E+00 | loss scale: 1.0 | grad norm: 0.092 | params norm: 11407.017 | params rms: 0.061765 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 0:47:11.046747 | finish at 2026-10-11 11:30:15 + [2026-10-07 10:44:28] iteration 3621/ 6855 | consumed samples: 463488 | elapsed time per iteration (ms): 84400.7 | throughput per GPU (TFLOP/s/GPU): 43.6 | MFU 13.97% | FLOPs estimator: workload-aware | logical tokens/s: 198780.4 | valid tokens/s: 198780.4 | physical tokens/s: 199085.3 | loss tokens/s: 147597.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.056628E-01 | lm-global-supervised-tokens: 1.245729E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.094645E-02 | lm-token-buffer-mib: 5.807496E+01 | global_load_balancing_loss: 1.000481E+00 | loss scale: 1.0 | grad norm: 0.099 | params norm: 11406.955 | params rms: 0.061765 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 3:49:11.970503 | finish at 2026-10-10 14:33:40 + [2026-10-07 10:45:57] iteration 3622/ 6855 | consumed samples: 463616 | elapsed time per iteration (ms): 88189.5 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.51% | FLOPs estimator: workload-aware | logical tokens/s: 190240.0 | valid tokens/s: 190240.0 | physical tokens/s: 190630.9 | loss tokens/s: 138520.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.128978E-01 | lm-global-supervised-tokens: 1.221601E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.137984E-02 | lm-token-buffer-mib: 5.806062E+01 | global_load_balancing_loss: 1.000191E+00 | loss scale: 1.0 | grad norm: 0.037 | params norm: 11406.895 | params rms: 0.061765 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 7:11:56.512744 | finish at 2026-10-10 17:57:53 + [2026-10-07 10:47:23] iteration 3623/ 6855 | consumed samples: 463744 | elapsed time per iteration (ms): 86813.8 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.63% | FLOPs estimator: workload-aware | logical tokens/s: 193254.7 | valid tokens/s: 193254.7 | physical tokens/s: 193974.0 | loss tokens/s: 145108.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.894152E-01 | lm-global-supervised-tokens: 1.259737E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.110363E-02 | lm-token-buffer-mib: 5.814691E+01 | global_load_balancing_loss: 9.993749E-01 | loss scale: 1.0 | grad norm: 0.093 | params norm: 11406.834 | params rms: 0.061764 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 5:56:22.112526 | finish at 2026-10-10 16:43:46 + [2026-10-07 10:48:50] iteration 3624/ 6855 | consumed samples: 463872 | elapsed time per iteration (ms): 87068.2 | throughput per GPU (TFLOP/s/GPU): 43.2 | MFU 13.85% | FLOPs estimator: workload-aware | logical tokens/s: 192690.6 | valid tokens/s: 192690.6 | physical tokens/s: 193151.5 | loss tokens/s: 142532.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.847524E-01 | lm-global-supervised-tokens: 1.241004E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.149498E-02 | lm-token-buffer-mib: 5.824754E+01 | global_load_balancing_loss: 1.000179E+00 | loss scale: 1.0 | grad norm: 0.038 | params norm: 11406.771 | params rms: 0.061764 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 6:08:37.236699 | finish at 2026-10-10 16:57:28 + [2026-10-07 10:50:16] iteration 3625/ 6855 | consumed samples: 464000 | elapsed time per iteration (ms): 85666.8 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.51% | FLOPs estimator: workload-aware | logical tokens/s: 195842.3 | valid tokens/s: 195842.3 | physical tokens/s: 196234.2 | loss tokens/s: 149333.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.385728E-01 | lm-global-supervised-tokens: 1.279294E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.356027E-02 | lm-token-buffer-mib: 5.804063E+01 | global_load_balancing_loss: 9.994305E-01 | loss scale: 1.0 | grad norm: 0.085 | params norm: 11406.706 | params rms: 0.061764 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 4:51:43.737888 | finish at 2026-10-10 15:42:00 + [2026-10-07 10:51:41] iteration 3626/ 6855 | consumed samples: 464128 | elapsed time per iteration (ms): 84973.4 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.69% | FLOPs estimator: workload-aware | logical tokens/s: 197440.4 | valid tokens/s: 197440.4 | physical tokens/s: 197869.7 | loss tokens/s: 143752.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.268710E-01 | lm-global-supervised-tokens: 1.221517E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.418427E-02 | lm-token-buffer-mib: 5.815339E+01 | global_load_balancing_loss: 9.999794E-01 | loss scale: 1.0 | grad norm: 0.035 | params norm: 11406.643 | params rms: 0.061763 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 4:12:59.214444 | finish at 2026-10-10 15:04:40 + [2026-10-07 10:53:07] iteration 3627/ 6855 | consumed samples: 464256 | elapsed time per iteration (ms): 86184.5 | throughput per GPU (TFLOP/s/GPU): 43.1 | MFU 13.83% | FLOPs estimator: workload-aware | logical tokens/s: 194666.3 | valid tokens/s: 194666.3 | physical tokens/s: 195124.8 | loss tokens/s: 147841.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.378424E-01 | lm-global-supervised-tokens: 1.274168E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.473234E-02 | lm-token-buffer-mib: 5.810125E+01 | global_load_balancing_loss: 9.995613E-01 | loss scale: 1.0 | grad norm: 0.062 | params norm: 11406.580 | params rms: 0.061763 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 5:16:43.560545 | finish at 2026-10-10 16:09:51 + [2026-10-07 10:54:35] iteration 3628/ 6855 | consumed samples: 464384 | elapsed time per iteration (ms): 87583.7 | throughput per GPU (TFLOP/s/GPU): 41.1 | MFU 13.18% | FLOPs estimator: workload-aware | logical tokens/s: 191555.6 | valid tokens/s: 191555.6 | physical tokens/s: 192007.4 | loss tokens/s: 140565.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.458030E-01 | lm-global-supervised-tokens: 1.231126E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.096552E-02 | lm-token-buffer-mib: 5.807560E+01 | global_load_balancing_loss: 1.001422E+00 | loss scale: 1.0 | grad norm: 0.055 | params norm: 11406.514 | params rms: 0.061763 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 6:30:32.679727 | finish at 2026-10-10 17:25:08 + [2026-10-07 10:56:00] iteration 3629/ 6855 | consumed samples: 464512 | elapsed time per iteration (ms): 85587.4 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.66% | FLOPs estimator: workload-aware | logical tokens/s: 196023.9 | valid tokens/s: 196023.9 | physical tokens/s: 196557.2 | loss tokens/s: 151214.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.187988E-01 | lm-global-supervised-tokens: 1.294208E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.320480E-02 | lm-token-buffer-mib: 5.809768E+01 | global_load_balancing_loss: 1.000818E+00 | loss scale: 1.0 | grad norm: 0.074 | params norm: 11406.448 | params rms: 0.061762 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 4:41:44.921504 | finish at 2026-10-10 15:37:45 + [2026-10-07 10:57:47] iteration 3630/ 6855 | consumed samples: 464640 | elapsed time per iteration (ms): 106256.3 | throughput per GPU (TFLOP/s/GPU): 34.3 | MFU 11.01% | FLOPs estimator: workload-aware | logical tokens/s: 157893.2 | valid tokens/s: 157893.2 | physical tokens/s: 158259.7 | loss tokens/s: 120496.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.943306E-01 | lm-global-supervised-tokens: 1.280354E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.265156E-02 | lm-token-buffer-mib: 5.810575E+01 | global_load_balancing_loss: 1.000811E+00 | loss scale: 1.0 | grad norm: 0.115 | params norm: 11406.379 | params rms: 0.061762 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 23:11:16.625079 | finish at 2026-10-11 10:09:03 + [2026-10-07 10:59:14] iteration 3631/ 6855 | consumed samples: 464768 | elapsed time per iteration (ms): 86988.8 | throughput per GPU (TFLOP/s/GPU): 41.5 | MFU 13.30% | FLOPs estimator: workload-aware | logical tokens/s: 192866.2 | valid tokens/s: 192866.2 | physical tokens/s: 193321.8 | loss tokens/s: 142057.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.298142E-01 | lm-global-supervised-tokens: 1.235737E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.880124E-02 | lm-token-buffer-mib: 5.823823E+01 | global_load_balancing_loss: 1.000036E+00 | loss scale: 1.0 | grad norm: 0.169 | params norm: 11406.314 | params rms: 0.061761 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 5:54:11.958231 | finish at 2026-10-10 16:53:26 + [2026-10-07 11:00:40] iteration 3632/ 6855 | consumed samples: 464896 | elapsed time per iteration (ms): 86579.5 | throughput per GPU (TFLOP/s/GPU): 41.7 | MFU 13.36% | FLOPs estimator: workload-aware | logical tokens/s: 193778.1 | valid tokens/s: 193778.1 | physical tokens/s: 194126.9 | loss tokens/s: 141836.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.009643E-01 | lm-global-supervised-tokens: 1.228016E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.318408E-02 | lm-token-buffer-mib: 5.805581E+01 | global_load_balancing_loss: 9.999645E-01 | loss scale: 1.0 | grad norm: 0.036 | params norm: 11406.249 | params rms: 0.061761 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 5:30:45.764487 | finish at 2026-10-10 16:31:26 + [2026-10-07 11:02:11] iteration 3633/ 6855 | consumed samples: 465024 | elapsed time per iteration (ms): 90587.0 | throughput per GPU (TFLOP/s/GPU): 41.4 | MFU 13.26% | FLOPs estimator: workload-aware | logical tokens/s: 185205.3 | valid tokens/s: 185205.3 | physical tokens/s: 185715.7 | loss tokens/s: 138179.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.793149E-01 | lm-global-supervised-tokens: 1.251727E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.287152E-02 | lm-token-buffer-mib: 5.805205E+01 | global_load_balancing_loss: 1.000123E+00 | loss scale: 1.0 | grad norm: 0.133 | params norm: 11406.184 | params rms: 0.061761 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 9:04:31.474975 | finish at 2026-10-10 20:06:42 + [2026-10-07 11:03:37] iteration 3634/ 6855 | consumed samples: 465152 | elapsed time per iteration (ms): 86066.0 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.62% | FLOPs estimator: workload-aware | logical tokens/s: 194934.1 | valid tokens/s: 194934.1 | physical tokens/s: 195551.2 | loss tokens/s: 146620.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.911283E-01 | lm-global-supervised-tokens: 1.261902E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.122750E-02 | lm-token-buffer-mib: 5.831013E+01 | global_load_balancing_loss: 9.995983E-01 | loss scale: 1.0 | grad norm: 0.137 | params norm: 11406.120 | params rms: 0.061760 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 5:00:18.724949 | finish at 2026-10-10 16:03:56 + [2026-10-07 11:05:04] iteration 3635/ 6855 | consumed samples: 465280 | elapsed time per iteration (ms): 86719.8 | throughput per GPU (TFLOP/s/GPU): 41.7 | MFU 13.35% | FLOPs estimator: workload-aware | logical tokens/s: 193464.6 | valid tokens/s: 193464.5 | physical tokens/s: 193902.2 | loss tokens/s: 145847.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.673359E-01 | lm-global-supervised-tokens: 1.264783E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.630203E-02 | lm-token-buffer-mib: 5.807537E+01 | global_load_balancing_loss: 1.000775E+00 | loss scale: 1.0 | grad norm: 0.047 | params norm: 11406.058 | params rms: 0.061760 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 5:33:57.617798 | finish at 2026-10-10 16:39:01 + [2026-10-07 11:06:35] iteration 3636/ 6855 | consumed samples: 465408 | elapsed time per iteration (ms): 91487.6 | throughput per GPU (TFLOP/s/GPU): 40.8 | MFU 13.07% | FLOPs estimator: workload-aware | logical tokens/s: 183382.3 | valid tokens/s: 183382.3 | physical tokens/s: 183837.8 | loss tokens/s: 132239.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.750039E-01 | lm-global-supervised-tokens: 1.209825E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.471412E-02 | lm-token-buffer-mib: 5.820033E+01 | global_load_balancing_loss: 9.992749E-01 | loss scale: 1.0 | grad norm: 0.054 | params norm: 11405.996 | params rms: 0.061760 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 9:48:18.527123 | finish at 2026-10-10 20:54:54 + [2026-10-07 11:08:02] iteration 3637/ 6855 | consumed samples: 465536 | elapsed time per iteration (ms): 87272.6 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.65% | FLOPs estimator: workload-aware | logical tokens/s: 192238.6 | valid tokens/s: 192238.6 | physical tokens/s: 192766.5 | loss tokens/s: 142014.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.324090E-01 | lm-global-supervised-tokens: 1.239398E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.093944E-02 | lm-token-buffer-mib: 5.831647E+01 | global_load_balancing_loss: 9.997401E-01 | loss scale: 1.0 | grad norm: 0.062 | params norm: 11405.933 | params rms: 0.061759 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 6:00:43.254980 | finish at 2026-10-10 17:08:46 + [2026-10-07 11:09:31] iteration 3638/ 6855 | consumed samples: 465664 | elapsed time per iteration (ms): 88147.4 | throughput per GPU (TFLOP/s/GPU): 40.9 | MFU 13.10% | FLOPs estimator: workload-aware | logical tokens/s: 190330.5 | valid tokens/s: 190330.5 | physical tokens/s: 190784.2 | loss tokens/s: 142086.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.263124E-01 | lm-global-supervised-tokens: 1.252456E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.230293E-02 | lm-token-buffer-mib: 5.810146E+01 | global_load_balancing_loss: 1.000791E+00 | loss scale: 1.0 | grad norm: 0.059 | params norm: 11405.871 | params rms: 0.061759 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 6:46:10.325078 | finish at 2026-10-10 17:55:41 + [2026-10-07 11:10:59] iteration 3639/ 6855 | consumed samples: 465792 | elapsed time per iteration (ms): 88081.1 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.79% | FLOPs estimator: workload-aware | logical tokens/s: 190474.6 | valid tokens/s: 190474.6 | physical tokens/s: 190811.7 | loss tokens/s: 140539.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.908935E-01 | lm-global-supervised-tokens: 1.237890E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.210351E-02 | lm-token-buffer-mib: 5.814923E+01 | global_load_balancing_loss: 1.000700E+00 | loss scale: 1.0 | grad norm: 0.040 | params norm: 11405.807 | params rms: 0.061759 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 6:41:08.881199 | finish at 2026-10-10 17:52:08 + [2026-10-07 11:12:50] iteration 3640/ 6855 | consumed samples: 465920 | elapsed time per iteration (ms): 111426.4 | throughput per GPU (TFLOP/s/GPU): 33.3 | MFU 10.68% | FLOPs estimator: workload-aware | logical tokens/s: 150567.4 | valid tokens/s: 150567.4 | physical tokens/s: 150856.6 | loss tokens/s: 109470.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.418503E-01 | lm-global-supervised-tokens: 1.219786E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.248855E-02 | lm-token-buffer-mib: 5.820684E+01 | global_load_balancing_loss: 9.998350E-01 | loss scale: 1.0 | grad norm: 0.050 | params norm: 11405.745 | params rms: 0.061758 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 3:30:35.896523 | finish at 2026-10-11 14:43:26 +times across ranks (ms): + save-checkpoint: + rank 0: 24810.53 + rank 1: 24810.48 + rank 2: 24810.57 + rank 3: 24810.56 + rank 4: 24810.54 + rank 5: 24810.38 + rank 6: 24810.46 + rank 7: 24810.60 + rank 8: 24810.85 + rank 9: 24810.82 + rank 10: 24810.83 + rank 11: 24810.79 + rank 12: 24810.82 + rank 13: 24810.79 + rank 14: 24810.81 + rank 15: 24810.78 + rank 16: 24810.70 + rank 17: 24810.69 + rank 18: 24810.67 + rank 19: 24810.70 + rank 20: 24810.68 + rank 21: 24810.69 + rank 22: 24810.68 + rank 23: 24810.68 + rank 24: 24810.78 + rank 25: 24810.81 + rank 26: 24810.81 + rank 27: 24810.80 + rank 28: 24810.82 + rank 29: 24810.77 + rank 30: 24810.71 + rank 31: 24810.77 + rank 32: 24810.63 + rank 33: 24810.60 + rank 34: 24810.62 + rank 35: 24810.63 + rank 36: 24810.61 + rank 37: 24810.48 + rank 38: 24810.56 + rank 39: 24810.46 + rank 40: 24810.48 + rank 41: 24810.46 + rank 42: 24810.51 + rank 43: 24810.49 + rank 44: 24810.52 + rank 45: 24810.47 + rank 46: 24810.47 + rank 47: 24810.46 + rank 48: 24810.45 + rank 49: 24810.65 + rank 50: 24810.44 + rank 51: 24810.60 + rank 52: 24810.56 + rank 53: 24810.56 + rank 54: 24810.60 + rank 55: 24810.65 + rank 56: 24811.29 + rank 57: 24811.33 + rank 58: 24811.24 + rank 59: 24811.24 + rank 60: 24811.25 + rank 61: 24811.28 + rank 62: 24811.25 + rank 63: 24811.28 + [2026-10-07 11:15:05] iteration 3641/ 6855 | consumed samples: 466048 | elapsed time per iteration (ms): 109957.7 | throughput per GPU (TFLOP/s/GPU): 34.2 | MFU 10.98% | FLOPs estimator: workload-aware | logical tokens/s: 152578.6 | valid tokens/s: 152578.6 | physical tokens/s: 152871.0 | loss tokens/s: 108766.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.380970E-01 | lm-global-supervised-tokens: 1.195975E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.680337E-01 | lm-token-buffer-mib: 5.812984E+01 | global_load_balancing_loss: 1.000945E+00 | loss scale: 1.0 | grad norm: 0.079 | params norm: 11405.685 | params rms: 0.061758 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 2:10:03.978880 | finish at 2026-10-11 13:25:09 + [2026-10-07 11:16:43] iteration 3642/ 6855 | consumed samples: 466176 | elapsed time per iteration (ms): 98215.7 | throughput per GPU (TFLOP/s/GPU): 39.2 | MFU 12.57% | FLOPs estimator: workload-aware | logical tokens/s: 170819.4 | valid tokens/s: 170819.4 | physical tokens/s: 171296.3 | loss tokens/s: 120631.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.457501E-01 | lm-global-supervised-tokens: 1.184790E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.042421E-02 | lm-token-buffer-mib: 5.825044E+01 | global_load_balancing_loss: 9.994860E-01 | loss scale: 1.0 | grad norm: 0.049 | params norm: 11405.624 | params rms: 0.061758 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 15:39:27.071392 | finish at 2026-10-11 02:56:10 + [2026-10-07 11:18:11] iteration 3643/ 6855 | consumed samples: 466304 | elapsed time per iteration (ms): 87453.0 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.70% | FLOPs estimator: workload-aware | logical tokens/s: 191842.0 | valid tokens/s: 191842.0 | physical tokens/s: 192262.7 | loss tokens/s: 142052.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.994369E-01 | lm-global-supervised-tokens: 1.242289E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.522782E-02 | lm-token-buffer-mib: 5.820665E+01 | global_load_balancing_loss: 9.999877E-01 | loss scale: 1.0 | grad norm: 0.190 | params norm: 11405.564 | params rms: 0.061757 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 6:01:39.100548 | finish at 2026-10-10 17:19:50 + [2026-10-07 11:19:38] iteration 3644/ 6855 | consumed samples: 466432 | elapsed time per iteration (ms): 87723.9 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.76% | FLOPs estimator: workload-aware | logical tokens/s: 191249.9 | valid tokens/s: 191249.8 | physical tokens/s: 191728.4 | loss tokens/s: 136133.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.456224E-01 | lm-global-supervised-tokens: 1.194213E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.780220E-02 | lm-token-buffer-mib: 5.814223E+01 | global_load_balancing_loss: 1.000103E+00 | loss scale: 1.0 | grad norm: 0.328 | params norm: 11405.504 | params rms: 0.061757 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 6:14:41.579426 | finish at 2026-10-10 17:34:20 + [2026-10-07 11:21:05] iteration 3645/ 6855 | consumed samples: 466560 | elapsed time per iteration (ms): 86997.6 | throughput per GPU (TFLOP/s/GPU): 43.1 | MFU 13.82% | FLOPs estimator: workload-aware | logical tokens/s: 192846.8 | valid tokens/s: 192846.8 | physical tokens/s: 193322.0 | loss tokens/s: 137063.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.583888E-01 | lm-global-supervised-tokens: 1.192415E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.305346E-02 | lm-token-buffer-mib: 5.809671E+01 | global_load_balancing_loss: 1.001283E+00 | loss scale: 1.0 | grad norm: 0.182 | params norm: 11405.445 | params rms: 0.061757 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 5:34:22.165382 | finish at 2026-10-10 16:55:28 + [2026-10-07 11:22:36] iteration 3646/ 6855 | consumed samples: 466688 | elapsed time per iteration (ms): 90725.9 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.38% | FLOPs estimator: workload-aware | logical tokens/s: 184921.5 | valid tokens/s: 184921.5 | physical tokens/s: 185359.1 | loss tokens/s: 130569.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.677837E-01 | lm-global-supervised-tokens: 1.184607E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.127930E-02 | lm-token-buffer-mib: 5.804781E+01 | global_load_balancing_loss: 1.000357E+00 | loss scale: 1.0 | grad norm: 0.178 | params norm: 11405.387 | params rms: 0.061756 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 8:52:19.346328 | finish at 2026-10-10 20:14:55 + [2026-10-07 11:24:03] iteration 3647/ 6855 | consumed samples: 466816 | elapsed time per iteration (ms): 86831.9 | throughput per GPU (TFLOP/s/GPU): 43.4 | MFU 13.91% | FLOPs estimator: workload-aware | logical tokens/s: 193213.8 | valid tokens/s: 193213.8 | physical tokens/s: 193701.9 | loss tokens/s: 138577.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.949677E-01 | lm-global-supervised-tokens: 1.203298E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.152374E-02 | lm-token-buffer-mib: 5.817312E+01 | global_load_balancing_loss: 9.998422E-01 | loss scale: 1.0 | grad norm: 0.179 | params norm: 11405.328 | params rms: 0.061756 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 5:22:36.760824 | finish at 2026-10-10 16:46:40 + [2026-10-07 11:25:33] iteration 3648/ 6855 | consumed samples: 466944 | elapsed time per iteration (ms): 89793.5 | throughput per GPU (TFLOP/s/GPU): 41.7 | MFU 13.37% | FLOPs estimator: workload-aware | logical tokens/s: 186841.6 | valid tokens/s: 186841.6 | physical tokens/s: 187353.0 | loss tokens/s: 142709.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.985273E-01 | lm-global-supervised-tokens: 1.281442E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.270211E-02 | lm-token-buffer-mib: 5.811262E+01 | global_load_balancing_loss: 9.997632E-01 | loss scale: 1.0 | grad norm: 0.152 | params norm: 11405.269 | params rms: 0.061756 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 7:59:27.828496 | finish at 2026-10-10 19:25:01 + [2026-10-07 11:27:01] iteration 3649/ 6855 | consumed samples: 467072 | elapsed time per iteration (ms): 87938.3 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.40% | FLOPs estimator: workload-aware | logical tokens/s: 190783.5 | valid tokens/s: 190783.4 | physical tokens/s: 191230.2 | loss tokens/s: 142538.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.582387E-01 | lm-global-supervised-tokens: 1.253457E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.157236E-02 | lm-token-buffer-mib: 5.803168E+01 | global_load_balancing_loss: 9.998536E-01 | loss scale: 1.0 | grad norm: 0.073 | params norm: 11405.212 | params rms: 0.061756 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 6:18:50.297237 | finish at 2026-10-10 17:45:51 + [2026-10-07 11:28:47] iteration 3650/ 6855 | consumed samples: 467200 | elapsed time per iteration (ms): 106045.7 | throughput per GPU (TFLOP/s/GPU): 34.0 | MFU 10.91% | FLOPs estimator: workload-aware | logical tokens/s: 158207.1 | valid tokens/s: 158207.0 | physical tokens/s: 158601.7 | loss tokens/s: 120293.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.710921E-01 | lm-global-supervised-tokens: 1.275665E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.350620E-02 | lm-token-buffer-mib: 5.819246E+01 | global_load_balancing_loss: 9.993228E-01 | loss scale: 1.0 | grad norm: 0.189 | params norm: 11405.154 | params rms: 0.061755 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 22:24:36.340361 | finish at 2026-10-11 09:53:23 + [2026-10-07 11:30:12] iteration 3651/ 6855 | consumed samples: 467328 | elapsed time per iteration (ms): 84817.8 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.66% | FLOPs estimator: workload-aware | logical tokens/s: 197802.5 | valid tokens/s: 197802.5 | physical tokens/s: 198402.2 | loss tokens/s: 148751.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.320561E-01 | lm-global-supervised-tokens: 1.261673E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.253965E-02 | lm-token-buffer-mib: 5.825546E+01 | global_load_balancing_loss: 1.000011E+00 | loss scale: 1.0 | grad norm: 0.063 | params norm: 11405.097 | params rms: 0.061755 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 3:29:16.112941 | finish at 2026-10-10 14:59:28 + [2026-10-07 11:31:40] iteration 3652/ 6855 | consumed samples: 467456 | elapsed time per iteration (ms): 88770.4 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.52% | FLOPs estimator: workload-aware | logical tokens/s: 188994.6 | valid tokens/s: 188994.6 | physical tokens/s: 189404.2 | loss tokens/s: 142686.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.040846E-01 | lm-global-supervised-tokens: 1.266634E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.375702E-02 | lm-token-buffer-mib: 5.802877E+01 | global_load_balancing_loss: 1.000969E+00 | loss scale: 1.0 | grad norm: 0.126 | params norm: 11405.037 | params rms: 0.061755 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 6:58:51.576842 | finish at 2026-10-10 18:30:32 + [2026-10-07 11:33:07] iteration 3653/ 6855 | consumed samples: 467584 | elapsed time per iteration (ms): 86617.0 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.62% | FLOPs estimator: workload-aware | logical tokens/s: 193694.1 | valid tokens/s: 193694.1 | physical tokens/s: 194221.9 | loss tokens/s: 139847.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.044556E-01 | lm-global-supervised-tokens: 1.211314E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.237877E-02 | lm-token-buffer-mib: 5.831171E+01 | global_load_balancing_loss: 1.001050E+00 | loss scale: 1.0 | grad norm: 0.100 | params norm: 11404.977 | params rms: 0.061754 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 5:02:27.528215 | finish at 2026-10-10 16:35:34 + [2026-10-07 11:34:32] iteration 3654/ 6855 | consumed samples: 467712 | elapsed time per iteration (ms): 85051.5 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.77% | FLOPs estimator: workload-aware | logical tokens/s: 197258.6 | valid tokens/s: 197258.5 | physical tokens/s: 197761.1 | loss tokens/s: 147496.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.951107E-01 | lm-global-supervised-tokens: 1.254484E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.154631E-02 | lm-token-buffer-mib: 5.823587E+01 | global_load_balancing_loss: 1.001974E+00 | loss scale: 1.0 | grad norm: 0.054 | params norm: 11404.917 | params rms: 0.061754 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 3:37:29.992187 | finish at 2026-10-10 15:12:02 + [2026-10-07 11:35:58] iteration 3655/ 6855 | consumed samples: 467840 | elapsed time per iteration (ms): 85954.9 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.79% | FLOPs estimator: workload-aware | logical tokens/s: 195186.3 | valid tokens/s: 195186.2 | physical tokens/s: 195802.6 | loss tokens/s: 145240.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.845810E-01 | lm-global-supervised-tokens: 1.248415E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.896203E-02 | lm-token-buffer-mib: 5.808479E+01 | global_load_balancing_loss: 1.001260E+00 | loss scale: 1.0 | grad norm: 0.114 | params norm: 11404.858 | params rms: 0.061754 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 4:24:15.675507 | finish at 2026-10-10 16:00:14 + [2026-10-07 11:37:23] iteration 3656/ 6855 | consumed samples: 467968 | elapsed time per iteration (ms): 85351.3 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.53% | FLOPs estimator: workload-aware | logical tokens/s: 196566.2 | valid tokens/s: 196566.2 | physical tokens/s: 197051.6 | loss tokens/s: 145115.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.783075E-01 | lm-global-supervised-tokens: 1.238581E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.408226E-02 | lm-token-buffer-mib: 5.815268E+01 | global_load_balancing_loss: 9.997041E-01 | loss scale: 1.0 | grad norm: 0.108 | params norm: 11404.800 | params rms: 0.061753 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 3:50:38.756077 | finish at 2026-10-10 15:28:02 + [2026-10-07 11:38:51] iteration 3657/ 6855 | consumed samples: 468096 | elapsed time per iteration (ms): 87325.7 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.69% | FLOPs estimator: workload-aware | logical tokens/s: 192122.0 | valid tokens/s: 192122.0 | physical tokens/s: 192458.9 | loss tokens/s: 141391.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.438727E-01 | lm-global-supervised-tokens: 1.234712E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.350139E-02 | lm-token-buffer-mib: 5.808829E+01 | global_load_balancing_loss: 9.993798E-01 | loss scale: 1.0 | grad norm: 0.069 | params norm: 11404.742 | params rms: 0.061753 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 5:34:27.651265 | finish at 2026-10-10 17:13:18 + [2026-10-07 11:40:16] iteration 3658/ 6855 | consumed samples: 468224 | elapsed time per iteration (ms): 85288.7 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.64% | FLOPs estimator: workload-aware | logical tokens/s: 196709.4 | valid tokens/s: 196709.4 | physical tokens/s: 197141.9 | loss tokens/s: 151589.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.694945E-01 | lm-global-supervised-tokens: 1.292886E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.768951E-02 | lm-token-buffer-mib: 5.826613E+01 | global_load_balancing_loss: 9.991184E-01 | loss scale: 1.0 | grad norm: 0.175 | params norm: 11404.684 | params rms: 0.061753 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 3:44:28.064174 | finish at 2026-10-10 15:24:44 + [2026-10-07 11:41:42] iteration 3659/ 6855 | consumed samples: 468352 | elapsed time per iteration (ms): 86331.4 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.80% | FLOPs estimator: workload-aware | logical tokens/s: 194334.4 | valid tokens/s: 194334.4 | physical tokens/s: 194759.7 | loss tokens/s: 145338.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.891627E-01 | lm-global-supervised-tokens: 1.254728E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.486521E-02 | lm-token-buffer-mib: 5.810989E+01 | global_load_balancing_loss: 9.993340E-01 | loss scale: 1.0 | grad norm: 0.124 | params norm: 11404.625 | params rms: 0.061752 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 4:38:35.036791 | finish at 2026-10-10 16:20:17 + [2026-10-07 11:43:36] iteration 3660/ 6855 | consumed samples: 468480 | elapsed time per iteration (ms): 113636.2 | throughput per GPU (TFLOP/s/GPU): 33.0 | MFU 10.58% | FLOPs estimator: workload-aware | logical tokens/s: 147639.0 | valid tokens/s: 147639.0 | physical tokens/s: 148177.2 | loss tokens/s: 109416.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.988302E-01 | lm-global-supervised-tokens: 1.243367E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.200818E-02 | lm-token-buffer-mib: 5.816016E+01 | global_load_balancing_loss: 9.992428E-01 | loss scale: 1.0 | grad norm: 0.122 | params norm: 11404.566 | params rms: 0.061752 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 4:51:07.799767 | finish at 2026-10-11 16:34:44 + [2026-10-07 11:45:02] iteration 3661/ 6855 | consumed samples: 468608 | elapsed time per iteration (ms): 86232.8 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.47% | FLOPs estimator: workload-aware | logical tokens/s: 194556.5 | valid tokens/s: 194556.5 | physical tokens/s: 195112.5 | loss tokens/s: 150303.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.354259E-01 | lm-global-supervised-tokens: 1.296111E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.428113E-02 | lm-token-buffer-mib: 5.812284E+01 | global_load_balancing_loss: 1.001911E+00 | loss scale: 1.0 | grad norm: 0.078 | params norm: 11404.507 | params rms: 0.061752 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 4:30:27.421581 | finish at 2026-10-10 16:15:30 + [2026-10-07 11:46:29] iteration 3662/ 6855 | consumed samples: 468736 | elapsed time per iteration (ms): 86997.2 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.56% | FLOPs estimator: workload-aware | logical tokens/s: 192847.0 | valid tokens/s: 192847.0 | physical tokens/s: 193246.4 | loss tokens/s: 143924.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.677280E-01 | lm-global-supervised-tokens: 1.252100E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.270238E-02 | lm-token-buffer-mib: 5.816263E+01 | global_load_balancing_loss: 1.001466E+00 | loss scale: 1.0 | grad norm: 0.113 | params norm: 11404.446 | params rms: 0.061751 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 5:09:42.194384 | finish at 2026-10-10 16:56:11 + [2026-10-07 11:47:57] iteration 3663/ 6855 | consumed samples: 468864 | elapsed time per iteration (ms): 87554.9 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.69% | FLOPs estimator: workload-aware | logical tokens/s: 191619.2 | valid tokens/s: 191619.2 | physical tokens/s: 192166.4 | loss tokens/s: 144878.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.965656E-01 | lm-global-supervised-tokens: 1.268477E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.512702E-02 | lm-token-buffer-mib: 5.827288E+01 | global_load_balancing_loss: 1.001404E+00 | loss scale: 1.0 | grad norm: 0.062 | params norm: 11404.384 | params rms: 0.061751 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 5:37:55.191874 | finish at 2026-10-10 17:25:52 + [2026-10-07 11:49:22] iteration 3664/ 6855 | consumed samples: 468992 | elapsed time per iteration (ms): 85744.6 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.58% | FLOPs estimator: workload-aware | logical tokens/s: 195664.5 | valid tokens/s: 195664.5 | physical tokens/s: 196047.1 | loss tokens/s: 147051.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.162360E-01 | lm-global-supervised-tokens: 1.260887E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.222996E-02 | lm-token-buffer-mib: 5.809800E+01 | global_load_balancing_loss: 1.000985E+00 | loss scale: 1.0 | grad norm: 0.065 | params norm: 11404.323 | params rms: 0.061751 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 4:00:11.070518 | finish at 2026-10-10 15:49:34 + [2026-10-07 11:50:48] iteration 3665/ 6855 | consumed samples: 469120 | elapsed time per iteration (ms): 85371.3 | throughput per GPU (TFLOP/s/GPU): 43.1 | MFU 13.80% | FLOPs estimator: workload-aware | logical tokens/s: 196519.8 | valid tokens/s: 196519.8 | physical tokens/s: 196947.6 | loss tokens/s: 148495.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.127684E-01 | lm-global-supervised-tokens: 1.267726E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.097145E-02 | lm-token-buffer-mib: 5.810943E+01 | global_load_balancing_loss: 9.995220E-01 | loss scale: 1.0 | grad norm: 0.144 | params norm: 11404.263 | params rms: 0.061750 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 3:38:54.364803 | finish at 2026-10-10 15:29:42 + [2026-10-07 11:52:19] iteration 3666/ 6855 | consumed samples: 469248 | elapsed time per iteration (ms): 90890.5 | throughput per GPU (TFLOP/s/GPU): 40.6 | MFU 13.02% | FLOPs estimator: workload-aware | logical tokens/s: 184587.2 | valid tokens/s: 184587.2 | physical tokens/s: 185046.8 | loss tokens/s: 134751.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.279290E-01 | lm-global-supervised-tokens: 1.224765E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.613219E-02 | lm-token-buffer-mib: 5.803551E+01 | global_load_balancing_loss: 9.996098E-01 | loss scale: 1.0 | grad norm: 0.045 | params norm: 11404.203 | params rms: 0.061750 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 8:30:49.667862 | finish at 2026-10-10 20:23:08 + [2026-10-07 11:53:45] iteration 3667/ 6855 | consumed samples: 469376 | elapsed time per iteration (ms): 86724.1 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.53% | FLOPs estimator: workload-aware | logical tokens/s: 193453.9 | valid tokens/s: 193453.9 | physical tokens/s: 193866.2 | loss tokens/s: 144687.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.915978E-01 | lm-global-supervised-tokens: 1.254788E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.126061E-02 | lm-token-buffer-mib: 5.809259E+01 | global_load_balancing_loss: 9.986706E-01 | loss scale: 1.0 | grad norm: 0.221 | params norm: 11404.141 | params rms: 0.061750 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 4:47:56.507928 | finish at 2026-10-10 16:41:42 + [2026-10-07 11:55:11] iteration 3668/ 6855 | consumed samples: 469504 | elapsed time per iteration (ms): 85852.8 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.51% | FLOPs estimator: workload-aware | logical tokens/s: 195418.2 | valid tokens/s: 195418.2 | physical tokens/s: 195880.9 | loss tokens/s: 145304.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.417927E-01 | lm-global-supervised-tokens: 1.247480E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.282812E-02 | lm-token-buffer-mib: 5.808598E+01 | global_load_balancing_loss: 9.989410E-01 | loss scale: 1.0 | grad norm: 0.121 | params norm: 11404.078 | params rms: 0.061749 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 4:00:12.883135 | finish at 2026-10-10 15:55:24 + [2026-10-07 11:56:38] iteration 3669/ 6855 | consumed samples: 469632 | elapsed time per iteration (ms): 86540.4 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.77% | FLOPs estimator: workload-aware | logical tokens/s: 193865.2 | valid tokens/s: 193865.2 | physical tokens/s: 194429.8 | loss tokens/s: 146019.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.708755E-01 | lm-global-supervised-tokens: 1.263657E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.614862E-02 | lm-token-buffer-mib: 5.830939E+01 | global_load_balancing_loss: 1.000610E+00 | loss scale: 1.0 | grad norm: 0.041 | params norm: 11404.015 | params rms: 0.061749 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 4:35:17.697779 | finish at 2026-10-10 16:31:56 + [2026-10-07 11:58:25] iteration 3670/ 6855 | consumed samples: 469760 | elapsed time per iteration (ms): 106734.1 | throughput per GPU (TFLOP/s/GPU): 34.1 | MFU 10.92% | FLOPs estimator: workload-aware | logical tokens/s: 157186.4 | valid tokens/s: 157186.4 | physical tokens/s: 157576.7 | loss tokens/s: 120903.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.718419E-01 | lm-global-supervised-tokens: 1.290454E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.480561E-02 | lm-token-buffer-mib: 5.821693E+01 | global_load_balancing_loss: 1.001182E+00 | loss scale: 1.0 | grad norm: 0.078 | params norm: 11403.951 | params rms: 0.061749 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 22:25:48.005556 | finish at 2026-10-11 10:24:13 + [2026-10-07 11:59:51] iteration 3671/ 6855 | consumed samples: 469888 | elapsed time per iteration (ms): 85983.4 | throughput per GPU (TFLOP/s/GPU): 41.0 | MFU 13.16% | FLOPs estimator: workload-aware | logical tokens/s: 195120.8 | valid tokens/s: 195120.8 | physical tokens/s: 195561.5 | loss tokens/s: 150469.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 5.902515E-01 | lm-global-supervised-tokens: 1.293792E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.452063E-02 | lm-token-buffer-mib: 5.811082E+01 | global_load_balancing_loss: 1.000407E+00 | loss scale: 1.0 | grad norm: 0.144 | params norm: 11403.887 | params rms: 0.061748 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 4:02:51.154289 | finish at 2026-10-10 16:02:42 + [2026-10-07 12:01:17] iteration 3672/ 6855 | consumed samples: 470016 | elapsed time per iteration (ms): 86080.9 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.60% | FLOPs estimator: workload-aware | logical tokens/s: 194900.2 | valid tokens/s: 194900.2 | physical tokens/s: 195346.1 | loss tokens/s: 146718.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.045556E-01 | lm-global-supervised-tokens: 1.262964E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.200337E-02 | lm-token-buffer-mib: 5.814010E+01 | global_load_balancing_loss: 1.001117E+00 | loss scale: 1.0 | grad norm: 0.156 | params norm: 11403.824 | params rms: 0.061748 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 4:06:35.547810 | finish at 2026-10-10 16:07:52 + [2026-10-07 12:02:41] iteration 3673/ 6855 | consumed samples: 470144 | elapsed time per iteration (ms): 84462.4 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.79% | FLOPs estimator: workload-aware | logical tokens/s: 198634.6 | valid tokens/s: 198634.6 | physical tokens/s: 199304.0 | loss tokens/s: 146864.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.797804E-01 | lm-global-supervised-tokens: 1.240452E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.244574E-02 | lm-token-buffer-mib: 5.824593E+01 | global_load_balancing_loss: 1.000096E+00 | loss scale: 1.0 | grad norm: 0.066 | params norm: 11403.761 | params rms: 0.061748 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 2:39:19.358947 | finish at 2026-10-10 14:42:00 + [2026-10-07 12:04:05] iteration 3674/ 6855 | consumed samples: 470272 | elapsed time per iteration (ms): 84284.0 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.75% | FLOPs estimator: workload-aware | logical tokens/s: 199054.8 | valid tokens/s: 199054.8 | physical tokens/s: 199686.8 | loss tokens/s: 148747.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.394942E-01 | lm-global-supervised-tokens: 1.253705E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.231608E-02 | lm-token-buffer-mib: 5.832113E+01 | global_load_balancing_loss: 1.001583E+00 | loss scale: 1.0 | grad norm: 0.058 | params norm: 11403.699 | params rms: 0.061747 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 2:28:27.380234 | finish at 2026-10-10 14:32:33 + [2026-10-07 12:05:32] iteration 3675/ 6855 | consumed samples: 470400 | elapsed time per iteration (ms): 86811.8 | throughput per GPU (TFLOP/s/GPU): 43.2 | MFU 13.83% | FLOPs estimator: workload-aware | logical tokens/s: 193259.2 | valid tokens/s: 193259.2 | physical tokens/s: 193952.4 | loss tokens/s: 138249.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.982270E-01 | lm-global-supervised-tokens: 1.200168E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.212495E-02 | lm-token-buffer-mib: 5.833438E+01 | global_load_balancing_loss: 9.998032E-01 | loss scale: 1.0 | grad norm: 0.198 | params norm: 11403.640 | params rms: 0.061747 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 4:41:01.614990 | finish at 2026-10-10 16:46:34 + [2026-10-07 12:06:58] iteration 3676/ 6855 | consumed samples: 470528 | elapsed time per iteration (ms): 86015.4 | throughput per GPU (TFLOP/s/GPU): 43.1 | MFU 13.82% | FLOPs estimator: workload-aware | logical tokens/s: 195048.9 | valid tokens/s: 195048.9 | physical tokens/s: 195708.9 | loss tokens/s: 146014.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.097310E-01 | lm-global-supervised-tokens: 1.255952E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.162817E-02 | lm-token-buffer-mib: 5.814301E+01 | global_load_balancing_loss: 9.999596E-01 | loss scale: 1.0 | grad norm: 0.249 | params norm: 11403.584 | params rms: 0.061747 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 3:57:22.994283 | finish at 2026-10-10 16:04:21 + [2026-10-07 12:08:23] iteration 3677/ 6855 | consumed samples: 470656 | elapsed time per iteration (ms): 84943.0 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.62% | FLOPs estimator: workload-aware | logical tokens/s: 197511.3 | valid tokens/s: 197511.3 | physical tokens/s: 198021.1 | loss tokens/s: 146743.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.742834E-01 | lm-global-supervised-tokens: 1.246481E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.151061E-02 | lm-token-buffer-mib: 5.823513E+01 | global_load_balancing_loss: 9.998696E-01 | loss scale: 1.0 | grad norm: 0.125 | params norm: 11403.530 | params rms: 0.061746 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 2:59:08.958053 | finish at 2026-10-10 15:07:32 + [2026-10-07 12:09:54] iteration 3678/ 6855 | consumed samples: 470784 | elapsed time per iteration (ms): 90583.8 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.48% | FLOPs estimator: workload-aware | logical tokens/s: 185211.3 | valid tokens/s: 185211.3 | physical tokens/s: 185664.3 | loss tokens/s: 134800.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.061344E-01 | lm-global-supervised-tokens: 1.221070E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.868285E-02 | lm-token-buffer-mib: 5.803778E+01 | global_load_balancing_loss: 9.997341E-01 | loss scale: 1.0 | grad norm: 0.077 | params norm: 11403.475 | params rms: 0.061746 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 7:56:24.673007 | finish at 2026-10-10 20:06:18 + [2026-10-07 12:11:20] iteration 3679/ 6855 | consumed samples: 470912 | elapsed time per iteration (ms): 86355.6 | throughput per GPU (TFLOP/s/GPU): 41.7 | MFU 13.37% | FLOPs estimator: workload-aware | logical tokens/s: 194280.5 | valid tokens/s: 194280.5 | physical tokens/s: 194911.7 | loss tokens/s: 144464.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.411692E-01 | lm-global-supervised-tokens: 1.247530E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.428727E-02 | lm-token-buffer-mib: 5.819590E+01 | global_load_balancing_loss: 9.996835E-01 | loss scale: 1.0 | grad norm: 0.253 | params norm: 11403.423 | params rms: 0.061746 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 4:11:05.523790 | finish at 2026-10-10 16:22:26 + [2026-10-07 12:13:12] iteration 3680/ 6855 | consumed samples: 471040 | elapsed time per iteration (ms): 111446.0 | throughput per GPU (TFLOP/s/GPU): 34.4 | MFU 11.03% | FLOPs estimator: workload-aware | logical tokens/s: 150540.6 | valid tokens/s: 150540.6 | physical tokens/s: 150829.2 | loss tokens/s: 108454.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.706675E-01 | lm-global-supervised-tokens: 1.208684E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.660622E-02 | lm-token-buffer-mib: 5.807234E+01 | global_load_balancing_loss: 9.993076E-01 | loss scale: 1.0 | grad norm: 0.152 | params norm: 11403.373 | params rms: 0.061746 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 2:17:21.053343 | finish at 2026-10-11 14:30:33 +times across ranks (ms): + save-checkpoint: + rank 0: 28134.56 + rank 1: 28134.54 + rank 2: 28134.58 + rank 3: 28134.57 + rank 4: 28134.57 + rank 5: 28134.55 + rank 6: 28134.62 + rank 7: 28134.56 + rank 8: 28132.45 + rank 9: 28132.34 + rank 10: 28132.36 + rank 11: 28132.32 + rank 12: 28132.32 + rank 13: 28132.36 + rank 14: 28132.38 + rank 15: 28132.33 + rank 16: 28135.05 + rank 17: 28135.03 + rank 18: 28135.01 + rank 19: 28135.04 + rank 20: 28135.08 + rank 21: 28135.07 + rank 22: 28135.02 + rank 23: 28135.00 + rank 24: 28133.96 + rank 25: 28133.92 + rank 26: 28133.94 + rank 27: 28134.20 + rank 28: 28134.10 + rank 29: 28133.94 + rank 30: 28133.93 + rank 31: 28133.97 + rank 32: 28134.77 + rank 33: 28134.83 + rank 34: 28134.78 + rank 35: 28134.71 + rank 36: 28134.68 + rank 37: 28134.69 + rank 38: 28134.73 + rank 39: 28134.69 + rank 40: 28134.52 + rank 41: 28134.51 + rank 42: 28134.53 + rank 43: 28134.53 + rank 44: 28134.50 + rank 45: 28134.52 + rank 46: 28134.51 + rank 47: 28134.52 + rank 48: 28134.13 + rank 49: 28134.32 + rank 50: 28134.28 + rank 51: 28134.47 + rank 52: 28134.36 + rank 53: 28134.35 + rank 54: 28134.31 + rank 55: 28134.38 + rank 56: 28134.78 + rank 57: 28134.76 + rank 58: 28134.76 + rank 59: 28134.76 + rank 60: 28134.72 + rank 61: 28134.76 + rank 62: 28134.81 + rank 63: 28134.79 + [2026-10-07 12:15:38] iteration 3681/ 6855 | consumed samples: 471168 | elapsed time per iteration (ms): 118377.7 | throughput per GPU (TFLOP/s/GPU): 30.9 | MFU 9.90% | FLOPs estimator: workload-aware | logical tokens/s: 141725.6 | valid tokens/s: 141725.6 | physical tokens/s: 141939.2 | loss tokens/s: 108162.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.325489E-01 | lm-global-supervised-tokens: 1.280406E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 4.925197E+00 | lm-token-buffer-mib: 5.810738E+01 | global_load_balancing_loss: 9.994665E-01 | loss scale: 1.0 | grad norm: 0.210 | params norm: 11403.322 | params rms: 0.061745 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 8:22:10.849600 | finish at 2026-10-11 20:37:49 + [2026-10-07 12:17:05] iteration 3682/ 6855 | consumed samples: 471296 | elapsed time per iteration (ms): 87212.6 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.41% | FLOPs estimator: workload-aware | logical tokens/s: 192371.5 | valid tokens/s: 192371.4 | physical tokens/s: 192896.3 | loss tokens/s: 145580.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.281414E-01 | lm-global-supervised-tokens: 1.269641E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.474571E-02 | lm-token-buffer-mib: 5.816552E+01 | global_load_balancing_loss: 1.000482E+00 | loss scale: 1.0 | grad norm: 0.141 | params norm: 11403.271 | params rms: 0.061745 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 4:52:05.455711 | finish at 2026-10-10 17:09:11 + [2026-10-07 12:18:30] iteration 3683/ 6855 | consumed samples: 471424 | elapsed time per iteration (ms): 84250.8 | throughput per GPU (TFLOP/s/GPU): 43.8 | MFU 14.04% | FLOPs estimator: workload-aware | logical tokens/s: 199133.6 | valid tokens/s: 199133.6 | physical tokens/s: 199460.6 | loss tokens/s: 153075.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.542689E-01 | lm-global-supervised-tokens: 1.289674E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.289077E-02 | lm-token-buffer-mib: 5.822572E+01 | global_load_balancing_loss: 1.000228E+00 | loss scale: 1.0 | grad norm: 0.198 | params norm: 11403.221 | params rms: 0.061745 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 2:14:03.600791 | finish at 2026-10-10 14:32:33 + [2026-10-07 12:19:55] iteration 3684/ 6855 | consumed samples: 471552 | elapsed time per iteration (ms): 85145.0 | throughput per GPU (TFLOP/s/GPU): 43.3 | MFU 13.89% | FLOPs estimator: workload-aware | logical tokens/s: 197041.9 | valid tokens/s: 197041.9 | physical tokens/s: 197772.5 | loss tokens/s: 147369.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.249731E-01 | lm-global-supervised-tokens: 1.254774E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.518113E-02 | lm-token-buffer-mib: 5.827048E+01 | global_load_balancing_loss: 1.000543E+00 | loss scale: 1.0 | grad norm: 0.098 | params norm: 11403.171 | params rms: 0.061744 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 2:59:54.818376 | finish at 2026-10-10 15:19:50 + [2026-10-07 12:21:19] iteration 3685/ 6855 | consumed samples: 471680 | elapsed time per iteration (ms): 84345.0 | throughput per GPU (TFLOP/s/GPU): 44.1 | MFU 14.14% | FLOPs estimator: workload-aware | logical tokens/s: 198911.1 | valid tokens/s: 198911.1 | physical tokens/s: 199360.3 | loss tokens/s: 147513.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.688461E-01 | lm-global-supervised-tokens: 1.244204E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.356059E-02 | lm-token-buffer-mib: 5.817210E+01 | global_load_balancing_loss: 1.002304E+00 | loss scale: 1.0 | grad norm: 0.164 | params norm: 11403.121 | params rms: 0.061744 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 2:16:13.618348 | finish at 2026-10-10 14:37:33 + [2026-10-07 12:22:46] iteration 3686/ 6855 | consumed samples: 471808 | elapsed time per iteration (ms): 87302.8 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.80% | FLOPs estimator: workload-aware | logical tokens/s: 192172.2 | valid tokens/s: 192172.2 | physical tokens/s: 192745.6 | loss tokens/s: 143404.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.709910E-01 | lm-global-supervised-tokens: 1.251960E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.603855E-02 | lm-token-buffer-mib: 5.818642E+01 | global_load_balancing_loss: 1.001301E+00 | loss scale: 1.0 | grad norm: 0.088 | params norm: 11403.069 | params rms: 0.061744 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 4:51:02.661409 | finish at 2026-10-10 17:13:49 + [2026-10-07 12:24:10] iteration 3687/ 6855 | consumed samples: 471936 | elapsed time per iteration (ms): 83624.6 | throughput per GPU (TFLOP/s/GPU): 43.8 | MFU 14.05% | FLOPs estimator: workload-aware | logical tokens/s: 200624.9 | valid tokens/s: 200624.9 | physical tokens/s: 201131.7 | loss tokens/s: 158351.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.499134E-01 | lm-global-supervised-tokens: 1.324212E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.234207E-02 | lm-token-buffer-mib: 5.816913E+01 | global_load_balancing_loss: 1.000256E+00 | loss scale: 1.0 | grad norm: 0.061 | params norm: 11403.015 | params rms: 0.061744 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 1:35:22.850418 | finish at 2026-10-10 13:59:33 + [2026-10-07 12:25:34] iteration 3688/ 6855 | consumed samples: 472064 | elapsed time per iteration (ms): 84223.4 | throughput per GPU (TFLOP/s/GPU): 43.1 | MFU 13.80% | FLOPs estimator: workload-aware | logical tokens/s: 199198.5 | valid tokens/s: 199198.5 | physical tokens/s: 200077.0 | loss tokens/s: 151481.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.409850E-01 | lm-global-supervised-tokens: 1.275827E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.831748E-02 | lm-token-buffer-mib: 5.834860E+01 | global_load_balancing_loss: 1.000788E+00 | loss scale: 1.0 | grad norm: 0.171 | params norm: 11402.959 | params rms: 0.061743 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 2:05:35.618408 | finish at 2026-10-10 14:31:10 + [2026-10-07 12:27:03] iteration 3689/ 6855 | consumed samples: 472192 | elapsed time per iteration (ms): 88521.7 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.69% | FLOPs estimator: workload-aware | logical tokens/s: 189526.6 | valid tokens/s: 189526.6 | physical tokens/s: 189840.8 | loss tokens/s: 137658.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.015284E-01 | lm-global-supervised-tokens: 1.218579E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.222731E-02 | lm-token-buffer-mib: 5.813193E+01 | global_load_balancing_loss: 9.998446E-01 | loss scale: 1.0 | grad norm: 0.047 | params norm: 11402.902 | params rms: 0.061743 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 5:50:59.599243 | finish at 2026-10-10 18:18:02 + [2026-10-07 12:28:54] iteration 3690/ 6855 | consumed samples: 472320 | elapsed time per iteration (ms): 111534.6 | throughput per GPU (TFLOP/s/GPU): 32.4 | MFU 10.40% | FLOPs estimator: workload-aware | logical tokens/s: 150420.8 | valid tokens/s: 150420.8 | physical tokens/s: 150923.5 | loss tokens/s: 115727.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.759819E-01 | lm-global-supervised-tokens: 1.290761E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.178894E-02 | lm-token-buffer-mib: 5.824339E+01 | global_load_balancing_loss: 9.990634E-01 | loss scale: 1.0 | grad norm: 0.185 | params norm: 11402.843 | params rms: 0.061743 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 2:03:26.941148 | finish at 2026-10-11 14:32:21 + [2026-10-07 12:30:25] iteration 3691/ 6855 | consumed samples: 472448 | elapsed time per iteration (ms): 90494.5 | throughput per GPU (TFLOP/s/GPU): 41.3 | MFU 13.22% | FLOPs estimator: workload-aware | logical tokens/s: 185394.5 | valid tokens/s: 185394.5 | physical tokens/s: 185906.1 | loss tokens/s: 131118.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.649175E-01 | lm-global-supervised-tokens: 1.186551E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.757419E-02 | lm-token-buffer-mib: 5.821729E+01 | global_load_balancing_loss: 9.993966E-01 | loss scale: 1.0 | grad norm: 0.320 | params norm: 11402.784 | params rms: 0.061742 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 7:32:04.720713 | finish at 2026-10-10 20:02:30 + [2026-10-07 12:31:58] iteration 3692/ 6855 | consumed samples: 472576 | elapsed time per iteration (ms): 92934.6 | throughput per GPU (TFLOP/s/GPU): 41.2 | MFU 13.20% | FLOPs estimator: workload-aware | logical tokens/s: 180527.1 | valid tokens/s: 180527.1 | physical tokens/s: 180862.8 | loss tokens/s: 136799.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.927508E-01 | lm-global-supervised-tokens: 1.271342E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.530337E-02 | lm-token-buffer-mib: 5.813579E+01 | global_load_balancing_loss: 9.990745E-01 | loss scale: 1.0 | grad norm: 0.054 | params norm: 11402.724 | params rms: 0.061742 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 9:39:12.183902 | finish at 2026-10-10 22:11:10 + [2026-10-07 12:33:26] iteration 3693/ 6855 | consumed samples: 472704 | elapsed time per iteration (ms): 88490.6 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.49% | FLOPs estimator: workload-aware | logical tokens/s: 189592.3 | valid tokens/s: 189592.3 | physical tokens/s: 189949.8 | loss tokens/s: 140555.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.228491E-01 | lm-global-supervised-tokens: 1.243780E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.188252E-02 | lm-token-buffer-mib: 5.824171E+01 | global_load_balancing_loss: 1.000234E+00 | loss scale: 1.0 | grad norm: 0.426 | params norm: 11402.662 | params rms: 0.061742 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 5:43:27.224020 | finish at 2026-10-10 18:16:54 + [2026-10-07 12:34:55] iteration 3694/ 6855 | consumed samples: 472832 | elapsed time per iteration (ms): 88979.3 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.39% | FLOPs estimator: workload-aware | logical tokens/s: 188551.6 | valid tokens/s: 188551.6 | physical tokens/s: 189045.1 | loss tokens/s: 143300.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.980941E-01 | lm-global-supervised-tokens: 1.275075E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.187428E-02 | lm-token-buffer-mib: 5.822143E+01 | global_load_balancing_loss: 1.000564E+00 | loss scale: 1.0 | grad norm: 0.152 | params norm: 11402.599 | params rms: 0.061741 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 6:07:43.426439 | finish at 2026-10-10 18:42:39 + [2026-10-07 12:36:20] iteration 3695/ 6855 | consumed samples: 472960 | elapsed time per iteration (ms): 85152.2 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.40% | FLOPs estimator: workload-aware | logical tokens/s: 197025.4 | valid tokens/s: 197025.4 | physical tokens/s: 197517.2 | loss tokens/s: 151196.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.479635E-01 | lm-global-supervised-tokens: 1.287469E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.342544E-02 | lm-token-buffer-mib: 5.809457E+01 | global_load_balancing_loss: 1.001765E+00 | loss scale: 1.0 | grad norm: 0.200 | params norm: 11402.536 | params rms: 0.061741 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 2:44:40.982084 | finish at 2026-10-10 15:21:01 + [2026-10-07 12:37:48] iteration 3696/ 6855 | consumed samples: 473088 | elapsed time per iteration (ms): 87292.1 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.45% | FLOPs estimator: workload-aware | logical tokens/s: 192196.2 | valid tokens/s: 192196.2 | physical tokens/s: 192685.9 | loss tokens/s: 136272.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.751985E-01 | lm-global-supervised-tokens: 1.189554E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.338685E-02 | lm-token-buffer-mib: 5.833026E+01 | global_load_balancing_loss: 1.001265E+00 | loss scale: 1.0 | grad norm: 0.090 | params norm: 11402.473 | params rms: 0.061741 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 4:35:55.697808 | finish at 2026-10-10 17:13:43 + [2026-10-07 12:39:13] iteration 3697/ 6855 | consumed samples: 473216 | elapsed time per iteration (ms): 85671.7 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.40% | FLOPs estimator: workload-aware | logical tokens/s: 195830.6 | valid tokens/s: 195830.6 | physical tokens/s: 196475.6 | loss tokens/s: 150591.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.474134E-01 | lm-global-supervised-tokens: 1.290143E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.198023E-02 | lm-token-buffer-mib: 5.829571E+01 | global_load_balancing_loss: 1.000450E+00 | loss scale: 1.0 | grad norm: 0.110 | params norm: 11402.411 | params rms: 0.061740 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 3:09:11.183427 | finish at 2026-10-10 15:48:25 + [2026-10-07 12:40:42] iteration 3698/ 6855 | consumed samples: 473344 | elapsed time per iteration (ms): 88697.6 | throughput per GPU (TFLOP/s/GPU): 41.7 | MFU 13.37% | FLOPs estimator: workload-aware | logical tokens/s: 189149.9 | valid tokens/s: 189149.9 | physical tokens/s: 189485.1 | loss tokens/s: 142574.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.741659E-01 | lm-global-supervised-tokens: 1.264602E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.302163E-02 | lm-token-buffer-mib: 5.807229E+01 | global_load_balancing_loss: 1.000398E+00 | loss scale: 1.0 | grad norm: 0.065 | params norm: 11402.348 | params rms: 0.061740 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 5:46:58.414674 | finish at 2026-10-10 18:27:40 + [2026-10-07 12:42:10] iteration 3699/ 6855 | consumed samples: 473472 | elapsed time per iteration (ms): 87677.1 | throughput per GPU (TFLOP/s/GPU): 43.3 | MFU 13.87% | FLOPs estimator: workload-aware | logical tokens/s: 191351.5 | valid tokens/s: 191351.5 | physical tokens/s: 191899.3 | loss tokens/s: 139654.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.258488E-01 | lm-global-supervised-tokens: 1.224451E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.488658E-02 | lm-token-buffer-mib: 5.814896E+01 | global_load_balancing_loss: 9.997473E-01 | loss scale: 1.0 | grad norm: 0.177 | params norm: 11402.285 | params rms: 0.061740 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 4:51:49.079415 | finish at 2026-10-10 17:33:59 + [2026-10-07 12:43:55] iteration 3700/ 6855 | consumed samples: 473600 | elapsed time per iteration (ms): 104962.9 | throughput per GPU (TFLOP/s/GPU): 34.2 | MFU 10.96% | FLOPs estimator: workload-aware | logical tokens/s: 159839.1 | valid tokens/s: 159839.0 | physical tokens/s: 160223.7 | loss tokens/s: 123885.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.365671E-01 | lm-global-supervised-tokens: 1.300338E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.862532E-02 | lm-token-buffer-mib: 5.818211E+01 | global_load_balancing_loss: 9.990216E-01 | loss scale: 1.0 | grad norm: 0.089 | params norm: 11402.220 | params rms: 0.061739 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 19:59:18.018461 | finish at 2026-10-11 08:43:13 + [2026-10-07 12:45:24] iteration 3701/ 6855 | consumed samples: 473728 | elapsed time per iteration (ms): 88825.4 | throughput per GPU (TFLOP/s/GPU): 41.2 | MFU 13.22% | FLOPs estimator: workload-aware | logical tokens/s: 188877.8 | valid tokens/s: 188877.7 | physical tokens/s: 189406.7 | loss tokens/s: 141586.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.448423E-01 | lm-global-supervised-tokens: 1.257650E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.283611E-02 | lm-token-buffer-mib: 5.822146E+01 | global_load_balancing_loss: 9.996242E-01 | loss scale: 1.0 | grad norm: 0.214 | params norm: 11402.155 | params rms: 0.061739 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 5:49:15.367606 | finish at 2026-10-10 18:34:39 + [2026-10-07 12:46:54] iteration 3702/ 6855 | consumed samples: 473856 | elapsed time per iteration (ms): 90946.6 | throughput per GPU (TFLOP/s/GPU): 41.3 | MFU 13.25% | FLOPs estimator: workload-aware | logical tokens/s: 184472.6 | valid tokens/s: 184472.6 | physical tokens/s: 184901.1 | loss tokens/s: 131874.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.620892E-01 | lm-global-supervised-tokens: 1.199352E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.505566E-02 | lm-token-buffer-mib: 5.809220E+01 | global_load_balancing_loss: 1.000932E+00 | loss scale: 1.0 | grad norm: 0.230 | params norm: 11402.091 | params rms: 0.061739 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 7:39:14.657489 | finish at 2026-10-10 20:26:09 + [2026-10-07 12:48:22] iteration 3703/ 6855 | consumed samples: 473984 | elapsed time per iteration (ms): 87244.3 | throughput per GPU (TFLOP/s/GPU): 43.4 | MFU 13.90% | FLOPs estimator: workload-aware | logical tokens/s: 192301.5 | valid tokens/s: 192301.5 | physical tokens/s: 192775.6 | loss tokens/s: 134965.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.080290E-01 | lm-global-supervised-tokens: 1.177494E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.589818E-02 | lm-token-buffer-mib: 5.824976E+01 | global_load_balancing_loss: 1.000745E+00 | loss scale: 1.0 | grad norm: 0.048 | params norm: 11402.025 | params rms: 0.061738 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 4:23:14.068569 | finish at 2026-10-10 17:11:36 + [2026-10-07 12:49:49] iteration 3704/ 6855 | consumed samples: 474112 | elapsed time per iteration (ms): 87256.1 | throughput per GPU (TFLOP/s/GPU): 43.2 | MFU 13.84% | FLOPs estimator: workload-aware | logical tokens/s: 192274.9 | valid tokens/s: 192274.9 | physical tokens/s: 192810.1 | loss tokens/s: 138660.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.664835E-01 | lm-global-supervised-tokens: 1.209900E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.481830E-02 | lm-token-buffer-mib: 5.813443E+01 | global_load_balancing_loss: 1.000605E+00 | loss scale: 1.0 | grad norm: 0.070 | params norm: 11401.957 | params rms: 0.061738 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 4:22:23.891276 | finish at 2026-10-10 17:12:13 + [2026-10-07 12:51:20] iteration 3705/ 6855 | consumed samples: 474240 | elapsed time per iteration (ms): 90757.5 | throughput per GPU (TFLOP/s/GPU): 41.7 | MFU 13.36% | FLOPs estimator: workload-aware | logical tokens/s: 184856.1 | valid tokens/s: 184856.1 | physical tokens/s: 185285.2 | loss tokens/s: 135521.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.672964E-01 | lm-global-supervised-tokens: 1.229956E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.171005E-02 | lm-token-buffer-mib: 5.808207E+01 | global_load_balancing_loss: 9.999322E-01 | loss scale: 1.0 | grad norm: 0.126 | params norm: 11401.891 | params rms: 0.061738 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 7:24:46.268234 | finish at 2026-10-10 20:16:06 + [2026-10-07 12:52:47] iteration 3706/ 6855 | consumed samples: 474368 | elapsed time per iteration (ms): 86977.7 | throughput per GPU (TFLOP/s/GPU): 42.8 | MFU 13.70% | FLOPs estimator: workload-aware | logical tokens/s: 192890.7 | valid tokens/s: 192890.7 | physical tokens/s: 193309.0 | loss tokens/s: 143668.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.067722E-01 | lm-global-supervised-tokens: 1.249596E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.552249E-02 | lm-token-buffer-mib: 5.802249E+01 | global_load_balancing_loss: 1.000179E+00 | loss scale: 1.0 | grad norm: 0.059 | params norm: 11401.825 | params rms: 0.061737 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 4:04:52.650403 | finish at 2026-10-10 16:57:39 + [2026-10-07 12:54:17] iteration 3707/ 6855 | consumed samples: 474496 | elapsed time per iteration (ms): 90055.8 | throughput per GPU (TFLOP/s/GPU): 41.1 | MFU 13.17% | FLOPs estimator: workload-aware | logical tokens/s: 186297.3 | valid tokens/s: 186297.3 | physical tokens/s: 186591.3 | loss tokens/s: 141858.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.652601E-01 | lm-global-supervised-tokens: 1.277518E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.133074E-02 | lm-token-buffer-mib: 5.820735E+01 | global_load_balancing_loss: 9.995076E-01 | loss scale: 1.0 | grad norm: 0.129 | params norm: 11401.759 | params rms: 0.061737 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 6:44:55.502915 | finish at 2026-10-10 19:39:12 + [2026-10-07 12:55:46] iteration 3708/ 6855 | consumed samples: 474624 | elapsed time per iteration (ms): 89680.1 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.58% | FLOPs estimator: workload-aware | logical tokens/s: 187077.8 | valid tokens/s: 187077.8 | physical tokens/s: 187450.0 | loss tokens/s: 138483.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.787408E-01 | lm-global-supervised-tokens: 1.241919E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.140845E-02 | lm-token-buffer-mib: 5.807438E+01 | global_load_balancing_loss: 9.987206E-01 | loss scale: 1.0 | grad norm: 0.051 | params norm: 11401.693 | params rms: 0.061736 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 6:23:43.134280 | finish at 2026-10-10 19:19:30 + [2026-10-07 12:57:15] iteration 3709/ 6855 | consumed samples: 474752 | elapsed time per iteration (ms): 88263.1 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.40% | FLOPs estimator: workload-aware | logical tokens/s: 190081.7 | valid tokens/s: 190081.7 | physical tokens/s: 190426.0 | loss tokens/s: 138985.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.341977E-01 | lm-global-supervised-tokens: 1.226725E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.666084E-02 | lm-token-buffer-mib: 5.806815E+01 | global_load_balancing_loss: 1.000511E+00 | loss scale: 1.0 | grad norm: 0.076 | params norm: 11401.628 | params rms: 0.061736 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 5:07:55.578802 | finish at 2026-10-10 18:05:10 + [2026-10-07 12:59:05] iteration 3710/ 6855 | consumed samples: 474880 | elapsed time per iteration (ms): 110124.0 | throughput per GPU (TFLOP/s/GPU): 33.9 | MFU 10.86% | FLOPs estimator: workload-aware | logical tokens/s: 152348.0 | valid tokens/s: 152348.0 | physical tokens/s: 152670.6 | loss tokens/s: 111514.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.412996E-01 | lm-global-supervised-tokens: 1.228039E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.280965E-02 | lm-token-buffer-mib: 5.808340E+01 | global_load_balancing_loss: 1.000847E+00 | loss scale: 1.0 | grad norm: 0.112 | params norm: 11401.564 | params rms: 0.061736 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 0:12:20.127194 | finish at 2026-10-11 13:11:25 + [2026-10-07 13:00:31] iteration 3711/ 6855 | consumed samples: 475008 | elapsed time per iteration (ms): 86448.3 | throughput per GPU (TFLOP/s/GPU): 42.8 | MFU 13.72% | FLOPs estimator: workload-aware | logical tokens/s: 194071.8 | valid tokens/s: 194071.8 | physical tokens/s: 194579.3 | loss tokens/s: 149514.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.775698E-01 | lm-global-supervised-tokens: 1.292524E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.165145E-02 | lm-token-buffer-mib: 5.813525E+01 | global_load_balancing_loss: 1.000720E+00 | loss scale: 1.0 | grad norm: 0.037 | params norm: 11401.500 | params rms: 0.061735 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 3:29:53.546288 | finish at 2026-10-10 16:30:25 + [2026-10-07 13:01:58] iteration 3712/ 6855 | consumed samples: 475136 | elapsed time per iteration (ms): 86483.9 | throughput per GPU (TFLOP/s/GPU): 42.8 | MFU 13.70% | FLOPs estimator: workload-aware | logical tokens/s: 193991.9 | valid tokens/s: 193991.9 | physical tokens/s: 194416.1 | loss tokens/s: 146493.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.238659E-01 | lm-global-supervised-tokens: 1.266936E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.786464E-02 | lm-token-buffer-mib: 5.812841E+01 | global_load_balancing_loss: 1.000386E+00 | loss scale: 1.0 | grad norm: 0.098 | params norm: 11401.435 | params rms: 0.061735 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 3:30:18.799006 | finish at 2026-10-10 16:32:17 + [2026-10-07 13:03:25] iteration 3713/ 6855 | consumed samples: 475264 | elapsed time per iteration (ms): 86990.1 | throughput per GPU (TFLOP/s/GPU): 43.1 | MFU 13.82% | FLOPs estimator: workload-aware | logical tokens/s: 192863.3 | valid tokens/s: 192863.2 | physical tokens/s: 193341.4 | loss tokens/s: 147284.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.309132E-01 | lm-global-supervised-tokens: 1.281225E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.514566E-02 | lm-token-buffer-mib: 5.820512E+01 | global_load_balancing_loss: 1.000711E+00 | loss scale: 1.0 | grad norm: 0.061 | params norm: 11401.368 | params rms: 0.061735 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 3:55:22.880424 | finish at 2026-10-10 16:58:48 + [2026-10-07 13:04:51] iteration 3714/ 6855 | consumed samples: 475392 | elapsed time per iteration (ms): 85783.2 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.75% | FLOPs estimator: workload-aware | logical tokens/s: 195575.7 | valid tokens/s: 195575.7 | physical tokens/s: 196041.5 | loss tokens/s: 146113.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.354267E-01 | lm-global-supervised-tokens: 1.253406E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.143030E-02 | lm-token-buffer-mib: 5.805054E+01 | global_load_balancing_loss: 1.000691E+00 | loss scale: 1.0 | grad norm: 0.040 | params norm: 11401.302 | params rms: 0.061734 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 2:50:45.052249 | finish at 2026-10-10 15:55:36 + [2026-10-07 13:06:20] iteration 3715/ 6855 | consumed samples: 475520 | elapsed time per iteration (ms): 89293.8 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.59% | FLOPs estimator: workload-aware | logical tokens/s: 187887.9 | valid tokens/s: 187887.9 | physical tokens/s: 188265.8 | loss tokens/s: 136799.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.656253E-01 | lm-global-supervised-tokens: 1.221534E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.231692E-02 | lm-token-buffer-mib: 5.811230E+01 | global_load_balancing_loss: 1.000642E+00 | loss scale: 1.0 | grad norm: 0.061 | params norm: 11401.236 | params rms: 0.061734 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 5:53:02.423811 | finish at 2026-10-10 18:59:22 + [2026-10-07 13:07:45] iteration 3716/ 6855 | consumed samples: 475648 | elapsed time per iteration (ms): 84639.4 | throughput per GPU (TFLOP/s/GPU): 43.2 | MFU 13.85% | FLOPs estimator: workload-aware | logical tokens/s: 198219.4 | valid tokens/s: 198219.4 | physical tokens/s: 198768.9 | loss tokens/s: 154180.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.477148E-01 | lm-global-supervised-tokens: 1.304971E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.661962E-02 | lm-token-buffer-mib: 5.824870E+01 | global_load_balancing_loss: 9.990395E-01 | loss scale: 1.0 | grad norm: 0.097 | params norm: 11401.168 | params rms: 0.061734 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 1:48:03.066139 | finish at 2026-10-10 14:55:48 + [2026-10-07 13:09:10] iteration 3717/ 6855 | consumed samples: 475776 | elapsed time per iteration (ms): 85229.2 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.69% | FLOPs estimator: workload-aware | logical tokens/s: 196847.8 | valid tokens/s: 196847.8 | physical tokens/s: 197470.9 | loss tokens/s: 146033.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.234267E-01 | lm-global-supervised-tokens: 1.244628E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.232680E-02 | lm-token-buffer-mib: 5.840884E+01 | global_load_balancing_loss: 9.993693E-01 | loss scale: 1.0 | grad norm: 0.099 | params norm: 11401.103 | params rms: 0.061733 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 2:17:29.299570 | finish at 2026-10-10 15:26:39 + [2026-10-07 13:10:36] iteration 3718/ 6855 | consumed samples: 475904 | elapsed time per iteration (ms): 86307.0 | throughput per GPU (TFLOP/s/GPU): 43.4 | MFU 13.90% | FLOPs estimator: workload-aware | logical tokens/s: 194389.2 | valid tokens/s: 194389.2 | physical tokens/s: 194759.7 | loss tokens/s: 138398.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.636453E-01 | lm-global-supervised-tokens: 1.194474E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.251317E-02 | lm-token-buffer-mib: 5.806149E+01 | global_load_balancing_loss: 9.999007E-01 | loss scale: 1.0 | grad norm: 0.041 | params norm: 11401.039 | params rms: 0.061733 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 3:12:25.170195 | finish at 2026-10-10 16:23:01 + [2026-10-07 13:12:02] iteration 3719/ 6855 | consumed samples: 476032 | elapsed time per iteration (ms): 85700.5 | throughput per GPU (TFLOP/s/GPU): 43.8 | MFU 14.04% | FLOPs estimator: workload-aware | logical tokens/s: 195765.7 | valid tokens/s: 195765.7 | physical tokens/s: 196391.2 | loss tokens/s: 142482.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.745457E-01 | lm-global-supervised-tokens: 1.221079E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.175875E-02 | lm-token-buffer-mib: 5.827837E+01 | global_load_balancing_loss: 1.000983E+00 | loss scale: 1.0 | grad norm: 0.106 | params norm: 11400.976 | params rms: 0.061733 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 2:39:16.670837 | finish at 2026-10-10 15:51:18 + [2026-10-07 13:13:53] iteration 3720/ 6855 | consumed samples: 476160 | elapsed time per iteration (ms): 111324.1 | throughput per GPU (TFLOP/s/GPU): 33.2 | MFU 10.64% | FLOPs estimator: workload-aware | logical tokens/s: 150705.5 | valid tokens/s: 150705.4 | physical tokens/s: 151140.6 | loss tokens/s: 112871.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.880429E-01 | lm-global-supervised-tokens: 1.256533E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.199043E-02 | lm-token-buffer-mib: 5.814175E+01 | global_load_balancing_loss: 9.995937E-01 | loss scale: 1.0 | grad norm: 0.187 | params norm: 11400.912 | params rms: 0.061732 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 0:56:41.126057 | finish at 2026-10-11 14:10:34 +times across ranks (ms): + save-checkpoint: + rank 0: 24197.71 + rank 1: 24197.69 + rank 2: 24197.76 + rank 3: 24197.88 + rank 4: 24197.72 + rank 5: 24197.70 + rank 6: 24197.70 + rank 7: 24197.74 + rank 8: 24197.95 + rank 9: 24197.93 + rank 10: 24197.93 + rank 11: 24197.93 + rank 12: 24197.94 + rank 13: 24197.92 + rank 14: 24197.92 + rank 15: 24197.91 + rank 16: 24198.25 + rank 17: 24198.21 + rank 18: 24198.19 + rank 19: 24198.21 + rank 20: 24198.20 + rank 21: 24198.22 + rank 22: 24198.20 + rank 23: 24198.19 + rank 24: 24198.17 + rank 25: 24198.09 + rank 26: 24198.14 + rank 27: 24198.14 + rank 28: 24198.14 + rank 29: 24198.15 + rank 30: 24198.14 + rank 31: 24198.14 + rank 32: 24197.20 + rank 33: 24197.24 + rank 34: 24197.21 + rank 35: 24197.20 + rank 36: 24197.17 + rank 37: 24197.18 + rank 38: 24197.17 + rank 39: 24197.16 + rank 40: 24197.48 + rank 41: 24197.44 + rank 42: 24197.45 + rank 43: 24197.45 + rank 44: 24197.43 + rank 45: 24197.44 + rank 46: 24197.45 + rank 47: 24197.43 + rank 48: 24198.22 + rank 49: 24198.21 + rank 50: 24198.19 + rank 51: 24198.20 + rank 52: 24198.19 + rank 53: 24198.22 + rank 54: 24198.20 + rank 55: 24198.21 + rank 56: 24197.68 + rank 57: 24197.68 + rank 58: 24197.67 + rank 59: 24197.65 + rank 60: 24197.65 + rank 61: 24197.66 + rank 62: 24197.64 + rank 63: 24197.69 + [2026-10-07 13:15:52] iteration 3721/ 6855 | consumed samples: 476288 | elapsed time per iteration (ms): 94369.5 | throughput per GPU (TFLOP/s/GPU): 40.2 | MFU 12.90% | FLOPs estimator: workload-aware | logical tokens/s: 177781.8 | valid tokens/s: 177781.8 | physical tokens/s: 178136.8 | loss tokens/s: 126684.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.815255E-01 | lm-global-supervised-tokens: 1.195515E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.787610E-02 | lm-token-buffer-mib: 5.807854E+01 | global_load_balancing_loss: 1.001667E+00 | loss scale: 1.0 | grad norm: 0.125 | params norm: 11400.847 | params rms: 0.061732 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 10:09:14.074773 | finish at 2026-10-10 23:25:06 + [2026-10-07 13:17:24] iteration 3722/ 6855 | consumed samples: 476416 | elapsed time per iteration (ms): 92291.4 | throughput per GPU (TFLOP/s/GPU): 39.6 | MFU 12.68% | FLOPs estimator: workload-aware | logical tokens/s: 181784.5 | valid tokens/s: 181784.5 | physical tokens/s: 182297.2 | loss tokens/s: 134848.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.670045E-01 | lm-global-supervised-tokens: 1.244540E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.357870E-02 | lm-token-buffer-mib: 5.811244E+01 | global_load_balancing_loss: 1.000224E+00 | loss scale: 1.0 | grad norm: 0.156 | params norm: 11400.783 | params rms: 0.061732 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 8:19:08.979217 | finish at 2026-10-10 21:36:33 + [2026-10-07 13:18:49] iteration 3723/ 6855 | consumed samples: 476544 | elapsed time per iteration (ms): 85157.9 | throughput per GPU (TFLOP/s/GPU): 43.5 | MFU 13.94% | FLOPs estimator: workload-aware | logical tokens/s: 197012.7 | valid tokens/s: 197012.7 | physical tokens/s: 197296.4 | loss tokens/s: 144273.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.094091E-01 | lm-global-supervised-tokens: 1.228603E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.376814E-02 | lm-token-buffer-mib: 5.808212E+01 | global_load_balancing_loss: 9.993775E-01 | loss scale: 1.0 | grad norm: 0.113 | params norm: 11400.719 | params rms: 0.061731 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 2:05:14.433329 | finish at 2026-10-10 15:24:04 + [2026-10-07 13:20:15] iteration 3724/ 6855 | consumed samples: 476672 | elapsed time per iteration (ms): 85699.1 | throughput per GPU (TFLOP/s/GPU): 43.4 | MFU 13.90% | FLOPs estimator: workload-aware | logical tokens/s: 195768.5 | valid tokens/s: 195768.5 | physical tokens/s: 196220.4 | loss tokens/s: 147492.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.998454E-01 | lm-global-supervised-tokens: 1.263994E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.397424E-02 | lm-token-buffer-mib: 5.806995E+01 | global_load_balancing_loss: 9.998363E-01 | loss scale: 1.0 | grad norm: 0.160 | params norm: 11400.655 | params rms: 0.061731 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 2:32:04.008312 | finish at 2026-10-10 15:52:19 + [2026-10-07 13:21:42] iteration 3725/ 6855 | consumed samples: 476800 | elapsed time per iteration (ms): 86905.0 | throughput per GPU (TFLOP/s/GPU): 43.2 | MFU 13.85% | FLOPs estimator: workload-aware | logical tokens/s: 193051.4 | valid tokens/s: 193051.3 | physical tokens/s: 193380.2 | loss tokens/s: 138265.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.450162E-01 | lm-global-supervised-tokens: 1.201593E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.386590E-02 | lm-token-buffer-mib: 5.807592E+01 | global_load_balancing_loss: 1.000824E+00 | loss scale: 1.0 | grad norm: 0.083 | params norm: 11400.593 | params rms: 0.061731 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 3:33:32.755132 | finish at 2026-10-10 16:55:15 + [2026-10-07 13:23:10] iteration 3726/ 6855 | consumed samples: 476928 | elapsed time per iteration (ms): 87558.1 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.52% | FLOPs estimator: workload-aware | logical tokens/s: 191611.6 | valid tokens/s: 191611.6 | physical tokens/s: 191962.6 | loss tokens/s: 138380.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.622809E-01 | lm-global-supervised-tokens: 1.211636E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.330291E-02 | lm-token-buffer-mib: 5.817457E+01 | global_load_balancing_loss: 1.000158E+00 | loss scale: 1.0 | grad norm: 0.136 | params norm: 11400.531 | params rms: 0.061730 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 4:06:09.347074 | finish at 2026-10-10 17:29:19 + [2026-10-07 13:24:36] iteration 3727/ 6855 | consumed samples: 477056 | elapsed time per iteration (ms): 86841.2 | throughput per GPU (TFLOP/s/GPU): 43.3 | MFU 13.87% | FLOPs estimator: workload-aware | logical tokens/s: 193194.0 | valid tokens/s: 193194.0 | physical tokens/s: 193579.4 | loss tokens/s: 143231.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.624231E-01 | lm-global-supervised-tokens: 1.243836E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.149015E-02 | lm-token-buffer-mib: 5.815721E+01 | global_load_balancing_loss: 1.000638E+00 | loss scale: 1.0 | grad norm: 0.082 | params norm: 11400.471 | params rms: 0.061730 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 3:27:19.353752 | finish at 2026-10-10 16:51:56 + [2026-10-07 13:26:07] iteration 3728/ 6855 | consumed samples: 477184 | elapsed time per iteration (ms): 90171.9 | throughput per GPU (TFLOP/s/GPU): 41.5 | MFU 13.30% | FLOPs estimator: workload-aware | logical tokens/s: 186057.5 | valid tokens/s: 186057.5 | physical tokens/s: 186415.2 | loss tokens/s: 136025.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.714898E-01 | lm-global-supervised-tokens: 1.226565E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.353328E-02 | lm-token-buffer-mib: 5.810275E+01 | global_load_balancing_loss: 9.998425E-01 | loss scale: 1.0 | grad norm: 0.093 | params norm: 11400.411 | params rms: 0.061730 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 6:19:27.388263 | finish at 2026-10-10 19:45:34 + [2026-10-07 13:27:35] iteration 3729/ 6855 | consumed samples: 477312 | elapsed time per iteration (ms): 88290.3 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.50% | FLOPs estimator: workload-aware | logical tokens/s: 190022.6 | valid tokens/s: 190022.6 | physical tokens/s: 190516.3 | loss tokens/s: 143096.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.602908E-01 | lm-global-supervised-tokens: 1.263404E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.746553E-02 | lm-token-buffer-mib: 5.821994E+01 | global_load_balancing_loss: 9.986427E-01 | loss scale: 1.0 | grad norm: 0.042 | params norm: 11400.351 | params rms: 0.061729 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 4:39:55.631740 | finish at 2026-10-10 18:07:30 + [2026-10-07 13:29:25] iteration 3730/ 6855 | consumed samples: 477440 | elapsed time per iteration (ms): 110530.7 | throughput per GPU (TFLOP/s/GPU): 33.5 | MFU 10.74% | FLOPs estimator: workload-aware | logical tokens/s: 151787.6 | valid tokens/s: 151787.6 | physical tokens/s: 152232.2 | loss tokens/s: 113302.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.072240E-01 | lm-global-supervised-tokens: 1.252341E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.159029E-02 | lm-token-buffer-mib: 5.811865E+01 | global_load_balancing_loss: 1.000152E+00 | loss scale: 1.0 | grad norm: 0.087 | params norm: 11400.292 | params rms: 0.061729 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 23:56:48.536494 | finish at 2026-10-11 13:26:14 + [2026-10-07 13:30:51] iteration 3731/ 6855 | consumed samples: 477568 | elapsed time per iteration (ms): 85940.4 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.66% | FLOPs estimator: workload-aware | logical tokens/s: 195218.3 | valid tokens/s: 195218.3 | physical tokens/s: 195807.9 | loss tokens/s: 144082.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.683229E-01 | lm-global-supervised-tokens: 1.238251E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.547061E-02 | lm-token-buffer-mib: 5.820400E+01 | global_load_balancing_loss: 1.001297E+00 | loss scale: 1.0 | grad norm: 0.060 | params norm: 11400.233 | params rms: 0.061729 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 2:34:37.815200 | finish at 2026-10-10 16:05:29 + [2026-10-07 13:32:18] iteration 3732/ 6855 | consumed samples: 477696 | elapsed time per iteration (ms): 86983.5 | throughput per GPU (TFLOP/s/GPU): 43.5 | MFU 13.94% | FLOPs estimator: workload-aware | logical tokens/s: 192877.8 | valid tokens/s: 192877.8 | physical tokens/s: 193262.4 | loss tokens/s: 137361.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.369966E-01 | lm-global-supervised-tokens: 1.194816E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.305798E-02 | lm-token-buffer-mib: 5.821651E+01 | global_load_balancing_loss: 1.001369E+00 | loss scale: 1.0 | grad norm: 0.110 | params norm: 11400.174 | params rms: 0.061728 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 3:27:29.499551 | finish at 2026-10-10 16:59:48 + [2026-10-07 13:33:44] iteration 3733/ 6855 | consumed samples: 477824 | elapsed time per iteration (ms): 85405.3 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.69% | FLOPs estimator: workload-aware | logical tokens/s: 196441.1 | valid tokens/s: 196441.1 | physical tokens/s: 196979.4 | loss tokens/s: 153097.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.885706E-01 | lm-global-supervised-tokens: 1.307536E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.162416E-02 | lm-token-buffer-mib: 5.835085E+01 | global_load_balancing_loss: 9.997504E-01 | loss scale: 1.0 | grad norm: 0.072 | params norm: 11400.111 | params rms: 0.061728 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 2:03:55.366391 | finish at 2026-10-10 15:37:39 + [2026-10-07 13:35:07] iteration 3734/ 6855 | consumed samples: 477952 | elapsed time per iteration (ms): 82986.4 | throughput per GPU (TFLOP/s/GPU): 43.4 | MFU 13.92% | FLOPs estimator: workload-aware | logical tokens/s: 202167.1 | valid tokens/s: 202167.0 | physical tokens/s: 202756.1 | loss tokens/s: 151327.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.774671E-01 | lm-global-supervised-tokens: 1.255809E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.255955E-02 | lm-token-buffer-mib: 5.812909E+01 | global_load_balancing_loss: 1.000564E+00 | loss scale: 1.0 | grad norm: 0.091 | params norm: 11400.050 | params rms: 0.061728 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 23:56:40.625488 | finish at 2026-10-10 13:31:47 + [2026-10-07 13:36:36] iteration 3735/ 6855 | consumed samples: 478080 | elapsed time per iteration (ms): 89761.4 | throughput per GPU (TFLOP/s/GPU): 41.7 | MFU 13.36% | FLOPs estimator: workload-aware | logical tokens/s: 186909.0 | valid tokens/s: 186909.0 | physical tokens/s: 187479.2 | loss tokens/s: 133788.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.434314E-01 | lm-global-supervised-tokens: 1.200906E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.167068E-02 | lm-token-buffer-mib: 5.853362E+01 | global_load_balancing_loss: 9.999667E-01 | loss scale: 1.0 | grad norm: 0.050 | params norm: 11399.990 | params rms: 0.061727 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 5:47:35.616302 | finish at 2026-10-10 19:24:12 + [2026-10-07 13:38:04] iteration 3736/ 6855 | consumed samples: 478208 | elapsed time per iteration (ms): 87262.9 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.54% | FLOPs estimator: workload-aware | logical tokens/s: 192259.4 | valid tokens/s: 192259.4 | physical tokens/s: 192959.2 | loss tokens/s: 138688.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.019681E-01 | lm-global-supervised-tokens: 1.210236E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.316952E-02 | lm-token-buffer-mib: 5.852384E+01 | global_load_balancing_loss: 1.000089E+00 | loss scale: 1.0 | grad norm: 0.080 | params norm: 11399.929 | params rms: 0.061727 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 3:36:12.986199 | finish at 2026-10-10 17:14:17 + [2026-10-07 13:39:33] iteration 3737/ 6855 | consumed samples: 478336 | elapsed time per iteration (ms): 89704.6 | throughput per GPU (TFLOP/s/GPU): 41.4 | MFU 13.26% | FLOPs estimator: workload-aware | logical tokens/s: 187027.3 | valid tokens/s: 187027.3 | physical tokens/s: 187349.8 | loss tokens/s: 131310.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.497437E-01 | lm-global-supervised-tokens: 1.177919E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.269016E-02 | lm-token-buffer-mib: 5.813074E+01 | global_load_balancing_loss: 9.994288E-01 | loss scale: 1.0 | grad norm: 0.078 | params norm: 11399.868 | params rms: 0.061727 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 5:41:39.030818 | finish at 2026-10-10 19:21:12 + [2026-10-07 13:41:03] iteration 3738/ 6855 | consumed samples: 478464 | elapsed time per iteration (ms): 89395.2 | throughput per GPU (TFLOP/s/GPU): 41.3 | MFU 13.24% | FLOPs estimator: workload-aware | logical tokens/s: 187674.7 | valid tokens/s: 187674.7 | physical tokens/s: 188091.9 | loss tokens/s: 140308.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.728492E-01 | lm-global-supervised-tokens: 1.254292E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.464772E-02 | lm-token-buffer-mib: 5.804695E+01 | global_load_balancing_loss: 1.000110E+00 | loss scale: 1.0 | grad norm: 0.058 | params norm: 11399.807 | params rms: 0.061726 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 5:24:04.818379 | finish at 2026-10-10 19:05:08 + [2026-10-07 13:42:30] iteration 3739/ 6855 | consumed samples: 478592 | elapsed time per iteration (ms): 87394.2 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.52% | FLOPs estimator: workload-aware | logical tokens/s: 191971.4 | valid tokens/s: 191971.4 | physical tokens/s: 192556.6 | loss tokens/s: 139710.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.608814E-01 | lm-global-supervised-tokens: 1.220991E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.278362E-02 | lm-token-buffer-mib: 5.824051E+01 | global_load_balancing_loss: 1.000891E+00 | loss scale: 1.0 | grad norm: 0.176 | params norm: 11399.746 | params rms: 0.061726 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 3:38:40.211576 | finish at 2026-10-10 17:21:10 + [2026-10-07 13:44:14] iteration 3740/ 6855 | consumed samples: 478720 | elapsed time per iteration (ms): 103813.1 | throughput per GPU (TFLOP/s/GPU): 34.7 | MFU 11.12% | FLOPs estimator: workload-aware | logical tokens/s: 161609.4 | valid tokens/s: 161609.4 | physical tokens/s: 162022.2 | loss tokens/s: 123433.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.371756E-01 | lm-global-supervised-tokens: 1.281403E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.329781E-02 | lm-token-buffer-mib: 5.811021E+01 | global_load_balancing_loss: 1.000826E+00 | loss scale: 1.0 | grad norm: 0.160 | params norm: 11399.686 | params rms: 0.061726 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 17:49:37.677585 | finish at 2026-10-11 07:33:52 + [2026-10-07 13:45:41] iteration 3741/ 6855 | consumed samples: 478848 | elapsed time per iteration (ms): 86818.8 | throughput per GPU (TFLOP/s/GPU): 43.2 | MFU 13.84% | FLOPs estimator: workload-aware | logical tokens/s: 193243.2 | valid tokens/s: 193243.2 | physical tokens/s: 193600.9 | loss tokens/s: 135482.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.170498E-01 | lm-global-supervised-tokens: 1.176241E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.102746E-02 | lm-token-buffer-mib: 5.810905E+01 | global_load_balancing_loss: 1.000080E+00 | loss scale: 1.0 | grad norm: 0.060 | params norm: 11399.626 | params rms: 0.061725 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 3:05:53.627295 | finish at 2026-10-10 16:51:34 + [2026-10-07 13:47:08] iteration 3742/ 6855 | consumed samples: 478976 | elapsed time per iteration (ms): 87024.6 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.55% | FLOPs estimator: workload-aware | logical tokens/s: 192787.1 | valid tokens/s: 192787.1 | physical tokens/s: 193278.9 | loss tokens/s: 145875.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.489756E-01 | lm-global-supervised-tokens: 1.269475E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.153060E-02 | lm-token-buffer-mib: 5.836267E+01 | global_load_balancing_loss: 9.998041E-01 | loss scale: 1.0 | grad norm: 0.109 | params norm: 11399.565 | params rms: 0.061725 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 3:15:07.495280 | finish at 2026-10-10 17:02:15 + [2026-10-07 13:48:34] iteration 3743/ 6855 | consumed samples: 479104 | elapsed time per iteration (ms): 85783.1 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.63% | FLOPs estimator: workload-aware | logical tokens/s: 195577.2 | valid tokens/s: 195577.2 | physical tokens/s: 195961.5 | loss tokens/s: 143917.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.832633E-01 | lm-global-supervised-tokens: 1.234571E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.138802E-02 | lm-token-buffer-mib: 5.807664E+01 | global_load_balancing_loss: 1.000113E+00 | loss scale: 1.0 | grad norm: 0.051 | params norm: 11399.502 | params rms: 0.061725 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 2:09:16.923758 | finish at 2026-10-10 15:57:51 + [2026-10-07 13:50:04] iteration 3744/ 6855 | consumed samples: 479232 | elapsed time per iteration (ms): 90207.3 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.42% | FLOPs estimator: workload-aware | logical tokens/s: 185984.9 | valid tokens/s: 185984.9 | physical tokens/s: 186924.2 | loss tokens/s: 136881.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.303202E-01 | lm-global-supervised-tokens: 1.234776E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.141699E-02 | lm-token-buffer-mib: 5.902916E+01 | global_load_balancing_loss: 1.000437E+00 | loss scale: 1.0 | grad norm: 0.095 | params norm: 11399.442 | params rms: 0.061724 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 5:57:15.025846 | finish at 2026-10-10 19:47:19 + [2026-10-07 13:51:33] iteration 3745/ 6855 | consumed samples: 479360 | elapsed time per iteration (ms): 89155.6 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.61% | FLOPs estimator: workload-aware | logical tokens/s: 188179.0 | valid tokens/s: 188179.0 | physical tokens/s: 188593.2 | loss tokens/s: 135267.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.518557E-01 | lm-global-supervised-tokens: 1.205984E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.529252E-02 | lm-token-buffer-mib: 5.813929E+01 | global_load_balancing_loss: 1.000452E+00 | loss scale: 1.0 | grad norm: 0.125 | params norm: 11399.380 | params rms: 0.061724 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 5:01:14.039307 | finish at 2026-10-10 18:52:47 + [2026-10-07 13:52:58] iteration 3746/ 6855 | consumed samples: 479488 | elapsed time per iteration (ms): 84941.4 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.63% | FLOPs estimator: workload-aware | logical tokens/s: 197514.6 | valid tokens/s: 197514.6 | physical tokens/s: 198000.0 | loss tokens/s: 152754.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.338022E-01 | lm-global-supervised-tokens: 1.297514E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.248022E-02 | lm-token-buffer-mib: 5.809126E+01 | global_load_balancing_loss: 9.996229E-01 | loss scale: 1.0 | grad norm: 0.041 | params norm: 11399.318 | params rms: 0.061724 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 1:21:22.902449 | finish at 2026-10-10 15:14:21 + [2026-10-07 13:54:28] iteration 3747/ 6855 | consumed samples: 479616 | elapsed time per iteration (ms): 89959.9 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.62% | FLOPs estimator: workload-aware | logical tokens/s: 186496.3 | valid tokens/s: 186496.3 | physical tokens/s: 186841.0 | loss tokens/s: 138709.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.096537E-01 | lm-global-supervised-tokens: 1.247826E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.416652E-02 | lm-token-buffer-mib: 5.804382E+01 | global_load_balancing_loss: 9.987402E-01 | loss scale: 1.0 | grad norm: 0.097 | params norm: 11399.256 | params rms: 0.061723 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 5:39:55.322954 | finish at 2026-10-10 19:34:23 + [2026-10-07 13:55:58] iteration 3748/ 6855 | consumed samples: 479744 | elapsed time per iteration (ms): 90313.8 | throughput per GPU (TFLOP/s/GPU): 41.4 | MFU 13.26% | FLOPs estimator: workload-aware | logical tokens/s: 185765.7 | valid tokens/s: 185765.7 | physical tokens/s: 186229.0 | loss tokens/s: 133764.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.405128E-01 | lm-global-supervised-tokens: 1.208077E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.212965E-02 | lm-token-buffer-mib: 5.805032E+01 | global_load_balancing_loss: 1.000420E+00 | loss scale: 1.0 | grad norm: 0.053 | params norm: 11399.194 | params rms: 0.061723 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 5:56:44.862081 | finish at 2026-10-10 19:52:43 + [2026-10-07 13:57:25] iteration 3749/ 6855 | consumed samples: 479872 | elapsed time per iteration (ms): 87209.7 | throughput per GPU (TFLOP/s/GPU): 43.3 | MFU 13.89% | FLOPs estimator: workload-aware | logical tokens/s: 192377.2 | valid tokens/s: 192377.1 | physical tokens/s: 192833.8 | loss tokens/s: 140237.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.027323E-01 | lm-global-supervised-tokens: 1.223010E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.162826E-02 | lm-token-buffer-mib: 5.822021E+01 | global_load_balancing_loss: 1.001771E+00 | loss scale: 1.0 | grad norm: 0.087 | params norm: 11399.132 | params rms: 0.061723 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 3:14:33.338161 | finish at 2026-10-10 17:11:59 + [2026-10-07 13:59:17] iteration 3750/ 6855 | consumed samples: 480000 | elapsed time per iteration (ms): 111521.1 | throughput per GPU (TFLOP/s/GPU): 32.9 | MFU 10.54% | FLOPs estimator: workload-aware | logical tokens/s: 150439.8 | valid tokens/s: 150439.8 | physical tokens/s: 150801.8 | loss tokens/s: 109469.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.041921E-01 | lm-global-supervised-tokens: 1.220811E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.542555E-02 | lm-token-buffer-mib: 5.822958E+01 | global_load_balancing_loss: 1.001579E+00 | loss scale: 1.0 | grad norm: 0.068 | params norm: 11399.071 | params rms: 0.061722 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 0:11:13.135564 | finish at 2026-10-11 14:10:30 + [2026-10-07 14:00:45] iteration 3751/ 6855 | consumed samples: 480128 | elapsed time per iteration (ms): 88019.3 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.48% | FLOPs estimator: workload-aware | logical tokens/s: 190608.4 | valid tokens/s: 190608.4 | physical tokens/s: 191157.3 | loss tokens/s: 141108.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.537706E-01 | lm-global-supervised-tokens: 1.242023E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.316399E-02 | lm-token-buffer-mib: 5.816190E+01 | global_load_balancing_loss: 1.002374E+00 | loss scale: 1.0 | grad norm: 0.074 | params norm: 11399.005 | params rms: 0.061722 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 3:53:31.773201 | finish at 2026-10-10 17:54:17 + [2026-10-07 14:02:12] iteration 3752/ 6855 | consumed samples: 480256 | elapsed time per iteration (ms): 86855.6 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.50% | FLOPs estimator: workload-aware | logical tokens/s: 193161.4 | valid tokens/s: 193161.4 | physical tokens/s: 193618.6 | loss tokens/s: 144926.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.165553E-01 | lm-global-supervised-tokens: 1.258769E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.433036E-02 | lm-token-buffer-mib: 5.812533E+01 | global_load_balancing_loss: 9.997362E-01 | loss scale: 1.0 | grad norm: 0.039 | params norm: 11398.944 | params rms: 0.061722 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 2:51:52.964899 | finish at 2026-10-10 16:54:05 + [2026-10-07 14:03:37] iteration 3753/ 6855 | consumed samples: 480384 | elapsed time per iteration (ms): 85552.5 | throughput per GPU (TFLOP/s/GPU): 44.4 | MFU 14.24% | FLOPs estimator: workload-aware | logical tokens/s: 196103.6 | valid tokens/s: 196103.6 | physical tokens/s: 196468.0 | loss tokens/s: 145821.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.233533E-01 | lm-global-supervised-tokens: 1.247541E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.868152E-02 | lm-token-buffer-mib: 5.807526E+01 | global_load_balancing_loss: 9.995322E-01 | loss scale: 1.0 | grad norm: 0.057 | params norm: 11398.885 | params rms: 0.061721 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 1:43:03.991851 | finish at 2026-10-10 15:46:41 + [2026-10-07 14:05:03] iteration 3754/ 6855 | consumed samples: 480512 | elapsed time per iteration (ms): 85598.0 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.57% | FLOPs estimator: workload-aware | logical tokens/s: 195999.6 | valid tokens/s: 195999.6 | physical tokens/s: 196459.7 | loss tokens/s: 148702.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.266513E-01 | lm-global-supervised-tokens: 1.272860E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.336731E-02 | lm-token-buffer-mib: 5.829963E+01 | global_load_balancing_loss: 9.991363E-01 | loss scale: 1.0 | grad norm: 0.039 | params norm: 11398.830 | params rms: 0.061721 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 1:43:59.400372 | finish at 2026-10-10 15:49:02 + [2026-10-07 14:06:28] iteration 3755/ 6855 | consumed samples: 480640 | elapsed time per iteration (ms): 84748.3 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.65% | FLOPs estimator: workload-aware | logical tokens/s: 197965.3 | valid tokens/s: 197965.3 | physical tokens/s: 198501.5 | loss tokens/s: 144404.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.789719E-01 | lm-global-supervised-tokens: 1.223805E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.866241E-02 | lm-token-buffer-mib: 5.808247E+01 | global_load_balancing_loss: 9.989308E-01 | loss scale: 1.0 | grad norm: 0.063 | params norm: 11398.778 | params rms: 0.061721 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 0:58:39.624543 | finish at 2026-10-10 15:05:07 + [2026-10-07 14:07:53] iteration 3756/ 6855 | consumed samples: 480768 | elapsed time per iteration (ms): 85778.8 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.75% | FLOPs estimator: workload-aware | logical tokens/s: 195586.6 | valid tokens/s: 195586.6 | physical tokens/s: 196009.1 | loss tokens/s: 144794.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.041811E-01 | lm-global-supervised-tokens: 1.242033E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.159376E-02 | lm-token-buffer-mib: 5.803376E+01 | global_load_balancing_loss: 9.994143E-01 | loss scale: 1.0 | grad norm: 0.052 | params norm: 11398.724 | params rms: 0.061720 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 1:50:28.441386 | finish at 2026-10-10 15:58:22 + [2026-10-07 14:09:22] iteration 3757/ 6855 | consumed samples: 480896 | elapsed time per iteration (ms): 88282.2 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.54% | FLOPs estimator: workload-aware | logical tokens/s: 190039.4 | valid tokens/s: 190039.4 | physical tokens/s: 190486.9 | loss tokens/s: 140735.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.613734E-01 | lm-global-supervised-tokens: 1.242441E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.126700E-02 | lm-token-buffer-mib: 5.809709E+01 | global_load_balancing_loss: 1.000106E+00 | loss scale: 1.0 | grad norm: 0.036 | params norm: 11398.673 | params rms: 0.061720 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 3:58:18.278062 | finish at 2026-10-10 18:07:40 + [2026-10-07 14:10:48] iteration 3758/ 6855 | consumed samples: 481024 | elapsed time per iteration (ms): 86340.2 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.49% | FLOPs estimator: workload-aware | logical tokens/s: 194314.5 | valid tokens/s: 194314.5 | physical tokens/s: 194918.0 | loss tokens/s: 151876.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.592622E-01 | lm-global-supervised-tokens: 1.311304E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.511360E-02 | lm-token-buffer-mib: 5.812723E+01 | global_load_balancing_loss: 1.000219E+00 | loss scale: 1.0 | grad norm: 0.073 | params norm: 11398.625 | params rms: 0.061720 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 2:16:35.721070 | finish at 2026-10-10 16:27:24 + [2026-10-07 14:12:17] iteration 3759/ 6855 | consumed samples: 481152 | elapsed time per iteration (ms): 89109.2 | throughput per GPU (TFLOP/s/GPU): 41.5 | MFU 13.31% | FLOPs estimator: workload-aware | logical tokens/s: 188276.4 | valid tokens/s: 188276.4 | physical tokens/s: 188833.5 | loss tokens/s: 138199.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.240337E-01 | lm-global-supervised-tokens: 1.231480E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.577620E-02 | lm-token-buffer-mib: 5.831391E+01 | global_load_balancing_loss: 1.000597E+00 | loss scale: 1.0 | grad norm: 0.095 | params norm: 11398.576 | params rms: 0.061720 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 4:38:02.047033 | finish at 2026-10-10 18:50:19 + [2026-10-07 14:14:09] iteration 3760/ 6855 | consumed samples: 481280 | elapsed time per iteration (ms): 111923.6 | throughput per GPU (TFLOP/s/GPU): 32.4 | MFU 10.39% | FLOPs estimator: workload-aware | logical tokens/s: 149898.3 | valid tokens/s: 149898.3 | physical tokens/s: 150434.0 | loss tokens/s: 109332.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.211163E-01 | lm-global-supervised-tokens: 1.223689E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.455717E-02 | lm-token-buffer-mib: 5.810780E+01 | global_load_balancing_loss: 1.000931E+00 | loss scale: 1.0 | grad norm: 0.102 | params norm: 11398.526 | params rms: 0.061719 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4 days, 0:13:23.417165 | finish at 2026-10-11 14:27:33 +times across ranks (ms): + save-checkpoint: + rank 0: 24514.20 + rank 1: 24514.17 + rank 2: 24514.23 + rank 3: 24514.26 + rank 4: 24514.19 + rank 5: 24514.17 + rank 6: 24514.16 + rank 7: 24514.20 + rank 8: 24514.43 + rank 9: 24514.37 + rank 10: 24514.38 + rank 11: 24514.38 + rank 12: 24514.38 + rank 13: 24514.38 + rank 14: 24514.40 + rank 15: 24514.38 + rank 16: 24513.65 + rank 17: 24513.47 + rank 18: 24513.61 + rank 19: 24513.62 + rank 20: 24513.61 + rank 21: 24513.65 + rank 22: 24513.62 + rank 23: 24513.60 + rank 24: 24514.59 + rank 25: 24514.52 + rank 26: 24514.62 + rank 27: 24514.60 + rank 28: 24514.53 + rank 29: 24514.56 + rank 30: 24514.59 + rank 31: 24514.54 + rank 32: 24514.22 + rank 33: 24514.17 + rank 34: 24514.29 + rank 35: 24514.17 + rank 36: 24514.14 + rank 37: 24514.17 + rank 38: 24514.16 + rank 39: 24514.23 + rank 40: 24514.45 + rank 41: 24514.43 + rank 42: 24514.42 + rank 43: 24514.44 + rank 44: 24514.40 + rank 45: 24514.44 + rank 46: 24514.41 + rank 47: 24514.39 + rank 48: 24513.75 + rank 49: 24513.72 + rank 50: 24513.55 + rank 51: 24513.64 + rank 52: 24513.72 + rank 53: 24513.66 + rank 54: 24513.72 + rank 55: 24513.69 + rank 56: 24514.26 + rank 57: 24514.16 + rank 58: 24514.14 + rank 59: 24514.16 + rank 60: 24514.21 + rank 61: 24514.16 + rank 62: 24514.17 + rank 63: 24514.15 + [2026-10-07 14:16:01] iteration 3761/ 6855 | consumed samples: 481408 | elapsed time per iteration (ms): 87062.6 | throughput per GPU (TFLOP/s/GPU): 42.8 | MFU 13.73% | FLOPs estimator: workload-aware | logical tokens/s: 192702.2 | valid tokens/s: 192702.2 | physical tokens/s: 193371.1 | loss tokens/s: 148001.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.585960E-01 | lm-global-supervised-tokens: 1.288540E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.609980E-01 | lm-token-buffer-mib: 5.819553E+01 | global_load_balancing_loss: 1.001293E+00 | loss scale: 1.0 | grad norm: 0.100 | params norm: 11398.476 | params rms: 0.061719 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 2:49:31.689984 | finish at 2026-10-10 17:05:33 + [2026-10-07 14:17:32] iteration 3762/ 6855 | consumed samples: 481536 | elapsed time per iteration (ms): 91523.3 | throughput per GPU (TFLOP/s/GPU): 40.1 | MFU 12.86% | FLOPs estimator: workload-aware | logical tokens/s: 183310.8 | valid tokens/s: 183310.8 | physical tokens/s: 184040.0 | loss tokens/s: 139470.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.539245E-01 | lm-global-supervised-tokens: 1.276481E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.319126E-02 | lm-token-buffer-mib: 5.810571E+01 | global_load_balancing_loss: 1.000828E+00 | loss scale: 1.0 | grad norm: 0.054 | params norm: 11398.426 | params rms: 0.061719 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 6:38:01.691317 | finish at 2026-10-10 20:55:34 + [2026-10-07 14:18:55] iteration 3763/ 6855 | consumed samples: 481664 | elapsed time per iteration (ms): 82910.2 | throughput per GPU (TFLOP/s/GPU): 43.3 | MFU 13.87% | FLOPs estimator: workload-aware | logical tokens/s: 202353.7 | valid tokens/s: 202353.7 | physical tokens/s: 202940.6 | loss tokens/s: 154109.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.890596E-01 | lm-global-supervised-tokens: 1.277722E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.149387E-02 | lm-token-buffer-mib: 5.828072E+01 | global_load_balancing_loss: 1.000378E+00 | loss scale: 1.0 | grad norm: 0.068 | params norm: 11398.375 | params rms: 0.061718 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 23:12:38.189856 | finish at 2026-10-10 13:31:34 + [2026-10-07 14:20:21] iteration 3764/ 6855 | consumed samples: 481792 | elapsed time per iteration (ms): 85660.4 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.65% | FLOPs estimator: workload-aware | logical tokens/s: 195856.4 | valid tokens/s: 195856.4 | physical tokens/s: 196381.1 | loss tokens/s: 147998.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.459150E-01 | lm-global-supervised-tokens: 1.267762E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.590074E-02 | lm-token-buffer-mib: 5.811162E+01 | global_load_balancing_loss: 9.995338E-01 | loss scale: 1.0 | grad norm: 0.107 | params norm: 11398.323 | params rms: 0.061718 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 1:32:56.282868 | finish at 2026-10-10 15:53:17 + [2026-10-07 14:21:47] iteration 3765/ 6855 | consumed samples: 481920 | elapsed time per iteration (ms): 85851.6 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.59% | FLOPs estimator: workload-aware | logical tokens/s: 195420.6 | valid tokens/s: 195420.6 | physical tokens/s: 195834.7 | loss tokens/s: 145577.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.525451E-01 | lm-global-supervised-tokens: 1.249802E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.272128E-02 | lm-token-buffer-mib: 5.831482E+01 | global_load_balancing_loss: 1.001585E+00 | loss scale: 1.0 | grad norm: 0.051 | params norm: 11398.270 | params rms: 0.061718 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 1:41:21.468837 | finish at 2026-10-10 16:03:08 + [2026-10-07 14:23:13] iteration 3766/ 6855 | consumed samples: 482048 | elapsed time per iteration (ms): 86327.6 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.63% | FLOPs estimator: workload-aware | logical tokens/s: 194342.5 | valid tokens/s: 194342.5 | physical tokens/s: 194720.0 | loss tokens/s: 150268.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.611143E-01 | lm-global-supervised-tokens: 1.297234E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.871680E-02 | lm-token-buffer-mib: 5.806798E+01 | global_load_balancing_loss: 9.990621E-01 | loss scale: 1.0 | grad norm: 0.067 | params norm: 11398.216 | params rms: 0.061718 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 2:04:26.027845 | finish at 2026-10-10 16:27:39 + [2026-10-07 14:24:40] iteration 3767/ 6855 | consumed samples: 482176 | elapsed time per iteration (ms): 86926.9 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.58% | FLOPs estimator: workload-aware | logical tokens/s: 193003.6 | valid tokens/s: 193003.6 | physical tokens/s: 193516.7 | loss tokens/s: 150848.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.680596E-01 | lm-global-supervised-tokens: 1.311276E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.367581E-02 | lm-token-buffer-mib: 5.811431E+01 | global_load_balancing_loss: 9.994506E-01 | loss scale: 1.0 | grad norm: 0.096 | params norm: 11398.160 | params rms: 0.061717 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 2:33:50.246307 | finish at 2026-10-10 16:58:30 + [2026-10-07 14:26:06] iteration 3768/ 6855 | consumed samples: 482304 | elapsed time per iteration (ms): 85413.8 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.41% | FLOPs estimator: workload-aware | logical tokens/s: 196422.1 | valid tokens/s: 196422.1 | physical tokens/s: 196912.7 | loss tokens/s: 147082.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.188375E-01 | lm-global-supervised-tokens: 1.256286E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.389935E-02 | lm-token-buffer-mib: 5.811252E+01 | global_load_balancing_loss: 9.995535E-01 | loss scale: 1.0 | grad norm: 0.148 | params norm: 11398.102 | params rms: 0.061717 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 1:14:32.540443 | finish at 2026-10-10 15:40:38 + [2026-10-07 14:27:37] iteration 3769/ 6855 | consumed samples: 482432 | elapsed time per iteration (ms): 91563.6 | throughput per GPU (TFLOP/s/GPU): 41.0 | MFU 13.13% | FLOPs estimator: workload-aware | logical tokens/s: 183229.8 | valid tokens/s: 183229.8 | physical tokens/s: 183652.7 | loss tokens/s: 134436.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.349265E-01 | lm-global-supervised-tokens: 1.230948E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.282093E-02 | lm-token-buffer-mib: 5.822529E+01 | global_load_balancing_loss: 9.995980E-01 | loss scale: 1.0 | grad norm: 0.302 | params norm: 11398.042 | params rms: 0.061717 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 6:29:25.341164 | finish at 2026-10-10 20:57:02 + [2026-10-07 14:29:29] iteration 3770/ 6855 | consumed samples: 482560 | elapsed time per iteration (ms): 111771.7 | throughput per GPU (TFLOP/s/GPU): 33.4 | MFU 10.69% | FLOPs estimator: workload-aware | logical tokens/s: 150102.0 | valid tokens/s: 150102.0 | physical tokens/s: 150648.0 | loss tokens/s: 111148.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.842033E-01 | lm-global-supervised-tokens: 1.242326E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.166765E-02 | lm-token-buffer-mib: 5.845586E+01 | global_load_balancing_loss: 9.996338E-01 | loss scale: 1.0 | grad norm: 0.482 | params norm: 11397.981 | params rms: 0.061716 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 23:46:55.550046 | finish at 2026-10-11 14:16:25 + [2026-10-07 14:30:54] iteration 3771/ 6855 | consumed samples: 482688 | elapsed time per iteration (ms): 85016.1 | throughput per GPU (TFLOP/s/GPU): 43.2 | MFU 13.86% | FLOPs estimator: workload-aware | logical tokens/s: 197341.6 | valid tokens/s: 197341.5 | physical tokens/s: 197690.5 | loss tokens/s: 149021.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.643351E-01 | lm-global-supervised-tokens: 1.266927E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.337857E-02 | lm-token-buffer-mib: 5.808423E+01 | global_load_balancing_loss: 1.001285E+00 | loss scale: 1.0 | grad norm: 0.233 | params norm: 11397.918 | params rms: 0.061716 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 0:49:49.756594 | finish at 2026-10-10 15:20:44 + [2026-10-07 14:32:25] iteration 3772/ 6855 | consumed samples: 482816 | elapsed time per iteration (ms): 90783.1 | throughput per GPU (TFLOP/s/GPU): 41.3 | MFU 13.22% | FLOPs estimator: workload-aware | logical tokens/s: 184805.5 | valid tokens/s: 184805.5 | physical tokens/s: 185085.8 | loss tokens/s: 137490.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.443810E-01 | lm-global-supervised-tokens: 1.248178E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.127759E-02 | lm-token-buffer-mib: 5.805386E+01 | global_load_balancing_loss: 9.997124E-01 | loss scale: 1.0 | grad norm: 0.152 | params norm: 11397.857 | params rms: 0.061716 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 5:44:44.148481 | finish at 2026-10-10 20:17:09 + [2026-10-07 14:33:53] iteration 3773/ 6855 | consumed samples: 482944 | elapsed time per iteration (ms): 87935.7 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.39% | FLOPs estimator: workload-aware | logical tokens/s: 190789.5 | valid tokens/s: 190789.5 | physical tokens/s: 191100.9 | loss tokens/s: 146433.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.768883E-01 | lm-global-supervised-tokens: 1.287677E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.534062E-02 | lm-token-buffer-mib: 5.812631E+01 | global_load_balancing_loss: 9.995222E-01 | loss scale: 1.0 | grad norm: 0.036 | params norm: 11397.796 | params rms: 0.061715 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 3:16:57.819866 | finish at 2026-10-10 17:50:51 + [2026-10-07 14:35:20] iteration 3774/ 6855 | consumed samples: 483072 | elapsed time per iteration (ms): 86716.8 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.38% | FLOPs estimator: workload-aware | logical tokens/s: 193471.0 | valid tokens/s: 193471.0 | physical tokens/s: 193969.8 | loss tokens/s: 145044.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.260297E-01 | lm-global-supervised-tokens: 1.257776E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.415401E-02 | lm-token-buffer-mib: 5.820056E+01 | global_load_balancing_loss: 1.000776E+00 | loss scale: 1.0 | grad norm: 0.155 | params norm: 11397.734 | params rms: 0.061715 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 2:12:54.385061 | finish at 2026-10-10 16:48:14 + [2026-10-07 14:36:49] iteration 3775/ 6855 | consumed samples: 483200 | elapsed time per iteration (ms): 89448.4 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.62% | FLOPs estimator: workload-aware | logical tokens/s: 187562.4 | valid tokens/s: 187562.4 | physical tokens/s: 188064.6 | loss tokens/s: 142031.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.061203E-01 | lm-global-supervised-tokens: 1.270448E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.195534E-02 | lm-token-buffer-mib: 5.813725E+01 | global_load_balancing_loss: 1.000332E+00 | loss scale: 1.0 | grad norm: 0.205 | params norm: 11397.671 | params rms: 0.061715 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 4:31:41.055908 | finish at 2026-10-10 19:08:30 + [2026-10-07 14:38:16] iteration 3776/ 6855 | consumed samples: 483328 | elapsed time per iteration (ms): 87358.0 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.55% | FLOPs estimator: workload-aware | logical tokens/s: 192051.2 | valid tokens/s: 192051.2 | physical tokens/s: 192414.0 | loss tokens/s: 145933.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.893026E-01 | lm-global-supervised-tokens: 1.274846E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.320974E-02 | lm-token-buffer-mib: 5.812962E+01 | global_load_balancing_loss: 1.000566E+00 | loss scale: 1.0 | grad norm: 0.190 | params norm: 11397.607 | params rms: 0.061714 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 2:42:55.419398 | finish at 2026-10-10 17:21:12 + [2026-10-07 14:39:44] iteration 3777/ 6855 | consumed samples: 483456 | elapsed time per iteration (ms): 87766.2 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.65% | FLOPs estimator: workload-aware | logical tokens/s: 191157.2 | valid tokens/s: 191157.2 | physical tokens/s: 191527.6 | loss tokens/s: 143487.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.504410E-01 | lm-global-supervised-tokens: 1.259334E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.314736E-02 | lm-token-buffer-mib: 5.811464E+01 | global_load_balancing_loss: 9.994974E-01 | loss scale: 1.0 | grad norm: 0.060 | params norm: 11397.544 | params rms: 0.061714 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 3:02:24.340319 | finish at 2026-10-10 17:42:08 + [2026-10-07 14:41:10] iteration 3778/ 6855 | consumed samples: 483584 | elapsed time per iteration (ms): 86293.0 | throughput per GPU (TFLOP/s/GPU): 42.8 | MFU 13.72% | FLOPs estimator: workload-aware | logical tokens/s: 194420.6 | valid tokens/s: 194420.6 | physical tokens/s: 194849.4 | loss tokens/s: 144037.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.595381E-01 | lm-global-supervised-tokens: 1.242941E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.131736E-02 | lm-token-buffer-mib: 5.808490E+01 | global_load_balancing_loss: 1.000553E+00 | loss scale: 1.0 | grad norm: 0.175 | params norm: 11397.478 | params rms: 0.061714 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 1:45:23.687129 | finish at 2026-10-10 16:26:34 + [2026-10-07 14:42:38] iteration 3779/ 6855 | consumed samples: 483712 | elapsed time per iteration (ms): 87788.5 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.52% | FLOPs estimator: workload-aware | logical tokens/s: 191109.0 | valid tokens/s: 191108.9 | physical tokens/s: 191525.0 | loss tokens/s: 137319.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.136729E-01 | lm-global-supervised-tokens: 1.205505E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.249832E-02 | lm-token-buffer-mib: 5.806116E+01 | global_load_balancing_loss: 1.001486E+00 | loss scale: 1.0 | grad norm: 0.070 | params norm: 11397.414 | params rms: 0.061713 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 3:00:37.504688 | finish at 2026-10-10 17:43:16 + [2026-10-07 14:44:28] iteration 3780/ 6855 | consumed samples: 483840 | elapsed time per iteration (ms): 110005.4 | throughput per GPU (TFLOP/s/GPU): 34.4 | MFU 11.03% | FLOPs estimator: workload-aware | logical tokens/s: 152512.3 | valid tokens/s: 152512.3 | physical tokens/s: 152771.7 | loss tokens/s: 106394.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.534754E-01 | lm-global-supervised-tokens: 1.170400E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.125821E-02 | lm-token-buffer-mib: 5.806786E+01 | global_load_balancing_loss: 1.000880E+00 | loss scale: 1.0 | grad norm: 0.097 | params norm: 11397.348 | params rms: 0.061713 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 21:57:46.631216 | finish at 2026-10-11 12:42:15 + [2026-10-07 14:45:57] iteration 3781/ 6855 | consumed samples: 483968 | elapsed time per iteration (ms): 88982.3 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.41% | FLOPs estimator: workload-aware | logical tokens/s: 188545.2 | valid tokens/s: 188545.2 | physical tokens/s: 188933.7 | loss tokens/s: 140811.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.114623E-01 | lm-global-supervised-tokens: 1.252977E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 3.364430E-02 | lm-token-buffer-mib: 5.813654E+01 | global_load_balancing_loss: 9.998003E-01 | loss scale: 1.0 | grad norm: 0.065 | params norm: 11397.284 | params rms: 0.061713 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 3:58:51.633574 | finish at 2026-10-10 18:44:49 + [2026-10-07 14:47:26] iteration 3782/ 6855 | consumed samples: 484096 | elapsed time per iteration (ms): 89197.6 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.54% | FLOPs estimator: workload-aware | logical tokens/s: 188090.3 | valid tokens/s: 188090.3 | physical tokens/s: 188607.5 | loss tokens/s: 134327.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.411027E-01 | lm-global-supervised-tokens: 1.198166E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.350541E-02 | lm-token-buffer-mib: 5.830139E+01 | global_load_balancing_loss: 9.996096E-01 | loss scale: 1.0 | grad norm: 0.060 | params norm: 11397.219 | params rms: 0.061712 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 4:08:24.118952 | finish at 2026-10-10 18:55:51 + [2026-10-07 14:48:55] iteration 3783/ 6855 | consumed samples: 484224 | elapsed time per iteration (ms): 88281.0 | throughput per GPU (TFLOP/s/GPU): 42.8 | MFU 13.72% | FLOPs estimator: workload-aware | logical tokens/s: 190042.7 | valid tokens/s: 190042.7 | physical tokens/s: 190647.5 | loss tokens/s: 139292.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.388393E-01 | lm-global-supervised-tokens: 1.229684E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.299408E-02 | lm-token-buffer-mib: 5.805021E+01 | global_load_balancing_loss: 9.990813E-01 | loss scale: 1.0 | grad norm: 0.078 | params norm: 11397.153 | params rms: 0.061712 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 3:19:59.355469 | finish at 2026-10-10 18:08:54 + [2026-10-07 14:50:25] iteration 3784/ 6855 | consumed samples: 484352 | elapsed time per iteration (ms): 90789.2 | throughput per GPU (TFLOP/s/GPU): 40.3 | MFU 12.93% | FLOPs estimator: workload-aware | logical tokens/s: 184793.1 | valid tokens/s: 184793.1 | physical tokens/s: 185296.9 | loss tokens/s: 134140.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.368164E-01 | lm-global-supervised-tokens: 1.217853E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.171766E-02 | lm-token-buffer-mib: 5.806317E+01 | global_load_balancing_loss: 1.000078E+00 | loss scale: 1.0 | grad norm: 0.040 | params norm: 11397.088 | params rms: 0.061712 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 5:26:53.481113 | finish at 2026-10-10 20:17:19 + [2026-10-07 14:51:55] iteration 3785/ 6855 | consumed samples: 484480 | elapsed time per iteration (ms): 89186.6 | throughput per GPU (TFLOP/s/GPU): 40.9 | MFU 13.12% | FLOPs estimator: workload-aware | logical tokens/s: 188113.3 | valid tokens/s: 188113.3 | physical tokens/s: 188635.4 | loss tokens/s: 140585.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.544807E-01 | lm-global-supervised-tokens: 1.253832E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.269725E-02 | lm-token-buffer-mib: 5.805383E+01 | global_load_balancing_loss: 1.000474E+00 | loss scale: 1.0 | grad norm: 0.075 | params norm: 11397.023 | params rms: 0.061711 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 4:03:22.747724 | finish at 2026-10-10 18:55:17 + [2026-10-07 14:53:21] iteration 3786/ 6855 | consumed samples: 484608 | elapsed time per iteration (ms): 86550.2 | throughput per GPU (TFLOP/s/GPU): 43.3 | MFU 13.87% | FLOPs estimator: workload-aware | logical tokens/s: 193843.2 | valid tokens/s: 193843.2 | physical tokens/s: 194226.6 | loss tokens/s: 138658.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.490541E-01 | lm-global-supervised-tokens: 1.200095E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.111245E-02 | lm-token-buffer-mib: 5.805764E+01 | global_load_balancing_loss: 1.001603E+00 | loss scale: 1.0 | grad norm: 0.045 | params norm: 11396.960 | params rms: 0.061711 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 1:47:02.523512 | finish at 2026-10-10 16:40:24 + [2026-10-07 14:54:48] iteration 3787/ 6855 | consumed samples: 484736 | elapsed time per iteration (ms): 86586.3 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.51% | FLOPs estimator: workload-aware | logical tokens/s: 193762.5 | valid tokens/s: 193762.5 | physical tokens/s: 194219.2 | loss tokens/s: 142547.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.952990E-01 | lm-global-supervised-tokens: 1.234268E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.447670E-02 | lm-token-buffer-mib: 5.803053E+01 | global_load_balancing_loss: 1.000167E+00 | loss scale: 1.0 | grad norm: 0.102 | params norm: 11396.896 | params rms: 0.061710 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 1:47:26.760036 | finish at 2026-10-10 16:42:15 + [2026-10-07 14:56:16] iteration 3788/ 6855 | consumed samples: 484864 | elapsed time per iteration (ms): 88098.7 | throughput per GPU (TFLOP/s/GPU): 42.8 | MFU 13.72% | FLOPs estimator: workload-aware | logical tokens/s: 190435.7 | valid tokens/s: 190435.7 | physical tokens/s: 191005.2 | loss tokens/s: 140593.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.801177E-01 | lm-global-supervised-tokens: 1.238606E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 4.033872E-02 | lm-token-buffer-mib: 5.816266E+01 | global_load_balancing_loss: 9.997498E-01 | loss scale: 1.0 | grad norm: 0.238 | params norm: 11396.833 | params rms: 0.061710 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 3:03:18.803715 | finish at 2026-10-10 17:59:35 + [2026-10-07 14:57:46] iteration 3789/ 6855 | consumed samples: 484992 | elapsed time per iteration (ms): 90016.2 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.40% | FLOPs estimator: workload-aware | logical tokens/s: 186379.6 | valid tokens/s: 186379.6 | physical tokens/s: 186687.1 | loss tokens/s: 139096.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.661596E-01 | lm-global-supervised-tokens: 1.252096E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.332300E-02 | lm-token-buffer-mib: 5.809460E+01 | global_load_balancing_loss: 1.000179E+00 | loss scale: 1.0 | grad norm: 0.128 | params norm: 11396.767 | params rms: 0.061710 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 4:39:49.670132 | finish at 2026-10-10 19:37:36 + [2026-10-07 14:59:35] iteration 3790/ 6855 | consumed samples: 485120 | elapsed time per iteration (ms): 109145.9 | throughput per GPU (TFLOP/s/GPU): 33.6 | MFU 10.76% | FLOPs estimator: workload-aware | logical tokens/s: 153713.3 | valid tokens/s: 153713.3 | physical tokens/s: 154097.8 | loss tokens/s: 116511.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.760105E-01 | lm-global-supervised-tokens: 1.271671E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.164651E-02 | lm-token-buffer-mib: 5.805055E+01 | global_load_balancing_loss: 9.991250E-01 | loss scale: 1.0 | grad norm: 0.247 | params norm: 11396.700 | params rms: 0.061709 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 20:55:32.288032 | finish at 2026-10-11 11:55:07 + [2026-10-07 15:00:58] iteration 3791/ 6855 | consumed samples: 485248 | elapsed time per iteration (ms): 83331.8 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.63% | FLOPs estimator: workload-aware | logical tokens/s: 201330.3 | valid tokens/s: 201330.3 | physical tokens/s: 201916.3 | loss tokens/s: 150203.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 5.837563E-01 | lm-global-supervised-tokens: 1.251672E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.413479E-02 | lm-token-buffer-mib: 5.820102E+01 | global_load_balancing_loss: 9.996720E-01 | loss scale: 1.0 | grad norm: 0.285 | params norm: 11396.635 | params rms: 0.061709 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 22:55:28.642456 | finish at 2026-10-10 13:56:27 + [2026-10-07 15:02:25] iteration 3792/ 6855 | consumed samples: 485376 | elapsed time per iteration (ms): 86632.0 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.63% | FLOPs estimator: workload-aware | logical tokens/s: 193660.3 | valid tokens/s: 193660.2 | physical tokens/s: 194334.7 | loss tokens/s: 137858.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.322419E-01 | lm-global-supervised-tokens: 1.194298E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.582442E-02 | lm-token-buffer-mib: 5.827624E+01 | global_load_balancing_loss: 1.000698E+00 | loss scale: 1.0 | grad norm: 0.060 | params norm: 11396.573 | params rms: 0.061709 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 1:42:33.775741 | finish at 2026-10-10 16:44:59 + [2026-10-07 15:03:50] iteration 3793/ 6855 | consumed samples: 485504 | elapsed time per iteration (ms): 85177.9 | throughput per GPU (TFLOP/s/GPU): 43.2 | MFU 13.85% | FLOPs estimator: workload-aware | logical tokens/s: 196966.3 | valid tokens/s: 196966.3 | physical tokens/s: 197438.2 | loss tokens/s: 151456.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.704133E-01 | lm-global-supervised-tokens: 1.290076E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.261235E-02 | lm-token-buffer-mib: 5.818549E+01 | global_load_balancing_loss: 1.001393E+00 | loss scale: 1.0 | grad norm: 0.159 | params norm: 11396.510 | params rms: 0.061708 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 0:26:54.795006 | finish at 2026-10-10 15:30:45 + [2026-10-07 15:05:18] iteration 3794/ 6855 | consumed samples: 485632 | elapsed time per iteration (ms): 87675.5 | throughput per GPU (TFLOP/s/GPU): 41.7 | MFU 13.37% | FLOPs estimator: workload-aware | logical tokens/s: 191354.8 | valid tokens/s: 191354.8 | physical tokens/s: 191736.9 | loss tokens/s: 138677.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.029856E-01 | lm-global-supervised-tokens: 1.215860E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.425342E-02 | lm-token-buffer-mib: 5.808506E+01 | global_load_balancing_loss: 1.001294E+00 | loss scale: 1.0 | grad norm: 0.149 | params norm: 11396.448 | params rms: 0.061708 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 2:32:54.640291 | finish at 2026-10-10 17:38:13 + [2026-10-07 15:06:46] iteration 3795/ 6855 | consumed samples: 485760 | elapsed time per iteration (ms): 88331.5 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.61% | FLOPs estimator: workload-aware | logical tokens/s: 189934.6 | valid tokens/s: 189934.6 | physical tokens/s: 190252.7 | loss tokens/s: 138177.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.741878E-01 | lm-global-supervised-tokens: 1.220539E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.362601E-02 | lm-token-buffer-mib: 5.807632E+01 | global_load_balancing_loss: 1.000940E+00 | loss scale: 1.0 | grad norm: 0.061 | params norm: 11396.384 | params rms: 0.061708 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 3:04:54.511271 | finish at 2026-10-10 18:11:41 + [2026-10-07 15:08:15] iteration 3796/ 6855 | consumed samples: 485888 | elapsed time per iteration (ms): 88487.8 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.50% | FLOPs estimator: workload-aware | logical tokens/s: 189598.9 | valid tokens/s: 189598.9 | physical tokens/s: 189807.4 | loss tokens/s: 137412.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.352172E-01 | lm-global-supervised-tokens: 1.215930E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.273104E-02 | lm-token-buffer-mib: 5.807147E+01 | global_load_balancing_loss: 9.990017E-01 | loss scale: 1.0 | grad norm: 0.082 | params norm: 11396.320 | params rms: 0.061707 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 3:11:24.124413 | finish at 2026-10-10 18:19:39 + [2026-10-07 15:09:44] iteration 3797/ 6855 | consumed samples: 486016 | elapsed time per iteration (ms): 88891.9 | throughput per GPU (TFLOP/s/GPU): 40.9 | MFU 13.09% | FLOPs estimator: workload-aware | logical tokens/s: 188736.8 | valid tokens/s: 188736.8 | physical tokens/s: 189309.4 | loss tokens/s: 139963.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.442991E-01 | lm-global-supervised-tokens: 1.244159E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.374009E-02 | lm-token-buffer-mib: 5.823307E+01 | global_load_balancing_loss: 9.996570E-01 | loss scale: 1.0 | grad norm: 0.146 | params norm: 11396.256 | params rms: 0.061707 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 3:30:31.578395 | finish at 2026-10-10 18:40:15 + [2026-10-07 15:11:11] iteration 3798/ 6855 | consumed samples: 486144 | elapsed time per iteration (ms): 87437.3 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.52% | FLOPs estimator: workload-aware | logical tokens/s: 191876.5 | valid tokens/s: 191876.5 | physical tokens/s: 192357.5 | loss tokens/s: 139987.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.302758E-01 | lm-global-supervised-tokens: 1.224013E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.295839E-02 | lm-token-buffer-mib: 5.806277E+01 | global_load_balancing_loss: 1.000804E+00 | loss scale: 1.0 | grad norm: 0.066 | params norm: 11396.192 | params rms: 0.061707 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 2:14:55.823812 | finish at 2026-10-10 17:26:07 + [2026-10-07 15:12:37] iteration 3799/ 6855 | consumed samples: 486272 | elapsed time per iteration (ms): 86230.3 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.49% | FLOPs estimator: workload-aware | logical tokens/s: 194562.2 | valid tokens/s: 194562.2 | physical tokens/s: 195006.5 | loss tokens/s: 146228.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.778964E-01 | lm-global-supervised-tokens: 1.260928E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.491004E-02 | lm-token-buffer-mib: 5.812975E+01 | global_load_balancing_loss: 9.996331E-01 | loss scale: 1.0 | grad norm: 0.142 | params norm: 11396.129 | params rms: 0.061706 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 1:11:59.848377 | finish at 2026-10-10 16:24:37 + [2026-10-07 15:14:28] iteration 3800/ 6855 | consumed samples: 486400 | elapsed time per iteration (ms): 110851.5 | throughput per GPU (TFLOP/s/GPU): 33.9 | MFU 10.87% | FLOPs estimator: workload-aware | logical tokens/s: 151348.6 | valid tokens/s: 151348.6 | physical tokens/s: 151607.4 | loss tokens/s: 108853.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.930938E-01 | lm-global-supervised-tokens: 1.206653E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.548648E-02 | lm-token-buffer-mib: 5.806973E+01 | global_load_balancing_loss: 9.998895E-01 | loss scale: 1.0 | grad norm: 0.060 | params norm: 11396.066 | params rms: 0.061706 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 22:04:11.248842 | finish at 2026-10-11 13:18:39 +times across ranks (ms): + save-checkpoint: + rank 0: 27039.76 + rank 1: 27039.77 + rank 2: 27039.79 + rank 3: 27039.79 + rank 4: 27039.79 + rank 5: 27039.75 + rank 6: 27039.75 + rank 7: 27039.76 + rank 8: 27040.07 + rank 9: 27040.05 + rank 10: 27040.07 + rank 11: 27040.04 + rank 12: 27040.06 + rank 13: 27040.04 + rank 14: 27040.10 + rank 15: 27040.12 + rank 16: 27039.78 + rank 17: 27039.75 + rank 18: 27039.76 + rank 19: 27039.80 + rank 20: 27039.75 + rank 21: 27039.78 + rank 22: 27039.74 + rank 23: 27039.74 + rank 24: 27040.05 + rank 25: 27040.01 + rank 26: 27040.03 + rank 27: 27040.08 + rank 28: 27040.00 + rank 29: 27040.03 + rank 30: 27039.94 + rank 31: 27040.01 + rank 32: 27040.14 + rank 33: 27040.11 + rank 34: 27040.15 + rank 35: 27040.13 + rank 36: 27040.04 + rank 37: 27040.04 + rank 38: 27040.04 + rank 39: 27039.92 + rank 40: 27039.82 + rank 41: 27039.80 + rank 42: 27039.81 + rank 43: 27039.83 + rank 44: 27039.79 + rank 45: 27039.81 + rank 46: 27039.81 + rank 47: 27039.79 + rank 48: 27039.52 + rank 49: 27039.67 + rank 50: 27039.56 + rank 51: 27039.61 + rank 52: 27039.67 + rank 53: 27039.64 + rank 54: 27039.63 + rank 55: 27039.69 + rank 56: 27039.24 + rank 57: 27039.21 + rank 58: 27039.15 + rank 59: 27039.16 + rank 60: 27039.18 + rank 61: 27039.20 + rank 62: 27039.17 + rank 63: 27039.20 + [2026-10-07 15:16:42] iteration 3801/ 6855 | consumed samples: 486528 | elapsed time per iteration (ms): 102607.4 | throughput per GPU (TFLOP/s/GPU): 35.4 | MFU 11.34% | FLOPs estimator: workload-aware | logical tokens/s: 163508.4 | valid tokens/s: 163508.4 | physical tokens/s: 163921.3 | loss tokens/s: 122155.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.562950E-01 | lm-global-supervised-tokens: 1.253401E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 4.844099E+00 | lm-token-buffer-mib: 5.805270E+01 | global_load_balancing_loss: 9.993768E-01 | loss scale: 1.0 | grad norm: 0.077 | params norm: 11396.002 | params rms: 0.061706 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 15:02:42.960099 | finish at 2026-10-11 06:19:25 + [2026-10-07 15:18:15] iteration 3802/ 6855 | consumed samples: 486656 | elapsed time per iteration (ms): 93578.6 | throughput per GPU (TFLOP/s/GPU): 40.1 | MFU 12.85% | FLOPs estimator: workload-aware | logical tokens/s: 179284.6 | valid tokens/s: 179284.6 | physical tokens/s: 179677.7 | loss tokens/s: 127476.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.554742E-01 | lm-global-supervised-tokens: 1.192903E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.514774E-02 | lm-token-buffer-mib: 5.811274E+01 | global_load_balancing_loss: 1.000115E+00 | loss scale: 1.0 | grad norm: 0.048 | params norm: 11395.937 | params rms: 0.061705 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 7:21:35.330926 | finish at 2026-10-10 22:39:51 + [2026-10-07 15:19:43] iteration 3803/ 6855 | consumed samples: 486784 | elapsed time per iteration (ms): 87416.2 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.52% | FLOPs estimator: workload-aware | logical tokens/s: 191923.2 | valid tokens/s: 191923.2 | physical tokens/s: 192408.0 | loss tokens/s: 135539.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.595474E-01 | lm-global-supervised-tokens: 1.184835E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.347915E-02 | lm-token-buffer-mib: 5.806902E+01 | global_load_balancing_loss: 1.000873E+00 | loss scale: 1.0 | grad norm: 0.077 | params norm: 11395.871 | params rms: 0.061705 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 2:06:34.092267 | finish at 2026-10-10 17:26:17 + [2026-10-07 15:21:10] iteration 3804/ 6855 | consumed samples: 486912 | elapsed time per iteration (ms): 87448.8 | throughput per GPU (TFLOP/s/GPU): 41.6 | MFU 13.33% | FLOPs estimator: workload-aware | logical tokens/s: 191851.3 | valid tokens/s: 191851.3 | physical tokens/s: 192380.9 | loss tokens/s: 147228.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.740840E-01 | lm-global-supervised-tokens: 1.287497E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.265858E-02 | lm-token-buffer-mib: 5.812868E+01 | global_load_balancing_loss: 1.000264E+00 | loss scale: 1.0 | grad norm: 0.044 | params norm: 11395.804 | params rms: 0.061705 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 2:06:46.166860 | finish at 2026-10-10 17:27:56 + [2026-10-07 15:22:38] iteration 3805/ 6855 | consumed samples: 487040 | elapsed time per iteration (ms): 87442.4 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.54% | FLOPs estimator: workload-aware | logical tokens/s: 191865.8 | valid tokens/s: 191865.8 | physical tokens/s: 192303.0 | loss tokens/s: 140367.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.687033E-01 | lm-global-supervised-tokens: 1.227407E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.538624E-02 | lm-token-buffer-mib: 5.804322E+01 | global_load_balancing_loss: 1.001003E+00 | loss scale: 1.0 | grad norm: 0.037 | params norm: 11395.737 | params rms: 0.061704 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 2:04:59.413741 | finish at 2026-10-10 17:27:37 + [2026-10-07 15:24:05] iteration 3806/ 6855 | consumed samples: 487168 | elapsed time per iteration (ms): 87644.2 | throughput per GPU (TFLOP/s/GPU): 43.4 | MFU 13.92% | FLOPs estimator: workload-aware | logical tokens/s: 191423.9 | valid tokens/s: 191423.9 | physical tokens/s: 191764.5 | loss tokens/s: 136248.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.230724E-01 | lm-global-supervised-tokens: 1.194138E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.166645E-02 | lm-token-buffer-mib: 5.813881E+01 | global_load_balancing_loss: 1.000609E+00 | loss scale: 1.0 | grad norm: 0.042 | params norm: 11395.671 | params rms: 0.061704 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 2:13:47.030127 | finish at 2026-10-10 17:37:52 + [2026-10-07 15:25:30] iteration 3807/ 6855 | consumed samples: 487296 | elapsed time per iteration (ms): 84280.2 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.75% | FLOPs estimator: workload-aware | logical tokens/s: 199064.0 | valid tokens/s: 199064.0 | physical tokens/s: 199713.6 | loss tokens/s: 155262.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.718266E-01 | lm-global-supervised-tokens: 1.308557E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.184784E-02 | lm-token-buffer-mib: 5.824407E+01 | global_load_balancing_loss: 9.995852E-01 | loss scale: 1.0 | grad norm: 0.095 | params norm: 11395.606 | params rms: 0.061704 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 23:21:26.090309 | finish at 2026-10-10 14:46:56 + [2026-10-07 15:26:57] iteration 3808/ 6855 | consumed samples: 487424 | elapsed time per iteration (ms): 87180.9 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.51% | FLOPs estimator: workload-aware | logical tokens/s: 192440.4 | valid tokens/s: 192440.4 | physical tokens/s: 193055.2 | loss tokens/s: 142074.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.487639E-01 | lm-global-supervised-tokens: 1.238620E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.227101E-02 | lm-token-buffer-mib: 5.834685E+01 | global_load_balancing_loss: 9.996144E-01 | loss scale: 1.0 | grad norm: 0.100 | params norm: 11395.542 | params rms: 0.061703 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 1:47:20.352246 | finish at 2026-10-10 17:14:17 + [2026-10-07 15:28:22] iteration 3809/ 6855 | consumed samples: 487552 | elapsed time per iteration (ms): 85211.9 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.65% | FLOPs estimator: workload-aware | logical tokens/s: 196887.6 | valid tokens/s: 196887.6 | physical tokens/s: 197457.8 | loss tokens/s: 152119.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.840174E-01 | lm-global-supervised-tokens: 1.296240E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.236567E-02 | lm-token-buffer-mib: 5.836392E+01 | global_load_balancing_loss: 9.986680E-01 | loss scale: 1.0 | grad norm: 0.072 | params norm: 11395.479 | params rms: 0.061703 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 0:05:55.323203 | finish at 2026-10-10 15:34:17 + [2026-10-07 15:30:09] iteration 3810/ 6855 | consumed samples: 487680 | elapsed time per iteration (ms): 106609.4 | throughput per GPU (TFLOP/s/GPU): 34.3 | MFU 10.98% | FLOPs estimator: workload-aware | logical tokens/s: 157370.5 | valid tokens/s: 157370.5 | physical tokens/s: 157805.7 | loss tokens/s: 116296.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.419072E-01 | lm-global-supervised-tokens: 1.239831E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.313184E-02 | lm-token-buffer-mib: 5.813055E+01 | global_load_balancing_loss: 1.000773E+00 | loss scale: 1.0 | grad norm: 0.112 | params norm: 11395.417 | params rms: 0.061702 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 18:10:25.545423 | finish at 2026-10-11 09:40:34 + [2026-10-07 15:31:35] iteration 3811/ 6855 | consumed samples: 487808 | elapsed time per iteration (ms): 86651.2 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.75% | FLOPs estimator: workload-aware | logical tokens/s: 193617.6 | valid tokens/s: 193617.6 | physical tokens/s: 194074.7 | loss tokens/s: 146136.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.405599E-01 | lm-global-supervised-tokens: 1.266293E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.227765E-02 | lm-token-buffer-mib: 5.813217E+01 | global_load_balancing_loss: 1.000382E+00 | loss scale: 1.0 | grad norm: 0.037 | params norm: 11395.352 | params rms: 0.061702 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 1:16:06.210877 | finish at 2026-10-10 16:47:41 + [2026-10-07 15:33:04] iteration 3812/ 6855 | consumed samples: 487936 | elapsed time per iteration (ms): 89005.0 | throughput per GPU (TFLOP/s/GPU): 41.6 | MFU 13.33% | FLOPs estimator: workload-aware | logical tokens/s: 188497.2 | valid tokens/s: 188497.2 | physical tokens/s: 189079.6 | loss tokens/s: 140760.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.244722E-01 | lm-global-supervised-tokens: 1.252841E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.308838E-02 | lm-token-buffer-mib: 5.807530E+01 | global_load_balancing_loss: 1.000870E+00 | loss scale: 1.0 | grad norm: 0.075 | params norm: 11395.286 | params rms: 0.061702 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 3:14:02.136268 | finish at 2026-10-10 18:47:06 + [2026-10-07 15:34:33] iteration 3813/ 6855 | consumed samples: 488064 | elapsed time per iteration (ms): 88299.0 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.54% | FLOPs estimator: workload-aware | logical tokens/s: 190003.8 | valid tokens/s: 190003.8 | physical tokens/s: 190295.2 | loss tokens/s: 140670.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.167999E-01 | lm-global-supervised-tokens: 1.242105E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.586977E-02 | lm-token-buffer-mib: 5.807217E+01 | global_load_balancing_loss: 1.000453E+00 | loss scale: 1.0 | grad norm: 0.089 | params norm: 11395.221 | params rms: 0.061701 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 2:36:45.633503 | finish at 2026-10-10 18:11:18 + [2026-10-07 15:36:00] iteration 3814/ 6855 | consumed samples: 488192 | elapsed time per iteration (ms): 87547.6 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.51% | FLOPs estimator: workload-aware | logical tokens/s: 191634.9 | valid tokens/s: 191634.9 | physical tokens/s: 192113.7 | loss tokens/s: 148050.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.834989E-01 | lm-global-supervised-tokens: 1.296144E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.342836E-02 | lm-token-buffer-mib: 5.817783E+01 | global_load_balancing_loss: 1.000224E+00 | loss scale: 1.0 | grad norm: 0.055 | params norm: 11395.155 | params rms: 0.061701 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 1:57:12.221243 | finish at 2026-10-10 17:33:12 + [2026-10-07 15:37:29] iteration 3815/ 6855 | consumed samples: 488320 | elapsed time per iteration (ms): 89044.1 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.63% | FLOPs estimator: workload-aware | logical tokens/s: 188414.0 | valid tokens/s: 188414.0 | physical tokens/s: 188730.8 | loss tokens/s: 137482.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.056161E-01 | lm-global-supervised-tokens: 1.224202E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.126819E-02 | lm-token-buffer-mib: 5.817622E+01 | global_load_balancing_loss: 1.000468E+00 | loss scale: 1.0 | grad norm: 0.056 | params norm: 11395.089 | params rms: 0.061701 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 3:11:34.205475 | finish at 2026-10-10 18:49:03 + [2026-10-07 15:39:02] iteration 3816/ 6855 | consumed samples: 488448 | elapsed time per iteration (ms): 92758.2 | throughput per GPU (TFLOP/s/GPU): 41.1 | MFU 13.16% | FLOPs estimator: workload-aware | logical tokens/s: 180869.9 | valid tokens/s: 180869.9 | physical tokens/s: 181258.3 | loss tokens/s: 130369.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.991284E-01 | lm-global-supervised-tokens: 1.209282E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.132323E-02 | lm-token-buffer-mib: 5.801694E+01 | global_load_balancing_loss: 1.000770E+00 | loss scale: 1.0 | grad norm: 0.040 | params norm: 11395.024 | params rms: 0.061700 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 6:18:12.307253 | finish at 2026-10-10 21:57:14 + [2026-10-07 15:40:30] iteration 3817/ 6855 | consumed samples: 488576 | elapsed time per iteration (ms): 88342.9 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.65% | FLOPs estimator: workload-aware | logical tokens/s: 189909.4 | valid tokens/s: 189909.4 | physical tokens/s: 190652.8 | loss tokens/s: 139096.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.738013E-01 | lm-global-supervised-tokens: 1.228817E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.871041E-02 | lm-token-buffer-mib: 5.806287E+01 | global_load_balancing_loss: 1.000019E+00 | loss scale: 1.0 | grad norm: 0.087 | params norm: 11394.959 | params rms: 0.061700 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 2:33:05.673094 | finish at 2026-10-10 18:13:36 + [2026-10-07 15:42:00] iteration 3818/ 6855 | consumed samples: 488704 | elapsed time per iteration (ms): 89667.1 | throughput per GPU (TFLOP/s/GPU): 40.3 | MFU 12.92% | FLOPs estimator: workload-aware | logical tokens/s: 187105.1 | valid tokens/s: 187105.1 | physical tokens/s: 187564.8 | loss tokens/s: 138278.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.296603E-01 | lm-global-supervised-tokens: 1.239907E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.541432E-02 | lm-token-buffer-mib: 5.822495E+01 | global_load_balancing_loss: 1.001064E+00 | loss scale: 1.0 | grad norm: 0.116 | params norm: 11394.894 | params rms: 0.061700 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 3:38:38.892047 | finish at 2026-10-10 19:20:39 + [2026-10-07 15:43:28] iteration 3819/ 6855 | consumed samples: 488832 | elapsed time per iteration (ms): 87869.1 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.58% | FLOPs estimator: workload-aware | logical tokens/s: 190933.4 | valid tokens/s: 190933.4 | physical tokens/s: 191442.2 | loss tokens/s: 135438.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.096460E-01 | lm-global-supervised-tokens: 1.190087E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.479855E-02 | lm-token-buffer-mib: 5.808052E+01 | global_load_balancing_loss: 9.992965E-01 | loss scale: 1.0 | grad norm: 0.162 | params norm: 11394.829 | params rms: 0.061699 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 2:06:10.606705 | finish at 2026-10-10 17:49:38 + [2026-10-07 15:45:19] iteration 3820/ 6855 | consumed samples: 488960 | elapsed time per iteration (ms): 110794.6 | throughput per GPU (TFLOP/s/GPU): 33.4 | MFU 10.71% | FLOPs estimator: workload-aware | logical tokens/s: 151426.0 | valid tokens/s: 151426.0 | physical tokens/s: 151817.2 | loss tokens/s: 112558.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.655768E-01 | lm-global-supervised-tokens: 1.247090E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.220930E-02 | lm-token-buffer-mib: 5.804416E+01 | global_load_balancing_loss: 1.000690E+00 | loss scale: 1.0 | grad norm: 0.066 | params norm: 11394.764 | params rms: 0.061699 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 21:24:21.648657 | finish at 2026-10-11 13:09:40 + [2026-10-07 15:46:48] iteration 3821/ 6855 | consumed samples: 489088 | elapsed time per iteration (ms): 89193.2 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.46% | FLOPs estimator: workload-aware | logical tokens/s: 188098.9 | valid tokens/s: 188098.9 | physical tokens/s: 188438.3 | loss tokens/s: 142545.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.149724E-01 | lm-global-supervised-tokens: 1.271413E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.196240E-02 | lm-token-buffer-mib: 5.811107E+01 | global_load_balancing_loss: 9.990620E-01 | loss scale: 1.0 | grad norm: 0.054 | params norm: 11394.699 | params rms: 0.061699 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 3:10:12.313810 | finish at 2026-10-10 18:57:00 + [2026-10-07 15:48:16] iteration 3822/ 6855 | consumed samples: 489216 | elapsed time per iteration (ms): 87757.9 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.44% | FLOPs estimator: workload-aware | logical tokens/s: 191175.5 | valid tokens/s: 191175.5 | physical tokens/s: 191742.5 | loss tokens/s: 135203.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.628165E-01 | lm-global-supervised-tokens: 1.186513E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.178426E-02 | lm-token-buffer-mib: 5.829267E+01 | global_load_balancing_loss: 9.997881E-01 | loss scale: 1.0 | grad norm: 0.080 | params norm: 11394.636 | params rms: 0.061698 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 1:56:09.649956 | finish at 2026-10-10 17:44:25 + [2026-10-07 15:49:44] iteration 3823/ 6855 | consumed samples: 489344 | elapsed time per iteration (ms): 88944.4 | throughput per GPU (TFLOP/s/GPU): 41.4 | MFU 13.27% | FLOPs estimator: workload-aware | logical tokens/s: 188626.0 | valid tokens/s: 188626.0 | physical tokens/s: 189189.0 | loss tokens/s: 141412.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.766899E-01 | lm-global-supervised-tokens: 1.257786E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.403899E-02 | lm-token-buffer-mib: 5.808512E+01 | global_load_balancing_loss: 9.997934E-01 | loss scale: 1.0 | grad norm: 0.153 | params norm: 11394.574 | params rms: 0.061698 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 2:54:39.277887 | finish at 2026-10-10 18:44:24 + [2026-10-07 15:51:11] iteration 3824/ 6855 | consumed samples: 489472 | elapsed time per iteration (ms): 86419.4 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.54% | FLOPs estimator: workload-aware | logical tokens/s: 194135.9 | valid tokens/s: 194135.9 | physical tokens/s: 194689.6 | loss tokens/s: 135259.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 5.827011E-01 | lm-global-supervised-tokens: 1.168908E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.487514E-02 | lm-token-buffer-mib: 5.836460E+01 | global_load_balancing_loss: 1.000986E+00 | loss scale: 1.0 | grad norm: 0.164 | params norm: 11394.513 | params rms: 0.061698 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 0:45:37.224454 | finish at 2026-10-10 16:36:48 + [2026-10-07 15:52:40] iteration 3825/ 6855 | consumed samples: 489600 | elapsed time per iteration (ms): 88930.5 | throughput per GPU (TFLOP/s/GPU): 41.7 | MFU 13.35% | FLOPs estimator: workload-aware | logical tokens/s: 188655.2 | valid tokens/s: 188655.2 | physical tokens/s: 189104.2 | loss tokens/s: 133573.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.754563E-01 | lm-global-supervised-tokens: 1.187879E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.146824E-02 | lm-token-buffer-mib: 5.812129E+01 | global_load_balancing_loss: 1.000044E+00 | loss scale: 1.0 | grad norm: 0.075 | params norm: 11394.451 | params rms: 0.061697 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 2:50:59.281447 | finish at 2026-10-10 18:43:39 + [2026-10-07 15:54:11] iteration 3826/ 6855 | consumed samples: 489728 | elapsed time per iteration (ms): 91604.4 | throughput per GPU (TFLOP/s/GPU): 41.7 | MFU 13.36% | FLOPs estimator: workload-aware | logical tokens/s: 183148.5 | valid tokens/s: 183148.5 | physical tokens/s: 183475.7 | loss tokens/s: 127160.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.367854E-01 | lm-global-supervised-tokens: 1.164846E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.134724E-02 | lm-token-buffer-mib: 5.813240E+01 | global_load_balancing_loss: 1.000831E+00 | loss scale: 1.0 | grad norm: 0.058 | params norm: 11394.389 | params rms: 0.061697 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 5:04:29.819794 | finish at 2026-10-10 20:58:41 + [2026-10-07 15:55:36] iteration 3827/ 6855 | consumed samples: 489856 | elapsed time per iteration (ms): 84303.3 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.49% | FLOPs estimator: workload-aware | logical tokens/s: 199010.0 | valid tokens/s: 199010.0 | physical tokens/s: 199521.7 | loss tokens/s: 149702.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.475039E-01 | lm-global-supervised-tokens: 1.262040E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.149470E-02 | lm-token-buffer-mib: 5.811884E+01 | global_load_balancing_loss: 1.000128E+00 | loss scale: 1.0 | grad norm: 0.083 | params norm: 11394.326 | params rms: 0.061697 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 22:54:30.335798 | finish at 2026-10-10 14:50:06 + [2026-10-07 15:57:03] iteration 3828/ 6855 | consumed samples: 489984 | elapsed time per iteration (ms): 86920.8 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.61% | FLOPs estimator: workload-aware | logical tokens/s: 193016.4 | valid tokens/s: 193016.4 | physical tokens/s: 193449.4 | loss tokens/s: 149348.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.967191E-01 | lm-global-supervised-tokens: 1.298148E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.802358E-02 | lm-token-buffer-mib: 5.826118E+01 | global_load_balancing_loss: 9.998646E-01 | loss scale: 1.0 | grad norm: 0.035 | params norm: 11394.265 | params rms: 0.061696 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 1:05:09.156866 | finish at 2026-10-10 17:02:12 + [2026-10-07 15:58:29] iteration 3829/ 6855 | consumed samples: 490112 | elapsed time per iteration (ms): 86507.4 | throughput per GPU (TFLOP/s/GPU): 41.4 | MFU 13.27% | FLOPs estimator: workload-aware | logical tokens/s: 193939.2 | valid tokens/s: 193939.2 | physical tokens/s: 194501.8 | loss tokens/s: 146440.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.619251E-01 | lm-global-supervised-tokens: 1.266822E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.152236E-02 | lm-token-buffer-mib: 5.813686E+01 | global_load_balancing_loss: 1.000478E+00 | loss scale: 1.0 | grad norm: 0.050 | params norm: 11394.203 | params rms: 0.061696 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 0:42:51.361486 | finish at 2026-10-10 16:41:21 + [2026-10-07 16:00:18] iteration 3830/ 6855 | consumed samples: 490240 | elapsed time per iteration (ms): 108861.6 | throughput per GPU (TFLOP/s/GPU): 33.6 | MFU 10.77% | FLOPs estimator: workload-aware | logical tokens/s: 154114.5 | valid tokens/s: 154114.5 | physical tokens/s: 154663.6 | loss tokens/s: 118004.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.930901E-01 | lm-global-supervised-tokens: 1.284616E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.339870E-02 | lm-token-buffer-mib: 5.820968E+01 | global_load_balancing_loss: 9.999434E-01 | loss scale: 1.0 | grad norm: 0.049 | params norm: 11394.141 | params rms: 0.061696 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 19:28:26.313080 | finish at 2026-10-11 11:28:44 + [2026-10-07 16:01:49] iteration 3831/ 6855 | consumed samples: 490368 | elapsed time per iteration (ms): 90957.9 | throughput per GPU (TFLOP/s/GPU): 41.7 | MFU 13.37% | FLOPs estimator: workload-aware | logical tokens/s: 184449.6 | valid tokens/s: 184449.6 | physical tokens/s: 184978.0 | loss tokens/s: 128250.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.850473E-01 | lm-global-supervised-tokens: 1.166540E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.889002E-02 | lm-token-buffer-mib: 5.809500E+01 | global_load_balancing_loss: 9.998427E-01 | loss scale: 1.0 | grad norm: 0.041 | params norm: 11394.079 | params rms: 0.061695 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 4:24:16.710651 | finish at 2026-10-10 20:26:06 + [2026-10-07 16:03:15] iteration 3832/ 6855 | consumed samples: 490496 | elapsed time per iteration (ms): 85693.3 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.42% | FLOPs estimator: workload-aware | logical tokens/s: 195781.3 | valid tokens/s: 195781.2 | physical tokens/s: 196228.0 | loss tokens/s: 144703.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.079699E-01 | lm-global-supervised-tokens: 1.240008E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.360884E-02 | lm-token-buffer-mib: 5.814404E+01 | global_load_balancing_loss: 1.001835E+00 | loss scale: 1.0 | grad norm: 0.046 | params norm: 11394.016 | params rms: 0.061695 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 23:57:30.836557 | finish at 2026-10-10 16:00:46 + [2026-10-07 16:04:39] iteration 3833/ 6855 | consumed samples: 490624 | elapsed time per iteration (ms): 84214.2 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.55% | FLOPs estimator: workload-aware | logical tokens/s: 199220.2 | valid tokens/s: 199220.2 | physical tokens/s: 199723.8 | loss tokens/s: 150334.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.025472E-01 | lm-global-supervised-tokens: 1.266031E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.127173E-02 | lm-token-buffer-mib: 5.821522E+01 | global_load_balancing_loss: 1.001356E+00 | loss scale: 1.0 | grad norm: 0.046 | params norm: 11393.954 | params rms: 0.061695 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 22:41:35.396038 | finish at 2026-10-10 14:46:14 + [2026-10-07 16:06:05] iteration 3834/ 6855 | consumed samples: 490752 | elapsed time per iteration (ms): 86076.1 | throughput per GPU (TFLOP/s/GPU): 41.4 | MFU 13.27% | FLOPs estimator: workload-aware | logical tokens/s: 194911.4 | valid tokens/s: 194911.3 | physical tokens/s: 195304.4 | loss tokens/s: 145830.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.178482E-01 | lm-global-supervised-tokens: 1.255256E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.067758E-02 | lm-token-buffer-mib: 5.818918E+01 | global_load_balancing_loss: 1.000105E+00 | loss scale: 1.0 | grad norm: 0.049 | params norm: 11393.894 | params rms: 0.061694 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 0:13:55.886191 | finish at 2026-10-10 16:20:01 + [2026-10-07 16:07:32] iteration 3835/ 6855 | consumed samples: 490880 | elapsed time per iteration (ms): 86642.1 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.55% | FLOPs estimator: workload-aware | logical tokens/s: 193638.0 | valid tokens/s: 193637.9 | physical tokens/s: 194034.7 | loss tokens/s: 147996.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.028000E-01 | lm-global-supervised-tokens: 1.282271E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.852947E-02 | lm-token-buffer-mib: 5.811507E+01 | global_load_balancing_loss: 9.991252E-01 | loss scale: 1.0 | grad norm: 0.064 | params norm: 11393.834 | params rms: 0.061694 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 0:40:59.244533 | finish at 2026-10-10 16:48:31 + [2026-10-07 16:08:58] iteration 3836/ 6855 | consumed samples: 491008 | elapsed time per iteration (ms): 86128.6 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.78% | FLOPs estimator: workload-aware | logical tokens/s: 194792.3 | valid tokens/s: 194792.3 | physical tokens/s: 195311.2 | loss tokens/s: 143117.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.409209E-01 | lm-global-supervised-tokens: 1.232651E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.455929E-02 | lm-token-buffer-mib: 5.809538E+01 | global_load_balancing_loss: 9.989632E-01 | loss scale: 1.0 | grad norm: 0.054 | params norm: 11393.772 | params rms: 0.061694 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 0:13:42.142274 | finish at 2026-10-10 16:22:40 + [2026-10-07 16:10:25] iteration 3837/ 6855 | consumed samples: 491136 | elapsed time per iteration (ms): 87254.7 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.78% | FLOPs estimator: workload-aware | logical tokens/s: 192278.0 | valid tokens/s: 192278.0 | physical tokens/s: 192724.9 | loss tokens/s: 139619.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.454069E-01 | lm-global-supervised-tokens: 1.218248E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.311206E-02 | lm-token-buffer-mib: 5.820085E+01 | global_load_balancing_loss: 9.994342E-01 | loss scale: 1.0 | grad norm: 0.178 | params norm: 11393.710 | params rms: 0.061693 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 1:08:54.821149 | finish at 2026-10-10 17:19:20 + [2026-10-07 16:11:52] iteration 3838/ 6855 | consumed samples: 491264 | elapsed time per iteration (ms): 87189.9 | throughput per GPU (TFLOP/s/GPU): 43.4 | MFU 13.91% | FLOPs estimator: workload-aware | logical tokens/s: 192421.5 | valid tokens/s: 192421.5 | physical tokens/s: 192799.0 | loss tokens/s: 144521.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.958205E-01 | lm-global-supervised-tokens: 1.260079E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.436608E-02 | lm-token-buffer-mib: 5.814008E+01 | global_load_balancing_loss: 1.000767E+00 | loss scale: 1.0 | grad norm: 0.228 | params norm: 11393.647 | params rms: 0.061693 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 1:04:11.923747 | finish at 2026-10-10 17:16:04 + [2026-10-07 16:13:16] iteration 3839/ 6855 | consumed samples: 491392 | elapsed time per iteration (ms): 84087.7 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.79% | FLOPs estimator: workload-aware | logical tokens/s: 199519.9 | valid tokens/s: 199519.9 | physical tokens/s: 200118.2 | loss tokens/s: 151328.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.680325E-01 | lm-global-supervised-tokens: 1.272489E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.760740E-02 | lm-token-buffer-mib: 5.811910E+01 | global_load_balancing_loss: 1.000495E+00 | loss scale: 1.0 | grad norm: 0.262 | params norm: 11393.583 | params rms: 0.061693 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 22:26:48.630144 | finish at 2026-10-10 14:40:05 + [2026-10-07 16:15:04] iteration 3840/ 6855 | consumed samples: 491520 | elapsed time per iteration (ms): 107272.7 | throughput per GPU (TFLOP/s/GPU): 34.1 | MFU 10.92% | FLOPs estimator: workload-aware | logical tokens/s: 156397.8 | valid tokens/s: 156397.8 | physical tokens/s: 156759.7 | loss tokens/s: 119284.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.951262E-01 | lm-global-supervised-tokens: 1.279596E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.159473E-02 | lm-token-buffer-mib: 5.812910E+01 | global_load_balancing_loss: 1.000526E+00 | loss scale: 1.0 | grad norm: 0.114 | params norm: 11393.520 | params rms: 0.061692 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 17:50:27.054856 | finish at 2026-10-11 10:05:31 +times across ranks (ms): + save-checkpoint: + rank 0: 24754.97 + rank 1: 24754.96 + rank 2: 24754.97 + rank 3: 24755.00 + rank 4: 24755.00 + rank 5: 24754.96 + rank 6: 24754.94 + rank 7: 24754.96 + rank 8: 24756.13 + rank 9: 24756.12 + rank 10: 24756.12 + rank 11: 24756.14 + rank 12: 24756.12 + rank 13: 24756.12 + rank 14: 24756.11 + rank 15: 24756.12 + rank 16: 24755.01 + rank 17: 24754.98 + rank 18: 24754.98 + rank 19: 24755.01 + rank 20: 24754.98 + rank 21: 24755.04 + rank 22: 24755.01 + rank 23: 24754.99 + rank 24: 24755.18 + rank 25: 24755.20 + rank 26: 24755.22 + rank 27: 24755.21 + rank 28: 24755.15 + rank 29: 24755.24 + rank 30: 24755.16 + rank 31: 24755.19 + rank 32: 24756.41 + rank 33: 24756.43 + rank 34: 24756.42 + rank 35: 24756.39 + rank 36: 24756.40 + rank 37: 24756.41 + rank 38: 24756.40 + rank 39: 24756.38 + rank 40: 24754.65 + rank 41: 24754.64 + rank 42: 24754.64 + rank 43: 24754.66 + rank 44: 24754.66 + rank 45: 24754.66 + rank 46: 24754.65 + rank 47: 24754.63 + rank 48: 24755.29 + rank 49: 24755.29 + rank 50: 24755.28 + rank 51: 24755.28 + rank 52: 24755.32 + rank 53: 24755.25 + rank 54: 24755.25 + rank 55: 24755.27 + rank 56: 24755.48 + rank 57: 24755.46 + rank 58: 24755.45 + rank 59: 24755.45 + rank 60: 24755.45 + rank 61: 24755.45 + rank 62: 24755.46 + rank 63: 24755.47 + [2026-10-07 16:17:10] iteration 3841/ 6855 | consumed samples: 491648 | elapsed time per iteration (ms): 101046.1 | throughput per GPU (TFLOP/s/GPU): 36.5 | MFU 11.69% | FLOPs estimator: workload-aware | logical tokens/s: 166034.4 | valid tokens/s: 166034.4 | physical tokens/s: 166448.3 | loss tokens/s: 122868.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.622779E-01 | lm-global-supervised-tokens: 1.241534E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 6.497020E-01 | lm-token-buffer-mib: 5.818526E+01 | global_load_balancing_loss: 1.001270E+00 | loss scale: 1.0 | grad norm: 0.314 | params norm: 11393.459 | params rms: 0.061692 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 12:35:53.080916 | finish at 2026-10-11 04:53:03 + [2026-10-07 16:18:42] iteration 3842/ 6855 | consumed samples: 491776 | elapsed time per iteration (ms): 91421.0 | throughput per GPU (TFLOP/s/GPU): 40.4 | MFU 12.94% | FLOPs estimator: workload-aware | logical tokens/s: 183515.1 | valid tokens/s: 183515.1 | physical tokens/s: 184130.0 | loss tokens/s: 132067.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.730881E-01 | lm-global-supervised-tokens: 1.207376E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.765813E-02 | lm-token-buffer-mib: 5.807498E+01 | global_load_balancing_loss: 1.000681E+00 | loss scale: 1.0 | grad norm: 0.513 | params norm: 11393.397 | params rms: 0.061692 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 4:30:51.411233 | finish at 2026-10-10 20:49:33 + [2026-10-07 16:20:08] iteration 3843/ 6855 | consumed samples: 491904 | elapsed time per iteration (ms): 86705.2 | throughput per GPU (TFLOP/s/GPU): 42.8 | MFU 13.72% | FLOPs estimator: workload-aware | logical tokens/s: 193497.2 | valid tokens/s: 193497.2 | physical tokens/s: 193853.7 | loss tokens/s: 138377.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.579884E-01 | lm-global-supervised-tokens: 1.199802E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.345581E-02 | lm-token-buffer-mib: 5.823760E+01 | global_load_balancing_loss: 1.001131E+00 | loss scale: 1.0 | grad norm: 0.154 | params norm: 11393.335 | params rms: 0.061691 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 0:32:36.105357 | finish at 2026-10-10 16:52:44 + [2026-10-07 16:21:33] iteration 3844/ 6855 | consumed samples: 492032 | elapsed time per iteration (ms): 84724.9 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.77% | FLOPs estimator: workload-aware | logical tokens/s: 198019.4 | valid tokens/s: 198019.4 | physical tokens/s: 198549.4 | loss tokens/s: 142296.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.348424E-01 | lm-global-supervised-tokens: 1.205607E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.334802E-02 | lm-token-buffer-mib: 5.815623E+01 | global_load_balancing_loss: 9.994321E-01 | loss scale: 1.0 | grad norm: 0.289 | params norm: 11393.273 | params rms: 0.061691 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 22:51:46.701201 | finish at 2026-10-10 15:13:20 + [2026-10-07 16:22:59] iteration 3845/ 6855 | consumed samples: 492160 | elapsed time per iteration (ms): 86018.4 | throughput per GPU (TFLOP/s/GPU): 43.4 | MFU 13.92% | FLOPs estimator: workload-aware | logical tokens/s: 195042.1 | valid tokens/s: 195042.1 | physical tokens/s: 195363.6 | loss tokens/s: 136464.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.560014E-01 | lm-global-supervised-tokens: 1.173842E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.488586E-02 | lm-token-buffer-mib: 5.811246E+01 | global_load_balancing_loss: 9.998568E-01 | loss scale: 1.0 | grad norm: 0.164 | params norm: 11393.211 | params rms: 0.061691 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 23:55:15.444143 | finish at 2026-10-10 16:18:14 + [2026-10-07 16:24:28] iteration 3846/ 6855 | consumed samples: 492288 | elapsed time per iteration (ms): 89271.6 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.51% | FLOPs estimator: workload-aware | logical tokens/s: 187933.7 | valid tokens/s: 187933.7 | physical tokens/s: 188422.9 | loss tokens/s: 130670.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.808501E-01 | lm-global-supervised-tokens: 1.166520E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.184598E-02 | lm-token-buffer-mib: 5.812376E+01 | global_load_balancing_loss: 9.987658E-01 | loss scale: 1.0 | grad norm: 0.151 | params norm: 11393.150 | params rms: 0.061690 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 2:36:58.324773 | finish at 2026-10-10 19:01:27 + [2026-10-07 16:25:58] iteration 3847/ 6855 | consumed samples: 492416 | elapsed time per iteration (ms): 89339.8 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.45% | FLOPs estimator: workload-aware | logical tokens/s: 187790.6 | valid tokens/s: 187790.6 | physical tokens/s: 188164.6 | loss tokens/s: 145968.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.254017E-01 | lm-global-supervised-tokens: 1.304083E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.274697E-02 | lm-token-buffer-mib: 5.815646E+01 | global_load_balancing_loss: 9.990340E-01 | loss scale: 1.0 | grad norm: 0.261 | params norm: 11393.089 | params rms: 0.061690 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 2:38:54.220596 | finish at 2026-10-10 19:04:52 + [2026-10-07 16:27:21] iteration 3848/ 6855 | consumed samples: 492544 | elapsed time per iteration (ms): 83564.0 | throughput per GPU (TFLOP/s/GPU): 44.4 | MFU 14.24% | FLOPs estimator: workload-aware | logical tokens/s: 200770.3 | valid tokens/s: 200770.3 | physical tokens/s: 201113.4 | loss tokens/s: 149263.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.933635E-01 | lm-global-supervised-tokens: 1.247307E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.806816E-02 | lm-token-buffer-mib: 5.823565E+01 | global_load_balancing_loss: 1.000384E+00 | loss scale: 1.0 | grad norm: 0.103 | params norm: 11393.025 | params rms: 0.061690 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 21:47:56.999292 | finish at 2026-10-10 14:15:18 + [2026-10-07 16:28:47] iteration 3849/ 6855 | consumed samples: 492672 | elapsed time per iteration (ms): 86321.0 | throughput per GPU (TFLOP/s/GPU): 43.3 | MFU 13.88% | FLOPs estimator: workload-aware | logical tokens/s: 194357.7 | valid tokens/s: 194357.7 | physical tokens/s: 194791.2 | loss tokens/s: 141508.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.197520E-01 | lm-global-supervised-tokens: 1.221518E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.249921E-02 | lm-token-buffer-mib: 5.822419E+01 | global_load_balancing_loss: 1.001031E+00 | loss scale: 1.0 | grad norm: 0.159 | params norm: 11392.963 | params rms: 0.061689 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 0:04:40.968583 | finish at 2026-10-10 16:33:28 + [2026-10-07 16:30:39] iteration 3850/ 6855 | consumed samples: 492800 | elapsed time per iteration (ms): 111400.9 | throughput per GPU (TFLOP/s/GPU): 33.1 | MFU 10.62% | FLOPs estimator: workload-aware | logical tokens/s: 150602.0 | valid tokens/s: 150602.0 | physical tokens/s: 150975.9 | loss tokens/s: 116067.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.714264E-01 | lm-global-supervised-tokens: 1.292999E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.734233E-02 | lm-token-buffer-mib: 5.819303E+01 | global_load_balancing_loss: 1.000849E+00 | loss scale: 1.0 | grad norm: 0.083 | params norm: 11392.902 | params rms: 0.061689 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 20:59:19.554423 | finish at 2026-10-11 13:29:58 + [2026-10-07 16:32:06] iteration 3851/ 6855 | consumed samples: 492928 | elapsed time per iteration (ms): 86930.2 | throughput per GPU (TFLOP/s/GPU): 43.1 | MFU 13.81% | FLOPs estimator: workload-aware | logical tokens/s: 192996.4 | valid tokens/s: 192996.4 | physical tokens/s: 193614.2 | loss tokens/s: 142353.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.355635E-01 | lm-global-supervised-tokens: 1.237482E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.554247E-02 | lm-token-buffer-mib: 5.821365E+01 | global_load_balancing_loss: 1.001178E+00 | loss scale: 1.0 | grad norm: 0.151 | params norm: 11392.841 | params rms: 0.061689 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 0:32:18.238736 | finish at 2026-10-10 17:04:24 + [2026-10-07 16:33:30] iteration 3852/ 6855 | consumed samples: 493056 | elapsed time per iteration (ms): 84414.0 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.59% | FLOPs estimator: workload-aware | logical tokens/s: 198749.0 | valid tokens/s: 198749.0 | physical tokens/s: 199136.1 | loss tokens/s: 152426.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.563131E-01 | lm-global-supervised-tokens: 1.286695E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.311384E-02 | lm-token-buffer-mib: 5.815164E+01 | global_load_balancing_loss: 1.000177E+00 | loss scale: 1.0 | grad norm: 0.077 | params norm: 11392.779 | params rms: 0.061688 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 22:24:55.164062 | finish at 2026-10-10 14:58:25 + [2026-10-07 16:34:58] iteration 3853/ 6855 | consumed samples: 493184 | elapsed time per iteration (ms): 87604.6 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.45% | FLOPs estimator: workload-aware | logical tokens/s: 191510.0 | valid tokens/s: 191510.0 | physical tokens/s: 191968.6 | loss tokens/s: 148117.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.099313E-01 | lm-global-supervised-tokens: 1.297576E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.160790E-02 | lm-token-buffer-mib: 5.814111E+01 | global_load_balancing_loss: 9.992581E-01 | loss scale: 1.0 | grad norm: 0.133 | params norm: 11392.717 | params rms: 0.061688 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 1:03:08.880226 | finish at 2026-10-10 17:38:07 + [2026-10-07 16:36:24] iteration 3854/ 6855 | consumed samples: 493312 | elapsed time per iteration (ms): 86685.7 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.45% | FLOPs estimator: workload-aware | logical tokens/s: 193540.2 | valid tokens/s: 193540.2 | physical tokens/s: 194116.2 | loss tokens/s: 147283.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.841386E-01 | lm-global-supervised-tokens: 1.276739E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.588324E-02 | lm-token-buffer-mib: 5.814756E+01 | global_load_balancing_loss: 1.000255E+00 | loss scale: 1.0 | grad norm: 0.050 | params norm: 11392.654 | params rms: 0.061688 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 0:15:43.864939 | finish at 2026-10-10 16:52:08 + [2026-10-07 16:37:51] iteration 3855/ 6855 | consumed samples: 493440 | elapsed time per iteration (ms): 86988.2 | throughput per GPU (TFLOP/s/GPU): 42.8 | MFU 13.73% | FLOPs estimator: workload-aware | logical tokens/s: 192867.3 | valid tokens/s: 192867.3 | physical tokens/s: 193310.5 | loss tokens/s: 145693.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.406128E-01 | lm-global-supervised-tokens: 1.267358E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.396361E-02 | lm-token-buffer-mib: 5.809010E+01 | global_load_balancing_loss: 1.000781E+00 | loss scale: 1.0 | grad norm: 0.061 | params norm: 11392.592 | params rms: 0.061687 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 0:29:24.534044 | finish at 2026-10-10 17:07:16 + [2026-10-07 16:39:22] iteration 3856/ 6855 | consumed samples: 493568 | elapsed time per iteration (ms): 90808.5 | throughput per GPU (TFLOP/s/GPU): 40.8 | MFU 13.09% | FLOPs estimator: workload-aware | logical tokens/s: 184753.6 | valid tokens/s: 184753.6 | physical tokens/s: 185277.7 | loss tokens/s: 139630.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.742063E-01 | lm-global-supervised-tokens: 1.267965E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.281270E-02 | lm-token-buffer-mib: 5.824323E+01 | global_load_balancing_loss: 1.001345E+00 | loss scale: 1.0 | grad norm: 0.126 | params norm: 11392.530 | params rms: 0.061687 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 3:38:54.558661 | finish at 2026-10-10 20:18:17 + [2026-10-07 16:40:46] iteration 3857/ 6855 | consumed samples: 493696 | elapsed time per iteration (ms): 83498.5 | throughput per GPU (TFLOP/s/GPU): 43.5 | MFU 13.93% | FLOPs estimator: workload-aware | logical tokens/s: 200926.9 | valid tokens/s: 200926.9 | physical tokens/s: 201317.3 | loss tokens/s: 154755.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.892981E-01 | lm-global-supervised-tokens: 1.292187E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.160354E-02 | lm-token-buffer-mib: 5.835315E+01 | global_load_balancing_loss: 1.000312E+00 | loss scale: 1.0 | grad norm: 0.054 | params norm: 11392.467 | params rms: 0.061687 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 21:32:08.600536 | finish at 2026-10-10 14:12:54 + [2026-10-07 16:42:16] iteration 3858/ 6855 | consumed samples: 493824 | elapsed time per iteration (ms): 90335.8 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.51% | FLOPs estimator: workload-aware | logical tokens/s: 185720.0 | valid tokens/s: 185720.0 | physical tokens/s: 186102.3 | loss tokens/s: 133333.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.773202E-01 | lm-global-supervised-tokens: 1.204477E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.280669E-02 | lm-token-buffer-mib: 5.817247E+01 | global_load_balancing_loss: 9.998654E-01 | loss scale: 1.0 | grad norm: 0.089 | params norm: 11392.406 | params rms: 0.061686 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 3:12:16.307367 | finish at 2026-10-10 19:54:32 + [2026-10-07 16:43:47] iteration 3859/ 6855 | consumed samples: 493952 | elapsed time per iteration (ms): 91267.4 | throughput per GPU (TFLOP/s/GPU): 40.8 | MFU 13.09% | FLOPs estimator: workload-aware | logical tokens/s: 183824.6 | valid tokens/s: 183824.5 | physical tokens/s: 184194.7 | loss tokens/s: 131095.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.614016E-01 | lm-global-supervised-tokens: 1.196477E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.195778E-02 | lm-token-buffer-mib: 5.834751E+01 | global_load_balancing_loss: 1.000609E+00 | loss scale: 1.0 | grad norm: 0.053 | params norm: 11392.345 | params rms: 0.061686 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 3:57:17.174766 | finish at 2026-10-10 20:41:05 + [2026-10-07 16:45:37] iteration 3860/ 6855 | consumed samples: 494080 | elapsed time per iteration (ms): 109389.7 | throughput per GPU (TFLOP/s/GPU): 33.8 | MFU 10.84% | FLOPs estimator: workload-aware | logical tokens/s: 153370.9 | valid tokens/s: 153370.9 | physical tokens/s: 153796.0 | loss tokens/s: 110968.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.988135E-01 | lm-global-supervised-tokens: 1.213881E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.161442E-02 | lm-token-buffer-mib: 5.802879E+01 | global_load_balancing_loss: 9.990067E-01 | loss scale: 1.0 | grad norm: 0.075 | params norm: 11392.284 | params rms: 0.061686 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 19:00:22.143453 | finish at 2026-10-11 11:45:59 + [2026-10-07 16:47:02] iteration 3861/ 6855 | consumed samples: 494208 | elapsed time per iteration (ms): 85368.0 | throughput per GPU (TFLOP/s/GPU): 43.2 | MFU 13.84% | FLOPs estimator: workload-aware | logical tokens/s: 196528.1 | valid tokens/s: 196528.1 | physical tokens/s: 197028.9 | loss tokens/s: 143296.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.280063E-01 | lm-global-supervised-tokens: 1.223289E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.549410E-02 | lm-token-buffer-mib: 5.826585E+01 | global_load_balancing_loss: 9.993529E-01 | loss scale: 1.0 | grad norm: 0.062 | params norm: 11392.223 | params rms: 0.061685 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 22:59:51.860619 | finish at 2026-10-10 15:46:54 + [2026-10-07 16:48:30] iteration 3862/ 6855 | consumed samples: 494336 | elapsed time per iteration (ms): 88106.0 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.57% | FLOPs estimator: workload-aware | logical tokens/s: 190420.7 | valid tokens/s: 190420.7 | physical tokens/s: 190902.7 | loss tokens/s: 139886.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.951700E-01 | lm-global-supervised-tokens: 1.232485E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.404065E-02 | lm-token-buffer-mib: 5.832138E+01 | global_load_balancing_loss: 9.995583E-01 | loss scale: 1.0 | grad norm: 0.108 | params norm: 11392.162 | params rms: 0.061685 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 1:15:01.175778 | finish at 2026-10-10 18:03:31 + [2026-10-07 16:49:57] iteration 3863/ 6855 | consumed samples: 494464 | elapsed time per iteration (ms): 86271.7 | throughput per GPU (TFLOP/s/GPU): 43.1 | MFU 13.82% | FLOPs estimator: workload-aware | logical tokens/s: 194468.7 | valid tokens/s: 194468.7 | physical tokens/s: 194909.4 | loss tokens/s: 147850.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.509189E-01 | lm-global-supervised-tokens: 1.275530E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.273188E-02 | lm-token-buffer-mib: 5.804110E+01 | global_load_balancing_loss: 1.000166E+00 | loss scale: 1.0 | grad norm: 0.033 | params norm: 11392.101 | params rms: 0.061685 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 23:42:04.924690 | finish at 2026-10-10 16:32:01 + [2026-10-07 16:51:23] iteration 3864/ 6855 | consumed samples: 494592 | elapsed time per iteration (ms): 86344.5 | throughput per GPU (TFLOP/s/GPU): 43.8 | MFU 14.05% | FLOPs estimator: workload-aware | logical tokens/s: 194305.1 | valid tokens/s: 194305.1 | physical tokens/s: 194696.8 | loss tokens/s: 143727.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.700866E-01 | lm-global-supervised-tokens: 1.241012E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.228051E-02 | lm-token-buffer-mib: 5.830133E+01 | global_load_balancing_loss: 1.000625E+00 | loss scale: 1.0 | grad norm: 0.050 | params norm: 11392.042 | params rms: 0.061684 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 23:44:16.523775 | finish at 2026-10-10 16:35:39 + [2026-10-07 16:52:48] iteration 3865/ 6855 | consumed samples: 494720 | elapsed time per iteration (ms): 85225.6 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.74% | FLOPs estimator: workload-aware | logical tokens/s: 196856.5 | valid tokens/s: 196856.5 | physical tokens/s: 197449.6 | loss tokens/s: 144191.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.300419E-01 | lm-global-supervised-tokens: 1.228877E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.155058E-02 | lm-token-buffer-mib: 5.828790E+01 | global_load_balancing_loss: 1.001342E+00 | loss scale: 1.0 | grad norm: 0.159 | params norm: 11391.982 | params rms: 0.061684 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 22:47:04.547577 | finish at 2026-10-10 15:39:53 + [2026-10-07 16:54:16] iteration 3866/ 6855 | consumed samples: 494848 | elapsed time per iteration (ms): 87458.2 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.57% | FLOPs estimator: workload-aware | logical tokens/s: 191830.5 | valid tokens/s: 191830.5 | physical tokens/s: 192302.3 | loss tokens/s: 142271.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.140694E-01 | lm-global-supervised-tokens: 1.244284E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.319218E-02 | lm-token-buffer-mib: 5.813834E+01 | global_load_balancing_loss: 1.001097E+00 | loss scale: 1.0 | grad norm: 0.143 | params norm: 11391.923 | params rms: 0.061684 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 0:36:52.540493 | finish at 2026-10-10 17:31:08 + [2026-10-07 16:55:45] iteration 3867/ 6855 | consumed samples: 494976 | elapsed time per iteration (ms): 89208.3 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.62% | FLOPs estimator: workload-aware | logical tokens/s: 188067.6 | valid tokens/s: 188067.5 | physical tokens/s: 188402.4 | loss tokens/s: 137033.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.558107E-01 | lm-global-supervised-tokens: 1.222449E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.363252E-02 | lm-token-buffer-mib: 5.820139E+01 | global_load_balancing_loss: 9.996234E-01 | loss scale: 1.0 | grad norm: 0.041 | params norm: 11391.864 | params rms: 0.061683 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 2:02:34.370819 | finish at 2026-10-10 18:58:19 + [2026-10-07 16:57:11] iteration 3868/ 6855 | consumed samples: 495104 | elapsed time per iteration (ms): 85921.4 | throughput per GPU (TFLOP/s/GPU): 43.7 | MFU 14.00% | FLOPs estimator: workload-aware | logical tokens/s: 195262.1 | valid tokens/s: 195262.1 | physical tokens/s: 195659.0 | loss tokens/s: 148192.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.904950E-01 | lm-global-supervised-tokens: 1.273292E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.314756E-02 | lm-token-buffer-mib: 5.806618E+01 | global_load_balancing_loss: 9.997440E-01 | loss scale: 1.0 | grad norm: 0.123 | params norm: 11391.805 | params rms: 0.061683 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 23:17:27.259042 | finish at 2026-10-10 16:14:38 + [2026-10-07 16:58:38] iteration 3869/ 6855 | consumed samples: 495232 | elapsed time per iteration (ms): 86943.9 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.51% | FLOPs estimator: workload-aware | logical tokens/s: 192965.1 | valid tokens/s: 192965.1 | physical tokens/s: 193341.8 | loss tokens/s: 142079.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.619210E-01 | lm-global-supervised-tokens: 1.235298E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.157502E-02 | lm-token-buffer-mib: 5.809673E+01 | global_load_balancing_loss: 9.998111E-01 | loss scale: 1.0 | grad norm: 0.086 | params norm: 11391.747 | params rms: 0.061683 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 0:06:54.511353 | finish at 2026-10-10 17:05:32 + [2026-10-07 17:00:25] iteration 3870/ 6855 | consumed samples: 495360 | elapsed time per iteration (ms): 107723.3 | throughput per GPU (TFLOP/s/GPU): 33.9 | MFU 10.87% | FLOPs estimator: workload-aware | logical tokens/s: 155743.4 | valid tokens/s: 155743.4 | physical tokens/s: 156139.2 | loss tokens/s: 113517.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.270116E-01 | lm-global-supervised-tokens: 1.222843E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.138658E-02 | lm-token-buffer-mib: 5.808938E+01 | global_load_balancing_loss: 1.000403E+00 | loss scale: 1.0 | grad norm: 0.188 | params norm: 11391.688 | params rms: 0.061682 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 17:19:14.154346 | finish at 2026-10-11 10:19:40 + [2026-10-07 17:01:56] iteration 3871/ 6855 | consumed samples: 495488 | elapsed time per iteration (ms): 90360.6 | throughput per GPU (TFLOP/s/GPU): 41.7 | MFU 13.36% | FLOPs estimator: workload-aware | logical tokens/s: 185669.6 | valid tokens/s: 185669.6 | physical tokens/s: 186039.9 | loss tokens/s: 142595.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.730123E-01 | lm-global-supervised-tokens: 1.288502E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.401036E-02 | lm-token-buffer-mib: 5.829179E+01 | global_load_balancing_loss: 9.988633E-01 | loss scale: 1.0 | grad norm: 0.057 | params norm: 11391.630 | params rms: 0.061682 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 2:53:55.971334 | finish at 2026-10-10 19:55:52 + [2026-10-07 17:03:24] iteration 3872/ 6855 | consumed samples: 495616 | elapsed time per iteration (ms): 88084.5 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.57% | FLOPs estimator: workload-aware | logical tokens/s: 190467.0 | valid tokens/s: 190467.0 | physical tokens/s: 190957.8 | loss tokens/s: 140117.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.909701E-01 | lm-global-supervised-tokens: 1.234214E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.485223E-02 | lm-token-buffer-mib: 5.808237E+01 | global_load_balancing_loss: 1.000490E+00 | loss scale: 1.0 | grad norm: 0.223 | params norm: 11391.571 | params rms: 0.061682 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 0:59:15.949929 | finish at 2026-10-10 18:02:40 + [2026-10-07 17:04:51] iteration 3873/ 6855 | consumed samples: 495744 | elapsed time per iteration (ms): 87528.8 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.61% | FLOPs estimator: workload-aware | logical tokens/s: 191675.6 | valid tokens/s: 191675.6 | physical tokens/s: 192095.2 | loss tokens/s: 143890.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.931078E-01 | lm-global-supervised-tokens: 1.259457E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.205087E-02 | lm-token-buffer-mib: 5.820318E+01 | global_load_balancing_loss: 1.000638E+00 | loss scale: 1.0 | grad norm: 0.063 | params norm: 11391.513 | params rms: 0.061681 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 0:30:10.979034 | finish at 2026-10-10 17:35:02 + [2026-10-07 17:06:21] iteration 3874/ 6855 | consumed samples: 495872 | elapsed time per iteration (ms): 89371.2 | throughput per GPU (TFLOP/s/GPU): 41.7 | MFU 13.38% | FLOPs estimator: workload-aware | logical tokens/s: 187724.1 | valid tokens/s: 187724.1 | physical tokens/s: 188095.8 | loss tokens/s: 136044.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.724216E-01 | lm-global-supervised-tokens: 1.215842E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.147108E-02 | lm-token-buffer-mib: 5.808314E+01 | global_load_balancing_loss: 1.002054E+00 | loss scale: 1.0 | grad norm: 0.109 | params norm: 11391.454 | params rms: 0.061681 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 2:00:15.652640 | finish at 2026-10-10 19:06:36 + [2026-10-07 17:07:47] iteration 3875/ 6855 | consumed samples: 496000 | elapsed time per iteration (ms): 85997.6 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.43% | FLOPs estimator: workload-aware | logical tokens/s: 195089.3 | valid tokens/s: 195089.3 | physical tokens/s: 195632.8 | loss tokens/s: 143707.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.030304E-01 | lm-global-supervised-tokens: 1.235846E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.178588E-02 | lm-token-buffer-mib: 5.817381E+01 | global_load_balancing_loss: 1.000955E+00 | loss scale: 1.0 | grad norm: 0.068 | params norm: 11391.396 | params rms: 0.061681 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 23:11:12.879496 | finish at 2026-10-10 16:19:00 + [2026-10-07 17:09:16] iteration 3876/ 6855 | consumed samples: 496128 | elapsed time per iteration (ms): 88906.6 | throughput per GPU (TFLOP/s/GPU): 41.7 | MFU 13.38% | FLOPs estimator: workload-aware | logical tokens/s: 188706.1 | valid tokens/s: 188706.1 | physical tokens/s: 189046.2 | loss tokens/s: 143186.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.580056E-01 | lm-global-supervised-tokens: 1.273020E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.157893E-02 | lm-token-buffer-mib: 5.808849E+01 | global_load_balancing_loss: 9.995260E-01 | loss scale: 1.0 | grad norm: 0.104 | params norm: 11391.339 | params rms: 0.061680 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 1:34:12.717360 | finish at 2026-10-10 18:43:28 + [2026-10-07 17:10:42] iteration 3877/ 6855 | consumed samples: 496256 | elapsed time per iteration (ms): 86118.9 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.41% | FLOPs estimator: workload-aware | logical tokens/s: 194813.3 | valid tokens/s: 194813.3 | physical tokens/s: 195299.5 | loss tokens/s: 147498.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.585250E-01 | lm-global-supervised-tokens: 1.270243E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.215728E-02 | lm-token-buffer-mib: 5.806915E+01 | global_load_balancing_loss: 9.992852E-01 | loss scale: 1.0 | grad norm: 0.098 | params norm: 11391.284 | params rms: 0.061680 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 23:14:22.107811 | finish at 2026-10-10 16:25:04 + [2026-10-07 17:12:09] iteration 3878/ 6855 | consumed samples: 496384 | elapsed time per iteration (ms): 86779.3 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.62% | FLOPs estimator: workload-aware | logical tokens/s: 193332.0 | valid tokens/s: 193332.0 | physical tokens/s: 193585.7 | loss tokens/s: 145757.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.043575E-01 | lm-global-supervised-tokens: 1.264877E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.636998E-02 | lm-token-buffer-mib: 5.807036E+01 | global_load_balancing_loss: 9.982970E-01 | loss scale: 1.0 | grad norm: 0.066 | params norm: 11391.227 | params rms: 0.061680 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 23:45:41.963248 | finish at 2026-10-10 16:57:50 + [2026-10-07 17:13:33] iteration 3879/ 6855 | consumed samples: 496512 | elapsed time per iteration (ms): 84519.5 | throughput per GPU (TFLOP/s/GPU): 43.6 | MFU 13.99% | FLOPs estimator: workload-aware | logical tokens/s: 198500.6 | valid tokens/s: 198500.6 | physical tokens/s: 198932.1 | loss tokens/s: 139873.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.354573E-01 | lm-global-supervised-tokens: 1.182203E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.840218E-02 | lm-token-buffer-mib: 5.823417E+01 | global_load_balancing_loss: 1.001023E+00 | loss scale: 1.0 | grad norm: 0.104 | params norm: 11391.171 | params rms: 0.061679 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 21:52:10.077461 | finish at 2026-10-10 15:05:43 + [2026-10-07 17:15:22] iteration 3880/ 6855 | consumed samples: 496640 | elapsed time per iteration (ms): 109178.9 | throughput per GPU (TFLOP/s/GPU): 33.8 | MFU 10.82% | FLOPs estimator: workload-aware | logical tokens/s: 153666.4 | valid tokens/s: 153666.4 | physical tokens/s: 154129.9 | loss tokens/s: 112553.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.876192E-01 | lm-global-supervised-tokens: 1.228842E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.550964E-02 | lm-token-buffer-mib: 5.833818E+01 | global_load_balancing_loss: 9.995812E-01 | loss scale: 1.0 | grad norm: 0.050 | params norm: 11391.114 | params rms: 0.061679 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 18:13:27.201266 | finish at 2026-10-11 11:28:49 +times across ranks (ms): + save-checkpoint: + rank 0: 24090.46 + rank 1: 24090.52 + rank 2: 24090.55 + rank 3: 24090.45 + rank 4: 24090.47 + rank 5: 24090.46 + rank 6: 24090.44 + rank 7: 24090.45 + rank 8: 24090.48 + rank 9: 24090.46 + rank 10: 24090.46 + rank 11: 24090.46 + rank 12: 24090.46 + rank 13: 24090.47 + rank 14: 24090.47 + rank 15: 24090.46 + rank 16: 24090.79 + rank 17: 24090.75 + rank 18: 24090.76 + rank 19: 24090.75 + rank 20: 24090.75 + rank 21: 24090.75 + rank 22: 24090.80 + rank 23: 24090.72 + rank 24: 24090.27 + rank 25: 24090.27 + rank 26: 24090.27 + rank 27: 24090.28 + rank 28: 24090.24 + rank 29: 24090.28 + rank 30: 24090.23 + rank 31: 24090.25 + rank 32: 24090.30 + rank 33: 24090.29 + rank 34: 24090.36 + rank 35: 24090.26 + rank 36: 24090.27 + rank 37: 24090.23 + rank 38: 24090.26 + rank 39: 24090.27 + rank 40: 24089.94 + rank 41: 24089.92 + rank 42: 24089.93 + rank 43: 24089.92 + rank 44: 24089.92 + rank 45: 24089.92 + rank 46: 24089.93 + rank 47: 24089.91 + rank 48: 24090.64 + rank 49: 24090.69 + rank 50: 24090.59 + rank 51: 24090.61 + rank 52: 24090.64 + rank 53: 24090.61 + rank 54: 24090.63 + rank 55: 24090.64 + rank 56: 24090.81 + rank 57: 24090.81 + rank 58: 24090.80 + rank 59: 24090.79 + rank 60: 24090.80 + rank 61: 24090.79 + rank 62: 24090.78 + rank 63: 24090.82 + [2026-10-07 17:17:30] iteration 3881/ 6855 | consumed samples: 496768 | elapsed time per iteration (ms): 100063.6 | throughput per GPU (TFLOP/s/GPU): 37.4 | MFU 11.98% | FLOPs estimator: workload-aware | logical tokens/s: 167665.5 | valid tokens/s: 167665.5 | physical tokens/s: 167941.9 | loss tokens/s: 116689.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.413185E-01 | lm-global-supervised-tokens: 1.167640E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.213978E-02 | lm-token-buffer-mib: 5.805349E+01 | global_load_balancing_loss: 1.000196E+00 | loss scale: 1.0 | grad norm: 0.050 | params norm: 11391.058 | params rms: 0.061679 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 10:39:49.017540 | finish at 2026-10-11 03:57:19 + [2026-10-07 17:19:03] iteration 3882/ 6855 | consumed samples: 496896 | elapsed time per iteration (ms): 93361.9 | throughput per GPU (TFLOP/s/GPU): 39.4 | MFU 12.62% | FLOPs estimator: workload-aware | logical tokens/s: 179700.6 | valid tokens/s: 179700.6 | physical tokens/s: 180140.4 | loss tokens/s: 130494.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.765997E-01 | lm-global-supervised-tokens: 1.218319E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.968263E-02 | lm-token-buffer-mib: 5.809838E+01 | global_load_balancing_loss: 1.000349E+00 | loss scale: 1.0 | grad norm: 0.211 | params norm: 11391.001 | params rms: 0.061679 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 5:06:04.780119 | finish at 2026-10-10 22:25:08 + [2026-10-07 17:20:27] iteration 3883/ 6855 | consumed samples: 497024 | elapsed time per iteration (ms): 84188.9 | throughput per GPU (TFLOP/s/GPU): 44.0 | MFU 14.11% | FLOPs estimator: workload-aware | logical tokens/s: 199280.7 | valid tokens/s: 199280.7 | physical tokens/s: 199604.5 | loss tokens/s: 145732.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.531532E-01 | lm-global-supervised-tokens: 1.226907E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.221280E-02 | lm-token-buffer-mib: 5.816598E+01 | global_load_balancing_loss: 1.000767E+00 | loss scale: 1.0 | grad norm: 0.064 | params norm: 11390.940 | params rms: 0.061678 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 21:30:09.336381 | finish at 2026-10-10 14:50:36 + [2026-10-07 17:21:52] iteration 3884/ 6855 | consumed samples: 497152 | elapsed time per iteration (ms): 85253.3 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.80% | FLOPs estimator: workload-aware | logical tokens/s: 196792.1 | valid tokens/s: 196792.1 | physical tokens/s: 197264.0 | loss tokens/s: 143826.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.308395E-01 | lm-global-supervised-tokens: 1.226166E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.218916E-02 | lm-token-buffer-mib: 5.822594E+01 | global_load_balancing_loss: 1.001347E+00 | loss scale: 1.0 | grad norm: 0.161 | params norm: 11390.883 | params rms: 0.061678 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 22:21:27.510947 | finish at 2026-10-10 15:43:20 + [2026-10-07 17:23:20] iteration 3885/ 6855 | consumed samples: 497280 | elapsed time per iteration (ms): 87587.2 | throughput per GPU (TFLOP/s/GPU): 42.8 | MFU 13.71% | FLOPs estimator: workload-aware | logical tokens/s: 191548.2 | valid tokens/s: 191548.2 | physical tokens/s: 192069.2 | loss tokens/s: 134284.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.513543E-01 | lm-global-supervised-tokens: 1.176164E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.497782E-02 | lm-token-buffer-mib: 5.823562E+01 | global_load_balancing_loss: 1.000094E+00 | loss scale: 1.0 | grad norm: 0.233 | params norm: 11390.831 | params rms: 0.061678 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 0:15:33.941295 | finish at 2026-10-10 17:38:54 + [2026-10-07 17:24:49] iteration 3886/ 6855 | consumed samples: 497408 | elapsed time per iteration (ms): 89201.3 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.52% | FLOPs estimator: workload-aware | logical tokens/s: 188082.4 | valid tokens/s: 188082.4 | physical tokens/s: 188741.8 | loss tokens/s: 130739.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.736571E-01 | lm-global-supervised-tokens: 1.166211E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.143986E-02 | lm-token-buffer-mib: 5.811938E+01 | global_load_balancing_loss: 9.994587E-01 | loss scale: 1.0 | grad norm: 0.053 | params norm: 11390.778 | params rms: 0.061677 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 1:33:58.640308 | finish at 2026-10-10 18:58:48 + [2026-10-07 17:26:20] iteration 3887/ 6855 | consumed samples: 497536 | elapsed time per iteration (ms): 90391.1 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.62% | FLOPs estimator: workload-aware | logical tokens/s: 185606.8 | valid tokens/s: 185606.8 | physical tokens/s: 186002.5 | loss tokens/s: 132684.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.618524E-01 | lm-global-supervised-tokens: 1.199344E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.782338E-02 | lm-token-buffer-mib: 5.820472E+01 | global_load_balancing_loss: 9.998706E-01 | loss scale: 1.0 | grad norm: 0.222 | params norm: 11390.728 | params rms: 0.061677 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 2:31:20.651558 | finish at 2026-10-10 19:57:40 + [2026-10-07 17:27:46] iteration 3888/ 6855 | consumed samples: 497664 | elapsed time per iteration (ms): 86579.1 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.63% | FLOPs estimator: workload-aware | logical tokens/s: 193778.8 | valid tokens/s: 193778.8 | physical tokens/s: 194316.7 | loss tokens/s: 140337.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.767790E-01 | lm-global-supervised-tokens: 1.215025E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.539173E-02 | lm-token-buffer-mib: 5.820798E+01 | global_load_balancing_loss: 1.000948E+00 | loss scale: 1.0 | grad norm: 0.146 | params norm: 11390.678 | params rms: 0.061677 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 23:21:20.113694 | finish at 2026-10-10 16:49:06 + [2026-10-07 17:29:15] iteration 3889/ 6855 | consumed samples: 497792 | elapsed time per iteration (ms): 89399.4 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.39% | FLOPs estimator: workload-aware | logical tokens/s: 187664.9 | valid tokens/s: 187664.9 | physical tokens/s: 188183.0 | loss tokens/s: 139873.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.898298E-01 | lm-global-supervised-tokens: 1.250463E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.229478E-02 | lm-token-buffer-mib: 5.833385E+01 | global_load_balancing_loss: 9.999900E-01 | loss scale: 1.0 | grad norm: 0.084 | params norm: 11390.629 | params rms: 0.061677 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 1:39:18.489565 | finish at 2026-10-10 19:08:34 + [2026-10-07 17:31:05] iteration 3890/ 6855 | consumed samples: 497920 | elapsed time per iteration (ms): 109838.5 | throughput per GPU (TFLOP/s/GPU): 34.3 | MFU 10.98% | FLOPs estimator: workload-aware | logical tokens/s: 152744.2 | valid tokens/s: 152744.2 | physical tokens/s: 153012.4 | loss tokens/s: 112170.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.649396E-01 | lm-global-supervised-tokens: 1.232067E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.169955E-02 | lm-token-buffer-mib: 5.807746E+01 | global_load_balancing_loss: 9.994220E-01 | loss scale: 1.0 | grad norm: 0.095 | params norm: 11390.580 | params rms: 0.061676 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 18:27:51.216897 | finish at 2026-10-11 11:58:57 + [2026-10-07 17:32:30] iteration 3891/ 6855 | consumed samples: 498048 | elapsed time per iteration (ms): 84575.4 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.57% | FLOPs estimator: workload-aware | logical tokens/s: 198369.5 | valid tokens/s: 198369.5 | physical tokens/s: 198973.8 | loss tokens/s: 149820.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.446244E-01 | lm-global-supervised-tokens: 1.267111E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.347771E-02 | lm-token-buffer-mib: 5.807815E+01 | global_load_balancing_loss: 1.000321E+00 | loss scale: 1.0 | grad norm: 0.166 | params norm: 11390.532 | params rms: 0.061676 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 21:38:01.521272 | finish at 2026-10-10 15:10:31 + [2026-10-07 17:33:57] iteration 3892/ 6855 | consumed samples: 498176 | elapsed time per iteration (ms): 87063.2 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.56% | FLOPs estimator: workload-aware | logical tokens/s: 192701.3 | valid tokens/s: 192700.6 | physical tokens/s: 193164.8 | loss tokens/s: 145982.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.788575E-01 | lm-global-supervised-tokens: 1.270965E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.163721E-02 | lm-token-buffer-mib: 5.807228E+01 | global_load_balancing_loss: 1.000083E+00 | loss scale: 1.0 | grad norm: 0.194 | params norm: 11390.484 | params rms: 0.061676 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 23:39:28.155626 | finish at 2026-10-10 17:13:25 + [2026-10-07 17:35:25] iteration 3893/ 6855 | consumed samples: 498304 | elapsed time per iteration (ms): 87638.8 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.43% | FLOPs estimator: workload-aware | logical tokens/s: 191435.9 | valid tokens/s: 191435.9 | physical tokens/s: 191768.0 | loss tokens/s: 145009.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.582509E-01 | lm-global-supervised-tokens: 1.270841E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.167488E-02 | lm-token-buffer-mib: 5.814192E+01 | global_load_balancing_loss: 1.001063E+00 | loss scale: 1.0 | grad norm: 0.088 | params norm: 11390.435 | params rms: 0.061676 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 0:06:26.101310 | finish at 2026-10-10 17:41:51 + [2026-10-07 17:36:52] iteration 3894/ 6855 | consumed samples: 498432 | elapsed time per iteration (ms): 87517.9 | throughput per GPU (TFLOP/s/GPU): 42.8 | MFU 13.71% | FLOPs estimator: workload-aware | logical tokens/s: 191698.7 | valid tokens/s: 191698.7 | physical tokens/s: 192089.8 | loss tokens/s: 138331.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.601096E-01 | lm-global-supervised-tokens: 1.210648E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.161109E-02 | lm-token-buffer-mib: 5.816565E+01 | global_load_balancing_loss: 1.000597E+00 | loss scale: 1.0 | grad norm: 0.073 | params norm: 11390.385 | params rms: 0.061675 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 23:59:00.499047 | finish at 2026-10-10 17:35:53 + [2026-10-07 17:38:20] iteration 3895/ 6855 | consumed samples: 498560 | elapsed time per iteration (ms): 87979.2 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.54% | FLOPs estimator: workload-aware | logical tokens/s: 190695.0 | valid tokens/s: 190695.0 | physical tokens/s: 191075.1 | loss tokens/s: 140792.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.892073E-01 | lm-global-supervised-tokens: 1.238679E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.427081E-02 | lm-token-buffer-mib: 5.810844E+01 | global_load_balancing_loss: 1.000565E+00 | loss scale: 1.0 | grad norm: 0.108 | params norm: 11390.335 | params rms: 0.061675 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 0:20:18.324394 | finish at 2026-10-10 17:58:38 + [2026-10-07 17:39:53] iteration 3896/ 6855 | consumed samples: 498688 | elapsed time per iteration (ms): 93129.0 | throughput per GPU (TFLOP/s/GPU): 40.2 | MFU 12.88% | FLOPs estimator: workload-aware | logical tokens/s: 180149.9 | valid tokens/s: 180149.9 | physical tokens/s: 180604.8 | loss tokens/s: 129630.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.909348E-01 | lm-global-supervised-tokens: 1.207233E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.220882E-02 | lm-token-buffer-mib: 5.821719E+01 | global_load_balancing_loss: 9.992119E-01 | loss scale: 1.0 | grad norm: 0.069 | params norm: 11390.284 | params rms: 0.061675 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 4:32:48.635361 | finish at 2026-10-10 22:12:42 + [2026-10-07 17:41:19] iteration 3897/ 6855 | consumed samples: 498816 | elapsed time per iteration (ms): 85764.1 | throughput per GPU (TFLOP/s/GPU): 43.3 | MFU 13.88% | FLOPs estimator: workload-aware | logical tokens/s: 195620.0 | valid tokens/s: 195619.9 | physical tokens/s: 196199.5 | loss tokens/s: 141490.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.950668E-01 | lm-global-supervised-tokens: 1.213479E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.305469E-02 | lm-token-buffer-mib: 5.819328E+01 | global_load_balancing_loss: 9.989045E-01 | loss scale: 1.0 | grad norm: 0.082 | params norm: 11390.231 | params rms: 0.061674 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 22:28:10.246104 | finish at 2026-10-10 16:09:29 + [2026-10-07 17:42:44] iteration 3898/ 6855 | consumed samples: 498944 | elapsed time per iteration (ms): 84704.5 | throughput per GPU (TFLOP/s/GPU): 43.4 | MFU 13.91% | FLOPs estimator: workload-aware | logical tokens/s: 198066.8 | valid tokens/s: 198066.8 | physical tokens/s: 198526.5 | loss tokens/s: 152107.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.890661E-01 | lm-global-supervised-tokens: 1.288414E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.311016E-02 | lm-token-buffer-mib: 5.813975E+01 | global_load_balancing_loss: 1.000204E+00 | loss scale: 1.0 | grad norm: 0.107 | params norm: 11390.176 | params rms: 0.061674 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 21:34:31.122486 | finish at 2026-10-10 15:17:15 + [2026-10-07 17:44:11] iteration 3899/ 6855 | consumed samples: 499072 | elapsed time per iteration (ms): 86819.4 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.52% | FLOPs estimator: workload-aware | logical tokens/s: 193242.3 | valid tokens/s: 193242.3 | physical tokens/s: 193672.5 | loss tokens/s: 147345.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.454977E-01 | lm-global-supervised-tokens: 1.279247E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.557215E-02 | lm-token-buffer-mib: 5.810809E+01 | global_load_balancing_loss: 9.991202E-01 | loss scale: 1.0 | grad norm: 0.109 | params norm: 11390.122 | params rms: 0.061674 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 23:17:18.076841 | finish at 2026-10-10 17:01:29 + [2026-10-07 17:46:01] iteration 3900/ 6855 | consumed samples: 499200 | elapsed time per iteration (ms): 110783.7 | throughput per GPU (TFLOP/s/GPU): 34.4 | MFU 11.03% | FLOPs estimator: workload-aware | logical tokens/s: 151441.1 | valid tokens/s: 151441.1 | physical tokens/s: 151849.6 | loss tokens/s: 111974.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.013581E-01 | lm-global-supervised-tokens: 1.240500E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.248907E-02 | lm-token-buffer-mib: 5.835423E+01 | global_load_balancing_loss: 1.000819E+00 | loss scale: 1.0 | grad norm: 0.155 | params norm: 11390.069 | params rms: 0.061674 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 18:56:05.922943 | finish at 2026-10-11 12:42:07 + [2026-10-07 17:47:28] iteration 3901/ 6855 | consumed samples: 499328 | elapsed time per iteration (ms): 86393.1 | throughput per GPU (TFLOP/s/GPU): 43.6 | MFU 13.99% | FLOPs estimator: workload-aware | logical tokens/s: 194196.3 | valid tokens/s: 194196.2 | physical tokens/s: 194540.4 | loss tokens/s: 141388.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.780929E-01 | lm-global-supervised-tokens: 1.221502E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.177335E-02 | lm-token-buffer-mib: 5.811013E+01 | global_load_balancing_loss: 1.001643E+00 | loss scale: 1.0 | grad norm: 0.049 | params norm: 11390.015 | params rms: 0.061673 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 22:53:25.189297 | finish at 2026-10-10 16:40:53 + [2026-10-07 17:48:58] iteration 3902/ 6855 | consumed samples: 499456 | elapsed time per iteration (ms): 89879.7 | throughput per GPU (TFLOP/s/GPU): 41.3 | MFU 13.23% | FLOPs estimator: workload-aware | logical tokens/s: 186663.0 | valid tokens/s: 186663.0 | physical tokens/s: 187348.9 | loss tokens/s: 137752.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.901262E-01 | lm-global-supervised-tokens: 1.238113E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.285735E-02 | lm-token-buffer-mib: 5.817448E+01 | global_load_balancing_loss: 1.001452E+00 | loss scale: 1.0 | grad norm: 0.131 | params norm: 11389.960 | params rms: 0.061673 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 1:43:34.746898 | finish at 2026-10-10 19:32:32 + [2026-10-07 17:50:21] iteration 3903/ 6855 | consumed samples: 499584 | elapsed time per iteration (ms): 83742.1 | throughput per GPU (TFLOP/s/GPU): 43.1 | MFU 13.83% | FLOPs estimator: workload-aware | logical tokens/s: 200342.8 | valid tokens/s: 200342.8 | physical tokens/s: 200946.8 | loss tokens/s: 150605.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.784061E-01 | lm-global-supervised-tokens: 1.261203E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.613409E-02 | lm-token-buffer-mib: 5.811421E+01 | global_load_balancing_loss: 1.000493E+00 | loss scale: 1.0 | grad norm: 0.067 | params norm: 11389.904 | params rms: 0.061673 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 20:40:06.779846 | finish at 2026-10-10 14:30:28 + [2026-10-07 17:51:52] iteration 3904/ 6855 | consumed samples: 499712 | elapsed time per iteration (ms): 90960.6 | throughput per GPU (TFLOP/s/GPU): 41.4 | MFU 13.25% | FLOPs estimator: workload-aware | logical tokens/s: 184444.5 | valid tokens/s: 184444.5 | physical tokens/s: 185042.1 | loss tokens/s: 136513.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.880273E-01 | lm-global-supervised-tokens: 1.241734E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.290077E-02 | lm-token-buffer-mib: 5.829732E+01 | global_load_balancing_loss: 1.000006E+00 | loss scale: 1.0 | grad norm: 0.139 | params norm: 11389.848 | params rms: 0.061672 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 2:33:44.663463 | finish at 2026-10-10 20:25:37 + [2026-10-07 17:53:19] iteration 3905/ 6855 | consumed samples: 499840 | elapsed time per iteration (ms): 86285.8 | throughput per GPU (TFLOP/s/GPU): 43.1 | MFU 13.82% | FLOPs estimator: workload-aware | logical tokens/s: 194437.7 | valid tokens/s: 194437.7 | physical tokens/s: 194741.7 | loss tokens/s: 143265.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.663128E-01 | lm-global-supervised-tokens: 1.236181E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.250047E-02 | lm-token-buffer-mib: 5.802711E+01 | global_load_balancing_loss: 9.994975E-01 | loss scale: 1.0 | grad norm: 0.038 | params norm: 11389.791 | params rms: 0.061672 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 22:42:23.063521 | finish at 2026-10-10 16:35:42 + [2026-10-07 17:54:45] iteration 3906/ 6855 | consumed samples: 499968 | elapsed time per iteration (ms): 85975.8 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.57% | FLOPs estimator: workload-aware | logical tokens/s: 195138.6 | valid tokens/s: 195138.5 | physical tokens/s: 195600.8 | loss tokens/s: 149771.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.423703E-01 | lm-global-supervised-tokens: 1.287674E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.431859E-02 | lm-token-buffer-mib: 5.816743E+01 | global_load_balancing_loss: 9.999791E-01 | loss scale: 1.0 | grad norm: 0.095 | params norm: 11389.733 | params rms: 0.061672 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 22:25:42.580875 | finish at 2026-10-10 16:20:27 + [2026-10-07 17:56:10] iteration 3907/ 6855 | consumed samples: 500096 | elapsed time per iteration (ms): 85719.3 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.56% | FLOPs estimator: workload-aware | logical tokens/s: 195722.4 | valid tokens/s: 195722.4 | physical tokens/s: 196308.4 | loss tokens/s: 146886.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.576977E-01 | lm-global-supervised-tokens: 1.259105E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.185445E-02 | lm-token-buffer-mib: 5.805573E+01 | global_load_balancing_loss: 9.998859E-01 | loss scale: 1.0 | grad norm: 0.100 | params norm: 11389.677 | params rms: 0.061671 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 22:11:40.592785 | finish at 2026-10-10 16:07:51 + [2026-10-07 17:57:40] iteration 3908/ 6855 | consumed samples: 500224 | elapsed time per iteration (ms): 89723.7 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.56% | FLOPs estimator: workload-aware | logical tokens/s: 186987.5 | valid tokens/s: 186987.4 | physical tokens/s: 187402.4 | loss tokens/s: 132337.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.435518E-01 | lm-global-supervised-tokens: 1.187385E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.555280E-02 | lm-token-buffer-mib: 5.827893E+01 | global_load_balancing_loss: 1.001119E+00 | loss scale: 1.0 | grad norm: 0.097 | params norm: 11389.619 | params rms: 0.061671 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 1:26:55.853646 | finish at 2026-10-10 19:24:36 + [2026-10-07 17:59:04] iteration 3909/ 6855 | consumed samples: 500352 | elapsed time per iteration (ms): 83891.6 | throughput per GPU (TFLOP/s/GPU): 43.3 | MFU 13.89% | FLOPs estimator: workload-aware | logical tokens/s: 199986.2 | valid tokens/s: 199986.2 | physical tokens/s: 200475.9 | loss tokens/s: 147628.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.117659E-01 | lm-global-supervised-tokens: 1.238477E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.448934E-02 | lm-token-buffer-mib: 5.806665E+01 | global_load_balancing_loss: 1.000707E+00 | loss scale: 1.0 | grad norm: 0.137 | params norm: 11389.559 | params rms: 0.061671 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 20:39:04.630810 | finish at 2026-10-10 14:38:09 + [2026-10-07 18:00:52] iteration 3910/ 6855 | consumed samples: 500480 | elapsed time per iteration (ms): 108585.5 | throughput per GPU (TFLOP/s/GPU): 34.5 | MFU 11.06% | FLOPs estimator: workload-aware | logical tokens/s: 154506.7 | valid tokens/s: 154506.7 | physical tokens/s: 154780.0 | loss tokens/s: 117181.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.571457E-01 | lm-global-supervised-tokens: 1.272417E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.981566E-02 | lm-token-buffer-mib: 5.805608E+01 | global_load_balancing_loss: 9.992776E-01 | loss scale: 1.0 | grad norm: 0.098 | params norm: 11389.501 | params rms: 0.061670 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 16:49:44.204823 | finish at 2026-10-11 10:50:37 + [2026-10-07 18:02:26] iteration 3911/ 6855 | consumed samples: 500608 | elapsed time per iteration (ms): 93179.9 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.44% | FLOPs estimator: workload-aware | logical tokens/s: 180051.5 | valid tokens/s: 180051.5 | physical tokens/s: 180453.1 | loss tokens/s: 132503.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.617671E-01 | lm-global-supervised-tokens: 1.234663E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.206340E-02 | lm-token-buffer-mib: 5.807854E+01 | global_load_balancing_loss: 9.996851E-01 | loss scale: 1.0 | grad norm: 0.151 | params norm: 11389.441 | params rms: 0.061670 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 4:12:01.706818 | finish at 2026-10-10 22:14:27 + [2026-10-07 18:03:53] iteration 3912/ 6855 | consumed samples: 500736 | elapsed time per iteration (ms): 86953.4 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.48% | FLOPs estimator: workload-aware | logical tokens/s: 192944.6 | valid tokens/s: 192944.6 | physical tokens/s: 193398.0 | loss tokens/s: 143812.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.659074E-01 | lm-global-supervised-tokens: 1.250495E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.422659E-02 | lm-token-buffer-mib: 5.805671E+01 | global_load_balancing_loss: 9.995601E-01 | loss scale: 1.0 | grad norm: 0.048 | params norm: 11389.379 | params rms: 0.061670 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 23:05:03.818006 | finish at 2026-10-10 17:08:56 + [2026-10-07 18:05:18] iteration 3913/ 6855 | consumed samples: 500864 | elapsed time per iteration (ms): 85571.8 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.70% | FLOPs estimator: workload-aware | logical tokens/s: 196059.0 | valid tokens/s: 196059.0 | physical tokens/s: 196421.6 | loss tokens/s: 144876.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.285331E-01 | lm-global-supervised-tokens: 1.239735E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.553584E-02 | lm-token-buffer-mib: 5.815723E+01 | global_load_balancing_loss: 1.000659E+00 | loss scale: 1.0 | grad norm: 0.095 | params norm: 11389.316 | params rms: 0.061669 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 21:55:52.348511 | finish at 2026-10-10 16:01:11 + [2026-10-07 18:06:47] iteration 3914/ 6855 | consumed samples: 500992 | elapsed time per iteration (ms): 88799.1 | throughput per GPU (TFLOP/s/GPU): 41.7 | MFU 13.38% | FLOPs estimator: workload-aware | logical tokens/s: 188934.3 | valid tokens/s: 188934.3 | physical tokens/s: 189489.6 | loss tokens/s: 136795.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.138120E-01 | lm-global-supervised-tokens: 1.214730E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.178113E-02 | lm-token-buffer-mib: 5.812739E+01 | global_load_balancing_loss: 1.001170E+00 | loss scale: 1.0 | grad norm: 0.280 | params norm: 11389.255 | params rms: 0.061669 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 0:32:38.054003 | finish at 2026-10-10 18:39:25 + [2026-10-07 18:08:15] iteration 3915/ 6855 | consumed samples: 501120 | elapsed time per iteration (ms): 88491.5 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.74% | FLOPs estimator: workload-aware | logical tokens/s: 189590.9 | valid tokens/s: 189590.9 | physical tokens/s: 189898.7 | loss tokens/s: 137477.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.596111E-01 | lm-global-supervised-tokens: 1.216555E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.164301E-02 | lm-token-buffer-mib: 5.809666E+01 | global_load_balancing_loss: 1.000251E+00 | loss scale: 1.0 | grad norm: 0.183 | params norm: 11389.194 | params rms: 0.061669 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 0:16:04.976764 | finish at 2026-10-10 18:24:20 + [2026-10-07 18:09:43] iteration 3916/ 6855 | consumed samples: 501248 | elapsed time per iteration (ms): 87472.9 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.77% | FLOPs estimator: workload-aware | logical tokens/s: 191798.8 | valid tokens/s: 191798.8 | physical tokens/s: 192472.8 | loss tokens/s: 142446.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.373510E-01 | lm-global-supervised-tokens: 1.246022E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.124842E-02 | lm-token-buffer-mib: 5.811720E+01 | global_load_balancing_loss: 1.000462E+00 | loss scale: 1.0 | grad norm: 0.335 | params norm: 11389.131 | params rms: 0.061668 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 23:24:42.744937 | finish at 2026-10-10 17:34:26 + [2026-10-07 18:11:07] iteration 3917/ 6855 | consumed samples: 501376 | elapsed time per iteration (ms): 84268.2 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.75% | FLOPs estimator: workload-aware | logical tokens/s: 199092.7 | valid tokens/s: 199092.7 | physical tokens/s: 199558.8 | loss tokens/s: 153292.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.702873E-01 | lm-global-supervised-tokens: 1.291767E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.226472E-02 | lm-token-buffer-mib: 5.808607E+01 | global_load_balancing_loss: 1.000470E+00 | loss scale: 1.0 | grad norm: 0.068 | params norm: 11389.067 | params rms: 0.061668 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 20:46:20.040714 | finish at 2026-10-10 14:57:27 + [2026-10-07 18:12:33] iteration 3918/ 6855 | consumed samples: 501504 | elapsed time per iteration (ms): 85847.5 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.62% | FLOPs estimator: workload-aware | logical tokens/s: 195430.0 | valid tokens/s: 195430.0 | physical tokens/s: 195902.3 | loss tokens/s: 145719.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.625444E-01 | lm-global-supervised-tokens: 1.250965E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.165422E-02 | lm-token-buffer-mib: 5.809129E+01 | global_load_balancing_loss: 9.997420E-01 | loss scale: 1.0 | grad norm: 0.329 | params norm: 11389.002 | params rms: 0.061668 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 22:02:14.248499 | finish at 2026-10-10 16:14:47 + [2026-10-07 18:14:00] iteration 3919/ 6855 | consumed samples: 501632 | elapsed time per iteration (ms): 86558.2 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.50% | FLOPs estimator: workload-aware | logical tokens/s: 193825.8 | valid tokens/s: 193825.8 | physical tokens/s: 194480.2 | loss tokens/s: 145680.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.829293E-01 | lm-global-supervised-tokens: 1.260984E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.164004E-02 | lm-token-buffer-mib: 5.819212E+01 | global_load_balancing_loss: 9.993155E-01 | loss scale: 1.0 | grad norm: 0.122 | params norm: 11388.938 | params rms: 0.061667 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 22:35:34.857355 | finish at 2026-10-10 16:49:34 + [2026-10-07 18:15:49] iteration 3920/ 6855 | consumed samples: 501760 | elapsed time per iteration (ms): 109329.0 | throughput per GPU (TFLOP/s/GPU): 33.1 | MFU 10.62% | FLOPs estimator: workload-aware | logical tokens/s: 153455.7 | valid tokens/s: 153455.7 | physical tokens/s: 153803.2 | loss tokens/s: 115604.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.762372E-01 | lm-global-supervised-tokens: 1.263891E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.538317E-02 | lm-token-buffer-mib: 5.814306E+01 | global_load_balancing_loss: 1.000415E+00 | loss scale: 1.0 | grad norm: 0.135 | params norm: 11388.874 | params rms: 0.061667 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 17:08:00.589797 | finish at 2026-10-11 11:23:50 +times across ranks (ms): + save-checkpoint: + rank 0: 25335.00 + rank 1: 25335.02 + rank 2: 25335.04 + rank 3: 25335.01 + rank 4: 25335.01 + rank 5: 25335.00 + rank 6: 25334.98 + rank 7: 25335.02 + rank 8: 25336.32 + rank 9: 25336.28 + rank 10: 25336.29 + rank 11: 25336.30 + rank 12: 25336.29 + rank 13: 25336.29 + rank 14: 25336.31 + rank 15: 25336.28 + rank 16: 25336.07 + rank 17: 25336.06 + rank 18: 25336.01 + rank 19: 25336.03 + rank 20: 25336.01 + rank 21: 25336.06 + rank 22: 25336.01 + rank 23: 25336.03 + rank 24: 25336.20 + rank 25: 25335.93 + rank 26: 25336.01 + rank 27: 25335.97 + rank 28: 25335.95 + rank 29: 25335.96 + rank 30: 25335.94 + rank 31: 25335.95 + rank 32: 25335.66 + rank 33: 25335.67 + rank 34: 25335.68 + rank 35: 25335.60 + rank 36: 25335.70 + rank 37: 25335.65 + rank 38: 25335.66 + rank 39: 25335.71 + rank 40: 25335.91 + rank 41: 25335.89 + rank 42: 25335.89 + rank 43: 25335.88 + rank 44: 25335.89 + rank 45: 25335.89 + rank 46: 25335.88 + rank 47: 25335.88 + rank 48: 25335.62 + rank 49: 25335.60 + rank 50: 25335.59 + rank 51: 25335.60 + rank 52: 25335.60 + rank 53: 25335.58 + rank 54: 25335.61 + rank 55: 25335.63 + rank 56: 25335.80 + rank 57: 25335.75 + rank 58: 25335.74 + rank 59: 25335.73 + rank 60: 25335.74 + rank 61: 25335.75 + rank 62: 25335.74 + rank 63: 25335.74 + [2026-10-07 18:17:44] iteration 3921/ 6855 | consumed samples: 501888 | elapsed time per iteration (ms): 89418.8 | throughput per GPU (TFLOP/s/GPU): 41.7 | MFU 13.36% | FLOPs estimator: workload-aware | logical tokens/s: 187625.0 | valid tokens/s: 187625.0 | physical tokens/s: 188103.0 | loss tokens/s: 136740.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.395375E-01 | lm-global-supervised-tokens: 1.222714E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.380754E+00 | lm-token-buffer-mib: 5.816894E+01 | global_load_balancing_loss: 1.001373E+00 | loss scale: 1.0 | grad norm: 0.219 | params norm: 11388.811 | params rms: 0.061667 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 0:52:34.860970 | finish at 2026-10-10 19:10:19 + [2026-10-07 18:19:18] iteration 3922/ 6855 | consumed samples: 502016 | elapsed time per iteration (ms): 93902.6 | throughput per GPU (TFLOP/s/GPU): 39.6 | MFU 12.70% | FLOPs estimator: workload-aware | logical tokens/s: 178666.0 | valid tokens/s: 178666.0 | physical tokens/s: 179035.0 | loss tokens/s: 129269.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.474249E-01 | lm-global-supervised-tokens: 1.213872E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.484985E-02 | lm-token-buffer-mib: 5.803655E+01 | global_load_balancing_loss: 1.000411E+00 | loss scale: 1.0 | grad norm: 0.064 | params norm: 11388.748 | params rms: 0.061666 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 4:30:16.203572 | finish at 2026-10-10 22:49:34 + [2026-10-07 18:20:40] iteration 3923/ 6855 | consumed samples: 502144 | elapsed time per iteration (ms): 82360.6 | throughput per GPU (TFLOP/s/GPU): 44.9 | MFU 14.38% | FLOPs estimator: workload-aware | logical tokens/s: 203704.3 | valid tokens/s: 203704.3 | physical tokens/s: 204111.0 | loss tokens/s: 149407.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.926859E-01 | lm-global-supervised-tokens: 1.230529E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.199116E-02 | lm-token-buffer-mib: 5.804692E+01 | global_load_balancing_loss: 1.000352E+00 | loss scale: 1.0 | grad norm: 0.066 | params norm: 11388.684 | params rms: 0.061666 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 19:04:41.244931 | finish at 2026-10-10 13:25:21 + [2026-10-07 18:22:05] iteration 3924/ 6855 | consumed samples: 502272 | elapsed time per iteration (ms): 85227.6 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.62% | FLOPs estimator: workload-aware | logical tokens/s: 196851.2 | valid tokens/s: 196851.2 | physical tokens/s: 197359.6 | loss tokens/s: 149325.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.449107E-01 | lm-global-supervised-tokens: 1.272663E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.157515E-02 | lm-token-buffer-mib: 5.820929E+01 | global_load_balancing_loss: 9.991470E-01 | loss scale: 1.0 | grad norm: 0.153 | params norm: 11388.618 | params rms: 0.061666 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 21:23:22.009933 | finish at 2026-10-10 15:45:27 + [2026-10-07 18:23:30] iteration 3925/ 6855 | consumed samples: 502400 | elapsed time per iteration (ms): 84448.2 | throughput per GPU (TFLOP/s/GPU): 44.0 | MFU 14.11% | FLOPs estimator: workload-aware | logical tokens/s: 198668.4 | valid tokens/s: 198668.4 | physical tokens/s: 199202.4 | loss tokens/s: 145976.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.852137E-01 | lm-global-supervised-tokens: 1.232744E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.689297E-02 | lm-token-buffer-mib: 5.810717E+01 | global_load_balancing_loss: 9.997604E-01 | loss scale: 1.0 | grad norm: 0.052 | params norm: 11388.552 | params rms: 0.061665 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 20:43:53.256700 | finish at 2026-10-10 15:07:23 + [2026-10-07 18:24:56] iteration 3926/ 6855 | consumed samples: 502528 | elapsed time per iteration (ms): 86628.2 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.76% | FLOPs estimator: workload-aware | logical tokens/s: 193669.0 | valid tokens/s: 193669.0 | physical tokens/s: 194052.2 | loss tokens/s: 141392.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.408927E-01 | lm-global-supervised-tokens: 1.224862E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.450800E-02 | lm-token-buffer-mib: 5.808424E+01 | global_load_balancing_loss: 9.997113E-01 | loss scale: 1.0 | grad norm: 0.084 | params norm: 11388.486 | params rms: 0.061665 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 22:28:54.102708 | finish at 2026-10-10 16:53:51 + [2026-10-07 18:26:19] iteration 3927/ 6855 | consumed samples: 502656 | elapsed time per iteration (ms): 82724.6 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.74% | FLOPs estimator: workload-aware | logical tokens/s: 202807.5 | valid tokens/s: 202807.5 | physical tokens/s: 203326.0 | loss tokens/s: 155310.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.451390E-01 | lm-global-supervised-tokens: 1.284802E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.553197E-02 | lm-token-buffer-mib: 5.809912E+01 | global_load_balancing_loss: 1.000416E+00 | loss scale: 1.0 | grad norm: 0.138 | params norm: 11388.419 | params rms: 0.061665 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 19:16:57.693249 | finish at 2026-10-10 13:43:17 + [2026-10-07 18:27:49] iteration 3928/ 6855 | consumed samples: 502784 | elapsed time per iteration (ms): 90058.3 | throughput per GPU (TFLOP/s/GPU): 41.4 | MFU 13.27% | FLOPs estimator: workload-aware | logical tokens/s: 186292.2 | valid tokens/s: 186292.2 | physical tokens/s: 186730.7 | loss tokens/s: 133518.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.417372E-01 | lm-global-supervised-tokens: 1.202442E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.475307E-02 | lm-token-buffer-mib: 5.831907E+01 | global_load_balancing_loss: 9.995402E-01 | loss scale: 1.0 | grad norm: 0.046 | params norm: 11388.353 | params rms: 0.061664 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 1:13:20.502490 | finish at 2026-10-10 19:41:10 + [2026-10-07 18:29:18] iteration 3929/ 6855 | consumed samples: 502912 | elapsed time per iteration (ms): 88704.1 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.43% | FLOPs estimator: workload-aware | logical tokens/s: 189136.4 | valid tokens/s: 189136.4 | physical tokens/s: 189652.9 | loss tokens/s: 134110.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.648669E-01 | lm-global-supervised-tokens: 1.189610E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.371729E-02 | lm-token-buffer-mib: 5.820017E+01 | global_load_balancing_loss: 1.000500E+00 | loss scale: 1.0 | grad norm: 0.122 | params norm: 11388.288 | params rms: 0.061664 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 0:05:48.194893 | finish at 2026-10-10 18:35:06 + [2026-10-07 18:31:09] iteration 3930/ 6855 | consumed samples: 503040 | elapsed time per iteration (ms): 111152.4 | throughput per GPU (TFLOP/s/GPU): 33.3 | MFU 10.68% | FLOPs estimator: workload-aware | logical tokens/s: 150938.4 | valid tokens/s: 150938.4 | physical tokens/s: 151216.3 | loss tokens/s: 112865.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.119355E-01 | lm-global-supervised-tokens: 1.254529E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.155406E-02 | lm-token-buffer-mib: 5.816354E+01 | global_load_balancing_loss: 1.001783E+00 | loss scale: 1.0 | grad norm: 0.066 | params norm: 11388.223 | params rms: 0.061664 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 18:18:40.904642 | finish at 2026-10-11 12:49:50 + [2026-10-07 18:32:38] iteration 3931/ 6855 | consumed samples: 503168 | elapsed time per iteration (ms): 88521.1 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.58% | FLOPs estimator: workload-aware | logical tokens/s: 189527.1 | valid tokens/s: 189527.1 | physical tokens/s: 190026.4 | loss tokens/s: 141620.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.224783E-01 | lm-global-supervised-tokens: 1.253637E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.194134E-02 | lm-token-buffer-mib: 5.804716E+01 | global_load_balancing_loss: 1.001228E+00 | loss scale: 1.0 | grad norm: 0.047 | params norm: 11388.158 | params rms: 0.061663 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 23:53:55.702804 | finish at 2026-10-10 18:26:33 + [2026-10-07 18:34:05] iteration 3932/ 6855 | consumed samples: 503296 | elapsed time per iteration (ms): 87182.2 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.49% | FLOPs estimator: workload-aware | logical tokens/s: 192438.5 | valid tokens/s: 192438.5 | physical tokens/s: 193034.8 | loss tokens/s: 145573.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.299429E-01 | lm-global-supervised-tokens: 1.269142E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.976573E-02 | lm-token-buffer-mib: 5.821828E+01 | global_load_balancing_loss: 1.001038E+00 | loss scale: 1.0 | grad norm: 0.056 | params norm: 11388.094 | params rms: 0.061663 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 22:47:13.629705 | finish at 2026-10-10 17:21:18 + [2026-10-07 18:35:35] iteration 3933/ 6855 | consumed samples: 503424 | elapsed time per iteration (ms): 89745.4 | throughput per GPU (TFLOP/s/GPU): 40.6 | MFU 13.02% | FLOPs estimator: workload-aware | logical tokens/s: 186942.4 | valid tokens/s: 186942.4 | physical tokens/s: 187384.7 | loss tokens/s: 133813.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.013084E-01 | lm-global-supervised-tokens: 1.200915E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.495671E-02 | lm-token-buffer-mib: 5.811139E+01 | global_load_balancing_loss: 1.000380E+00 | loss scale: 1.0 | grad norm: 0.041 | params norm: 11388.030 | params rms: 0.061662 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 0:50:36.022433 | finish at 2026-10-10 19:26:11 + [2026-10-07 18:37:01] iteration 3934/ 6855 | consumed samples: 503552 | elapsed time per iteration (ms): 86245.7 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.57% | FLOPs estimator: workload-aware | logical tokens/s: 194527.5 | valid tokens/s: 194527.5 | physical tokens/s: 195111.5 | loss tokens/s: 142062.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.401106E-01 | lm-global-supervised-tokens: 1.225225E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.232066E-02 | lm-token-buffer-mib: 5.815054E+01 | global_load_balancing_loss: 1.000569E+00 | loss scale: 1.0 | grad norm: 0.088 | params norm: 11387.966 | params rms: 0.061662 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 21:58:43.770838 | finish at 2026-10-10 16:35:45 + [2026-10-07 18:38:29] iteration 3935/ 6855 | consumed samples: 503680 | elapsed time per iteration (ms): 87960.8 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.56% | FLOPs estimator: workload-aware | logical tokens/s: 190734.8 | valid tokens/s: 190734.8 | physical tokens/s: 191283.7 | loss tokens/s: 137605.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.683300E-01 | lm-global-supervised-tokens: 1.210387E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.296382E-02 | lm-token-buffer-mib: 5.817623E+01 | global_load_balancing_loss: 9.987705E-01 | loss scale: 1.0 | grad norm: 0.058 | params norm: 11387.906 | params rms: 0.061662 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 23:20:45.664597 | finish at 2026-10-10 17:59:14 + [2026-10-07 18:39:56] iteration 3936/ 6855 | consumed samples: 503808 | elapsed time per iteration (ms): 87659.9 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.44% | FLOPs estimator: workload-aware | logical tokens/s: 191389.3 | valid tokens/s: 191389.3 | physical tokens/s: 192153.5 | loss tokens/s: 141809.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.324372E-01 | lm-global-supervised-tokens: 1.243102E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.206768E-02 | lm-token-buffer-mib: 5.818606E+01 | global_load_balancing_loss: 9.981095E-01 | loss scale: 1.0 | grad norm: 0.080 | params norm: 11387.845 | params rms: 0.061661 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 23:04:39.267441 | finish at 2026-10-10 17:44:36 + [2026-10-07 18:41:23] iteration 3937/ 6855 | consumed samples: 503936 | elapsed time per iteration (ms): 87072.0 | throughput per GPU (TFLOP/s/GPU): 41.5 | MFU 13.31% | FLOPs estimator: workload-aware | logical tokens/s: 192682.2 | valid tokens/s: 192682.2 | physical tokens/s: 193294.6 | loss tokens/s: 146867.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.397013E-01 | lm-global-supervised-tokens: 1.278801E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.325918E-02 | lm-token-buffer-mib: 5.815997E+01 | global_load_balancing_loss: 9.983664E-01 | loss scale: 1.0 | grad norm: 0.155 | params norm: 11387.782 | params rms: 0.061661 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 22:34:36.016072 | finish at 2026-10-10 17:15:59 + [2026-10-07 18:42:51] iteration 3938/ 6855 | consumed samples: 504064 | elapsed time per iteration (ms): 87657.5 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.47% | FLOPs estimator: workload-aware | logical tokens/s: 191395.0 | valid tokens/s: 191395.0 | physical tokens/s: 191946.1 | loss tokens/s: 140915.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.192421E-01 | lm-global-supervised-tokens: 1.235232E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.169268E-02 | lm-token-buffer-mib: 5.817352E+01 | global_load_balancing_loss: 9.995986E-01 | loss scale: 1.0 | grad norm: 0.091 | params norm: 11387.720 | params rms: 0.061661 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 23:01:36.994348 | finish at 2026-10-10 17:44:28 + [2026-10-07 18:44:19] iteration 3939/ 6855 | consumed samples: 504192 | elapsed time per iteration (ms): 87986.9 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.70% | FLOPs estimator: workload-aware | logical tokens/s: 190678.0 | valid tokens/s: 190678.0 | physical tokens/s: 191032.8 | loss tokens/s: 142985.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.039331E-01 | lm-global-supervised-tokens: 1.258084E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.660283E-02 | lm-token-buffer-mib: 5.812530E+01 | global_load_balancing_loss: 1.001745E+00 | loss scale: 1.0 | grad norm: 0.063 | params norm: 11387.658 | params rms: 0.061660 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 23:16:09.855589 | finish at 2026-10-10 18:00:29 + [2026-10-07 18:46:10] iteration 3940/ 6855 | consumed samples: 504320 | elapsed time per iteration (ms): 110926.5 | throughput per GPU (TFLOP/s/GPU): 34.3 | MFU 10.98% | FLOPs estimator: workload-aware | logical tokens/s: 151246.2 | valid tokens/s: 151246.2 | physical tokens/s: 151582.9 | loss tokens/s: 107655.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.893333E-01 | lm-global-supervised-tokens: 1.194189E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.903929E-02 | lm-token-buffer-mib: 5.804508E+01 | global_load_balancing_loss: 1.001616E+00 | loss scale: 1.0 | grad norm: 0.043 | params norm: 11387.597 | params rms: 0.061660 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 17:49:10.694225 | finish at 2026-10-11 12:35:21 + [2026-10-07 18:47:33] iteration 3941/ 6855 | consumed samples: 504448 | elapsed time per iteration (ms): 83446.1 | throughput per GPU (TFLOP/s/GPU): 44.1 | MFU 14.15% | FLOPs estimator: workload-aware | logical tokens/s: 201053.4 | valid tokens/s: 201053.4 | physical tokens/s: 201751.8 | loss tokens/s: 151327.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.396088E-01 | lm-global-supervised-tokens: 1.262772E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.294602E-02 | lm-token-buffer-mib: 5.825222E+01 | global_load_balancing_loss: 1.001365E+00 | loss scale: 1.0 | grad norm: 0.116 | params norm: 11387.534 | params rms: 0.061660 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 19:32:42.012507 | finish at 2026-10-10 14:20:15 + [2026-10-07 18:49:05] iteration 3942/ 6855 | consumed samples: 504576 | elapsed time per iteration (ms): 91402.4 | throughput per GPU (TFLOP/s/GPU): 41.7 | MFU 13.35% | FLOPs estimator: workload-aware | logical tokens/s: 183553.3 | valid tokens/s: 183553.3 | physical tokens/s: 183845.7 | loss tokens/s: 132233.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.845877E-01 | lm-global-supervised-tokens: 1.208648E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.383007E-02 | lm-token-buffer-mib: 5.810578E+01 | global_load_balancing_loss: 1.002143E+00 | loss scale: 1.0 | grad norm: 0.071 | params norm: 11387.471 | params rms: 0.061659 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 1:57:35.085683 | finish at 2026-10-10 20:46:40 + [2026-10-07 18:50:29] iteration 3943/ 6855 | consumed samples: 504704 | elapsed time per iteration (ms): 83691.2 | throughput per GPU (TFLOP/s/GPU): 43.8 | MFU 14.04% | FLOPs estimator: workload-aware | logical tokens/s: 200465.4 | valid tokens/s: 200465.4 | physical tokens/s: 200822.6 | loss tokens/s: 157680.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.857737E-01 | lm-global-supervised-tokens: 1.319649E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.274567E-02 | lm-token-buffer-mib: 5.811678E+01 | global_load_balancing_loss: 9.990529E-01 | loss scale: 1.0 | grad norm: 0.112 | params norm: 11387.408 | params rms: 0.061659 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 19:41:48.837631 | finish at 2026-10-10 14:32:17 + [2026-10-07 18:51:59] iteration 3944/ 6855 | consumed samples: 504832 | elapsed time per iteration (ms): 90359.3 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.54% | FLOPs estimator: workload-aware | logical tokens/s: 185671.8 | valid tokens/s: 185671.8 | physical tokens/s: 185931.6 | loss tokens/s: 130613.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.264535E-01 | lm-global-supervised-tokens: 1.180211E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.334214E-02 | lm-token-buffer-mib: 5.808185E+01 | global_load_balancing_loss: 9.987751E-01 | loss scale: 1.0 | grad norm: 0.054 | params norm: 11387.344 | params rms: 0.061659 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 1:03:55.779032 | finish at 2026-10-10 19:55:55 + [2026-10-07 18:53:26] iteration 3945/ 6855 | consumed samples: 504960 | elapsed time per iteration (ms): 86700.4 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.56% | FLOPs estimator: workload-aware | logical tokens/s: 193507.2 | valid tokens/s: 193507.2 | physical tokens/s: 193878.1 | loss tokens/s: 149593.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.649987E-01 | lm-global-supervised-tokens: 1.296984E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.145626E-02 | lm-token-buffer-mib: 5.820406E+01 | global_load_balancing_loss: 9.990864E-01 | loss scale: 1.0 | grad norm: 0.184 | params norm: 11387.280 | params rms: 0.061658 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 22:04:58.168495 | finish at 2026-10-10 16:58:24 + [2026-10-07 18:54:51] iteration 3946/ 6855 | consumed samples: 505088 | elapsed time per iteration (ms): 85313.9 | throughput per GPU (TFLOP/s/GPU): 43.6 | MFU 13.98% | FLOPs estimator: workload-aware | logical tokens/s: 196652.2 | valid tokens/s: 196652.1 | physical tokens/s: 197087.9 | loss tokens/s: 139475.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.490592E-01 | lm-global-supervised-tokens: 1.189919E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.426990E-02 | lm-token-buffer-mib: 5.819308E+01 | global_load_balancing_loss: 1.000912E+00 | loss scale: 1.0 | grad norm: 0.130 | params norm: 11387.220 | params rms: 0.061658 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 20:56:18.059484 | finish at 2026-10-10 15:51:09 + [2026-10-07 18:56:21] iteration 3947/ 6855 | consumed samples: 505216 | elapsed time per iteration (ms): 90355.8 | throughput per GPU (TFLOP/s/GPU): 41.1 | MFU 13.17% | FLOPs estimator: workload-aware | logical tokens/s: 185679.1 | valid tokens/s: 185679.1 | physical tokens/s: 186191.0 | loss tokens/s: 132654.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.559998E-01 | lm-global-supervised-tokens: 1.198611E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.141520E-02 | lm-token-buffer-mib: 5.809345E+01 | global_load_balancing_loss: 1.001027E+00 | loss scale: 1.0 | grad norm: 0.080 | params norm: 11387.161 | params rms: 0.061658 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 0:59:14.537190 | finish at 2026-10-10 19:55:36 + [2026-10-07 18:57:49] iteration 3948/ 6855 | consumed samples: 505344 | elapsed time per iteration (ms): 87757.5 | throughput per GPU (TFLOP/s/GPU): 41.5 | MFU 13.31% | FLOPs estimator: workload-aware | logical tokens/s: 191176.8 | valid tokens/s: 191176.8 | physical tokens/s: 191687.6 | loss tokens/s: 143282.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.650929E-01 | lm-global-supervised-tokens: 1.257407E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.297724E-02 | lm-token-buffer-mib: 5.807309E+01 | global_load_balancing_loss: 1.000544E+00 | loss scale: 1.0 | grad norm: 0.050 | params norm: 11387.101 | params rms: 0.061657 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 22:51:50.956661 | finish at 2026-10-10 17:49:40 + [2026-10-07 18:59:15] iteration 3949/ 6855 | consumed samples: 505472 | elapsed time per iteration (ms): 86278.3 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.76% | FLOPs estimator: workload-aware | logical tokens/s: 194453.7 | valid tokens/s: 194453.7 | physical tokens/s: 194945.9 | loss tokens/s: 144959.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.647630E-01 | lm-global-supervised-tokens: 1.250686E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.147879E-02 | lm-token-buffer-mib: 5.811909E+01 | global_load_balancing_loss: 1.000134E+00 | loss scale: 1.0 | grad norm: 0.090 | params norm: 11387.041 | params rms: 0.061657 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 21:38:44.699059 | finish at 2026-10-10 16:38:00 + [2026-10-07 19:01:04] iteration 3950/ 6855 | consumed samples: 505600 | elapsed time per iteration (ms): 109112.3 | throughput per GPU (TFLOP/s/GPU): 34.0 | MFU 10.89% | FLOPs estimator: workload-aware | logical tokens/s: 153760.7 | valid tokens/s: 153760.7 | physical tokens/s: 154234.1 | loss tokens/s: 116908.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.403909E-01 | lm-global-supervised-tokens: 1.275618E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.866998E-02 | lm-token-buffer-mib: 5.815428E+01 | global_load_balancing_loss: 9.992611E-01 | loss scale: 1.0 | grad norm: 0.057 | params norm: 11386.982 | params rms: 0.061657 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 16:02:51.153693 | finish at 2026-10-11 11:03:56 + [2026-10-07 19:02:34] iteration 3951/ 6855 | consumed samples: 505728 | elapsed time per iteration (ms): 89555.5 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.53% | FLOPs estimator: workload-aware | logical tokens/s: 187338.2 | valid tokens/s: 187338.2 | physical tokens/s: 187649.4 | loss tokens/s: 137665.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.657772E-01 | lm-global-supervised-tokens: 1.232870E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.264657E-02 | lm-token-buffer-mib: 5.815621E+01 | global_load_balancing_loss: 1.000241E+00 | loss scale: 1.0 | grad norm: 0.058 | params norm: 11386.925 | params rms: 0.061656 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 0:14:29.264866 | finish at 2026-10-10 19:17:03 + [2026-10-07 19:04:03] iteration 3952/ 6855 | consumed samples: 505856 | elapsed time per iteration (ms): 88840.8 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.64% | FLOPs estimator: workload-aware | logical tokens/s: 188845.3 | valid tokens/s: 188845.3 | physical tokens/s: 189233.0 | loss tokens/s: 132229.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.199037E-01 | lm-global-supervised-tokens: 1.174737E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.140720E-02 | lm-token-buffer-mib: 5.806747E+01 | global_load_balancing_loss: 1.000347E+00 | loss scale: 1.0 | grad norm: 0.109 | params norm: 11386.867 | params rms: 0.061656 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 23:38:24.708263 | finish at 2026-10-10 18:42:28 + [2026-10-07 19:05:33] iteration 3953/ 6855 | consumed samples: 505984 | elapsed time per iteration (ms): 90559.7 | throughput per GPU (TFLOP/s/GPU): 41.5 | MFU 13.29% | FLOPs estimator: workload-aware | logical tokens/s: 185260.7 | valid tokens/s: 185260.7 | physical tokens/s: 185702.7 | loss tokens/s: 131732.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.444610E-01 | lm-global-supervised-tokens: 1.192963E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.415165E-02 | lm-token-buffer-mib: 5.808146E+01 | global_load_balancing_loss: 1.001149E+00 | loss scale: 1.0 | grad norm: 0.103 | params norm: 11386.807 | params rms: 0.061656 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 1:00:04.330387 | finish at 2026-10-10 20:05:38 + [2026-10-07 19:07:01] iteration 3954/ 6855 | consumed samples: 506112 | elapsed time per iteration (ms): 87156.9 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.65% | FLOPs estimator: workload-aware | logical tokens/s: 192494.0 | valid tokens/s: 192494.0 | physical tokens/s: 192813.9 | loss tokens/s: 146573.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.981871E-01 | lm-global-supervised-tokens: 1.277488E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.150765E-02 | lm-token-buffer-mib: 5.816484E+01 | global_load_balancing_loss: 9.991913E-01 | loss scale: 1.0 | grad norm: 0.063 | params norm: 11386.744 | params rms: 0.061656 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 22:14:02.054646 | finish at 2026-10-10 17:21:03 + [2026-10-07 19:08:26] iteration 3955/ 6855 | consumed samples: 506240 | elapsed time per iteration (ms): 85329.9 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.74% | FLOPs estimator: workload-aware | logical tokens/s: 196615.7 | valid tokens/s: 196615.7 | physical tokens/s: 197076.5 | loss tokens/s: 139821.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 5.912570E-01 | lm-global-supervised-tokens: 1.193092E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.161654E-02 | lm-token-buffer-mib: 5.804787E+01 | global_load_balancing_loss: 1.000377E+00 | loss scale: 1.0 | grad norm: 0.062 | params norm: 11386.685 | params rms: 0.061655 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 20:44:16.845593 | finish at 2026-10-10 15:52:43 + [2026-10-07 19:09:50] iteration 3956/ 6855 | consumed samples: 506368 | elapsed time per iteration (ms): 84242.5 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.68% | FLOPs estimator: workload-aware | logical tokens/s: 199153.7 | valid tokens/s: 199153.7 | physical tokens/s: 199649.7 | loss tokens/s: 149208.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.061000E-01 | lm-global-supervised-tokens: 1.256966E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.371665E-02 | lm-token-buffer-mib: 5.813792E+01 | global_load_balancing_loss: 9.997173E-01 | loss scale: 1.0 | grad norm: 0.090 | params norm: 11386.627 | params rms: 0.061655 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 19:50:19.096164 | finish at 2026-10-10 15:00:09 + [2026-10-07 19:11:21] iteration 3957/ 6855 | consumed samples: 506496 | elapsed time per iteration (ms): 91021.5 | throughput per GPU (TFLOP/s/GPU): 41.2 | MFU 13.19% | FLOPs estimator: workload-aware | logical tokens/s: 184320.6 | valid tokens/s: 184320.6 | physical tokens/s: 184717.1 | loss tokens/s: 140189.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.167477E-01 | lm-global-supervised-tokens: 1.276023E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.260696E-02 | lm-token-buffer-mib: 5.816217E+01 | global_load_balancing_loss: 9.983698E-01 | loss scale: 1.0 | grad norm: 0.099 | params norm: 11386.570 | params rms: 0.061655 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 1:16:20.350159 | finish at 2026-10-10 20:27:42 + [2026-10-07 19:12:48] iteration 3958/ 6855 | consumed samples: 506624 | elapsed time per iteration (ms): 86674.7 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.65% | FLOPs estimator: workload-aware | logical tokens/s: 193565.0 | valid tokens/s: 193565.0 | physical tokens/s: 194058.5 | loss tokens/s: 145698.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.216024E-01 | lm-global-supervised-tokens: 1.262838E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.293994E-02 | lm-token-buffer-mib: 5.805099E+01 | global_load_balancing_loss: 9.996625E-01 | loss scale: 1.0 | grad norm: 0.043 | params norm: 11386.512 | params rms: 0.061654 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 21:44:56.467823 | finish at 2026-10-10 16:57:44 + [2026-10-07 19:14:20] iteration 3959/ 6855 | consumed samples: 506752 | elapsed time per iteration (ms): 92218.5 | throughput per GPU (TFLOP/s/GPU): 40.7 | MFU 13.03% | FLOPs estimator: workload-aware | logical tokens/s: 181928.9 | valid tokens/s: 181928.9 | physical tokens/s: 182332.7 | loss tokens/s: 128603.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.568328E-01 | lm-global-supervised-tokens: 1.185964E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.427091E-02 | lm-token-buffer-mib: 5.809553E+01 | global_load_balancing_loss: 1.000809E+00 | loss scale: 1.0 | grad norm: 0.083 | params norm: 11386.457 | params rms: 0.061654 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 2:11:04.777088 | finish at 2026-10-10 21:25:25 + [2026-10-07 19:16:11] iteration 3960/ 6855 | consumed samples: 506880 | elapsed time per iteration (ms): 110465.5 | throughput per GPU (TFLOP/s/GPU): 33.7 | MFU 10.79% | FLOPs estimator: workload-aware | logical tokens/s: 151877.5 | valid tokens/s: 151877.5 | physical tokens/s: 152285.6 | loss tokens/s: 113949.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.481511E-01 | lm-global-supervised-tokens: 1.258748E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.612025E-02 | lm-token-buffer-mib: 5.811104E+01 | global_load_balancing_loss: 1.001913E+00 | loss scale: 1.0 | grad norm: 0.037 | params norm: 11386.402 | params rms: 0.061654 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 16:49:57.517233 | finish at 2026-10-11 12:06:08 +times across ranks (ms): + save-checkpoint: + rank 0: 24632.59 + rank 1: 24632.64 + rank 2: 24632.63 + rank 3: 24632.63 + rank 4: 24632.60 + rank 5: 24632.61 + rank 6: 24632.57 + rank 7: 24632.61 + rank 8: 24631.27 + rank 9: 24631.25 + rank 10: 24631.25 + rank 11: 24631.26 + rank 12: 24631.25 + rank 13: 24631.27 + rank 14: 24631.27 + rank 15: 24631.25 + rank 16: 24630.53 + rank 17: 24630.51 + rank 18: 24630.50 + rank 19: 24630.56 + rank 20: 24630.50 + rank 21: 24630.54 + rank 22: 24630.51 + rank 23: 24630.50 + rank 24: 24631.15 + rank 25: 24631.13 + rank 26: 24631.11 + rank 27: 24632.37 + rank 28: 24631.09 + rank 29: 24631.11 + rank 30: 24631.08 + rank 31: 24631.11 + rank 32: 24631.22 + rank 33: 24631.20 + rank 34: 24631.18 + rank 35: 24631.19 + rank 36: 24631.15 + rank 37: 24631.17 + rank 38: 24631.17 + rank 39: 24631.16 + rank 40: 24631.12 + rank 41: 24631.12 + rank 42: 24631.11 + rank 43: 24631.11 + rank 44: 24631.10 + rank 45: 24631.11 + rank 46: 24631.13 + rank 47: 24631.09 + rank 48: 24631.59 + rank 49: 24631.57 + rank 50: 24631.54 + rank 51: 24631.55 + rank 52: 24631.58 + rank 53: 24631.56 + rank 54: 24631.56 + rank 55: 24631.57 + rank 56: 24631.05 + rank 57: 24631.05 + rank 58: 24631.02 + rank 59: 24631.03 + rank 60: 24631.02 + rank 61: 24631.04 + rank 62: 24631.07 + rank 63: 24631.04 + [2026-10-07 19:18:10] iteration 3961/ 6855 | consumed samples: 507008 | elapsed time per iteration (ms): 94670.2 | throughput per GPU (TFLOP/s/GPU): 40.3 | MFU 12.90% | FLOPs estimator: workload-aware | logical tokens/s: 177217.4 | valid tokens/s: 177217.4 | physical tokens/s: 177561.8 | loss tokens/s: 130464.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.706752E-01 | lm-global-supervised-tokens: 1.235106E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.048894E+00 | lm-token-buffer-mib: 5.813718E+01 | global_load_balancing_loss: 1.000912E+00 | loss scale: 1.0 | grad norm: 0.063 | params norm: 11386.348 | params rms: 0.061653 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 4:06:15.701943 | finish at 2026-10-10 23:24:26 + [2026-10-07 19:19:42] iteration 3962/ 6855 | consumed samples: 507136 | elapsed time per iteration (ms): 91661.7 | throughput per GPU (TFLOP/s/GPU): 40.5 | MFU 12.98% | FLOPs estimator: workload-aware | logical tokens/s: 183033.5 | valid tokens/s: 183033.5 | physical tokens/s: 183507.7 | loss tokens/s: 142885.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.310495E-01 | lm-global-supervised-tokens: 1.309712E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.602228E-02 | lm-token-buffer-mib: 5.820312E+01 | global_load_balancing_loss: 1.000405E+00 | loss scale: 1.0 | grad norm: 0.038 | params norm: 11386.292 | params rms: 0.061653 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 1:39:37.196048 | finish at 2026-10-10 20:59:19 + [2026-10-07 19:21:11] iteration 3963/ 6855 | consumed samples: 507264 | elapsed time per iteration (ms): 89137.0 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.59% | FLOPs estimator: workload-aware | logical tokens/s: 188218.2 | valid tokens/s: 188218.2 | physical tokens/s: 188580.4 | loss tokens/s: 132406.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.554857E-01 | lm-global-supervised-tokens: 1.180227E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.173231E-02 | lm-token-buffer-mib: 5.811143E+01 | global_load_balancing_loss: 1.000629E+00 | loss scale: 1.0 | grad norm: 0.057 | params norm: 11386.235 | params rms: 0.061653 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 23:36:24.211827 | finish at 2026-10-10 18:57:35 + [2026-10-07 19:22:39] iteration 3964/ 6855 | consumed samples: 507392 | elapsed time per iteration (ms): 87744.1 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.60% | FLOPs estimator: workload-aware | logical tokens/s: 191206.0 | valid tokens/s: 191206.0 | physical tokens/s: 191781.3 | loss tokens/s: 138817.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.528062E-01 | lm-global-supervised-tokens: 1.218038E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.276966E-02 | lm-token-buffer-mib: 5.822500E+01 | global_load_balancing_loss: 1.000211E+00 | loss scale: 1.0 | grad norm: 0.049 | params norm: 11386.178 | params rms: 0.061652 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 22:27:48.054139 | finish at 2026-10-10 17:50:27 + [2026-10-07 19:24:04] iteration 3965/ 6855 | consumed samples: 507520 | elapsed time per iteration (ms): 85519.2 | throughput per GPU (TFLOP/s/GPU): 44.3 | MFU 14.19% | FLOPs estimator: workload-aware | logical tokens/s: 196179.5 | valid tokens/s: 196179.5 | physical tokens/s: 196639.9 | loss tokens/s: 139236.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.205329E-01 | lm-global-supervised-tokens: 1.190739E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.109306E-02 | lm-token-buffer-mib: 5.814127E+01 | global_load_balancing_loss: 9.992579E-01 | loss scale: 1.0 | grad norm: 0.043 | params norm: 11386.120 | params rms: 0.061652 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 20:39:10.406256 | finish at 2026-10-10 16:03:14 + [2026-10-07 19:25:31] iteration 3966/ 6855 | consumed samples: 507648 | elapsed time per iteration (ms): 87277.6 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.55% | FLOPs estimator: workload-aware | logical tokens/s: 192227.6 | valid tokens/s: 192227.6 | physical tokens/s: 192626.5 | loss tokens/s: 140219.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.458274E-01 | lm-global-supervised-tokens: 1.223802E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.458575E-02 | lm-token-buffer-mib: 5.818655E+01 | global_load_balancing_loss: 1.000166E+00 | loss scale: 1.0 | grad norm: 0.038 | params norm: 11386.063 | params rms: 0.061652 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 22:02:25.056112 | finish at 2026-10-10 17:27:56 + [2026-10-07 19:26:57] iteration 3967/ 6855 | consumed samples: 507776 | elapsed time per iteration (ms): 86083.3 | throughput per GPU (TFLOP/s/GPU): 43.6 | MFU 13.97% | FLOPs estimator: workload-aware | logical tokens/s: 194894.8 | valid tokens/s: 194894.8 | physical tokens/s: 195232.3 | loss tokens/s: 136950.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.165841E-01 | lm-global-supervised-tokens: 1.178916E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.625681E-02 | lm-token-buffer-mib: 5.813955E+01 | global_load_balancing_loss: 1.000079E+00 | loss scale: 1.0 | grad norm: 0.043 | params norm: 11386.005 | params rms: 0.061652 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 21:03:28.498430 | finish at 2026-10-10 16:30:26 + [2026-10-07 19:28:26] iteration 3968/ 6855 | consumed samples: 507904 | elapsed time per iteration (ms): 88082.2 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.45% | FLOPs estimator: workload-aware | logical tokens/s: 190472.2 | valid tokens/s: 190472.2 | physical tokens/s: 190994.1 | loss tokens/s: 140646.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.218021E-01 | lm-global-supervised-tokens: 1.238847E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.748740E-02 | lm-token-buffer-mib: 5.813251E+01 | global_load_balancing_loss: 9.985894E-01 | loss scale: 1.0 | grad norm: 0.047 | params norm: 11385.948 | params rms: 0.061651 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 22:38:13.372117 | finish at 2026-10-10 18:06:39 + [2026-10-07 19:29:50] iteration 3969/ 6855 | consumed samples: 508032 | elapsed time per iteration (ms): 84359.9 | throughput per GPU (TFLOP/s/GPU): 44.0 | MFU 14.10% | FLOPs estimator: workload-aware | logical tokens/s: 198876.1 | valid tokens/s: 198876.1 | physical tokens/s: 199296.5 | loss tokens/s: 148541.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.144852E-01 | lm-global-supervised-tokens: 1.253096E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.249089E-02 | lm-token-buffer-mib: 5.802833E+01 | global_load_balancing_loss: 9.990907E-01 | loss scale: 1.0 | grad norm: 0.040 | params norm: 11385.890 | params rms: 0.061651 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 19:37:42.781486 | finish at 2026-10-10 15:07:33 + [2026-10-07 19:31:40] iteration 3970/ 6855 | consumed samples: 508160 | elapsed time per iteration (ms): 109908.3 | throughput per GPU (TFLOP/s/GPU): 33.7 | MFU 10.81% | FLOPs estimator: workload-aware | logical tokens/s: 152647.2 | valid tokens/s: 152647.2 | physical tokens/s: 152938.2 | loss tokens/s: 116267.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.141256E-01 | lm-global-supervised-tokens: 1.277876E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.160915E-02 | lm-token-buffer-mib: 5.805472E+01 | global_load_balancing_loss: 9.997971E-01 | loss scale: 1.0 | grad norm: 0.042 | params norm: 11385.832 | params rms: 0.061651 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 16:04:45.526443 | finish at 2026-10-11 11:36:25 + [2026-10-07 19:33:07] iteration 3971/ 6855 | consumed samples: 508288 | elapsed time per iteration (ms): 87336.9 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.39% | FLOPs estimator: workload-aware | logical tokens/s: 192096.7 | valid tokens/s: 192096.7 | physical tokens/s: 193067.3 | loss tokens/s: 138583.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.309710E-01 | lm-global-supervised-tokens: 1.210343E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.307990E-02 | lm-token-buffer-mib: 5.813760E+01 | global_load_balancing_loss: 1.001599E+00 | loss scale: 1.0 | grad norm: 0.052 | params norm: 11385.776 | params rms: 0.061650 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 21:57:59.678721 | finish at 2026-10-10 17:31:07 + [2026-10-07 19:34:38] iteration 3972/ 6855 | consumed samples: 508416 | elapsed time per iteration (ms): 90805.4 | throughput per GPU (TFLOP/s/GPU): 41.2 | MFU 13.21% | FLOPs estimator: workload-aware | logical tokens/s: 184759.9 | valid tokens/s: 184759.9 | physical tokens/s: 185123.1 | loss tokens/s: 132405.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.885365E-01 | lm-global-supervised-tokens: 1.202314E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.560663E-02 | lm-token-buffer-mib: 5.808233E+01 | global_load_balancing_loss: 1.001463E+00 | loss scale: 1.0 | grad norm: 0.050 | params norm: 11385.718 | params rms: 0.061650 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 0:43:11.973395 | finish at 2026-10-10 20:17:50 + [2026-10-07 19:36:02] iteration 3973/ 6855 | consumed samples: 508544 | elapsed time per iteration (ms): 84134.0 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.77% | FLOPs estimator: workload-aware | logical tokens/s: 199410.0 | valid tokens/s: 199410.0 | physical tokens/s: 199871.6 | loss tokens/s: 151806.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.381940E-01 | lm-global-supervised-tokens: 1.277206E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.346230E-02 | lm-token-buffer-mib: 5.806985E+01 | global_load_balancing_loss: 1.001121E+00 | loss scale: 1.0 | grad norm: 0.081 | params norm: 11385.658 | params rms: 0.061650 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 19:21:14.236280 | finish at 2026-10-10 14:57:16 + [2026-10-07 19:37:29] iteration 3974/ 6855 | consumed samples: 508672 | elapsed time per iteration (ms): 86481.4 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.38% | FLOPs estimator: workload-aware | logical tokens/s: 193997.5 | valid tokens/s: 193997.5 | physical tokens/s: 194540.2 | loss tokens/s: 142626.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.696998E-01 | lm-global-supervised-tokens: 1.233456E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.530677E-02 | lm-token-buffer-mib: 5.823595E+01 | global_load_balancing_loss: 1.001406E+00 | loss scale: 1.0 | grad norm: 0.166 | params norm: 11385.598 | params rms: 0.061649 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 21:12:32.916185 | finish at 2026-10-10 16:50:01 + [2026-10-07 19:38:56] iteration 3975/ 6855 | consumed samples: 508800 | elapsed time per iteration (ms): 87573.3 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.56% | FLOPs estimator: workload-aware | logical tokens/s: 191578.8 | valid tokens/s: 191578.7 | physical tokens/s: 192044.6 | loss tokens/s: 150753.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.571221E-01 | lm-global-supervised-tokens: 1.320196E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.653612E-02 | lm-token-buffer-mib: 5.810250E+01 | global_load_balancing_loss: 9.999974E-01 | loss scale: 1.0 | grad norm: 0.340 | params norm: 11385.538 | params rms: 0.061649 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 22:03:31.116028 | finish at 2026-10-10 17:42:27 + [2026-10-07 19:40:24] iteration 3976/ 6855 | consumed samples: 508928 | elapsed time per iteration (ms): 88079.0 | throughput per GPU (TFLOP/s/GPU): 43.4 | MFU 13.92% | FLOPs estimator: workload-aware | logical tokens/s: 190478.4 | valid tokens/s: 190478.4 | physical tokens/s: 191121.1 | loss tokens/s: 139613.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.758583E-01 | lm-global-supervised-tokens: 1.229704E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.279604E-02 | lm-token-buffer-mib: 5.864702E+01 | global_load_balancing_loss: 9.984084E-01 | loss scale: 1.0 | grad norm: 0.445 | params norm: 11385.479 | params rms: 0.061649 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 22:26:19.475309 | finish at 2026-10-10 18:06:44 + [2026-10-07 19:41:50] iteration 3977/ 6855 | consumed samples: 509056 | elapsed time per iteration (ms): 85938.8 | throughput per GPU (TFLOP/s/GPU): 43.6 | MFU 13.97% | FLOPs estimator: workload-aware | logical tokens/s: 195222.4 | valid tokens/s: 195222.4 | physical tokens/s: 195826.0 | loss tokens/s: 144721.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.653795E-01 | lm-global-supervised-tokens: 1.243717E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.170533E-02 | lm-token-buffer-mib: 5.827323E+01 | global_load_balancing_loss: 9.996878E-01 | loss scale: 1.0 | grad norm: 0.192 | params norm: 11385.416 | params rms: 0.061648 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 20:42:11.871481 | finish at 2026-10-10 16:24:02 + [2026-10-07 19:43:17] iteration 3978/ 6855 | consumed samples: 509184 | elapsed time per iteration (ms): 87130.5 | throughput per GPU (TFLOP/s/GPU): 41.2 | MFU 13.22% | FLOPs estimator: workload-aware | logical tokens/s: 192552.0 | valid tokens/s: 192552.0 | physical tokens/s: 193024.7 | loss tokens/s: 144498.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.323299E-01 | lm-global-supervised-tokens: 1.259021E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.281885E-02 | lm-token-buffer-mib: 5.806267E+01 | global_load_balancing_loss: 9.996916E-01 | loss scale: 1.0 | grad norm: 0.285 | params norm: 11385.351 | params rms: 0.061648 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 21:37:54.420602 | finish at 2026-10-10 17:21:12 + [2026-10-07 19:44:47] iteration 3979/ 6855 | consumed samples: 509312 | elapsed time per iteration (ms): 89269.0 | throughput per GPU (TFLOP/s/GPU): 43.2 | MFU 13.84% | FLOPs estimator: workload-aware | logical tokens/s: 187939.4 | valid tokens/s: 187939.4 | physical tokens/s: 188498.4 | loss tokens/s: 136940.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.711104E-01 | lm-global-supervised-tokens: 1.222454E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.140523E-02 | lm-token-buffer-mib: 5.811494E+01 | global_load_balancing_loss: 9.986107E-01 | loss scale: 1.0 | grad norm: 0.146 | params norm: 11385.287 | params rms: 0.061648 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 23:18:57.741522 | finish at 2026-10-10 19:03:44 + [2026-10-07 19:46:33] iteration 3980/ 6855 | consumed samples: 509440 | elapsed time per iteration (ms): 106108.6 | throughput per GPU (TFLOP/s/GPU): 34.0 | MFU 10.90% | FLOPs estimator: workload-aware | logical tokens/s: 158113.6 | valid tokens/s: 158113.6 | physical tokens/s: 158554.3 | loss tokens/s: 116506.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.420155E-01 | lm-global-supervised-tokens: 1.236232E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.289571E-02 | lm-token-buffer-mib: 5.830669E+01 | global_load_balancing_loss: 1.001400E+00 | loss scale: 1.0 | grad norm: 0.144 | params norm: 11385.224 | params rms: 0.061647 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 12:44:22.155485 | finish at 2026-10-11 08:30:55 + [2026-10-07 19:48:00] iteration 3981/ 6855 | consumed samples: 509568 | elapsed time per iteration (ms): 87003.4 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.65% | FLOPs estimator: workload-aware | logical tokens/s: 192833.7 | valid tokens/s: 192833.7 | physical tokens/s: 193118.8 | loss tokens/s: 141589.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.060612E-01 | lm-global-supervised-tokens: 1.231878E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.646980E-02 | lm-token-buffer-mib: 5.804905E+01 | global_load_balancing_loss: 1.001003E+00 | loss scale: 1.0 | grad norm: 0.102 | params norm: 11385.159 | params rms: 0.061647 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 21:27:27.665643 | finish at 2026-10-10 17:15:27 + [2026-10-07 19:49:24] iteration 3982/ 6855 | consumed samples: 509696 | elapsed time per iteration (ms): 84342.7 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.63% | FLOPs estimator: workload-aware | logical tokens/s: 198917.0 | valid tokens/s: 198917.0 | physical tokens/s: 199455.0 | loss tokens/s: 145426.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.702011E-01 | lm-global-supervised-tokens: 1.226566E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.166164E-02 | lm-token-buffer-mib: 5.806345E+01 | global_load_balancing_loss: 1.001256E+00 | loss scale: 1.0 | grad norm: 0.147 | params norm: 11385.093 | params rms: 0.061647 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 19:18:36.623007 | finish at 2026-10-10 15:08:01 + [2026-10-07 19:50:53] iteration 3983/ 6855 | consumed samples: 509824 | elapsed time per iteration (ms): 88741.5 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.53% | FLOPs estimator: workload-aware | logical tokens/s: 189057.0 | valid tokens/s: 189057.0 | physical tokens/s: 189476.3 | loss tokens/s: 128218.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.549737E-01 | lm-global-supervised-tokens: 1.137830E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.255235E-02 | lm-token-buffer-mib: 5.809038E+01 | global_load_balancing_loss: 1.000629E+00 | loss scale: 1.0 | grad norm: 0.073 | params norm: 11385.030 | params rms: 0.061646 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 22:47:45.607573 | finish at 2026-10-10 18:38:38 + [2026-10-07 19:52:19] iteration 3984/ 6855 | consumed samples: 509952 | elapsed time per iteration (ms): 85813.7 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.54% | FLOPs estimator: workload-aware | logical tokens/s: 195507.1 | valid tokens/s: 195507.1 | physical tokens/s: 195965.8 | loss tokens/s: 143207.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.718583E-01 | lm-global-supervised-tokens: 1.228920E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.044795E-02 | lm-token-buffer-mib: 5.826929E+01 | global_load_balancing_loss: 1.000265E+00 | loss scale: 1.0 | grad norm: 0.056 | params norm: 11384.969 | params rms: 0.061646 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 20:26:11.263327 | finish at 2026-10-10 16:18:30 + [2026-10-07 19:53:45] iteration 3985/ 6855 | consumed samples: 510080 | elapsed time per iteration (ms): 86257.1 | throughput per GPU (TFLOP/s/GPU): 43.1 | MFU 13.81% | FLOPs estimator: workload-aware | logical tokens/s: 194501.6 | valid tokens/s: 194501.5 | physical tokens/s: 194817.5 | loss tokens/s: 140222.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.625416E-01 | lm-global-supervised-tokens: 1.209519E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.415122E-02 | lm-token-buffer-mib: 5.808083E+01 | global_load_balancing_loss: 9.999977E-01 | loss scale: 1.0 | grad norm: 0.074 | params norm: 11384.906 | params rms: 0.061646 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 20:45:57.771926 | finish at 2026-10-10 16:39:43 + [2026-10-07 19:55:14] iteration 3986/ 6855 | consumed samples: 510208 | elapsed time per iteration (ms): 88820.2 | throughput per GPU (TFLOP/s/GPU): 41.5 | MFU 13.29% | FLOPs estimator: workload-aware | logical tokens/s: 188889.2 | valid tokens/s: 188889.2 | physical tokens/s: 189400.5 | loss tokens/s: 142919.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.966739E-01 | lm-global-supervised-tokens: 1.269411E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.465629E-02 | lm-token-buffer-mib: 5.807889E+01 | global_load_balancing_loss: 9.987727E-01 | loss scale: 1.0 | grad norm: 0.125 | params norm: 11384.842 | params rms: 0.061645 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 22:47:05.199533 | finish at 2026-10-10 18:42:19 + [2026-10-07 19:56:42] iteration 3987/ 6855 | consumed samples: 510336 | elapsed time per iteration (ms): 88557.7 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.56% | FLOPs estimator: workload-aware | logical tokens/s: 189448.7 | valid tokens/s: 189448.7 | physical tokens/s: 189977.9 | loss tokens/s: 139915.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.450119E-01 | lm-global-supervised-tokens: 1.239063E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.667240E-02 | lm-token-buffer-mib: 5.806531E+01 | global_load_balancing_loss: 1.000120E+00 | loss scale: 1.0 | grad norm: 0.074 | params norm: 11384.778 | params rms: 0.061645 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 22:33:03.589354 | finish at 2026-10-10 18:29:46 + [2026-10-07 19:58:11] iteration 3988/ 6855 | consumed samples: 510464 | elapsed time per iteration (ms): 88990.0 | throughput per GPU (TFLOP/s/GPU): 41.4 | MFU 13.26% | FLOPs estimator: workload-aware | logical tokens/s: 188529.3 | valid tokens/s: 188529.3 | physical tokens/s: 188942.1 | loss tokens/s: 138479.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.645662E-01 | lm-global-supervised-tokens: 1.232324E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.538031E-02 | lm-token-buffer-mib: 5.809968E+01 | global_load_balancing_loss: 1.000202E+00 | loss scale: 1.0 | grad norm: 0.066 | params norm: 11384.716 | params rms: 0.061645 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 22:52:14.207673 | finish at 2026-10-10 18:50:25 + [2026-10-07 19:59:39] iteration 3989/ 6855 | consumed samples: 510592 | elapsed time per iteration (ms): 88029.2 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.42% | FLOPs estimator: workload-aware | logical tokens/s: 190586.6 | valid tokens/s: 190586.6 | physical tokens/s: 191163.8 | loss tokens/s: 137786.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.583094E-01 | lm-global-supervised-tokens: 1.212923E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.186536E-02 | lm-token-buffer-mib: 5.804350E+01 | global_load_balancing_loss: 1.001192E+00 | loss scale: 1.0 | grad norm: 0.063 | params norm: 11384.654 | params rms: 0.061644 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 22:04:51.681271 | finish at 2026-10-10 18:04:31 + [2026-10-07 20:01:32] iteration 3990/ 6855 | consumed samples: 510720 | elapsed time per iteration (ms): 112606.1 | throughput per GPU (TFLOP/s/GPU): 33.4 | MFU 10.72% | FLOPs estimator: workload-aware | logical tokens/s: 148989.6 | valid tokens/s: 148989.6 | physical tokens/s: 149498.5 | loss tokens/s: 106337.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 5.931358E-01 | lm-global-supervised-tokens: 1.197425E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.237199E-02 | lm-token-buffer-mib: 5.875406E+01 | global_load_balancing_loss: 1.000304E+00 | loss scale: 1.0 | grad norm: 0.054 | params norm: 11384.590 | params rms: 0.061644 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 17:36:56.467173 | finish at 2026-10-11 13:38:28 + [2026-10-07 20:03:01] iteration 3991/ 6855 | consumed samples: 510848 | elapsed time per iteration (ms): 89343.2 | throughput per GPU (TFLOP/s/GPU): 41.2 | MFU 13.19% | FLOPs estimator: workload-aware | logical tokens/s: 187783.1 | valid tokens/s: 187783.1 | physical tokens/s: 188072.1 | loss tokens/s: 132536.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 5.922207E-01 | lm-global-supervised-tokens: 1.184123E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.745979E-02 | lm-token-buffer-mib: 5.806084E+01 | global_load_balancing_loss: 1.000844E+00 | loss scale: 1.0 | grad norm: 0.069 | params norm: 11384.527 | params rms: 0.061644 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 23:04:39.056496 | finish at 2026-10-10 19:07:40 + [2026-10-07 20:04:29] iteration 3992/ 6855 | consumed samples: 510976 | elapsed time per iteration (ms): 87951.2 | throughput per GPU (TFLOP/s/GPU): 42.8 | MFU 13.72% | FLOPs estimator: workload-aware | logical tokens/s: 190755.8 | valid tokens/s: 190755.8 | physical tokens/s: 191138.2 | loss tokens/s: 140915.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.016502E-01 | lm-global-supervised-tokens: 1.239366E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.239218E-02 | lm-token-buffer-mib: 5.813037E+01 | global_load_balancing_loss: 9.995688E-01 | loss scale: 1.0 | grad norm: 0.079 | params norm: 11384.465 | params rms: 0.061643 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 21:56:44.246034 | finish at 2026-10-10 18:01:13 + [2026-10-07 20:05:57] iteration 3993/ 6855 | consumed samples: 511104 | elapsed time per iteration (ms): 87795.0 | throughput per GPU (TFLOP/s/GPU): 43.6 | MFU 13.96% | FLOPs estimator: workload-aware | logical tokens/s: 191095.3 | valid tokens/s: 191095.3 | physical tokens/s: 191481.4 | loss tokens/s: 137677.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.021143E-01 | lm-global-supervised-tokens: 1.208738E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.250768E-02 | lm-token-buffer-mib: 5.808709E+01 | global_load_balancing_loss: 9.999038E-01 | loss scale: 1.0 | grad norm: 0.056 | params norm: 11384.402 | params rms: 0.061643 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 21:47:49.408948 | finish at 2026-10-10 17:53:46 + [2026-10-07 20:07:23] iteration 3994/ 6855 | consumed samples: 511232 | elapsed time per iteration (ms): 85777.2 | throughput per GPU (TFLOP/s/GPU): 43.3 | MFU 13.89% | FLOPs estimator: workload-aware | logical tokens/s: 195590.1 | valid tokens/s: 195590.1 | physical tokens/s: 196037.9 | loss tokens/s: 147175.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.728929E-01 | lm-global-supervised-tokens: 1.262426E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.276607E-02 | lm-token-buffer-mib: 5.804933E+01 | global_load_balancing_loss: 9.992625E-01 | loss scale: 1.0 | grad norm: 0.090 | params norm: 11384.338 | params rms: 0.061642 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 20:10:08.637365 | finish at 2026-10-10 16:17:31 + [2026-10-07 20:08:47] iteration 3995/ 6855 | consumed samples: 511360 | elapsed time per iteration (ms): 84635.9 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.69% | FLOPs estimator: workload-aware | logical tokens/s: 198227.5 | valid tokens/s: 198227.5 | physical tokens/s: 198821.6 | loss tokens/s: 147125.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.512232E-01 | lm-global-supervised-tokens: 1.245210E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.262548E-02 | lm-token-buffer-mib: 5.850705E+01 | global_load_balancing_loss: 9.998305E-01 | loss scale: 1.0 | grad norm: 0.086 | params norm: 11384.277 | params rms: 0.061642 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 19:14:18.803616 | finish at 2026-10-10 15:23:06 + [2026-10-07 20:10:17] iteration 3996/ 6855 | consumed samples: 511488 | elapsed time per iteration (ms): 89710.1 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.74% | FLOPs estimator: workload-aware | logical tokens/s: 187015.1 | valid tokens/s: 187015.1 | physical tokens/s: 187396.1 | loss tokens/s: 135824.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.911860E-01 | lm-global-supervised-tokens: 1.218484E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.107504E-02 | lm-token-buffer-mib: 5.833424E+01 | global_load_balancing_loss: 9.997067E-01 | loss scale: 1.0 | grad norm: 0.123 | params norm: 11384.215 | params rms: 0.061642 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 23:14:41.305909 | finish at 2026-10-10 19:24:58 + [2026-10-07 20:11:44] iteration 3997/ 6855 | consumed samples: 511616 | elapsed time per iteration (ms): 87293.1 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.64% | FLOPs estimator: workload-aware | logical tokens/s: 192193.7 | valid tokens/s: 192193.7 | physical tokens/s: 192620.0 | loss tokens/s: 138398.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.543525E-01 | lm-global-supervised-tokens: 1.208121E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.135876E-02 | lm-token-buffer-mib: 5.808772E+01 | global_load_balancing_loss: 1.000867E+00 | loss scale: 1.0 | grad norm: 0.068 | params norm: 11384.152 | params rms: 0.061641 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 21:18:03.776898 | finish at 2026-10-10 17:29:48 + [2026-10-07 20:13:07] iteration 3998/ 6855 | consumed samples: 511744 | elapsed time per iteration (ms): 83045.9 | throughput per GPU (TFLOP/s/GPU): 43.1 | MFU 13.82% | FLOPs estimator: workload-aware | logical tokens/s: 202022.7 | valid tokens/s: 202022.7 | physical tokens/s: 202643.1 | loss tokens/s: 156556.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.337553E-01 | lm-global-supervised-tokens: 1.300134E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.137961E-02 | lm-token-buffer-mib: 5.816752E+01 | global_load_balancing_loss: 1.000790E+00 | loss scale: 1.0 | grad norm: 0.056 | params norm: 11384.091 | params rms: 0.061641 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 17:54:22.047372 | finish at 2026-10-10 14:07:29 + [2026-10-07 20:14:37] iteration 3999/ 6855 | consumed samples: 511872 | elapsed time per iteration (ms): 89402.5 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.53% | FLOPs estimator: workload-aware | logical tokens/s: 187659.1 | valid tokens/s: 187659.1 | physical tokens/s: 188125.2 | loss tokens/s: 137071.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.817058E-01 | lm-global-supervised-tokens: 1.225456E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.114138E-02 | lm-token-buffer-mib: 5.819486E+01 | global_load_balancing_loss: 9.995637E-01 | loss scale: 1.0 | grad norm: 0.063 | params norm: 11384.032 | params rms: 0.061641 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 22:55:33.537031 | finish at 2026-10-10 19:10:10 + [2026-10-07 20:16:24] iteration 4000/ 6855 | consumed samples: 512000 | elapsed time per iteration (ms): 107111.4 | throughput per GPU (TFLOP/s/GPU): 33.6 | MFU 10.78% | FLOPs estimator: workload-aware | logical tokens/s: 156633.2 | valid tokens/s: 156633.2 | physical tokens/s: 157015.2 | loss tokens/s: 122778.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.593511E-01 | lm-global-supervised-tokens: 1.315092E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.938098E-02 | lm-token-buffer-mib: 5.821770E+01 | global_load_balancing_loss: 9.999986E-01 | loss scale: 1.0 | grad norm: 0.135 | params norm: 11383.972 | params rms: 0.061641 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 12:56:42.925521 | finish at 2026-10-11 09:13:07 +times across ranks (ms): + save-checkpoint: + rank 0: 24469.22 + rank 1: 24469.20 + rank 2: 24469.21 + rank 3: 24469.18 + rank 4: 24469.20 + rank 5: 24469.15 + rank 6: 24469.15 + rank 7: 24469.19 + rank 8: 24469.79 + rank 9: 24469.80 + rank 10: 24469.75 + rank 11: 24469.78 + rank 12: 24469.76 + rank 13: 24469.75 + rank 14: 24469.76 + rank 15: 24469.74 + rank 16: 24468.94 + rank 17: 24468.93 + rank 18: 24468.88 + rank 19: 24468.98 + rank 20: 24468.91 + rank 21: 24468.92 + rank 22: 24468.91 + rank 23: 24468.91 + rank 24: 24469.32 + rank 25: 24469.35 + rank 26: 24469.38 + rank 27: 24469.25 + rank 28: 24469.16 + rank 29: 24469.20 + rank 30: 24469.18 + rank 31: 24469.05 + rank 32: 24469.99 + rank 33: 24470.14 + rank 34: 24470.07 + rank 35: 24470.11 + rank 36: 24469.95 + rank 37: 24470.08 + rank 38: 24469.94 + rank 39: 24469.95 + rank 40: 24469.16 + rank 41: 24469.16 + rank 42: 24469.15 + rank 43: 24469.20 + rank 44: 24469.15 + rank 45: 24469.15 + rank 46: 24469.17 + rank 47: 24469.15 + rank 48: 24469.21 + rank 49: 24469.23 + rank 50: 24469.21 + rank 51: 24469.20 + rank 52: 24469.17 + rank 53: 24469.18 + rank 54: 24469.21 + rank 55: 24469.18 + rank 56: 24468.79 + rank 57: 24468.77 + rank 58: 24468.72 + rank 59: 24468.72 + rank 60: 24468.72 + rank 61: 24468.74 + rank 62: 24468.73 + rank 63: 24468.74 + [2026-10-07 20:18:31] iteration 4001/ 6855 | consumed samples: 512128 | elapsed time per iteration (ms): 102054.8 | throughput per GPU (TFLOP/s/GPU): 36.2 | MFU 11.61% | FLOPs estimator: workload-aware | logical tokens/s: 164394.0 | valid tokens/s: 164394.0 | physical tokens/s: 164783.6 | loss tokens/s: 122438.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.472065E-01 | lm-global-supervised-tokens: 1.249544E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 4.117721E-01 | lm-token-buffer-mib: 5.816592E+01 | global_load_balancing_loss: 1.001125E+00 | loss scale: 1.0 | grad norm: 0.181 | params norm: 11383.912 | params rms: 0.061640 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 8:54:24.509528 | finish at 2026-10-11 05:12:55 +times across ranks (ms): + forward-backward: + rank 0: 101535.56 + rank 1: 101537.27 + rank 2: 101537.55 + rank 3: 101535.23 + rank 4: 101536.83 + rank 5: 101534.82 + rank 6: 101537.01 + rank 7: 101536.66 + rank 8: 101527.10 + rank 9: 101529.38 + rank 10: 101529.23 + rank 11: 101529.41 + rank 12: 101530.16 + rank 13: 101529.13 + rank 14: 101527.80 + rank 15: 101528.63 + rank 16: 101520.62 + rank 17: 101522.77 + rank 18: 101523.01 + rank 19: 101521.95 + rank 20: 101522.77 + rank 21: 101523.03 + rank 22: 101521.55 + rank 23: 101522.19 + rank 24: 101519.28 + rank 25: 101522.66 + rank 26: 101523.20 + rank 27: 101520.08 + rank 28: 101523.01 + rank 29: 101522.56 + rank 30: 101520.40 + rank 31: 101520.48 + rank 32: 101518.95 + rank 33: 101521.01 + rank 34: 101520.99 + rank 35: 101519.36 + rank 36: 101522.47 + rank 37: 101520.42 + rank 38: 101522.14 + rank 39: 101521.34 + rank 40: 101520.81 + rank 41: 101520.92 + rank 42: 101520.91 + rank 43: 101518.59 + rank 44: 101519.52 + rank 45: 101516.48 + rank 46: 101523.12 + rank 47: 101521.46 + rank 48: 101526.14 + rank 49: 101527.26 + rank 50: 101525.41 + rank 51: 101525.14 + rank 52: 101526.04 + rank 53: 101523.88 + rank 54: 101528.77 + rank 55: 101528.09 + rank 56: 101527.95 + rank 57: 101528.38 + rank 58: 101528.23 + rank 59: 101525.26 + rank 60: 101526.49 + rank 61: 101523.69 + rank 62: 101529.59 + rank 63: 101528.80 + forward-compute: + rank 0: 45829.27 + rank 1: 45826.30 + rank 2: 45829.66 + rank 3: 45825.44 + rank 4: 45827.79 + rank 5: 45830.06 + rank 6: 45829.59 + rank 7: 45833.49 + rank 8: 45684.71 + rank 9: 45688.21 + rank 10: 45684.04 + rank 11: 45682.55 + rank 12: 45687.67 + rank 13: 45693.73 + rank 14: 45688.14 + rank 15: 45687.71 + rank 16: 50314.42 + rank 17: 50316.79 + rank 18: 50317.15 + rank 19: 50315.90 + rank 20: 50316.34 + rank 21: 50351.40 + rank 22: 50315.73 + rank 23: 50316.19 + rank 24: 44119.91 + rank 25: 44117.47 + rank 26: 44124.29 + rank 27: 44116.82 + rank 28: 44123.65 + rank 29: 44121.36 + rank 30: 44133.58 + rank 31: 44127.05 + rank 32: 50590.77 + rank 33: 50592.21 + rank 34: 50593.32 + rank 35: 50591.05 + rank 36: 50592.37 + rank 37: 50589.42 + rank 38: 50592.12 + rank 39: 50591.45 + rank 40: 49667.99 + rank 41: 49669.49 + rank 42: 49671.28 + rank 43: 49669.15 + rank 44: 49670.67 + rank 45: 49669.47 + rank 46: 49671.95 + rank 47: 49670.45 + rank 48: 45073.23 + rank 49: 45076.39 + rank 50: 45076.23 + rank 51: 45075.76 + rank 52: 45074.89 + rank 53: 45070.67 + rank 54: 45071.47 + rank 55: 45070.80 + rank 56: 50906.21 + rank 57: 50909.63 + rank 58: 50906.70 + rank 59: 50907.25 + rank 60: 50914.67 + rank 61: 50914.29 + rank 62: 50910.02 + rank 63: 50915.15 + backward-compute: + rank 0: 55610.15 + rank 1: 55623.14 + rank 2: 55617.46 + rank 3: 55616.29 + rank 4: 55620.88 + rank 5: 55619.21 + rank 6: 55611.04 + rank 7: 55611.43 + rank 8: 55804.90 + rank 9: 55804.64 + rank 10: 55808.75 + rank 11: 55808.67 + rank 12: 55806.23 + rank 13: 55799.90 + rank 14: 55803.61 + rank 15: 55805.23 + rank 16: 51167.39 + rank 17: 51169.84 + rank 18: 51169.03 + rank 19: 51167.66 + rank 20: 51170.13 + rank 21: 51135.05 + rank 22: 51168.30 + rank 23: 51167.65 + rank 24: 57357.73 + rank 25: 57365.18 + rank 26: 57357.50 + rank 27: 57363.98 + rank 28: 57358.10 + rank 29: 57356.54 + rank 30: 57344.25 + rank 31: 57351.04 + rank 32: 50886.36 + rank 33: 50885.43 + rank 34: 50887.89 + rank 35: 50886.28 + rank 36: 50889.52 + rank 37: 50889.41 + rank 38: 50888.41 + rank 39: 50889.64 + rank 40: 51813.21 + rank 41: 51810.93 + rank 42: 51810.93 + rank 43: 51809.06 + rank 44: 51808.15 + rank 45: 51808.43 + rank 46: 51812.80 + rank 47: 51812.47 + rank 48: 56413.83 + rank 49: 56414.88 + rank 50: 56416.04 + rank 51: 56413.92 + rank 52: 56410.74 + rank 53: 56416.64 + rank 54: 56416.37 + rank 55: 56421.94 + rank 56: 50585.64 + rank 57: 50578.62 + rank 58: 50584.49 + rank 59: 50578.86 + rank 60: 50572.88 + rank 61: 50572.23 + rank 62: 50582.22 + rank 63: 50574.37 + batch-generator: + rank 0: 40.56 + rank 1: 38.38 + rank 2: 39.32 + rank 3: 40.65 + rank 4: 38.35 + rank 5: 36.83 + rank 6: 39.55 + rank 7: 38.34 + rank 8: 23.37 + rank 9: 23.34 + rank 10: 24.16 + rank 11: 24.31 + rank 12: 24.60 + rank 13: 23.54 + rank 14: 23.57 + rank 15: 23.04 + rank 16: 24.92 + rank 17: 24.53 + rank 18: 24.49 + rank 19: 24.67 + rank 20: 24.48 + rank 21: 25.21 + rank 22: 24.07 + rank 23: 24.33 + rank 24: 33.03 + rank 25: 34.03 + rank 26: 35.17 + rank 27: 34.74 + rank 28: 33.36 + rank 29: 39.48 + rank 30: 34.64 + rank 31: 32.99 + rank 32: 34.77 + rank 33: 34.18 + rank 34: 33.74 + rank 35: 31.98 + rank 36: 32.42 + rank 37: 31.84 + rank 38: 33.04 + rank 39: 30.91 + rank 40: 24.10 + rank 41: 23.13 + rank 42: 24.13 + rank 43: 24.32 + rank 44: 24.52 + rank 45: 23.75 + rank 46: 23.68 + rank 47: 24.18 + rank 48: 24.76 + rank 49: 23.11 + rank 50: 25.01 + rank 51: 28.37 + rank 52: 32.59 + rank 53: 29.65 + rank 54: 31.65 + rank 55: 27.37 + rank 56: 26.10 + rank 57: 27.04 + rank 58: 27.36 + rank 59: 24.76 + rank 60: 24.83 + rank 61: 24.86 + rank 62: 25.71 + rank 63: 28.36 + embedding-grads-all-reduce: + rank 0: 0.03 + rank 1: 0.02 + rank 2: 0.03 + rank 3: 0.03 + rank 4: 0.02 + rank 5: 0.03 + rank 6: 0.03 + rank 7: 0.03 + rank 8: 0.03 + rank 9: 0.03 + rank 10: 0.02 + rank 11: 0.03 + rank 12: 0.03 + rank 13: 0.02 + rank 14: 0.02 + rank 15: 0.03 + rank 16: 0.03 + rank 17: 0.02 + rank 18: 0.03 + rank 19: 0.03 + rank 20: 0.03 + rank 21: 0.03 + rank 22: 0.03 + rank 23: 0.03 + rank 24: 0.04 + rank 25: 0.03 + rank 26: 0.04 + rank 27: 0.05 + rank 28: 0.04 + rank 29: 0.04 + rank 30: 0.04 + rank 31: 0.04 + rank 32: 0.03 + rank 33: 0.03 + rank 34: 0.03 + rank 35: 0.03 + rank 36: 0.03 + rank 37: 0.03 + rank 38: 0.03 + rank 39: 0.03 + rank 40: 0.03 + rank 41: 0.03 + rank 42: 0.02 + rank 43: 0.03 + rank 44: 0.02 + rank 45: 0.03 + rank 46: 0.03 + rank 47: 0.03 + rank 48: 0.04 + rank 49: 0.03 + rank 50: 0.03 + rank 51: 0.03 + rank 52: 0.03 + rank 53: 0.03 + rank 54: 0.03 + rank 55: 0.03 + rank 56: 0.03 + rank 57: 0.02 + rank 58: 0.02 + rank 59: 0.03 + rank 60: 0.02 + rank 61: 0.03 + rank 62: 0.03 + rank 63: 0.03 + all-grads-sync: + rank 0: 0.78 + rank 1: 0.79 + rank 2: 0.80 + rank 3: 0.80 + rank 4: 0.77 + rank 5: 0.79 + rank 6: 0.79 + rank 7: 0.86 + rank 8: 0.78 + rank 9: 0.79 + rank 10: 0.75 + rank 11: 1.07 + rank 12: 0.79 + rank 13: 1.00 + rank 14: 0.75 + rank 15: 1.20 + rank 16: 0.76 + rank 17: 0.84 + rank 18: 0.80 + rank 19: 0.79 + rank 20: 0.77 + rank 21: 0.79 + rank 22: 0.78 + rank 23: 1.22 + rank 24: 1.23 + rank 25: 1.27 + rank 26: 1.19 + rank 27: 1.29 + rank 28: 1.02 + rank 29: 1.29 + rank 30: 1.31 + rank 31: 1.47 + rank 32: 1.17 + rank 33: 1.27 + rank 34: 1.15 + rank 35: 1.10 + rank 36: 1.12 + rank 37: 1.27 + rank 38: 1.21 + rank 39: 1.20 + rank 40: 0.79 + rank 41: 0.88 + rank 42: 0.83 + rank 43: 0.76 + rank 44: 0.75 + rank 45: 0.75 + rank 46: 0.79 + rank 47: 0.90 + rank 48: 0.82 + rank 49: 1.09 + rank 50: 0.83 + rank 51: 1.11 + rank 52: 1.07 + rank 53: 1.07 + rank 54: 1.03 + rank 55: 1.04 + rank 56: 0.96 + rank 57: 0.80 + rank 58: 0.72 + rank 59: 0.79 + rank 60: 0.74 + rank 61: 0.76 + rank 62: 0.77 + rank 63: 0.77 + params-all-gather: + rank 0: 0.02 + rank 1: 0.02 + rank 2: 0.02 + rank 3: 0.02 + rank 4: 0.02 + rank 5: 0.02 + rank 6: 0.02 + rank 7: 0.02 + rank 8: 0.02 + rank 9: 0.02 + rank 10: 0.02 + rank 11: 0.02 + rank 12: 0.02 + rank 13: 0.02 + rank 14: 0.02 + rank 15: 0.02 + rank 16: 0.02 + rank 17: 0.02 + rank 18: 0.02 + rank 19: 0.02 + rank 20: 0.02 + rank 21: 0.02 + rank 22: 0.02 + rank 23: 0.02 + rank 24: 0.04 + rank 25: 0.03 + rank 26: 0.04 + rank 27: 0.03 + rank 28: 0.03 + rank 29: 0.03 + rank 30: 0.03 + rank 31: 0.04 + rank 32: 0.02 + rank 33: 0.02 + rank 34: 0.02 + rank 35: 0.03 + rank 36: 0.02 + rank 37: 0.02 + rank 38: 0.02 + rank 39: 0.02 + rank 40: 0.02 + rank 41: 0.02 + rank 42: 0.02 + rank 43: 0.02 + rank 44: 0.02 + rank 45: 0.02 + rank 46: 0.02 + rank 47: 0.02 + rank 48: 0.03 + rank 49: 0.03 + rank 50: 0.03 + rank 51: 0.02 + rank 52: 0.03 + rank 53: 0.02 + rank 54: 0.03 + rank 55: 0.03 + rank 56: 0.03 + rank 57: 0.02 + rank 58: 0.02 + rank 59: 0.02 + rank 60: 0.03 + rank 61: 0.02 + rank 62: 0.02 + rank 63: 0.02 + optimizer-copy-to-main-grad: + rank 0: 2.28 + rank 1: 1.85 + rank 2: 1.72 + rank 3: 2.19 + rank 4: 1.71 + rank 5: 1.78 + rank 6: 1.88 + rank 7: 1.93 + rank 8: 1.72 + rank 9: 2.09 + rank 10: 1.72 + rank 11: 1.74 + rank 12: 1.80 + rank 13: 1.70 + rank 14: 1.92 + rank 15: 2.01 + rank 16: 2.40 + rank 17: 2.08 + rank 18: 2.88 + rank 19: 2.16 + rank 20: 2.12 + rank 21: 2.39 + rank 22: 2.57 + rank 23: 2.36 + rank 24: 1.93 + rank 25: 2.27 + rank 26: 1.98 + rank 27: 2.07 + rank 28: 2.21 + rank 29: 2.60 + rank 30: 2.32 + rank 31: 2.13 + rank 32: 2.03 + rank 33: 1.89 + rank 34: 2.06 + rank 35: 2.22 + rank 36: 2.11 + rank 37: 2.06 + rank 38: 2.35 + rank 39: 1.88 + rank 40: 3.05 + rank 41: 2.06 + rank 42: 2.93 + rank 43: 2.11 + rank 44: 2.04 + rank 45: 2.25 + rank 46: 2.86 + rank 47: 2.68 + rank 48: 1.94 + rank 49: 1.59 + rank 50: 2.07 + rank 51: 1.80 + rank 52: 2.05 + rank 53: 1.78 + rank 54: 1.80 + rank 55: 1.60 + rank 56: 1.73 + rank 57: 2.43 + rank 58: 1.79 + rank 59: 1.73 + rank 60: 1.67 + rank 61: 1.63 + rank 62: 1.87 + rank 63: 1.70 + optimizer-inner-step: + rank 0: 45.18 + rank 1: 41.98 + rank 2: 41.76 + rank 3: 41.88 + rank 4: 41.71 + rank 5: 41.82 + rank 6: 42.55 + rank 7: 42.46 + rank 8: 41.91 + rank 9: 41.94 + rank 10: 41.83 + rank 11: 41.89 + rank 12: 41.95 + rank 13: 42.12 + rank 14: 41.95 + rank 15: 41.67 + rank 16: 42.27 + rank 17: 41.98 + rank 18: 42.01 + rank 19: 42.02 + rank 20: 42.07 + rank 21: 41.91 + rank 22: 42.36 + rank 23: 42.07 + rank 24: 41.62 + rank 25: 41.90 + rank 26: 41.88 + rank 27: 42.17 + rank 28: 46.07 + rank 29: 41.83 + rank 30: 41.77 + rank 31: 42.03 + rank 32: 42.00 + rank 33: 41.71 + rank 34: 41.87 + rank 35: 48.45 + rank 36: 42.03 + rank 37: 42.06 + rank 38: 42.11 + rank 39: 41.80 + rank 40: 41.88 + rank 41: 42.00 + rank 42: 42.07 + rank 43: 42.13 + rank 44: 41.71 + rank 45: 42.28 + rank 46: 41.98 + rank 47: 41.59 + rank 48: 42.52 + rank 49: 41.80 + rank 50: 53.21 + rank 51: 41.63 + rank 52: 41.94 + rank 53: 41.78 + rank 54: 42.07 + rank 55: 41.95 + rank 56: 41.56 + rank 57: 48.32 + rank 58: 41.92 + rank 59: 41.75 + rank 60: 42.10 + rank 61: 41.72 + rank 62: 41.86 + rank 63: 41.65 + optimizer-copy-main-to-model-params: + rank 0: 4.48 + rank 1: 3.92 + rank 2: 3.79 + rank 3: 4.35 + rank 4: 3.76 + rank 5: 3.94 + rank 6: 3.92 + rank 7: 4.04 + rank 8: 3.88 + rank 9: 4.47 + rank 10: 3.76 + rank 11: 3.75 + rank 12: 3.92 + rank 13: 3.76 + rank 14: 3.99 + rank 15: 4.21 + rank 16: 4.43 + rank 17: 3.99 + rank 18: 4.08 + rank 19: 4.06 + rank 20: 4.08 + rank 21: 4.36 + rank 22: 4.61 + rank 23: 4.22 + rank 24: 3.88 + rank 25: 4.17 + rank 26: 3.90 + rank 27: 4.13 + rank 28: 4.35 + rank 29: 4.49 + rank 30: 4.29 + rank 31: 3.99 + rank 32: 4.10 + rank 33: 3.90 + rank 34: 4.04 + rank 35: 4.54 + rank 36: 4.26 + rank 37: 4.10 + rank 38: 4.61 + rank 39: 3.93 + rank 40: 3.76 + rank 41: 4.01 + rank 42: 3.89 + rank 43: 4.01 + rank 44: 4.05 + rank 45: 4.28 + rank 46: 3.80 + rank 47: 3.77 + rank 48: 4.07 + rank 49: 3.93 + rank 50: 4.87 + rank 51: 3.94 + rank 52: 4.26 + rank 53: 3.85 + rank 54: 3.88 + rank 55: 3.86 + rank 56: 3.83 + rank 57: 4.79 + rank 58: 3.90 + rank 59: 3.79 + rank 60: 3.74 + rank 61: 3.66 + rank 62: 3.93 + rank 63: 3.72 + optimizer: + rank 0: 57.57 + rank 1: 53.59 + rank 2: 53.17 + rank 3: 54.15 + rank 4: 53.09 + rank 5: 53.49 + rank 6: 54.21 + rank 7: 54.24 + rank 8: 53.50 + rank 9: 54.33 + rank 10: 53.29 + rank 11: 53.31 + rank 12: 53.58 + rank 13: 53.58 + rank 14: 53.65 + rank 15: 53.79 + rank 16: 54.81 + rank 17: 53.94 + rank 18: 54.12 + rank 19: 54.02 + rank 20: 54.06 + rank 21: 54.33 + rank 22: 55.16 + rank 23: 54.32 + rank 24: 53.21 + rank 25: 53.88 + rank 26: 53.50 + rank 27: 54.04 + rank 28: 58.25 + rank 29: 54.15 + rank 30: 53.91 + rank 31: 53.77 + rank 32: 53.83 + rank 33: 53.24 + rank 34: 53.59 + rank 35: 60.93 + rank 36: 54.01 + rank 37: 53.83 + rank 38: 54.55 + rank 39: 53.37 + rank 40: 53.30 + rank 41: 53.76 + rank 42: 53.72 + rank 43: 53.89 + rank 44: 53.52 + rank 45: 54.47 + rank 46: 53.43 + rank 47: 53.04 + rank 48: 54.34 + rank 49: 53.39 + rank 50: 66.16 + rank 51: 53.22 + rank 52: 53.96 + rank 53: 53.28 + rank 54: 53.59 + rank 55: 53.45 + rank 56: 53.06 + rank 57: 61.17 + rank 58: 53.51 + rank 59: 53.16 + rank 60: 53.50 + rank 61: 53.00 + rank 62: 53.46 + rank 63: 53.00 + interval-time: + rank 0: 0.83 + rank 1: 0.87 + rank 2: 0.91 + rank 3: 0.87 + rank 4: 0.86 + rank 5: 0.84 + rank 6: 0.81 + rank 7: 0.98 + rank 8: 0.74 + rank 9: 0.73 + rank 10: 0.73 + rank 11: 0.74 + rank 12: 0.74 + rank 13: 0.72 + rank 14: 0.71 + rank 15: 0.74 + rank 16: 0.75 + rank 17: 0.74 + rank 18: 0.75 + rank 19: 0.76 + rank 20: 0.73 + rank 21: 0.75 + rank 22: 0.73 + rank 23: 0.84 + rank 24: 0.86 + rank 25: 0.86 + rank 26: 0.90 + rank 27: 0.89 + rank 28: 0.83 + rank 29: 0.86 + rank 30: 0.83 + rank 31: 0.92 + rank 32: 0.87 + rank 33: 0.97 + rank 34: 0.90 + rank 35: 0.87 + rank 36: 0.80 + rank 37: 0.80 + rank 38: 0.98 + rank 39: 1.04 + rank 40: 0.75 + rank 41: 0.75 + rank 42: 0.91 + rank 43: 0.75 + rank 44: 0.74 + rank 45: 0.77 + rank 46: 0.76 + rank 47: 0.82 + rank 48: 0.82 + rank 49: 0.83 + rank 50: 0.78 + rank 51: 0.80 + rank 52: 0.84 + rank 53: 0.83 + rank 54: 0.82 + rank 55: 0.81 + rank 56: 0.75 + rank 57: 0.73 + rank 58: 0.74 + rank 59: 0.74 + rank 60: 0.73 + rank 61: 0.73 + rank 62: 0.73 + rank 63: 3.72 + [2026-10-07 20:20:08] iteration 4002/ 6855 | consumed samples: 512256 | elapsed time per iteration (ms): 97274.2 | throughput per GPU (TFLOP/s/GPU): 38.2 | MFU 12.25% | FLOPs estimator: workload-aware | logical tokens/s: 172472.9 | valid tokens/s: 172472.9 | physical tokens/s: 172839.8 | loss tokens/s: 129756.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.977109E-01 | lm-global-supervised-tokens: 1.262194E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.933207E-02 | lm-token-buffer-mib: 5.824619E+01 | global_load_balancing_loss: 1.000805E+00 | loss scale: 1.0 | grad norm: 0.182 | params norm: 11383.852 | params rms: 0.061640 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 5:05:23.158492 | finish at 2026-10-11 01:25:31 +times across ranks (ms): + forward-backward: + rank 0: 91102.44 + rank 1: 91175.68 + rank 2: 91175.73 + rank 3: 91176.93 + rank 4: 91176.22 + rank 5: 91176.70 + rank 6: 91173.93 + rank 7: 91174.97 + rank 8: 91100.17 + rank 9: 91174.16 + rank 10: 91173.64 + rank 11: 91174.81 + rank 12: 91175.26 + rank 13: 91176.45 + rank 14: 91171.34 + rank 15: 91172.98 + rank 16: 91094.05 + rank 17: 91167.89 + rank 18: 91167.62 + rank 19: 91168.38 + rank 20: 91168.57 + rank 21: 91170.85 + rank 22: 91166.17 + rank 23: 91167.39 + rank 24: 91092.67 + rank 25: 91167.72 + rank 26: 91168.51 + rank 27: 91166.94 + rank 28: 91169.85 + rank 29: 91170.48 + rank 30: 91165.20 + rank 31: 91166.13 + rank 32: 91092.55 + rank 33: 91165.75 + rank 34: 91165.44 + rank 35: 91165.94 + rank 36: 91166.65 + rank 37: 91165.67 + rank 38: 91164.94 + rank 39: 91165.73 + rank 40: 91093.96 + rank 41: 91166.02 + rank 42: 91165.95 + rank 43: 91164.59 + rank 44: 91164.94 + rank 45: 91163.52 + rank 46: 91166.92 + rank 47: 91167.24 + rank 48: 91099.50 + rank 49: 91172.71 + rank 50: 91170.02 + rank 51: 91171.41 + rank 52: 91169.70 + rank 53: 91172.04 + rank 54: 91173.20 + rank 55: 91174.09 + rank 56: 91102.10 + rank 57: 91173.62 + rank 58: 91173.52 + rank 59: 91172.39 + rank 60: 91172.12 + rank 61: 91171.09 + rank 62: 91174.96 + rank 63: 91173.87 + forward-compute: + rank 0: 36792.67 + rank 1: 36868.35 + rank 2: 36872.93 + rank 3: 36861.89 + rank 4: 36862.56 + rank 5: 36861.41 + rank 6: 36860.50 + rank 7: 36869.13 + rank 8: 39435.77 + rank 9: 39509.82 + rank 10: 39512.11 + rank 11: 39513.04 + rank 12: 39513.23 + rank 13: 39514.50 + rank 14: 39512.69 + rank 15: 39512.24 + rank 16: 33266.73 + rank 17: 33323.70 + rank 18: 33328.29 + rank 19: 33338.61 + rank 20: 33342.29 + rank 21: 33333.08 + rank 22: 33330.46 + rank 23: 33326.26 + rank 24: 33906.54 + rank 25: 33982.56 + rank 26: 33989.72 + rank 27: 33986.90 + rank 28: 33989.25 + rank 29: 33992.70 + rank 30: 33993.03 + rank 31: 33984.43 + rank 32: 37686.15 + rank 33: 37760.14 + rank 34: 37763.07 + rank 35: 37764.57 + rank 36: 37759.61 + rank 37: 37757.74 + rank 38: 37751.48 + rank 39: 37752.46 + rank 40: 39942.49 + rank 41: 40011.44 + rank 42: 40013.67 + rank 43: 40012.02 + rank 44: 40012.51 + rank 45: 40015.57 + rank 46: 40011.83 + rank 47: 40015.26 + rank 48: 33666.65 + rank 49: 33742.81 + rank 50: 33741.43 + rank 51: 33742.86 + rank 52: 33755.36 + rank 53: 33753.01 + rank 54: 33747.75 + rank 55: 33747.90 + rank 56: 37863.66 + rank 57: 37938.09 + rank 58: 37939.33 + rank 59: 37937.30 + rank 60: 37940.14 + rank 61: 37941.24 + rank 62: 37939.30 + rank 63: 37940.43 + backward-compute: + rank 0: 54273.48 + rank 1: 54270.62 + rank 2: 54262.57 + rank 3: 54277.01 + rank 4: 54279.76 + rank 5: 54278.88 + rank 6: 54277.85 + rank 7: 54267.44 + rank 8: 51628.96 + rank 9: 51628.43 + rank 10: 51624.94 + rank 11: 51622.92 + rank 12: 51624.72 + rank 13: 51625.27 + rank 14: 51623.94 + rank 15: 51622.66 + rank 16: 57794.56 + rank 17: 57812.32 + rank 18: 57807.93 + rank 19: 57795.50 + rank 20: 57793.71 + rank 21: 57803.51 + rank 22: 57801.93 + rank 23: 57808.63 + rank 24: 57144.79 + rank 25: 57147.13 + rank 26: 57137.07 + rank 27: 57138.20 + rank 28: 57135.94 + rank 29: 57124.79 + rank 30: 57129.54 + rank 31: 57141.36 + rank 32: 53365.59 + rank 33: 53365.91 + rank 34: 53359.64 + rank 35: 53361.31 + rank 36: 53369.55 + rank 37: 53367.58 + rank 38: 53371.12 + rank 39: 53371.83 + rank 40: 51114.98 + rank 41: 51120.35 + rank 42: 51114.20 + rank 43: 51115.14 + rank 44: 51116.16 + rank 45: 51111.81 + rank 46: 51119.16 + rank 47: 51113.50 + rank 48: 57399.08 + rank 49: 57395.16 + rank 50: 57396.89 + rank 51: 57394.49 + rank 52: 57377.31 + rank 53: 57381.14 + rank 54: 57388.35 + rank 55: 57389.24 + rank 56: 53199.10 + rank 57: 53196.58 + rank 58: 53190.28 + rank 59: 53196.57 + rank 60: 53193.15 + rank 61: 53191.27 + rank 62: 53196.97 + rank 63: 53193.31 + batch-generator: + rank 0: 23.80 + rank 1: 23.17 + rank 2: 24.84 + rank 3: 25.01 + rank 4: 24.58 + rank 5: 22.24 + rank 6: 25.60 + rank 7: 25.48 + rank 8: 22.96 + rank 9: 22.95 + rank 10: 22.45 + rank 11: 22.56 + rank 12: 24.16 + rank 13: 23.14 + rank 14: 23.61 + rank 15: 22.70 + rank 16: 23.94 + rank 17: 24.08 + rank 18: 22.85 + rank 19: 22.86 + rank 20: 22.41 + rank 21: 25.89 + rank 22: 24.07 + rank 23: 23.71 + rank 24: 32.86 + rank 25: 32.51 + rank 26: 33.04 + rank 27: 33.36 + rank 28: 37.49 + rank 29: 35.23 + rank 30: 33.80 + rank 31: 32.27 + rank 32: 36.26 + rank 33: 32.56 + rank 34: 35.70 + rank 35: 33.18 + rank 36: 33.48 + rank 37: 32.74 + rank 38: 34.17 + rank 39: 33.12 + rank 40: 24.19 + rank 41: 22.52 + rank 42: 22.39 + rank 43: 23.86 + rank 44: 23.15 + rank 45: 22.19 + rank 46: 21.94 + rank 47: 22.22 + rank 48: 30.56 + rank 49: 29.36 + rank 50: 20.65 + rank 51: 24.40 + rank 52: 28.96 + rank 53: 29.40 + rank 54: 26.75 + rank 55: 30.18 + rank 56: 26.29 + rank 57: 27.27 + rank 58: 26.72 + rank 59: 25.57 + rank 60: 25.92 + rank 61: 28.62 + rank 62: 26.29 + rank 63: 29.98 + embedding-grads-all-reduce: + rank 0: 0.02 + rank 1: 0.02 + rank 2: 0.03 + rank 3: 0.03 + rank 4: 0.02 + rank 5: 0.03 + rank 6: 0.03 + rank 7: 0.03 + rank 8: 0.02 + rank 9: 0.03 + rank 10: 0.02 + rank 11: 0.03 + rank 12: 0.03 + rank 13: 0.03 + rank 14: 0.02 + rank 15: 0.03 + rank 16: 0.03 + rank 17: 0.03 + rank 18: 0.03 + rank 19: 0.03 + rank 20: 0.03 + rank 21: 0.02 + rank 22: 0.03 + rank 23: 0.03 + rank 24: 0.04 + rank 25: 0.03 + rank 26: 0.04 + rank 27: 0.04 + rank 28: 0.04 + rank 29: 0.04 + rank 30: 0.04 + rank 31: 0.04 + rank 32: 0.04 + rank 33: 0.03 + rank 34: 0.03 + rank 35: 0.03 + rank 36: 0.03 + rank 37: 0.03 + rank 38: 0.03 + rank 39: 0.03 + rank 40: 0.03 + rank 41: 0.02 + rank 42: 0.02 + rank 43: 0.02 + rank 44: 0.03 + rank 45: 0.03 + rank 46: 0.02 + rank 47: 0.03 + rank 48: 0.03 + rank 49: 0.03 + rank 50: 0.02 + rank 51: 0.03 + rank 52: 0.03 + rank 53: 0.03 + rank 54: 0.03 + rank 55: 0.03 + rank 56: 0.03 + rank 57: 0.02 + rank 58: 0.02 + rank 59: 0.03 + rank 60: 0.03 + rank 61: 0.02 + rank 62: 0.03 + rank 63: 0.03 + all-grads-sync: + rank 0: 1.22 + rank 1: 0.77 + rank 2: 0.78 + rank 3: 0.79 + rank 4: 0.79 + rank 5: 0.77 + rank 6: 0.76 + rank 7: 0.79 + rank 8: 0.76 + rank 9: 0.79 + rank 10: 0.83 + rank 11: 0.82 + rank 12: 0.86 + rank 13: 0.82 + rank 14: 0.84 + rank 15: 0.81 + rank 16: 0.81 + rank 17: 0.76 + rank 18: 0.76 + rank 19: 0.78 + rank 20: 1.01 + rank 21: 0.79 + rank 22: 0.76 + rank 23: 0.75 + rank 24: 1.28 + rank 25: 1.23 + rank 26: 1.30 + rank 27: 1.26 + rank 28: 1.19 + rank 29: 1.72 + rank 30: 1.23 + rank 31: 1.42 + rank 32: 1.22 + rank 33: 1.25 + rank 34: 1.40 + rank 35: 1.21 + rank 36: 1.11 + rank 37: 1.14 + rank 38: 1.21 + rank 39: 1.38 + rank 40: 0.84 + rank 41: 0.79 + rank 42: 0.80 + rank 43: 0.91 + rank 44: 0.96 + rank 45: 0.92 + rank 46: 0.87 + rank 47: 0.80 + rank 48: 0.80 + rank 49: 1.10 + rank 50: 0.80 + rank 51: 0.93 + rank 52: 0.96 + rank 53: 1.32 + rank 54: 1.02 + rank 55: 1.15 + rank 56: 0.84 + rank 57: 1.06 + rank 58: 0.86 + rank 59: 0.80 + rank 60: 0.94 + rank 61: 0.89 + rank 62: 1.01 + rank 63: 0.77 + params-all-gather: + rank 0: 0.02 + rank 1: 0.02 + rank 2: 0.02 + rank 3: 0.02 + rank 4: 0.02 + rank 5: 0.03 + rank 6: 0.03 + rank 7: 0.02 + rank 8: 0.02 + rank 9: 0.02 + rank 10: 0.02 + rank 11: 0.02 + rank 12: 0.02 + rank 13: 0.02 + rank 14: 0.03 + rank 15: 0.02 + rank 16: 0.02 + rank 17: 0.02 + rank 18: 0.02 + rank 19: 0.02 + rank 20: 0.02 + rank 21: 0.03 + rank 22: 0.02 + rank 23: 0.02 + rank 24: 0.03 + rank 25: 0.04 + rank 26: 0.04 + rank 27: 0.03 + rank 28: 0.03 + rank 29: 0.04 + rank 30: 0.03 + rank 31: 0.03 + rank 32: 0.02 + rank 33: 0.03 + rank 34: 0.03 + rank 35: 0.03 + rank 36: 0.02 + rank 37: 0.03 + rank 38: 0.02 + rank 39: 0.02 + rank 40: 0.02 + rank 41: 0.02 + rank 42: 0.02 + rank 43: 0.02 + rank 44: 0.02 + rank 45: 0.02 + rank 46: 0.02 + rank 47: 0.02 + rank 48: 0.03 + rank 49: 0.03 + rank 50: 0.03 + rank 51: 0.02 + rank 52: 0.03 + rank 53: 0.02 + rank 54: 0.03 + rank 55: 0.03 + rank 56: 0.02 + rank 57: 0.02 + rank 58: 0.02 + rank 59: 0.02 + rank 60: 0.02 + rank 61: 0.02 + rank 62: 0.02 + rank 63: 0.02 + optimizer-copy-to-main-grad: + rank 0: 2.25 + rank 1: 1.88 + rank 2: 1.75 + rank 3: 2.26 + rank 4: 1.64 + rank 5: 1.83 + rank 6: 1.93 + rank 7: 1.92 + rank 8: 1.72 + rank 9: 2.10 + rank 10: 1.71 + rank 11: 1.68 + rank 12: 1.80 + rank 13: 1.75 + rank 14: 1.81 + rank 15: 2.00 + rank 16: 2.39 + rank 17: 2.07 + rank 18: 2.14 + rank 19: 2.13 + rank 20: 2.15 + rank 21: 2.32 + rank 22: 2.55 + rank 23: 2.34 + rank 24: 2.85 + rank 25: 2.48 + rank 26: 3.17 + rank 27: 2.34 + rank 28: 2.44 + rank 29: 3.29 + rank 30: 2.53 + rank 31: 2.31 + rank 32: 2.11 + rank 33: 1.94 + rank 34: 2.06 + rank 35: 2.26 + rank 36: 2.07 + rank 37: 1.97 + rank 38: 2.19 + rank 39: 1.93 + rank 40: 1.69 + rank 41: 1.78 + rank 42: 1.85 + rank 43: 1.93 + rank 44: 1.93 + rank 45: 1.98 + rank 46: 1.76 + rank 47: 1.74 + rank 48: 1.92 + rank 49: 1.79 + rank 50: 2.25 + rank 51: 1.77 + rank 52: 1.94 + rank 53: 1.78 + rank 54: 1.77 + rank 55: 1.75 + rank 56: 1.76 + rank 57: 2.35 + rank 58: 1.83 + rank 59: 1.67 + rank 60: 1.80 + rank 61: 1.63 + rank 62: 1.85 + rank 63: 1.69 + optimizer-inner-step: + rank 0: 41.64 + rank 1: 41.55 + rank 2: 41.34 + rank 3: 41.36 + rank 4: 41.28 + rank 5: 41.42 + rank 6: 41.73 + rank 7: 41.75 + rank 8: 41.42 + rank 9: 41.52 + rank 10: 41.37 + rank 11: 41.44 + rank 12: 41.45 + rank 13: 41.68 + rank 14: 41.43 + rank 15: 41.35 + rank 16: 41.82 + rank 17: 41.62 + rank 18: 41.62 + rank 19: 41.58 + rank 20: 41.58 + rank 21: 41.45 + rank 22: 41.91 + rank 23: 41.63 + rank 24: 41.14 + rank 25: 41.61 + rank 26: 41.40 + rank 27: 41.90 + rank 28: 45.67 + rank 29: 41.26 + rank 30: 41.17 + rank 31: 41.61 + rank 32: 41.54 + rank 33: 41.31 + rank 34: 41.47 + rank 35: 47.73 + rank 36: 41.58 + rank 37: 41.50 + rank 38: 41.41 + rank 39: 41.38 + rank 40: 41.42 + rank 41: 41.61 + rank 42: 41.68 + rank 43: 41.73 + rank 44: 41.28 + rank 45: 41.84 + rank 46: 41.53 + rank 47: 41.18 + rank 48: 42.43 + rank 49: 41.33 + rank 50: 52.91 + rank 51: 41.17 + rank 52: 41.49 + rank 53: 41.34 + rank 54: 41.56 + rank 55: 41.45 + rank 56: 40.61 + rank 57: 47.19 + rank 58: 40.91 + rank 59: 40.80 + rank 60: 41.11 + rank 61: 40.74 + rank 62: 40.89 + rank 63: 40.75 + optimizer-copy-main-to-model-params: + rank 0: 4.38 + rank 1: 3.95 + rank 2: 3.74 + rank 3: 4.44 + rank 4: 3.68 + rank 5: 3.92 + rank 6: 3.93 + rank 7: 4.11 + rank 8: 3.92 + rank 9: 4.49 + rank 10: 3.78 + rank 11: 3.81 + rank 12: 3.97 + rank 13: 3.88 + rank 14: 4.02 + rank 15: 4.28 + rank 16: 4.41 + rank 17: 4.02 + rank 18: 4.02 + rank 19: 4.03 + rank 20: 4.03 + rank 21: 4.34 + rank 22: 4.60 + rank 23: 4.21 + rank 24: 3.90 + rank 25: 4.27 + rank 26: 3.99 + rank 27: 4.49 + rank 28: 4.38 + rank 29: 4.37 + rank 30: 4.45 + rank 31: 4.08 + rank 32: 4.21 + rank 33: 3.94 + rank 34: 4.00 + rank 35: 4.56 + rank 36: 4.21 + rank 37: 4.07 + rank 38: 4.44 + rank 39: 3.91 + rank 40: 3.76 + rank 41: 4.03 + rank 42: 3.94 + rank 43: 4.02 + rank 44: 4.05 + rank 45: 4.28 + rank 46: 3.77 + rank 47: 3.77 + rank 48: 4.09 + rank 49: 3.95 + rank 50: 4.89 + rank 51: 4.00 + rank 52: 4.18 + rank 53: 3.92 + rank 54: 3.92 + rank 55: 3.90 + rank 56: 3.78 + rank 57: 4.67 + rank 58: 3.91 + rank 59: 3.76 + rank 60: 3.70 + rank 61: 3.65 + rank 62: 3.92 + rank 63: 3.67 + optimizer: + rank 0: 55.99 + rank 1: 55.29 + rank 2: 54.80 + rank 3: 55.83 + rank 4: 54.68 + rank 5: 55.15 + rank 6: 55.53 + rank 7: 55.72 + rank 8: 55.13 + rank 9: 56.01 + rank 10: 54.93 + rank 11: 55.02 + rank 12: 55.21 + rank 13: 55.31 + rank 14: 55.29 + rank 15: 55.56 + rank 16: 56.44 + rank 17: 55.69 + rank 18: 55.71 + rank 19: 55.66 + rank 20: 55.64 + rank 21: 55.97 + rank 22: 56.81 + rank 23: 55.96 + rank 24: 54.82 + rank 25: 55.77 + rank 26: 55.18 + rank 27: 56.35 + rank 28: 60.15 + rank 29: 55.62 + rank 30: 55.68 + rank 31: 55.50 + rank 32: 55.53 + rank 33: 54.96 + rank 34: 55.24 + rank 35: 62.23 + rank 36: 55.62 + rank 37: 55.35 + rank 38: 55.77 + rank 39: 55.00 + rank 40: 54.93 + rank 41: 55.45 + rank 42: 55.41 + rank 43: 55.58 + rank 44: 55.18 + rank 45: 56.09 + rank 46: 55.00 + rank 47: 54.69 + rank 48: 56.35 + rank 49: 55.00 + rank 50: 67.92 + rank 51: 54.90 + rank 52: 55.53 + rank 53: 54.97 + rank 54: 55.20 + rank 55: 55.06 + rank 56: 54.64 + rank 57: 62.56 + rank 58: 55.13 + rank 59: 54.78 + rank 60: 55.06 + rank 61: 54.62 + rank 62: 55.11 + rank 63: 54.64 + interval-time: + rank 0: 0.85 + rank 1: 0.76 + rank 2: 0.86 + rank 3: 0.76 + rank 4: 0.74 + rank 5: 0.73 + rank 6: 0.73 + rank 7: 0.74 + rank 8: 0.84 + rank 9: 0.83 + rank 10: 0.79 + rank 11: 0.82 + rank 12: 0.84 + rank 13: 0.81 + rank 14: 0.74 + rank 15: 0.76 + rank 16: 0.85 + rank 17: 0.84 + rank 18: 0.77 + rank 19: 0.86 + rank 20: 0.73 + rank 21: 0.86 + rank 22: 0.75 + rank 23: 0.83 + rank 24: 1.31 + rank 25: 1.36 + rank 26: 1.34 + rank 27: 1.28 + rank 28: 1.16 + rank 29: 1.37 + rank 30: 1.08 + rank 31: 1.13 + rank 32: 0.92 + rank 33: 0.92 + rank 34: 0.93 + rank 35: 0.81 + rank 36: 0.88 + rank 37: 0.89 + rank 38: 0.80 + rank 39: 0.82 + rank 40: 0.84 + rank 41: 0.84 + rank 42: 0.82 + rank 43: 0.86 + rank 44: 0.76 + rank 45: 0.76 + rank 46: 0.75 + rank 47: 0.85 + rank 48: 0.92 + rank 49: 0.83 + rank 50: 0.87 + rank 51: 0.88 + rank 52: 0.84 + rank 53: 0.94 + rank 54: 0.92 + rank 55: 0.83 + rank 56: 0.82 + rank 57: 0.74 + rank 58: 0.83 + rank 59: 0.81 + rank 60: 0.81 + rank 61: 0.73 + rank 62: 0.81 + rank 63: 3.54 + [2026-10-07 20:21:33] iteration 4003/ 6855 | consumed samples: 512384 | elapsed time per iteration (ms): 85319.2 | throughput per GPU (TFLOP/s/GPU): 43.3 | MFU 13.87% | FLOPs estimator: workload-aware | logical tokens/s: 196640.5 | valid tokens/s: 196640.5 | physical tokens/s: 197077.7 | loss tokens/s: 148839.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.763831E-01 | lm-global-supervised-tokens: 1.269883E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.097276E-02 | lm-token-buffer-mib: 5.814968E+01 | global_load_balancing_loss: 1.000962E+00 | loss scale: 1.0 | grad norm: 0.056 | params norm: 11383.791 | params rms: 0.061640 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 19:35:30.316353 | finish at 2026-10-10 15:57:03 +times across ranks (ms): + forward-backward: + rank 0: 85102.46 + rank 1: 85102.77 + rank 2: 85103.54 + rank 3: 85103.83 + rank 4: 85104.14 + rank 5: 85104.21 + rank 6: 85101.31 + rank 7: 85102.41 + rank 8: 85109.35 + rank 9: 85109.17 + rank 10: 85109.57 + rank 11: 85110.87 + rank 12: 85110.91 + rank 13: 85110.89 + rank 14: 85108.83 + rank 15: 85108.75 + rank 16: 85102.85 + rank 17: 85103.75 + rank 18: 85104.31 + rank 19: 85104.25 + rank 20: 85105.42 + rank 21: 85106.34 + rank 22: 85102.48 + rank 23: 85103.27 + rank 24: 85101.71 + rank 25: 85104.02 + rank 26: 85103.63 + rank 27: 85103.62 + rank 28: 85105.46 + rank 29: 85104.89 + rank 30: 85100.85 + rank 31: 85101.20 + rank 32: 85101.41 + rank 33: 85101.14 + rank 34: 85101.25 + rank 35: 85101.39 + rank 36: 85102.46 + rank 37: 85102.66 + rank 38: 85100.35 + rank 39: 85102.78 + rank 40: 85103.45 + rank 41: 85101.48 + rank 42: 85101.75 + rank 43: 85101.63 + rank 44: 85101.38 + rank 45: 85100.99 + rank 46: 85103.74 + rank 47: 85103.45 + rank 48: 85108.02 + rank 49: 85109.34 + rank 50: 85108.22 + rank 51: 85107.64 + rank 52: 85107.14 + rank 53: 85107.20 + rank 54: 85109.06 + rank 55: 85110.00 + rank 56: 85110.20 + rank 57: 85110.27 + rank 58: 85109.58 + rank 59: 85110.18 + rank 60: 85108.56 + rank 61: 85107.61 + rank 62: 85110.98 + rank 63: 85110.70 + forward-compute: + rank 0: 29113.98 + rank 1: 29115.48 + rank 2: 29123.58 + rank 3: 29117.48 + rank 4: 29116.79 + rank 5: 29115.41 + rank 6: 29113.23 + rank 7: 29123.50 + rank 8: 34763.50 + rank 9: 34765.02 + rank 10: 34766.64 + rank 11: 34767.27 + rank 12: 34766.36 + rank 13: 34770.39 + rank 14: 34767.10 + rank 15: 34766.55 + rank 16: 33760.15 + rank 17: 33753.79 + rank 18: 33758.89 + rank 19: 33757.31 + rank 20: 33762.43 + rank 21: 33759.86 + rank 22: 33757.62 + rank 23: 33758.52 + rank 24: 27443.55 + rank 25: 27448.72 + rank 26: 27451.83 + rank 27: 27447.07 + rank 28: 27475.15 + rank 29: 27469.73 + rank 30: 27467.39 + rank 31: 27453.87 + rank 32: 32481.16 + rank 33: 32485.55 + rank 34: 32488.92 + rank 35: 32490.83 + rank 36: 32494.73 + rank 37: 32496.74 + rank 38: 32495.46 + rank 39: 32491.28 + rank 40: 33342.91 + rank 41: 33341.62 + rank 42: 33344.12 + rank 43: 33345.91 + rank 44: 33346.35 + rank 45: 33347.76 + rank 46: 33347.91 + rank 47: 33347.32 + rank 48: 31006.10 + rank 49: 30999.16 + rank 50: 31004.77 + rank 51: 31001.20 + rank 52: 31010.47 + rank 53: 31011.12 + rank 54: 31005.55 + rank 55: 31007.64 + rank 56: 30898.72 + rank 57: 30893.08 + rank 58: 30896.14 + rank 59: 30896.27 + rank 60: 30898.65 + rank 61: 30900.77 + rank 62: 30894.65 + rank 63: 30897.25 + backward-compute: + rank 0: 55956.24 + rank 1: 55954.54 + rank 2: 55945.29 + rank 3: 55953.37 + rank 4: 55953.59 + rank 5: 55956.81 + rank 6: 55956.01 + rank 7: 55945.39 + rank 8: 50310.85 + rank 9: 50311.73 + rank 10: 50310.35 + rank 11: 50308.83 + rank 12: 50310.36 + rank 13: 50305.93 + rank 14: 50308.02 + rank 15: 50307.06 + rank 16: 51309.25 + rank 17: 51316.02 + rank 18: 51313.23 + rank 19: 51314.53 + rank 20: 51309.37 + rank 21: 51311.27 + rank 22: 51311.98 + rank 23: 51311.52 + rank 24: 57622.06 + rank 25: 57616.45 + rank 26: 57614.57 + rank 27: 57618.85 + rank 28: 57588.40 + rank 29: 57598.29 + rank 30: 57598.31 + rank 31: 57612.12 + rank 32: 52577.64 + rank 33: 52577.93 + rank 34: 52573.41 + rank 35: 52572.43 + rank 36: 52570.99 + rank 37: 52568.11 + rank 38: 52564.82 + rank 39: 52574.23 + rank 40: 51724.48 + rank 41: 51727.28 + rank 42: 51722.40 + rank 43: 51720.35 + rank 44: 51721.55 + rank 45: 51719.31 + rank 46: 51721.90 + rank 47: 51721.97 + rank 48: 54069.66 + rank 49: 54072.70 + rank 50: 54070.46 + rank 51: 54072.74 + rank 52: 54059.73 + rank 53: 54058.93 + rank 54: 54068.80 + rank 55: 54068.89 + rank 56: 54177.60 + rank 57: 54183.68 + rank 58: 54179.94 + rank 59: 54180.09 + rank 60: 54177.35 + rank 61: 54174.62 + rank 62: 54184.28 + rank 63: 54178.91 + batch-generator: + rank 0: 23.25 + rank 1: 23.12 + rank 2: 22.78 + rank 3: 22.04 + rank 4: 22.05 + rank 5: 21.91 + rank 6: 23.29 + rank 7: 23.51 + rank 8: 24.17 + rank 9: 22.66 + rank 10: 22.39 + rank 11: 22.98 + rank 12: 23.40 + rank 13: 22.29 + rank 14: 24.43 + rank 15: 22.58 + rank 16: 22.34 + rank 17: 21.73 + rank 18: 23.07 + rank 19: 22.67 + rank 20: 21.62 + rank 21: 22.66 + rank 22: 23.43 + rank 23: 23.90 + rank 24: 35.82 + rank 25: 30.96 + rank 26: 33.36 + rank 27: 36.78 + rank 28: 34.20 + rank 29: 26.36 + rank 30: 32.08 + rank 31: 31.10 + rank 32: 32.98 + rank 33: 28.18 + rank 34: 34.51 + rank 35: 33.22 + rank 36: 32.45 + rank 37: 32.43 + rank 38: 34.98 + rank 39: 32.60 + rank 40: 22.12 + rank 41: 21.81 + rank 42: 21.75 + rank 43: 22.27 + rank 44: 23.04 + rank 45: 22.73 + rank 46: 22.30 + rank 47: 22.04 + rank 48: 22.76 + rank 49: 30.97 + rank 50: 26.72 + rank 51: 21.27 + rank 52: 27.34 + rank 53: 29.43 + rank 54: 22.41 + rank 55: 27.84 + rank 56: 22.30 + rank 57: 22.26 + rank 58: 22.56 + rank 59: 21.87 + rank 60: 21.91 + rank 61: 21.85 + rank 62: 22.70 + rank 63: 24.28 + embedding-grads-all-reduce: + rank 0: 0.02 + rank 1: 0.02 + rank 2: 0.02 + rank 3: 0.03 + rank 4: 0.02 + rank 5: 0.03 + rank 6: 0.03 + rank 7: 0.03 + rank 8: 0.02 + rank 9: 0.03 + rank 10: 0.02 + rank 11: 0.03 + rank 12: 0.03 + rank 13: 0.02 + rank 14: 0.02 + rank 15: 0.02 + rank 16: 0.03 + rank 17: 0.03 + rank 18: 0.03 + rank 19: 0.03 + rank 20: 0.03 + rank 21: 0.03 + rank 22: 0.03 + rank 23: 0.02 + rank 24: 0.04 + rank 25: 0.04 + rank 26: 0.04 + rank 27: 0.04 + rank 28: 0.04 + rank 29: 0.03 + rank 30: 0.04 + rank 31: 0.04 + rank 32: 0.03 + rank 33: 0.03 + rank 34: 0.03 + rank 35: 0.03 + rank 36: 0.03 + rank 37: 0.03 + rank 38: 0.03 + rank 39: 0.03 + rank 40: 0.03 + rank 41: 0.03 + rank 42: 0.02 + rank 43: 0.03 + rank 44: 0.03 + rank 45: 0.03 + rank 46: 0.02 + rank 47: 0.03 + rank 48: 0.03 + rank 49: 0.03 + rank 50: 0.03 + rank 51: 0.03 + rank 52: 0.04 + rank 53: 0.03 + rank 54: 0.03 + rank 55: 0.03 + rank 56: 0.02 + rank 57: 0.03 + rank 58: 0.03 + rank 59: 0.03 + rank 60: 0.03 + rank 61: 0.03 + rank 62: 0.03 + rank 63: 0.03 + all-grads-sync: + rank 0: 1.00 + rank 1: 0.74 + rank 2: 0.77 + rank 3: 0.78 + rank 4: 0.75 + rank 5: 0.78 + rank 6: 0.76 + rank 7: 0.76 + rank 8: 1.14 + rank 9: 0.75 + rank 10: 0.82 + rank 11: 0.78 + rank 12: 0.88 + rank 13: 0.80 + rank 14: 1.15 + rank 15: 0.95 + rank 16: 0.77 + rank 17: 0.77 + rank 18: 1.02 + rank 19: 0.79 + rank 20: 1.06 + rank 21: 0.91 + rank 22: 0.89 + rank 23: 0.98 + rank 24: 1.29 + rank 25: 1.24 + rank 26: 1.37 + rank 27: 1.33 + rank 28: 1.22 + rank 29: 1.10 + rank 30: 1.02 + rank 31: 1.23 + rank 32: 1.27 + rank 33: 1.20 + rank 34: 1.23 + rank 35: 1.20 + rank 36: 1.16 + rank 37: 1.50 + rank 38: 1.24 + rank 39: 1.09 + rank 40: 1.36 + rank 41: 0.83 + rank 42: 0.78 + rank 43: 0.81 + rank 44: 0.89 + rank 45: 1.31 + rank 46: 1.16 + rank 47: 0.80 + rank 48: 0.87 + rank 49: 1.49 + rank 50: 1.27 + rank 51: 0.82 + rank 52: 1.02 + rank 53: 1.10 + rank 54: 0.84 + rank 55: 1.03 + rank 56: 0.91 + rank 57: 1.13 + rank 58: 1.20 + rank 59: 1.02 + rank 60: 0.92 + rank 61: 0.77 + rank 62: 0.94 + rank 63: 0.78 + params-all-gather: + rank 0: 0.02 + rank 1: 0.02 + rank 2: 0.02 + rank 3: 0.02 + rank 4: 0.02 + rank 5: 0.02 + rank 6: 0.02 + rank 7: 0.02 + rank 8: 0.02 + rank 9: 0.02 + rank 10: 0.02 + rank 11: 0.02 + rank 12: 0.02 + rank 13: 0.02 + rank 14: 0.02 + rank 15: 0.02 + rank 16: 0.02 + rank 17: 0.02 + rank 18: 0.02 + rank 19: 0.02 + rank 20: 0.02 + rank 21: 0.02 + rank 22: 0.02 + rank 23: 0.02 + rank 24: 0.04 + rank 25: 0.03 + rank 26: 0.04 + rank 27: 0.04 + rank 28: 0.04 + rank 29: 0.03 + rank 30: 0.03 + rank 31: 0.03 + rank 32: 0.02 + rank 33: 0.02 + rank 34: 0.02 + rank 35: 0.02 + rank 36: 0.02 + rank 37: 0.02 + rank 38: 0.02 + rank 39: 0.02 + rank 40: 0.02 + rank 41: 0.02 + rank 42: 0.02 + rank 43: 0.02 + rank 44: 0.02 + rank 45: 0.02 + rank 46: 0.02 + rank 47: 0.02 + rank 48: 0.03 + rank 49: 0.03 + rank 50: 0.02 + rank 51: 0.03 + rank 52: 0.03 + rank 53: 0.03 + rank 54: 0.03 + rank 55: 0.03 + rank 56: 0.02 + rank 57: 0.02 + rank 58: 0.02 + rank 59: 0.02 + rank 60: 0.02 + rank 61: 0.02 + rank 62: 0.02 + rank 63: 0.02 + optimizer-copy-to-main-grad: + rank 0: 2.16 + rank 1: 1.83 + rank 2: 1.72 + rank 3: 2.06 + rank 4: 1.62 + rank 5: 1.80 + rank 6: 1.84 + rank 7: 1.89 + rank 8: 1.78 + rank 9: 2.06 + rank 10: 1.73 + rank 11: 1.67 + rank 12: 1.77 + rank 13: 1.70 + rank 14: 1.84 + rank 15: 2.01 + rank 16: 2.34 + rank 17: 2.09 + rank 18: 2.16 + rank 19: 2.06 + rank 20: 2.29 + rank 21: 2.30 + rank 22: 2.48 + rank 23: 2.39 + rank 24: 1.93 + rank 25: 2.24 + rank 26: 1.89 + rank 27: 2.11 + rank 28: 2.33 + rank 29: 2.31 + rank 30: 2.27 + rank 31: 1.94 + rank 32: 2.16 + rank 33: 1.81 + rank 34: 1.98 + rank 35: 2.10 + rank 36: 2.03 + rank 37: 1.98 + rank 38: 2.27 + rank 39: 1.92 + rank 40: 1.75 + rank 41: 1.75 + rank 42: 1.82 + rank 43: 1.86 + rank 44: 1.81 + rank 45: 2.01 + rank 46: 1.75 + rank 47: 1.73 + rank 48: 1.90 + rank 49: 1.82 + rank 50: 2.32 + rank 51: 1.79 + rank 52: 1.93 + rank 53: 1.77 + rank 54: 1.76 + rank 55: 1.77 + rank 56: 1.75 + rank 57: 2.33 + rank 58: 1.75 + rank 59: 1.68 + rank 60: 1.70 + rank 61: 1.64 + rank 62: 1.78 + rank 63: 1.73 + optimizer-inner-step: + rank 0: 41.79 + rank 1: 41.61 + rank 2: 41.39 + rank 3: 41.54 + rank 4: 41.43 + rank 5: 41.56 + rank 6: 41.91 + rank 7: 41.81 + rank 8: 41.66 + rank 9: 41.65 + rank 10: 41.54 + rank 11: 41.67 + rank 12: 41.64 + rank 13: 41.91 + rank 14: 41.58 + rank 15: 41.54 + rank 16: 41.96 + rank 17: 41.67 + rank 18: 41.66 + rank 19: 41.69 + rank 20: 41.75 + rank 21: 41.64 + rank 22: 42.10 + rank 23: 41.75 + rank 24: 41.21 + rank 25: 41.71 + rank 26: 41.54 + rank 27: 41.74 + rank 28: 45.74 + rank 29: 41.27 + rank 30: 41.39 + rank 31: 41.63 + rank 32: 41.74 + rank 33: 41.49 + rank 34: 41.60 + rank 35: 47.16 + rank 36: 41.75 + rank 37: 41.70 + rank 38: 41.58 + rank 39: 41.54 + rank 40: 41.64 + rank 41: 41.77 + rank 42: 41.80 + rank 43: 41.91 + rank 44: 41.41 + rank 45: 41.95 + rank 46: 41.68 + rank 47: 41.31 + rank 48: 42.28 + rank 49: 41.55 + rank 50: 53.23 + rank 51: 41.41 + rank 52: 41.71 + rank 53: 41.58 + rank 54: 41.78 + rank 55: 41.63 + rank 56: 41.34 + rank 57: 47.08 + rank 58: 41.64 + rank 59: 41.48 + rank 60: 41.82 + rank 61: 41.43 + rank 62: 41.53 + rank 63: 41.43 + optimizer-copy-main-to-model-params: + rank 0: 4.39 + rank 1: 3.87 + rank 2: 3.80 + rank 3: 4.33 + rank 4: 3.75 + rank 5: 3.90 + rank 6: 3.95 + rank 7: 4.00 + rank 8: 3.85 + rank 9: 4.43 + rank 10: 3.76 + rank 11: 3.78 + rank 12: 3.89 + rank 13: 3.77 + rank 14: 4.01 + rank 15: 4.25 + rank 16: 4.36 + rank 17: 4.00 + rank 18: 4.09 + rank 19: 3.98 + rank 20: 4.02 + rank 21: 4.21 + rank 22: 4.61 + rank 23: 4.22 + rank 24: 3.86 + rank 25: 4.22 + rank 26: 3.89 + rank 27: 4.05 + rank 28: 4.31 + rank 29: 4.37 + rank 30: 4.30 + rank 31: 3.96 + rank 32: 4.13 + rank 33: 3.96 + rank 34: 4.01 + rank 35: 4.51 + rank 36: 4.25 + rank 37: 4.08 + rank 38: 4.46 + rank 39: 3.93 + rank 40: 3.80 + rank 41: 3.94 + rank 42: 3.95 + rank 43: 4.03 + rank 44: 3.90 + rank 45: 4.28 + rank 46: 3.79 + rank 47: 3.71 + rank 48: 4.09 + rank 49: 3.88 + rank 50: 4.87 + rank 51: 3.95 + rank 52: 4.20 + rank 53: 3.88 + rank 54: 3.88 + rank 55: 3.85 + rank 56: 3.78 + rank 57: 4.71 + rank 58: 3.87 + rank 59: 3.77 + rank 60: 3.78 + rank 61: 3.67 + rank 62: 3.90 + rank 63: 3.68 + optimizer: + rank 0: 53.34 + rank 1: 52.46 + rank 2: 52.14 + rank 3: 52.98 + rank 4: 52.05 + rank 5: 52.43 + rank 6: 52.84 + rank 7: 52.80 + rank 8: 52.38 + rank 9: 53.21 + rank 10: 52.15 + rank 11: 52.28 + rank 12: 52.43 + rank 13: 52.54 + rank 14: 52.53 + rank 15: 52.82 + rank 16: 53.63 + rank 17: 52.87 + rank 18: 53.05 + rank 19: 52.86 + rank 20: 52.95 + rank 21: 53.10 + rank 22: 54.10 + rank 23: 53.27 + rank 24: 52.09 + rank 25: 52.94 + rank 26: 52.37 + rank 27: 52.91 + rank 28: 57.15 + rank 29: 52.89 + rank 30: 52.82 + rank 31: 52.69 + rank 32: 52.76 + rank 33: 52.26 + rank 34: 52.46 + rank 35: 58.71 + rank 36: 52.94 + rank 37: 52.69 + rank 38: 53.07 + rank 39: 52.29 + rank 40: 52.31 + rank 41: 52.64 + rank 42: 52.69 + rank 43: 52.88 + rank 44: 52.25 + rank 45: 53.31 + rank 46: 52.32 + rank 47: 51.87 + rank 48: 53.29 + rank 49: 52.29 + rank 50: 65.33 + rank 51: 52.21 + rank 52: 52.85 + rank 53: 52.29 + rank 54: 52.48 + rank 55: 52.32 + rank 56: 51.97 + rank 57: 59.04 + rank 58: 52.37 + rank 59: 52.09 + rank 60: 52.45 + rank 61: 51.90 + rank 62: 52.32 + rank 63: 51.89 + interval-time: + rank 0: 0.82 + rank 1: 0.75 + rank 2: 0.82 + rank 3: 0.73 + rank 4: 0.74 + rank 5: 0.73 + rank 6: 0.73 + rank 7: 0.73 + rank 8: 0.82 + rank 9: 0.80 + rank 10: 0.74 + rank 11: 0.79 + rank 12: 0.82 + rank 13: 0.79 + rank 14: 0.71 + rank 15: 0.81 + rank 16: 0.99 + rank 17: 1.05 + rank 18: 0.99 + rank 19: 1.03 + rank 20: 1.06 + rank 21: 1.02 + rank 22: 0.99 + rank 23: 1.06 + rank 24: 0.89 + rank 25: 0.97 + rank 26: 0.99 + rank 27: 0.91 + rank 28: 0.84 + rank 29: 0.96 + rank 30: 0.85 + rank 31: 0.88 + rank 32: 0.97 + rank 33: 0.99 + rank 34: 0.97 + rank 35: 0.84 + rank 36: 0.89 + rank 37: 0.89 + rank 38: 0.82 + rank 39: 0.81 + rank 40: 0.82 + rank 41: 0.82 + rank 42: 0.79 + rank 43: 0.81 + rank 44: 0.72 + rank 45: 0.80 + rank 46: 0.74 + rank 47: 0.71 + rank 48: 1.15 + rank 49: 1.02 + rank 50: 1.09 + rank 51: 1.09 + rank 52: 1.02 + rank 53: 1.09 + rank 54: 1.09 + rank 55: 1.02 + rank 56: 0.83 + rank 57: 0.75 + rank 58: 0.84 + rank 59: 0.82 + rank 60: 0.82 + rank 61: 0.80 + rank 62: 0.80 + rank 63: 3.67 + [2026-10-07 20:22:56] iteration 4004/ 6855 | consumed samples: 512512 | elapsed time per iteration (ms): 83286.8 | throughput per GPU (TFLOP/s/GPU): 43.6 | MFU 13.98% | FLOPs estimator: workload-aware | logical tokens/s: 201438.5 | valid tokens/s: 201438.5 | physical tokens/s: 201987.6 | loss tokens/s: 151048.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.309228E-01 | lm-global-supervised-tokens: 1.258038E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.237355E-02 | lm-token-buffer-mib: 5.819750E+01 | global_load_balancing_loss: 9.998272E-01 | loss scale: 1.0 | grad norm: 0.142 | params norm: 11383.731 | params rms: 0.061639 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 17:57:30.781411 | finish at 2026-10-10 14:20:27 +times across ranks (ms): + forward-backward: + rank 0: 83073.46 + rank 1: 83074.99 + rank 2: 83075.75 + rank 3: 83074.92 + rank 4: 83075.91 + rank 5: 83076.18 + rank 6: 83073.73 + rank 7: 83073.49 + rank 8: 83080.78 + rank 9: 83081.05 + rank 10: 83080.30 + rank 11: 83081.59 + rank 12: 83082.10 + rank 13: 83082.27 + rank 14: 83079.56 + rank 15: 83079.02 + rank 16: 83076.01 + rank 17: 83077.22 + rank 18: 83076.98 + rank 19: 83077.23 + rank 20: 83078.05 + rank 21: 83078.41 + rank 22: 83075.91 + rank 23: 83075.08 + rank 24: 83072.21 + rank 25: 83073.03 + rank 26: 83073.55 + rank 27: 83072.58 + rank 28: 83075.31 + rank 29: 83075.03 + rank 30: 83073.52 + rank 31: 83071.27 + rank 32: 83071.84 + rank 33: 83071.71 + rank 34: 83069.93 + rank 35: 83071.70 + rank 36: 83070.14 + rank 37: 83070.26 + rank 38: 83071.62 + rank 39: 83070.19 + rank 40: 83073.13 + rank 41: 83072.20 + rank 42: 83072.16 + rank 43: 83071.47 + rank 44: 83070.37 + rank 45: 83069.88 + rank 46: 83074.23 + rank 47: 83072.09 + rank 48: 83078.16 + rank 49: 83078.19 + rank 50: 83077.77 + rank 51: 83077.59 + rank 52: 83076.05 + rank 53: 83076.55 + rank 54: 83079.27 + rank 55: 83079.17 + rank 56: 83080.72 + rank 57: 83079.89 + rank 58: 83081.46 + rank 59: 83078.86 + rank 60: 83078.94 + rank 61: 83078.37 + rank 62: 83082.84 + rank 63: 83080.79 + forward-compute: + rank 0: 28832.29 + rank 1: 28837.14 + rank 2: 28844.82 + rank 3: 28842.84 + rank 4: 28844.98 + rank 5: 28841.68 + rank 6: 28841.37 + rank 7: 28848.07 + rank 8: 31354.82 + rank 9: 31356.40 + rank 10: 31358.59 + rank 11: 31359.41 + rank 12: 31359.39 + rank 13: 31359.76 + rank 14: 31355.55 + rank 15: 31357.57 + rank 16: 29240.54 + rank 17: 29240.39 + rank 18: 29242.96 + rank 19: 29245.31 + rank 20: 29244.67 + rank 21: 29242.06 + rank 22: 29238.42 + rank 23: 29243.37 + rank 24: 31350.11 + rank 25: 31352.40 + rank 26: 31353.27 + rank 27: 31352.46 + rank 28: 31354.21 + rank 29: 31353.51 + rank 30: 31352.65 + rank 31: 31351.11 + rank 32: 30865.43 + rank 33: 30870.79 + rank 34: 30872.21 + rank 35: 30873.48 + rank 36: 30878.01 + rank 37: 30881.24 + rank 38: 30878.89 + rank 39: 30873.46 + rank 40: 29741.66 + rank 41: 29748.26 + rank 42: 29748.27 + rank 43: 29748.90 + rank 44: 29746.59 + rank 45: 29746.85 + rank 46: 29745.52 + rank 47: 29748.27 + rank 48: 29512.23 + rank 49: 29518.51 + rank 50: 29522.71 + rank 51: 29521.54 + rank 52: 29530.64 + rank 53: 29531.63 + rank 54: 29522.98 + rank 55: 29520.97 + rank 56: 31843.67 + rank 57: 31845.59 + rank 58: 31844.65 + rank 59: 31841.84 + rank 60: 31844.39 + rank 61: 31845.27 + rank 62: 31842.85 + rank 63: 31848.95 + backward-compute: + rank 0: 54207.56 + rank 1: 54203.94 + rank 2: 54196.12 + rank 3: 54198.54 + rank 4: 54196.64 + rank 5: 54200.29 + rank 6: 54199.93 + rank 7: 54191.47 + rank 8: 51670.62 + rank 9: 51671.81 + rank 10: 51665.53 + rank 11: 51666.53 + rank 12: 51667.86 + rank 13: 51668.24 + rank 14: 51669.69 + rank 15: 51667.87 + rank 16: 53802.37 + rank 17: 53805.26 + rank 18: 53801.59 + rank 19: 53799.84 + rank 20: 53799.89 + rank 21: 53804.09 + rank 22: 53805.23 + rank 23: 53799.67 + rank 24: 51684.87 + rank 25: 51673.71 + rank 26: 51681.03 + rank 27: 51680.96 + rank 28: 51680.76 + rank 29: 51681.40 + rank 30: 51677.25 + rank 31: 51684.33 + rank 32: 52166.99 + rank 33: 52162.87 + rank 34: 52160.77 + rank 35: 52160.29 + rank 36: 52155.45 + rank 37: 52152.71 + rank 38: 52153.94 + rank 39: 52158.07 + rank 40: 53298.22 + rank 41: 53287.66 + rank 42: 53291.46 + rank 43: 53287.79 + rank 44: 53288.59 + rank 45: 53290.17 + rank 46: 53295.10 + rank 47: 53289.01 + rank 48: 53533.57 + rank 49: 53526.07 + rank 50: 53522.76 + rank 51: 53524.09 + rank 52: 53511.29 + rank 53: 53507.06 + rank 54: 53521.03 + rank 55: 53524.94 + rank 56: 51203.64 + rank 57: 51201.48 + rank 58: 51201.95 + rank 59: 51203.65 + rank 60: 51201.74 + rank 61: 51199.82 + rank 62: 51205.75 + rank 63: 51194.73 + batch-generator: + rank 0: 22.50 + rank 1: 22.74 + rank 2: 22.60 + rank 3: 21.77 + rank 4: 23.61 + rank 5: 20.92 + rank 6: 21.93 + rank 7: 22.08 + rank 8: 23.06 + rank 9: 22.19 + rank 10: 23.23 + rank 11: 23.11 + rank 12: 23.51 + rank 13: 22.01 + rank 14: 22.17 + rank 15: 22.05 + rank 16: 21.31 + rank 17: 22.22 + rank 18: 21.08 + rank 19: 23.05 + rank 20: 20.36 + rank 21: 23.01 + rank 22: 22.66 + rank 23: 22.93 + rank 24: 32.28 + rank 25: 33.98 + rank 26: 36.30 + rank 27: 36.01 + rank 28: 31.27 + rank 29: 33.63 + rank 30: 32.05 + rank 31: 27.32 + rank 32: 35.90 + rank 33: 24.30 + rank 34: 30.20 + rank 35: 33.52 + rank 36: 32.16 + rank 37: 33.24 + rank 38: 34.37 + rank 39: 28.38 + rank 40: 22.33 + rank 41: 22.01 + rank 42: 22.76 + rank 43: 22.72 + rank 44: 21.17 + rank 45: 22.31 + rank 46: 21.57 + rank 47: 21.08 + rank 48: 23.81 + rank 49: 29.17 + rank 50: 27.29 + rank 51: 21.53 + rank 52: 26.55 + rank 53: 29.87 + rank 54: 27.04 + rank 55: 30.92 + rank 56: 22.45 + rank 57: 23.12 + rank 58: 22.47 + rank 59: 21.26 + rank 60: 21.61 + rank 61: 21.69 + rank 62: 21.66 + rank 63: 22.98 + embedding-grads-all-reduce: + rank 0: 0.03 + rank 1: 0.03 + rank 2: 0.03 + rank 3: 0.03 + rank 4: 0.03 + rank 5: 0.03 + rank 6: 0.02 + rank 7: 0.03 + rank 8: 0.02 + rank 9: 0.03 + rank 10: 0.02 + rank 11: 0.03 + rank 12: 0.03 + rank 13: 0.02 + rank 14: 0.02 + rank 15: 0.02 + rank 16: 0.03 + rank 17: 0.02 + rank 18: 0.03 + rank 19: 0.02 + rank 20: 0.03 + rank 21: 0.02 + rank 22: 0.03 + rank 23: 0.03 + rank 24: 0.04 + rank 25: 0.04 + rank 26: 0.04 + rank 27: 0.04 + rank 28: 0.03 + rank 29: 0.04 + rank 30: 0.04 + rank 31: 0.03 + rank 32: 0.03 + rank 33: 0.04 + rank 34: 0.03 + rank 35: 0.03 + rank 36: 0.03 + rank 37: 0.03 + rank 38: 0.03 + rank 39: 0.03 + rank 40: 0.03 + rank 41: 0.03 + rank 42: 0.02 + rank 43: 0.03 + rank 44: 0.03 + rank 45: 0.03 + rank 46: 0.02 + rank 47: 0.03 + rank 48: 0.03 + rank 49: 0.03 + rank 50: 0.03 + rank 51: 0.03 + rank 52: 0.03 + rank 53: 0.04 + rank 54: 0.03 + rank 55: 0.03 + rank 56: 0.03 + rank 57: 0.03 + rank 58: 0.02 + rank 59: 0.02 + rank 60: 0.03 + rank 61: 0.02 + rank 62: 0.02 + rank 63: 0.03 + all-grads-sync: + rank 0: 0.87 + rank 1: 0.79 + rank 2: 0.77 + rank 3: 0.80 + rank 4: 1.01 + rank 5: 1.30 + rank 6: 0.80 + rank 7: 0.83 + rank 8: 1.06 + rank 9: 1.19 + rank 10: 0.82 + rank 11: 0.81 + rank 12: 0.78 + rank 13: 0.95 + rank 14: 0.81 + rank 15: 0.82 + rank 16: 0.79 + rank 17: 0.78 + rank 18: 0.77 + rank 19: 0.82 + rank 20: 1.13 + rank 21: 0.79 + rank 22: 0.77 + rank 23: 0.81 + rank 24: 1.20 + rank 25: 1.21 + rank 26: 1.33 + rank 27: 1.20 + rank 28: 1.19 + rank 29: 1.28 + rank 30: 1.25 + rank 31: 1.12 + rank 32: 1.68 + rank 33: 1.22 + rank 34: 1.29 + rank 35: 1.30 + rank 36: 1.19 + rank 37: 1.16 + rank 38: 1.49 + rank 39: 1.17 + rank 40: 0.82 + rank 41: 0.75 + rank 42: 0.77 + rank 43: 0.79 + rank 44: 1.13 + rank 45: 0.80 + rank 46: 0.79 + rank 47: 0.76 + rank 48: 0.81 + rank 49: 0.99 + rank 50: 1.06 + rank 51: 1.01 + rank 52: 0.80 + rank 53: 0.97 + rank 54: 0.84 + rank 55: 1.03 + rank 56: 0.82 + rank 57: 0.77 + rank 58: 1.10 + rank 59: 0.95 + rank 60: 0.81 + rank 61: 1.23 + rank 62: 1.18 + rank 63: 0.80 + params-all-gather: + rank 0: 0.02 + rank 1: 0.02 + rank 2: 0.02 + rank 3: 0.02 + rank 4: 0.02 + rank 5: 0.02 + rank 6: 0.02 + rank 7: 0.02 + rank 8: 0.02 + rank 9: 0.02 + rank 10: 0.02 + rank 11: 0.02 + rank 12: 0.02 + rank 13: 0.02 + rank 14: 0.02 + rank 15: 0.02 + rank 16: 0.02 + rank 17: 0.02 + rank 18: 0.02 + rank 19: 0.02 + rank 20: 0.02 + rank 21: 0.02 + rank 22: 0.02 + rank 23: 0.02 + rank 24: 0.03 + rank 25: 0.03 + rank 26: 0.04 + rank 27: 0.03 + rank 28: 0.03 + rank 29: 0.03 + rank 30: 0.03 + rank 31: 0.04 + rank 32: 0.02 + rank 33: 0.02 + rank 34: 0.03 + rank 35: 0.02 + rank 36: 0.02 + rank 37: 0.03 + rank 38: 0.02 + rank 39: 0.02 + rank 40: 0.02 + rank 41: 0.02 + rank 42: 0.02 + rank 43: 0.02 + rank 44: 0.02 + rank 45: 0.02 + rank 46: 0.02 + rank 47: 0.02 + rank 48: 0.03 + rank 49: 0.03 + rank 50: 0.02 + rank 51: 0.02 + rank 52: 0.03 + rank 53: 0.03 + rank 54: 0.03 + rank 55: 0.03 + rank 56: 0.02 + rank 57: 0.02 + rank 58: 0.02 + rank 59: 0.02 + rank 60: 0.02 + rank 61: 0.02 + rank 62: 0.02 + rank 63: 0.02 + optimizer-copy-to-main-grad: + rank 0: 2.17 + rank 1: 1.85 + rank 2: 1.71 + rank 3: 2.11 + rank 4: 1.72 + rank 5: 1.87 + rank 6: 1.90 + rank 7: 1.92 + rank 8: 1.83 + rank 9: 2.12 + rank 10: 1.73 + rank 11: 1.72 + rank 12: 1.75 + rank 13: 1.75 + rank 14: 1.85 + rank 15: 1.99 + rank 16: 2.37 + rank 17: 2.11 + rank 18: 2.13 + rank 19: 2.12 + rank 20: 2.10 + rank 21: 2.27 + rank 22: 2.48 + rank 23: 2.23 + rank 24: 2.08 + rank 25: 2.37 + rank 26: 2.10 + rank 27: 2.28 + rank 28: 2.44 + rank 29: 2.55 + rank 30: 2.51 + rank 31: 2.14 + rank 32: 2.07 + rank 33: 1.89 + rank 34: 2.14 + rank 35: 2.18 + rank 36: 1.98 + rank 37: 2.01 + rank 38: 2.25 + rank 39: 1.93 + rank 40: 1.87 + rank 41: 1.83 + rank 42: 1.85 + rank 43: 1.96 + rank 44: 1.87 + rank 45: 2.03 + rank 46: 1.81 + rank 47: 1.67 + rank 48: 1.92 + rank 49: 1.75 + rank 50: 2.43 + rank 51: 1.82 + rank 52: 1.93 + rank 53: 1.73 + rank 54: 1.73 + rank 55: 1.76 + rank 56: 1.73 + rank 57: 2.30 + rank 58: 1.80 + rank 59: 1.66 + rank 60: 1.71 + rank 61: 1.65 + rank 62: 1.82 + rank 63: 1.68 + optimizer-inner-step: + rank 0: 41.87 + rank 1: 41.77 + rank 2: 41.56 + rank 3: 41.67 + rank 4: 41.46 + rank 5: 41.52 + rank 6: 41.95 + rank 7: 41.88 + rank 8: 41.60 + rank 9: 41.59 + rank 10: 41.46 + rank 11: 41.62 + rank 12: 41.59 + rank 13: 41.83 + rank 14: 41.61 + rank 15: 41.53 + rank 16: 42.01 + rank 17: 41.74 + rank 18: 41.70 + rank 19: 41.78 + rank 20: 41.75 + rank 21: 41.65 + rank 22: 42.14 + rank 23: 41.80 + rank 24: 41.40 + rank 25: 41.66 + rank 26: 41.56 + rank 27: 41.75 + rank 28: 45.49 + rank 29: 41.54 + rank 30: 41.52 + rank 31: 41.80 + rank 32: 41.66 + rank 33: 41.54 + rank 34: 41.69 + rank 35: 47.09 + rank 36: 41.70 + rank 37: 41.71 + rank 38: 41.56 + rank 39: 41.56 + rank 40: 41.66 + rank 41: 41.71 + rank 42: 41.85 + rank 43: 41.82 + rank 44: 41.43 + rank 45: 42.03 + rank 46: 41.72 + rank 47: 41.31 + rank 48: 42.20 + rank 49: 41.57 + rank 50: 53.18 + rank 51: 41.38 + rank 52: 41.71 + rank 53: 41.58 + rank 54: 41.82 + rank 55: 41.62 + rank 56: 41.28 + rank 57: 47.20 + rank 58: 41.59 + rank 59: 41.46 + rank 60: 41.79 + rank 61: 41.41 + rank 62: 41.57 + rank 63: 41.47 + optimizer-copy-main-to-model-params: + rank 0: 4.65 + rank 1: 3.94 + rank 2: 3.76 + rank 3: 4.27 + rank 4: 3.77 + rank 5: 3.97 + rank 6: 4.03 + rank 7: 4.08 + rank 8: 3.86 + rank 9: 4.45 + rank 10: 3.78 + rank 11: 3.80 + rank 12: 3.94 + rank 13: 3.77 + rank 14: 4.01 + rank 15: 4.27 + rank 16: 4.43 + rank 17: 4.02 + rank 18: 4.03 + rank 19: 4.01 + rank 20: 4.00 + rank 21: 4.21 + rank 22: 4.48 + rank 23: 4.16 + rank 24: 3.88 + rank 25: 4.19 + rank 26: 3.94 + rank 27: 4.06 + rank 28: 4.45 + rank 29: 4.42 + rank 30: 4.34 + rank 31: 3.95 + rank 32: 4.15 + rank 33: 3.91 + rank 34: 4.01 + rank 35: 4.51 + rank 36: 4.19 + rank 37: 4.05 + rank 38: 4.51 + rank 39: 3.91 + rank 40: 3.78 + rank 41: 4.01 + rank 42: 3.94 + rank 43: 4.08 + rank 44: 3.92 + rank 45: 4.27 + rank 46: 3.82 + rank 47: 3.71 + rank 48: 4.08 + rank 49: 3.89 + rank 50: 4.88 + rank 51: 3.97 + rank 52: 4.24 + rank 53: 3.83 + rank 54: 3.87 + rank 55: 3.87 + rank 56: 3.81 + rank 57: 4.74 + rank 58: 3.90 + rank 59: 3.73 + rank 60: 3.76 + rank 61: 3.68 + rank 62: 3.87 + rank 63: 3.67 + optimizer: + rank 0: 53.57 + rank 1: 52.54 + rank 2: 52.07 + rank 3: 52.94 + rank 4: 52.01 + rank 5: 52.36 + rank 6: 52.86 + rank 7: 52.85 + rank 8: 52.32 + rank 9: 53.12 + rank 10: 52.12 + rank 11: 52.25 + rank 12: 52.42 + rank 13: 52.44 + rank 14: 52.49 + rank 15: 52.82 + rank 16: 53.71 + rank 17: 52.83 + rank 18: 52.89 + rank 19: 52.88 + rank 20: 52.82 + rank 21: 53.03 + rank 22: 53.94 + rank 23: 53.12 + rank 24: 52.09 + rank 25: 52.77 + rank 26: 52.32 + rank 27: 52.68 + rank 28: 56.91 + rank 29: 52.93 + rank 30: 52.81 + rank 31: 52.60 + rank 32: 52.63 + rank 33: 52.19 + rank 34: 52.48 + rank 35: 58.57 + rank 36: 52.75 + rank 37: 52.59 + rank 38: 53.06 + rank 39: 52.21 + rank 40: 52.22 + rank 41: 52.56 + rank 42: 52.66 + rank 43: 52.78 + rank 44: 52.21 + rank 45: 53.28 + rank 46: 52.30 + rank 47: 51.79 + rank 48: 53.15 + rank 49: 52.24 + rank 50: 65.24 + rank 51: 52.14 + rank 52: 52.83 + rank 53: 52.16 + rank 54: 52.45 + rank 55: 52.25 + rank 56: 51.85 + rank 57: 59.11 + rank 58: 52.33 + rank 59: 51.95 + rank 60: 52.31 + rank 61: 51.83 + rank 62: 52.25 + rank 63: 51.87 + interval-time: + rank 0: 0.83 + rank 1: 0.74 + rank 2: 0.84 + rank 3: 0.73 + rank 4: 0.77 + rank 5: 0.74 + rank 6: 0.74 + rank 7: 0.75 + rank 8: 0.98 + rank 9: 1.00 + rank 10: 0.91 + rank 11: 0.96 + rank 12: 0.95 + rank 13: 0.98 + rank 14: 0.86 + rank 15: 0.89 + rank 16: 0.84 + rank 17: 0.82 + rank 18: 0.74 + rank 19: 0.83 + rank 20: 0.79 + rank 21: 0.83 + rank 22: 0.74 + rank 23: 0.80 + rank 24: 0.87 + rank 25: 0.96 + rank 26: 0.97 + rank 27: 0.89 + rank 28: 0.85 + rank 29: 0.97 + rank 30: 0.85 + rank 31: 0.89 + rank 32: 0.91 + rank 33: 0.82 + rank 34: 0.93 + rank 35: 0.80 + rank 36: 0.88 + rank 37: 0.87 + rank 38: 0.80 + rank 39: 0.79 + rank 40: 0.83 + rank 41: 0.80 + rank 42: 0.79 + rank 43: 0.84 + rank 44: 0.72 + rank 45: 0.72 + rank 46: 0.73 + rank 47: 0.73 + rank 48: 0.94 + rank 49: 0.84 + rank 50: 0.91 + rank 51: 0.91 + rank 52: 0.84 + rank 53: 0.95 + rank 54: 0.92 + rank 55: 0.83 + rank 56: 0.82 + rank 57: 0.73 + rank 58: 0.83 + rank 59: 0.82 + rank 60: 0.80 + rank 61: 0.72 + rank 62: 0.79 + rank 63: 3.58 + [2026-10-07 20:24:24] iteration 4005/ 6855 | consumed samples: 512640 | elapsed time per iteration (ms): 87769.8 | throughput per GPU (TFLOP/s/GPU): 43.5 | MFU 13.94% | FLOPs estimator: workload-aware | logical tokens/s: 191149.5 | valid tokens/s: 191149.5 | physical tokens/s: 191509.6 | loss tokens/s: 139102.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.777703E-01 | lm-global-supervised-tokens: 1.220902E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.161652E-02 | lm-token-buffer-mib: 5.815288E+01 | global_load_balancing_loss: 9.995040E-01 | loss scale: 1.0 | grad norm: 0.045 | params norm: 11383.669 | params rms: 0.061639 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 21:29:04.044685 | finish at 2026-10-10 17:53:28 +times across ranks (ms): + forward-backward: + rank 0: 87538.16 + rank 1: 87539.27 + rank 2: 87539.16 + rank 3: 87539.97 + rank 4: 87540.37 + rank 5: 87540.45 + rank 6: 87537.74 + rank 7: 87538.42 + rank 8: 87541.94 + rank 9: 87543.02 + rank 10: 87543.09 + rank 11: 87544.91 + rank 12: 87544.40 + rank 13: 87544.88 + rank 14: 87541.20 + rank 15: 87542.14 + rank 16: 87541.02 + rank 17: 87541.70 + rank 18: 87542.04 + rank 19: 87542.54 + rank 20: 87542.73 + rank 21: 87544.22 + rank 22: 87540.70 + rank 23: 87541.60 + rank 24: 87539.04 + rank 25: 87537.23 + rank 26: 87538.70 + rank 27: 87541.74 + rank 28: 87542.15 + rank 29: 87542.19 + rank 30: 87536.99 + rank 31: 87537.10 + rank 32: 87535.60 + rank 33: 87533.91 + rank 34: 87533.03 + rank 35: 87533.83 + rank 36: 87535.82 + rank 37: 87534.91 + rank 38: 87534.77 + rank 39: 87536.54 + rank 40: 87536.43 + rank 41: 87535.82 + rank 42: 87535.00 + rank 43: 87535.59 + rank 44: 87534.98 + rank 45: 87533.54 + rank 46: 87537.17 + rank 47: 87537.02 + rank 48: 87542.47 + rank 49: 87541.12 + rank 50: 87540.08 + rank 51: 87541.48 + rank 52: 87539.98 + rank 53: 87540.91 + rank 54: 87542.60 + rank 55: 87544.02 + rank 56: 87544.59 + rank 57: 87543.44 + rank 58: 87543.60 + rank 59: 87543.41 + rank 60: 87542.46 + rank 61: 87541.39 + rank 62: 87546.13 + rank 63: 87544.99 + forward-compute: + rank 0: 31633.02 + rank 1: 31636.53 + rank 2: 31634.68 + rank 3: 31636.87 + rank 4: 31649.22 + rank 5: 31642.27 + rank 6: 31640.98 + rank 7: 31640.30 + rank 8: 32502.37 + rank 9: 32503.00 + rank 10: 32505.44 + rank 11: 32505.28 + rank 12: 32506.14 + rank 13: 32507.34 + rank 14: 32503.92 + rank 15: 32503.86 + rank 16: 28508.56 + rank 17: 28499.43 + rank 18: 28504.33 + rank 19: 28506.10 + rank 20: 28505.20 + rank 21: 28505.80 + rank 22: 28501.28 + rank 23: 28505.60 + rank 24: 30428.43 + rank 25: 30435.89 + rank 26: 30436.91 + rank 27: 30433.32 + rank 28: 30447.12 + rank 29: 30443.78 + rank 30: 30440.31 + rank 31: 30442.38 + rank 32: 32238.39 + rank 33: 32237.45 + rank 34: 32235.27 + rank 35: 32235.71 + rank 36: 32244.92 + rank 37: 32244.69 + rank 38: 32246.85 + rank 39: 32239.15 + rank 40: 32682.12 + rank 41: 32682.09 + rank 42: 32683.34 + rank 43: 32682.98 + rank 44: 32682.20 + rank 45: 32683.88 + rank 46: 32681.83 + rank 47: 32681.99 + rank 48: 30932.07 + rank 49: 30936.80 + rank 50: 30933.55 + rank 51: 30931.59 + rank 52: 30937.21 + rank 53: 30939.03 + rank 54: 30938.85 + rank 55: 30939.33 + rank 56: 32664.58 + rank 57: 32665.77 + rank 58: 32664.78 + rank 59: 32666.58 + rank 60: 32669.22 + rank 61: 32667.48 + rank 62: 32667.13 + rank 63: 32672.13 + backward-compute: + rank 0: 55871.88 + rank 1: 55868.22 + rank 2: 55870.19 + rank 3: 55869.12 + rank 4: 55857.06 + rank 5: 55864.75 + rank 6: 55862.07 + rank 7: 55863.59 + rank 8: 55006.60 + rank 9: 55005.69 + rank 10: 55003.69 + rank 11: 55005.08 + rank 12: 55004.66 + rank 13: 55004.70 + rank 14: 55003.85 + rank 15: 55005.04 + rank 16: 59000.25 + rank 17: 59010.41 + rank 18: 59006.68 + rank 19: 59004.50 + rank 20: 59007.01 + rank 21: 59005.07 + rank 22: 59006.93 + rank 23: 59003.12 + rank 24: 57071.04 + rank 25: 57058.15 + rank 26: 57059.73 + rank 27: 57064.87 + rank 28: 57052.60 + rank 29: 57054.57 + rank 30: 57053.41 + rank 31: 57056.24 + rank 32: 55256.35 + rank 33: 55258.27 + rank 34: 55258.77 + rank 35: 55260.17 + rank 36: 55253.06 + rank 37: 55250.67 + rank 38: 55247.00 + rank 39: 55258.23 + rank 40: 54819.00 + rank 41: 54820.50 + rank 42: 54817.83 + rank 43: 54815.84 + rank 44: 54817.91 + rank 45: 54814.47 + rank 46: 54821.92 + rank 47: 54820.29 + rank 48: 56573.50 + rank 49: 56571.19 + rank 50: 56571.88 + rank 51: 56577.84 + rank 52: 56566.53 + rank 53: 56565.75 + rank 54: 56570.37 + rank 55: 56572.07 + rank 56: 54843.87 + rank 57: 54843.75 + rank 58: 54844.48 + rank 59: 54843.14 + rank 60: 54839.50 + rank 61: 54840.59 + rank 62: 54844.87 + rank 63: 54835.79 + batch-generator: + rank 0: 23.72 + rank 1: 24.15 + rank 2: 22.42 + rank 3: 23.75 + rank 4: 24.00 + rank 5: 22.31 + rank 6: 22.82 + rank 7: 23.29 + rank 8: 23.64 + rank 9: 22.01 + rank 10: 22.60 + rank 11: 22.69 + rank 12: 24.76 + rank 13: 21.86 + rank 14: 22.66 + rank 15: 23.25 + rank 16: 22.45 + rank 17: 21.68 + rank 18: 21.78 + rank 19: 21.23 + rank 20: 21.68 + rank 21: 22.21 + rank 22: 22.10 + rank 23: 21.34 + rank 24: 31.47 + rank 25: 34.94 + rank 26: 33.64 + rank 27: 36.95 + rank 28: 31.84 + rank 29: 35.59 + rank 30: 36.41 + rank 31: 32.13 + rank 32: 33.90 + rank 33: 32.42 + rank 34: 32.36 + rank 35: 32.16 + rank 36: 32.17 + rank 37: 32.11 + rank 38: 35.62 + rank 39: 33.12 + rank 40: 22.62 + rank 41: 21.04 + rank 42: 21.96 + rank 43: 21.97 + rank 44: 21.51 + rank 45: 21.74 + rank 46: 21.98 + rank 47: 21.50 + rank 48: 30.56 + rank 49: 29.26 + rank 50: 28.26 + rank 51: 21.97 + rank 52: 24.07 + rank 53: 26.93 + rank 54: 23.87 + rank 55: 29.15 + rank 56: 24.49 + rank 57: 22.94 + rank 58: 23.12 + rank 59: 22.05 + rank 60: 21.48 + rank 61: 22.02 + rank 62: 22.83 + rank 63: 24.58 + embedding-grads-all-reduce: + rank 0: 0.03 + rank 1: 0.03 + rank 2: 0.03 + rank 3: 0.03 + rank 4: 0.02 + rank 5: 0.02 + rank 6: 0.03 + rank 7: 0.03 + rank 8: 0.02 + rank 9: 0.02 + rank 10: 0.02 + rank 11: 0.03 + rank 12: 0.03 + rank 13: 0.03 + rank 14: 0.03 + rank 15: 0.02 + rank 16: 0.03 + rank 17: 0.03 + rank 18: 0.02 + rank 19: 0.02 + rank 20: 0.03 + rank 21: 0.03 + rank 22: 0.03 + rank 23: 0.02 + rank 24: 0.04 + rank 25: 0.04 + rank 26: 0.04 + rank 27: 0.05 + rank 28: 0.04 + rank 29: 0.05 + rank 30: 0.04 + rank 31: 0.04 + rank 32: 0.03 + rank 33: 0.03 + rank 34: 0.03 + rank 35: 0.03 + rank 36: 0.03 + rank 37: 0.03 + rank 38: 0.03 + rank 39: 0.03 + rank 40: 0.03 + rank 41: 0.03 + rank 42: 0.02 + rank 43: 0.03 + rank 44: 0.03 + rank 45: 0.03 + rank 46: 0.02 + rank 47: 0.03 + rank 48: 0.03 + rank 49: 0.03 + rank 50: 0.03 + rank 51: 0.03 + rank 52: 0.03 + rank 53: 0.03 + rank 54: 0.03 + rank 55: 0.03 + rank 56: 0.03 + rank 57: 0.03 + rank 58: 0.02 + rank 59: 0.02 + rank 60: 0.03 + rank 61: 0.02 + rank 62: 0.03 + rank 63: 0.03 + all-grads-sync: + rank 0: 0.82 + rank 1: 0.80 + rank 2: 0.80 + rank 3: 1.00 + rank 4: 0.82 + rank 5: 1.03 + rank 6: 0.83 + rank 7: 0.79 + rank 8: 0.82 + rank 9: 0.87 + rank 10: 0.81 + rank 11: 1.10 + rank 12: 0.84 + rank 13: 0.80 + rank 14: 0.98 + rank 15: 0.88 + rank 16: 0.81 + rank 17: 0.85 + rank 18: 1.12 + rank 19: 0.93 + rank 20: 0.97 + rank 21: 0.81 + rank 22: 1.08 + rank 23: 1.16 + rank 24: 1.32 + rank 25: 1.26 + rank 26: 1.30 + rank 27: 1.88 + rank 28: 1.90 + rank 29: 1.83 + rank 30: 1.28 + rank 31: 1.33 + rank 32: 1.43 + rank 33: 1.16 + rank 34: 0.93 + rank 35: 1.22 + rank 36: 1.43 + rank 37: 1.22 + rank 38: 1.52 + rank 39: 1.24 + rank 40: 1.00 + rank 41: 0.84 + rank 42: 1.00 + rank 43: 0.82 + rank 44: 1.15 + rank 45: 0.79 + rank 46: 0.77 + rank 47: 0.88 + rank 48: 1.08 + rank 49: 0.93 + rank 50: 0.86 + rank 51: 0.93 + rank 52: 0.88 + rank 53: 1.09 + rank 54: 0.84 + rank 55: 1.01 + rank 56: 0.81 + rank 57: 0.80 + rank 58: 0.84 + rank 59: 1.22 + rank 60: 0.78 + rank 61: 0.91 + rank 62: 1.10 + rank 63: 0.79 + params-all-gather: + rank 0: 0.02 + rank 1: 0.02 + rank 2: 0.02 + rank 3: 0.02 + rank 4: 0.02 + rank 5: 0.02 + rank 6: 0.02 + rank 7: 0.03 + rank 8: 0.02 + rank 9: 0.02 + rank 10: 0.02 + rank 11: 0.02 + rank 12: 0.02 + rank 13: 0.02 + rank 14: 0.02 + rank 15: 0.02 + rank 16: 0.03 + rank 17: 0.02 + rank 18: 0.03 + rank 19: 0.02 + rank 20: 0.02 + rank 21: 0.02 + rank 22: 0.02 + rank 23: 0.02 + rank 24: 0.03 + rank 25: 0.03 + rank 26: 0.03 + rank 27: 0.03 + rank 28: 0.04 + rank 29: 0.03 + rank 30: 0.03 + rank 31: 0.03 + rank 32: 0.02 + rank 33: 0.02 + rank 34: 0.02 + rank 35: 0.02 + rank 36: 0.02 + rank 37: 0.03 + rank 38: 0.03 + rank 39: 0.02 + rank 40: 0.02 + rank 41: 0.02 + rank 42: 0.02 + rank 43: 0.02 + rank 44: 0.02 + rank 45: 0.02 + rank 46: 0.02 + rank 47: 0.02 + rank 48: 0.03 + rank 49: 0.03 + rank 50: 0.03 + rank 51: 0.20 + rank 52: 0.03 + rank 53: 0.03 + rank 54: 0.03 + rank 55: 0.03 + rank 56: 0.02 + rank 57: 0.02 + rank 58: 0.02 + rank 59: 0.02 + rank 60: 0.02 + rank 61: 0.02 + rank 62: 0.02 + rank 63: 0.02 + optimizer-copy-to-main-grad: + rank 0: 2.19 + rank 1: 1.84 + rank 2: 1.70 + rank 3: 2.09 + rank 4: 1.72 + rank 5: 1.85 + rank 6: 1.88 + rank 7: 1.95 + rank 8: 1.75 + rank 9: 2.09 + rank 10: 1.76 + rank 11: 1.74 + rank 12: 1.74 + rank 13: 1.76 + rank 14: 1.82 + rank 15: 2.14 + rank 16: 4.22 + rank 17: 3.04 + rank 18: 2.64 + rank 19: 3.37 + rank 20: 2.86 + rank 21: 3.24 + rank 22: 2.68 + rank 23: 3.48 + rank 24: 1.87 + rank 25: 2.14 + rank 26: 1.79 + rank 27: 2.03 + rank 28: 2.04 + rank 29: 2.15 + rank 30: 2.12 + rank 31: 1.85 + rank 32: 2.18 + rank 33: 1.68 + rank 34: 2.14 + rank 35: 2.17 + rank 36: 2.15 + rank 37: 2.22 + rank 38: 2.23 + rank 39: 1.86 + rank 40: 1.69 + rank 41: 1.87 + rank 42: 1.80 + rank 43: 1.85 + rank 44: 1.91 + rank 45: 2.03 + rank 46: 1.74 + rank 47: 1.73 + rank 48: 1.94 + rank 49: 1.60 + rank 50: 2.24 + rank 51: 1.76 + rank 52: 1.88 + rank 53: 1.73 + rank 54: 1.74 + rank 55: 1.74 + rank 56: 1.78 + rank 57: 2.32 + rank 58: 1.80 + rank 59: 1.70 + rank 60: 1.72 + rank 61: 1.63 + rank 62: 1.83 + rank 63: 1.68 + optimizer-inner-step: + rank 0: 44.71 + rank 1: 44.63 + rank 2: 44.35 + rank 3: 44.52 + rank 4: 44.42 + rank 5: 44.43 + rank 6: 44.85 + rank 7: 44.78 + rank 8: 44.58 + rank 9: 44.57 + rank 10: 44.44 + rank 11: 44.51 + rank 12: 44.53 + rank 13: 44.75 + rank 14: 44.52 + rank 15: 44.43 + rank 16: 56.59 + rank 17: 44.59 + rank 18: 44.65 + rank 19: 44.38 + rank 20: 44.60 + rank 21: 44.52 + rank 22: 44.99 + rank 23: 44.34 + rank 24: 44.22 + rank 25: 44.58 + rank 26: 44.42 + rank 27: 44.74 + rank 28: 47.18 + rank 29: 44.41 + rank 30: 44.41 + rank 31: 44.71 + rank 32: 44.67 + rank 33: 44.39 + rank 34: 44.46 + rank 35: 47.27 + rank 36: 44.63 + rank 37: 44.62 + rank 38: 44.49 + rank 39: 44.45 + rank 40: 44.53 + rank 41: 44.66 + rank 42: 44.74 + rank 43: 44.76 + rank 44: 44.28 + rank 45: 44.92 + rank 46: 44.54 + rank 47: 44.25 + rank 48: 44.81 + rank 49: 44.66 + rank 50: 55.32 + rank 51: 44.47 + rank 52: 44.74 + rank 53: 44.61 + rank 54: 44.84 + rank 55: 43.37 + rank 56: 44.16 + rank 57: 51.54 + rank 58: 44.52 + rank 59: 44.36 + rank 60: 44.72 + rank 61: 44.33 + rank 62: 44.47 + rank 63: 44.35 + optimizer-copy-main-to-model-params: + rank 0: 4.45 + rank 1: 3.93 + rank 2: 3.83 + rank 3: 4.35 + rank 4: 3.77 + rank 5: 4.01 + rank 6: 4.03 + rank 7: 4.14 + rank 8: 3.89 + rank 9: 4.47 + rank 10: 3.84 + rank 11: 3.80 + rank 12: 3.97 + rank 13: 3.81 + rank 14: 4.02 + rank 15: 4.29 + rank 16: 6.41 + rank 17: 4.04 + rank 18: 4.38 + rank 19: 4.03 + rank 20: 3.98 + rank 21: 4.25 + rank 22: 4.53 + rank 23: 4.19 + rank 24: 3.91 + rank 25: 4.25 + rank 26: 3.93 + rank 27: 4.34 + rank 28: 4.31 + rank 29: 4.38 + rank 30: 4.37 + rank 31: 4.04 + rank 32: 4.52 + rank 33: 3.90 + rank 34: 4.00 + rank 35: 4.51 + rank 36: 4.19 + rank 37: 4.06 + rank 38: 4.47 + rank 39: 3.96 + rank 40: 3.79 + rank 41: 4.02 + rank 42: 3.89 + rank 43: 4.03 + rank 44: 4.03 + rank 45: 4.29 + rank 46: 3.81 + rank 47: 3.78 + rank 48: 4.35 + rank 49: 4.11 + rank 50: 5.03 + rank 51: 3.93 + rank 52: 4.41 + rank 53: 4.07 + rank 54: 4.07 + rank 55: 4.30 + rank 56: 3.87 + rank 57: 4.79 + rank 58: 4.00 + rank 59: 3.83 + rank 60: 3.89 + rank 61: 3.75 + rank 62: 4.03 + rank 63: 3.80 + optimizer: + rank 0: 58.70 + rank 1: 57.89 + rank 2: 57.48 + rank 3: 58.37 + rank 4: 57.46 + rank 5: 57.84 + rank 6: 58.27 + rank 7: 58.33 + rank 8: 57.79 + rank 9: 58.59 + rank 10: 57.59 + rank 11: 57.61 + rank 12: 57.84 + rank 13: 57.87 + rank 14: 57.89 + rank 15: 58.21 + rank 16: 73.44 + rank 17: 58.22 + rank 18: 58.72 + rank 19: 58.23 + rank 20: 58.12 + rank 21: 58.34 + rank 22: 59.35 + rank 23: 58.51 + rank 24: 57.44 + rank 25: 58.22 + rank 26: 57.67 + rank 27: 58.49 + rank 28: 60.96 + rank 29: 58.25 + rank 30: 58.22 + rank 31: 58.09 + rank 32: 58.58 + rank 33: 57.53 + rank 34: 57.80 + rank 35: 61.27 + rank 36: 58.20 + rank 37: 58.01 + rank 38: 58.42 + rank 39: 57.66 + rank 40: 57.60 + rank 41: 58.03 + rank 42: 57.97 + rank 43: 58.17 + rank 44: 57.70 + rank 45: 58.72 + rank 46: 57.62 + rank 47: 57.28 + rank 48: 58.56 + rank 49: 58.06 + rank 50: 70.04 + rank 51: 57.88 + rank 52: 58.56 + rank 53: 57.94 + rank 54: 58.19 + rank 55: 58.27 + rank 56: 57.37 + rank 57: 66.02 + rank 58: 57.88 + rank 59: 57.48 + rank 60: 57.91 + rank 61: 57.35 + rank 62: 57.85 + rank 63: 57.43 + interval-time: + rank 0: 0.83 + rank 1: 0.75 + rank 2: 0.82 + rank 3: 0.73 + rank 4: 0.78 + rank 5: 0.75 + rank 6: 0.75 + rank 7: 0.75 + rank 8: 0.83 + rank 9: 0.84 + rank 10: 0.75 + rank 11: 0.82 + rank 12: 0.83 + rank 13: 0.81 + rank 14: 0.74 + rank 15: 0.85 + rank 16: 0.84 + rank 17: 0.83 + rank 18: 0.73 + rank 19: 0.83 + rank 20: 0.81 + rank 21: 0.83 + rank 22: 0.72 + rank 23: 0.80 + rank 24: 3.12 + rank 25: 3.10 + rank 26: 3.13 + rank 27: 3.18 + rank 28: 3.09 + rank 29: 3.14 + rank 30: 3.08 + rank 31: 3.13 + rank 32: 0.98 + rank 33: 0.89 + rank 34: 0.94 + rank 35: 0.80 + rank 36: 0.92 + rank 37: 0.90 + rank 38: 0.91 + rank 39: 0.82 + rank 40: 0.84 + rank 41: 0.85 + rank 42: 0.81 + rank 43: 0.86 + rank 44: 0.76 + rank 45: 0.82 + rank 46: 0.75 + rank 47: 0.75 + rank 48: 0.93 + rank 49: 0.90 + rank 50: 0.90 + rank 51: 0.89 + rank 52: 0.86 + rank 53: 0.91 + rank 54: 0.90 + rank 55: 0.83 + rank 56: 0.85 + rank 57: 0.76 + rank 58: 0.87 + rank 59: 0.84 + rank 60: 0.82 + rank 61: 0.79 + rank 62: 0.84 + rank 63: 4.34 + [2026-10-07 20:25:55] iteration 4006/ 6855 | consumed samples: 512768 | elapsed time per iteration (ms): 90998.5 | throughput per GPU (TFLOP/s/GPU): 40.8 | MFU 13.07% | FLOPs estimator: workload-aware | logical tokens/s: 184367.9 | valid tokens/s: 184367.9 | physical tokens/s: 184788.3 | loss tokens/s: 136138.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.217287E-01 | lm-global-supervised-tokens: 1.238839E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.457385E-02 | lm-token-buffer-mib: 5.818457E+01 | global_load_balancing_loss: 9.998321E-01 | loss scale: 1.0 | grad norm: 0.131 | params norm: 11383.606 | params rms: 0.061639 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 0:00:54.669073 | finish at 2026-10-10 20:26:50 + [2026-10-07 20:27:23] iteration 4007/ 6855 | consumed samples: 512896 | elapsed time per iteration (ms): 87467.9 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.62% | FLOPs estimator: workload-aware | logical tokens/s: 191809.6 | valid tokens/s: 191809.6 | physical tokens/s: 192267.5 | loss tokens/s: 145719.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.975750E-01 | lm-global-supervised-tokens: 1.274577E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.976397E-02 | lm-token-buffer-mib: 5.814773E+01 | global_load_balancing_loss: 1.000575E+00 | loss scale: 1.0 | grad norm: 0.062 | params norm: 11383.545 | params rms: 0.061638 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 21:11:48.577271 | finish at 2026-10-10 17:39:11 + [2026-10-07 20:28:50] iteration 4008/ 6855 | consumed samples: 513024 | elapsed time per iteration (ms): 87305.4 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.54% | FLOPs estimator: workload-aware | logical tokens/s: 192167.1 | valid tokens/s: 192167.1 | physical tokens/s: 192590.1 | loss tokens/s: 146840.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.005166E-01 | lm-global-supervised-tokens: 1.281994E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.708696E-02 | lm-token-buffer-mib: 5.813036E+01 | global_load_balancing_loss: 1.000288E+00 | loss scale: 1.0 | grad norm: 0.038 | params norm: 11383.486 | params rms: 0.061638 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 21:02:38.358787 | finish at 2026-10-10 17:31:28 + [2026-10-07 20:30:20] iteration 4009/ 6855 | consumed samples: 513152 | elapsed time per iteration (ms): 90129.2 | throughput per GPU (TFLOP/s/GPU): 40.9 | MFU 13.10% | FLOPs estimator: workload-aware | logical tokens/s: 186146.1 | valid tokens/s: 186146.1 | physical tokens/s: 186505.1 | loss tokens/s: 136378.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.564867E-01 | lm-global-supervised-tokens: 1.229170E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.508951E-02 | lm-token-buffer-mib: 5.807353E+01 | global_load_balancing_loss: 1.001349E+00 | loss scale: 1.0 | grad norm: 0.085 | params norm: 11383.427 | params rms: 0.061638 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 23:15:07.671257 | finish at 2026-10-10 19:45:28 + [2026-10-07 20:32:09] iteration 4010/ 6855 | consumed samples: 513280 | elapsed time per iteration (ms): 108504.3 | throughput per GPU (TFLOP/s/GPU): 33.7 | MFU 10.81% | FLOPs estimator: workload-aware | logical tokens/s: 154621.7 | valid tokens/s: 154621.7 | physical tokens/s: 155004.8 | loss tokens/s: 114362.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.933831E-01 | lm-global-supervised-tokens: 1.240885E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.158955E-02 | lm-token-buffer-mib: 5.825702E+01 | global_load_balancing_loss: 9.996626E-01 | loss scale: 1.0 | grad norm: 0.046 | params norm: 11383.367 | params rms: 0.061637 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 13:44:54.825405 | finish at 2026-10-11 10:17:03 + [2026-10-07 20:33:39] iteration 4011/ 6855 | consumed samples: 513408 | elapsed time per iteration (ms): 90205.4 | throughput per GPU (TFLOP/s/GPU): 41.1 | MFU 13.18% | FLOPs estimator: workload-aware | logical tokens/s: 185988.9 | valid tokens/s: 185988.9 | physical tokens/s: 186473.7 | loss tokens/s: 139348.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.804016E-01 | lm-global-supervised-tokens: 1.257002E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.150290E-02 | lm-token-buffer-mib: 5.815899E+01 | global_load_balancing_loss: 9.998459E-01 | loss scale: 1.0 | grad norm: 0.086 | params norm: 11383.306 | params rms: 0.061637 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 23:15:44.290472 | finish at 2026-10-10 19:49:23 + [2026-10-07 20:35:04] iteration 4012/ 6855 | consumed samples: 513536 | elapsed time per iteration (ms): 85614.3 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.74% | FLOPs estimator: workload-aware | logical tokens/s: 195961.6 | valid tokens/s: 195961.5 | physical tokens/s: 196433.5 | loss tokens/s: 151106.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.860545E-01 | lm-global-supervised-tokens: 1.293689E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.925461E-02 | lm-token-buffer-mib: 5.806347E+01 | global_load_balancing_loss: 9.995209E-01 | loss scale: 1.0 | grad norm: 0.070 | params norm: 11383.248 | params rms: 0.061637 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 19:36:41.545764 | finish at 2026-10-10 16:11:46 + [2026-10-07 20:36:32] iteration 4013/ 6855 | consumed samples: 513664 | elapsed time per iteration (ms): 87162.9 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.51% | FLOPs estimator: workload-aware | logical tokens/s: 192480.8 | valid tokens/s: 192480.8 | physical tokens/s: 192998.2 | loss tokens/s: 146098.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.665201E-01 | lm-global-supervised-tokens: 1.273433E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.182498E-02 | lm-token-buffer-mib: 5.815221E+01 | global_load_balancing_loss: 1.000027E+00 | loss scale: 1.0 | grad norm: 0.077 | params norm: 11383.191 | params rms: 0.061636 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 20:48:37.090459 | finish at 2026-10-10 17:25:09 + [2026-10-07 20:37:59] iteration 4014/ 6855 | consumed samples: 513792 | elapsed time per iteration (ms): 87311.2 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.53% | FLOPs estimator: workload-aware | logical tokens/s: 192154.1 | valid tokens/s: 192154.1 | physical tokens/s: 192593.0 | loss tokens/s: 145347.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.141416E-01 | lm-global-supervised-tokens: 1.269047E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.576386E-02 | lm-token-buffer-mib: 5.824622E+01 | global_load_balancing_loss: 9.999058E-01 | loss scale: 1.0 | grad norm: 0.194 | params norm: 11383.134 | params rms: 0.061636 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 20:54:10.987520 | finish at 2026-10-10 17:32:10 + [2026-10-07 20:39:28] iteration 4015/ 6855 | consumed samples: 513920 | elapsed time per iteration (ms): 88671.3 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.51% | FLOPs estimator: workload-aware | logical tokens/s: 189206.4 | valid tokens/s: 189206.4 | physical tokens/s: 189493.5 | loss tokens/s: 140394.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.697122E-01 | lm-global-supervised-tokens: 1.244898E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.340530E-02 | lm-token-buffer-mib: 5.820984E+01 | global_load_balancing_loss: 1.000244E+00 | loss scale: 1.0 | grad norm: 0.078 | params norm: 11383.076 | params rms: 0.061636 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 21:57:06.541243 | finish at 2026-10-10 18:36:34 + [2026-10-07 20:40:57] iteration 4016/ 6855 | consumed samples: 514048 | elapsed time per iteration (ms): 89081.6 | throughput per GPU (TFLOP/s/GPU): 41.2 | MFU 13.21% | FLOPs estimator: workload-aware | logical tokens/s: 188334.2 | valid tokens/s: 188334.2 | physical tokens/s: 188899.4 | loss tokens/s: 147269.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.870795E-01 | lm-global-supervised-tokens: 1.311896E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.266892E-02 | lm-token-buffer-mib: 5.826022E+01 | global_load_balancing_loss: 9.998582E-01 | loss scale: 1.0 | grad norm: 0.151 | params norm: 11383.019 | params rms: 0.061635 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 22:15:02.543808 | finish at 2026-10-10 18:55:59 + [2026-10-07 20:42:26] iteration 4017/ 6855 | consumed samples: 514176 | elapsed time per iteration (ms): 89731.1 | throughput per GPU (TFLOP/s/GPU): 41.5 | MFU 13.30% | FLOPs estimator: workload-aware | logical tokens/s: 186972.1 | valid tokens/s: 186972.1 | physical tokens/s: 187526.4 | loss tokens/s: 141776.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.308931E-01 | lm-global-supervised-tokens: 1.272178E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.469245E-02 | lm-token-buffer-mib: 5.814066E+01 | global_load_balancing_loss: 1.001882E+00 | loss scale: 1.0 | grad norm: 0.096 | params norm: 11382.962 | params rms: 0.061635 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 22:44:16.780838 | finish at 2026-10-10 19:26:43 + [2026-10-07 20:43:52] iteration 4018/ 6855 | consumed samples: 514304 | elapsed time per iteration (ms): 85679.6 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.61% | FLOPs estimator: workload-aware | logical tokens/s: 195813.0 | valid tokens/s: 195813.0 | physical tokens/s: 196394.3 | loss tokens/s: 148449.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.297197E-01 | lm-global-supervised-tokens: 1.271908E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.114857E-02 | lm-token-buffer-mib: 5.821993E+01 | global_load_balancing_loss: 1.000879E+00 | loss scale: 1.0 | grad norm: 0.117 | params norm: 11382.905 | params rms: 0.061635 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 19:31:13.124631 | finish at 2026-10-10 16:15:05 + [2026-10-07 20:45:16] iteration 4019/ 6855 | consumed samples: 514432 | elapsed time per iteration (ms): 83764.8 | throughput per GPU (TFLOP/s/GPU): 43.2 | MFU 13.86% | FLOPs estimator: workload-aware | logical tokens/s: 200289.6 | valid tokens/s: 200289.6 | physical tokens/s: 200696.3 | loss tokens/s: 153327.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.515483E-01 | lm-global-supervised-tokens: 1.284341E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.418183E-02 | lm-token-buffer-mib: 5.808430E+01 | global_load_balancing_loss: 9.995716E-01 | loss scale: 1.0 | grad norm: 0.064 | params norm: 11382.847 | params rms: 0.061634 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 17:59:16.923644 | finish at 2026-10-10 14:44:33 + [2026-10-07 20:47:04] iteration 4020/ 6855 | consumed samples: 514560 | elapsed time per iteration (ms): 108145.6 | throughput per GPU (TFLOP/s/GPU): 34.4 | MFU 11.04% | FLOPs estimator: workload-aware | logical tokens/s: 155135.0 | valid tokens/s: 155135.0 | physical tokens/s: 155542.3 | loss tokens/s: 112808.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.843945E-01 | lm-global-supervised-tokens: 1.219974E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.587728E-02 | lm-token-buffer-mib: 5.804386E+01 | global_load_balancing_loss: 9.993458E-01 | loss scale: 1.0 | grad norm: 0.153 | params norm: 11382.789 | params rms: 0.061634 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 13:09:52.756627 | finish at 2026-10-11 09:56:57 + [2026-10-07 20:48:30] iteration 4021/ 6855 | consumed samples: 514688 | elapsed time per iteration (ms): 86387.6 | throughput per GPU (TFLOP/s/GPU): 43.1 | MFU 13.82% | FLOPs estimator: workload-aware | logical tokens/s: 194207.3 | valid tokens/s: 194207.3 | physical tokens/s: 194930.6 | loss tokens/s: 143597.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.800033E-01 | lm-global-supervised-tokens: 1.240504E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.584645E-02 | lm-token-buffer-mib: 5.822894E+01 | global_load_balancing_loss: 1.000432E+00 | loss scale: 1.0 | grad norm: 0.105 | params norm: 11382.731 | params rms: 0.061634 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 20:00:22.429866 | finish at 2026-10-10 16:48:53 + [2026-10-07 20:49:57] iteration 4022/ 6855 | consumed samples: 514816 | elapsed time per iteration (ms): 86591.4 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.74% | FLOPs estimator: workload-aware | logical tokens/s: 193751.5 | valid tokens/s: 193751.5 | physical tokens/s: 194234.2 | loss tokens/s: 145840.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.084252E-01 | lm-global-supervised-tokens: 1.262850E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.337522E-02 | lm-token-buffer-mib: 5.819179E+01 | global_load_balancing_loss: 1.000220E+00 | loss scale: 1.0 | grad norm: 0.120 | params norm: 11382.674 | params rms: 0.061633 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 20:08:33.373124 | finish at 2026-10-10 16:58:30 + [2026-10-07 20:51:27] iteration 4023/ 6855 | consumed samples: 514944 | elapsed time per iteration (ms): 90149.3 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.50% | FLOPs estimator: workload-aware | logical tokens/s: 186104.1 | valid tokens/s: 186104.1 | physical tokens/s: 186493.6 | loss tokens/s: 136798.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.573821E-01 | lm-global-supervised-tokens: 1.233232E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.159173E-02 | lm-token-buffer-mib: 5.805866E+01 | global_load_balancing_loss: 1.000653E+00 | loss scale: 1.0 | grad norm: 0.066 | params norm: 11382.616 | params rms: 0.061633 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 22:55:02.680138 | finish at 2026-10-10 19:46:30 + [2026-10-07 20:52:55] iteration 4024/ 6855 | consumed samples: 515072 | elapsed time per iteration (ms): 88055.2 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.69% | FLOPs estimator: workload-aware | logical tokens/s: 190530.5 | valid tokens/s: 190530.5 | physical tokens/s: 190877.3 | loss tokens/s: 140956.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.377446E-01 | lm-global-supervised-tokens: 1.241199E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.253808E-02 | lm-token-buffer-mib: 5.809280E+01 | global_load_balancing_loss: 1.001396E+00 | loss scale: 1.0 | grad norm: 0.079 | params norm: 11382.559 | params rms: 0.061633 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 21:14:44.402426 | finish at 2026-10-10 18:07:40 + [2026-10-07 20:54:23] iteration 4025/ 6855 | consumed samples: 515200 | elapsed time per iteration (ms): 87989.2 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.47% | FLOPs estimator: workload-aware | logical tokens/s: 190672.8 | valid tokens/s: 190672.8 | physical tokens/s: 191144.7 | loss tokens/s: 138154.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.865241E-01 | lm-global-supervised-tokens: 1.215607E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.364166E-02 | lm-token-buffer-mib: 5.806548E+01 | global_load_balancing_loss: 1.000693E+00 | loss scale: 1.0 | grad norm: 0.096 | params norm: 11382.505 | params rms: 0.061633 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 21:10:09.447796 | finish at 2026-10-10 18:04:33 + [2026-10-07 20:55:52] iteration 4026/ 6855 | consumed samples: 515328 | elapsed time per iteration (ms): 88309.6 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.61% | FLOPs estimator: workload-aware | logical tokens/s: 189981.7 | valid tokens/s: 189981.7 | physical tokens/s: 190296.3 | loss tokens/s: 133556.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.030184E-01 | lm-global-supervised-tokens: 1.179436E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.480224E-02 | lm-token-buffer-mib: 5.814253E+01 | global_load_balancing_loss: 1.000720E+00 | loss scale: 1.0 | grad norm: 0.066 | params norm: 11382.451 | params rms: 0.061632 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 21:23:47.859399 | finish at 2026-10-10 18:19:39 + [2026-10-07 20:57:24] iteration 4027/ 6855 | consumed samples: 515456 | elapsed time per iteration (ms): 92293.0 | throughput per GPU (TFLOP/s/GPU): 41.4 | MFU 13.27% | FLOPs estimator: workload-aware | logical tokens/s: 181781.4 | valid tokens/s: 181781.4 | physical tokens/s: 182228.1 | loss tokens/s: 135462.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.605341E-01 | lm-global-supervised-tokens: 1.250220E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.153353E-02 | lm-token-buffer-mib: 5.818218E+01 | global_load_balancing_loss: 9.990909E-01 | loss scale: 1.0 | grad norm: 0.132 | params norm: 11382.396 | params rms: 0.061632 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 0:30:04.679467 | finish at 2026-10-10 21:27:28 + [2026-10-07 20:58:50] iteration 4028/ 6855 | consumed samples: 515584 | elapsed time per iteration (ms): 86006.9 | throughput per GPU (TFLOP/s/GPU): 43.1 | MFU 13.81% | FLOPs estimator: workload-aware | logical tokens/s: 195068.3 | valid tokens/s: 195068.3 | physical tokens/s: 195460.6 | loss tokens/s: 141775.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.062302E-01 | lm-global-supervised-tokens: 1.219369E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.279089E-02 | lm-token-buffer-mib: 5.808227E+01 | global_load_balancing_loss: 9.992265E-01 | loss scale: 1.0 | grad norm: 0.060 | params norm: 11382.340 | params rms: 0.061632 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 19:32:21.468085 | finish at 2026-10-10 16:31:11 + [2026-10-07 21:00:16] iteration 4029/ 6855 | consumed samples: 515712 | elapsed time per iteration (ms): 85946.6 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.70% | FLOPs estimator: workload-aware | logical tokens/s: 195205.1 | valid tokens/s: 195205.1 | physical tokens/s: 195556.3 | loss tokens/s: 144345.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.625803E-01 | lm-global-supervised-tokens: 1.240603E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.347994E-02 | lm-token-buffer-mib: 5.805068E+01 | global_load_balancing_loss: 9.994508E-01 | loss scale: 1.0 | grad norm: 0.106 | params norm: 11382.284 | params rms: 0.061631 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 19:28:04.985032 | finish at 2026-10-10 16:28:21 + [2026-10-07 21:02:06] iteration 4030/ 6855 | consumed samples: 515840 | elapsed time per iteration (ms): 109972.8 | throughput per GPU (TFLOP/s/GPU): 33.6 | MFU 10.76% | FLOPs estimator: workload-aware | logical tokens/s: 152557.4 | valid tokens/s: 152557.4 | physical tokens/s: 152939.5 | loss tokens/s: 110335.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.842862E-01 | lm-global-supervised-tokens: 1.213395E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.136876E-02 | lm-token-buffer-mib: 5.814109E+01 | global_load_balancing_loss: 9.995613E-01 | loss scale: 1.0 | grad norm: 0.069 | params norm: 11382.228 | params rms: 0.061631 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 14:17:53.174191 | finish at 2026-10-11 11:19:59 + [2026-10-07 21:03:33] iteration 4031/ 6855 | consumed samples: 515968 | elapsed time per iteration (ms): 87342.3 | throughput per GPU (TFLOP/s/GPU): 43.5 | MFU 13.96% | FLOPs estimator: workload-aware | logical tokens/s: 192085.1 | valid tokens/s: 192085.1 | physical tokens/s: 192537.9 | loss tokens/s: 141145.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.685753E-01 | lm-global-supervised-tokens: 1.232800E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.330438E-02 | lm-token-buffer-mib: 5.813960E+01 | global_load_balancing_loss: 1.001569E+00 | loss scale: 1.0 | grad norm: 0.094 | params norm: 11382.172 | params rms: 0.061631 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 20:30:54.677244 | finish at 2026-10-10 17:34:28 + [2026-10-07 21:05:01] iteration 4032/ 6855 | consumed samples: 516096 | elapsed time per iteration (ms): 87556.4 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.75% | FLOPs estimator: workload-aware | logical tokens/s: 191615.9 | valid tokens/s: 191615.9 | physical tokens/s: 192128.7 | loss tokens/s: 136686.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.943423E-01 | lm-global-supervised-tokens: 1.196779E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.258265E-02 | lm-token-buffer-mib: 5.809820E+01 | global_load_balancing_loss: 1.000273E+00 | loss scale: 1.0 | grad norm: 0.066 | params norm: 11382.114 | params rms: 0.061630 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 20:39:31.653431 | finish at 2026-10-10 17:44:32 + [2026-10-07 21:06:27] iteration 4033/ 6855 | consumed samples: 516224 | elapsed time per iteration (ms): 86290.3 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.44% | FLOPs estimator: workload-aware | logical tokens/s: 194427.5 | valid tokens/s: 194427.5 | physical tokens/s: 194784.4 | loss tokens/s: 143878.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.560590E-01 | lm-global-supervised-tokens: 1.241527E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.211184E-02 | lm-token-buffer-mib: 5.812557E+01 | global_load_balancing_loss: 1.001060E+00 | loss scale: 1.0 | grad norm: 0.092 | params norm: 11382.055 | params rms: 0.061630 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 19:38:31.217550 | finish at 2026-10-10 16:44:58 + [2026-10-07 21:07:54] iteration 4034/ 6855 | consumed samples: 516352 | elapsed time per iteration (ms): 86978.9 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.49% | FLOPs estimator: workload-aware | logical tokens/s: 192888.3 | valid tokens/s: 192888.3 | physical tokens/s: 193260.0 | loss tokens/s: 143995.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.251508E-01 | lm-global-supervised-tokens: 1.252459E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.443096E-02 | lm-token-buffer-mib: 5.814352E+01 | global_load_balancing_loss: 1.000708E+00 | loss scale: 1.0 | grad norm: 0.074 | params norm: 11381.997 | params rms: 0.061630 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 20:09:27.579007 | finish at 2026-10-10 17:17:21 + [2026-10-07 21:09:20] iteration 4035/ 6855 | consumed samples: 516480 | elapsed time per iteration (ms): 86551.4 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.76% | FLOPs estimator: workload-aware | logical tokens/s: 193840.8 | valid tokens/s: 193840.8 | physical tokens/s: 194181.2 | loss tokens/s: 138523.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.467114E-01 | lm-global-supervised-tokens: 1.198942E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.176277E-02 | lm-token-buffer-mib: 5.808919E+01 | global_load_balancing_loss: 9.991443E-01 | loss scale: 1.0 | grad norm: 0.058 | params norm: 11381.940 | params rms: 0.061630 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 19:47:55.014410 | finish at 2026-10-10 16:57:15 + [2026-10-07 21:10:48] iteration 4036/ 6855 | consumed samples: 516608 | elapsed time per iteration (ms): 87932.3 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.56% | FLOPs estimator: workload-aware | logical tokens/s: 190796.6 | valid tokens/s: 190796.6 | physical tokens/s: 191400.1 | loss tokens/s: 133466.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.385693E-01 | lm-global-supervised-tokens: 1.173600E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.165499E-02 | lm-token-buffer-mib: 5.816773E+01 | global_load_balancing_loss: 1.000197E+00 | loss scale: 1.0 | grad norm: 0.124 | params norm: 11381.881 | params rms: 0.061629 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 20:51:21.024240 | finish at 2026-10-10 18:02:09 + [2026-10-07 21:12:15] iteration 4037/ 6855 | consumed samples: 516736 | elapsed time per iteration (ms): 86804.7 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.63% | FLOPs estimator: workload-aware | logical tokens/s: 193274.8 | valid tokens/s: 193274.8 | physical tokens/s: 193878.3 | loss tokens/s: 142514.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.600595E-01 | lm-global-supervised-tokens: 1.237091E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.142151E-02 | lm-token-buffer-mib: 5.842485E+01 | global_load_balancing_loss: 9.993340E-01 | loss scale: 1.0 | grad norm: 0.109 | params norm: 11381.823 | params rms: 0.061629 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 19:56:55.704108 | finish at 2026-10-10 17:09:11 + [2026-10-07 21:13:44] iteration 4038/ 6855 | consumed samples: 516864 | elapsed time per iteration (ms): 88584.1 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.41% | FLOPs estimator: workload-aware | logical tokens/s: 189392.5 | valid tokens/s: 189392.5 | physical tokens/s: 189785.3 | loss tokens/s: 138962.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.122103E-01 | lm-global-supervised-tokens: 1.230983E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.265542E-02 | lm-token-buffer-mib: 5.805211E+01 | global_load_balancing_loss: 1.000309E+00 | loss scale: 1.0 | grad norm: 0.137 | params norm: 11381.764 | params rms: 0.061629 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 21:19:01.496362 | finish at 2026-10-10 18:32:45 + [2026-10-07 21:15:08] iteration 4039/ 6855 | consumed samples: 516992 | elapsed time per iteration (ms): 84390.5 | throughput per GPU (TFLOP/s/GPU): 43.7 | MFU 14.02% | FLOPs estimator: workload-aware | logical tokens/s: 198804.6 | valid tokens/s: 198804.6 | physical tokens/s: 199191.8 | loss tokens/s: 150314.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.747290E-01 | lm-global-supervised-tokens: 1.268510E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.152796E-02 | lm-token-buffer-mib: 5.806936E+01 | global_load_balancing_loss: 9.999678E-01 | loss scale: 1.0 | grad norm: 0.037 | params norm: 11381.705 | params rms: 0.061628 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 18:00:43.560242 | finish at 2026-10-10 15:15:52 + [2026-10-07 21:17:01] iteration 4040/ 6855 | consumed samples: 517120 | elapsed time per iteration (ms): 112777.6 | throughput per GPU (TFLOP/s/GPU): 33.2 | MFU 10.63% | FLOPs estimator: workload-aware | logical tokens/s: 148763.7 | valid tokens/s: 148763.7 | physical tokens/s: 149081.3 | loss tokens/s: 112799.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.769710E-01 | lm-global-supervised-tokens: 1.272124E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.723247E-02 | lm-token-buffer-mib: 5.818931E+01 | global_load_balancing_loss: 9.998375E-01 | loss scale: 1.0 | grad norm: 0.065 | params norm: 11381.647 | params rms: 0.061628 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 16:11:09.044813 | finish at 2026-10-11 13:28:10 +times across ranks (ms): + save-checkpoint: + rank 0: 25019.59 + rank 1: 25019.61 + rank 2: 25019.62 + rank 3: 25019.61 + rank 4: 25019.60 + rank 5: 25019.58 + rank 6: 25019.58 + rank 7: 25019.60 + rank 8: 25019.37 + rank 9: 25019.33 + rank 10: 25019.32 + rank 11: 25019.36 + rank 12: 25019.32 + rank 13: 25019.33 + rank 14: 25019.32 + rank 15: 25019.32 + rank 16: 25019.48 + rank 17: 25019.51 + rank 18: 25019.46 + rank 19: 25019.57 + rank 20: 25019.45 + rank 21: 25019.49 + rank 22: 25019.46 + rank 23: 25019.43 + rank 24: 25019.22 + rank 25: 25019.29 + rank 26: 25019.35 + rank 27: 25019.21 + rank 28: 25019.26 + rank 29: 25019.53 + rank 30: 25019.29 + rank 31: 25019.35 + rank 32: 25019.21 + rank 33: 25019.04 + rank 34: 25019.19 + rank 35: 25019.16 + rank 36: 25019.12 + rank 37: 25019.19 + rank 38: 25019.14 + rank 39: 25019.14 + rank 40: 25019.86 + rank 41: 25019.86 + rank 42: 25019.88 + rank 43: 25019.88 + rank 44: 25019.96 + rank 45: 25019.88 + rank 46: 25019.86 + rank 47: 25019.86 + rank 48: 25019.59 + rank 49: 25019.59 + rank 50: 25019.36 + rank 51: 25019.32 + rank 52: 25019.53 + rank 53: 25019.48 + rank 54: 25019.22 + rank 55: 25019.49 + rank 56: 25019.26 + rank 57: 25019.26 + rank 58: 25019.23 + rank 59: 25019.22 + rank 60: 25019.22 + rank 61: 25019.26 + rank 62: 25019.25 + rank 63: 25019.12 + [2026-10-07 21:18:53] iteration 4041/ 6855 | consumed samples: 517248 | elapsed time per iteration (ms): 86858.5 | throughput per GPU (TFLOP/s/GPU): 42.8 | MFU 13.70% | FLOPs estimator: workload-aware | logical tokens/s: 193155.2 | valid tokens/s: 193155.2 | physical tokens/s: 193587.4 | loss tokens/s: 144430.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.451275E-01 | lm-global-supervised-tokens: 1.254500E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.424673E-01 | lm-token-buffer-mib: 5.805974E+01 | global_load_balancing_loss: 1.000966E+00 | loss scale: 1.0 | grad norm: 0.049 | params norm: 11381.587 | params rms: 0.061628 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 19:53:39.894153 | finish at 2026-10-10 17:12:33 + [2026-10-07 21:20:25] iteration 4042/ 6855 | consumed samples: 517376 | elapsed time per iteration (ms): 92359.4 | throughput per GPU (TFLOP/s/GPU): 39.4 | MFU 12.61% | FLOPs estimator: workload-aware | logical tokens/s: 181651.0 | valid tokens/s: 181651.0 | physical tokens/s: 182089.5 | loss tokens/s: 135228.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.237725E-01 | lm-global-supervised-tokens: 1.248959E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.417801E-02 | lm-token-buffer-mib: 5.821734E+01 | global_load_balancing_loss: 1.001371E+00 | loss scale: 1.0 | grad norm: 0.053 | params norm: 11381.528 | params rms: 0.061627 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 0:10:07.025158 | finish at 2026-10-10 21:30:32 + [2026-10-07 21:21:52] iteration 4043/ 6855 | consumed samples: 517504 | elapsed time per iteration (ms): 87074.9 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.49% | FLOPs estimator: workload-aware | logical tokens/s: 192674.9 | valid tokens/s: 192674.9 | physical tokens/s: 193167.9 | loss tokens/s: 143685.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.804793E-01 | lm-global-supervised-tokens: 1.251143E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.140296E-02 | lm-token-buffer-mib: 5.814346E+01 | global_load_balancing_loss: 1.001465E+00 | loss scale: 1.0 | grad norm: 0.042 | params norm: 11381.468 | params rms: 0.061627 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 20:00:54.691629 | finish at 2026-10-10 17:22:47 + [2026-10-07 21:23:21] iteration 4044/ 6855 | consumed samples: 517632 | elapsed time per iteration (ms): 88990.1 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.61% | FLOPs estimator: workload-aware | logical tokens/s: 188528.4 | valid tokens/s: 188528.4 | physical tokens/s: 188867.0 | loss tokens/s: 141120.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.215236E-01 | lm-global-supervised-tokens: 1.255829E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.168491E-02 | lm-token-buffer-mib: 5.805375E+01 | global_load_balancing_loss: 1.000851E+00 | loss scale: 1.0 | grad norm: 0.045 | params norm: 11381.407 | params rms: 0.061627 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 21:29:11.185583 | finish at 2026-10-10 18:52:33 + [2026-10-07 21:24:47] iteration 4045/ 6855 | consumed samples: 517760 | elapsed time per iteration (ms): 85570.9 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.73% | FLOPs estimator: workload-aware | logical tokens/s: 196061.3 | valid tokens/s: 196061.3 | physical tokens/s: 196536.8 | loss tokens/s: 145157.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.714315E-01 | lm-global-supervised-tokens: 1.242122E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.760639E-02 | lm-token-buffer-mib: 5.813629E+01 | global_load_balancing_loss: 9.997439E-01 | loss scale: 1.0 | grad norm: 0.090 | params norm: 11381.347 | params rms: 0.061626 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 18:47:34.357529 | finish at 2026-10-10 16:12:21 + [2026-10-07 21:26:11] iteration 4046/ 6855 | consumed samples: 517888 | elapsed time per iteration (ms): 84147.2 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.76% | FLOPs estimator: workload-aware | logical tokens/s: 199379.2 | valid tokens/s: 199379.2 | physical tokens/s: 199959.8 | loss tokens/s: 150582.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.273320E-01 | lm-global-supervised-tokens: 1.267107E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.207677E-02 | lm-token-buffer-mib: 5.811880E+01 | global_load_balancing_loss: 9.993699E-01 | loss scale: 1.0 | grad norm: 0.098 | params norm: 11381.290 | params rms: 0.061626 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 17:39:29.591587 | finish at 2026-10-10 15:05:41 + [2026-10-07 21:27:35] iteration 4047/ 6855 | consumed samples: 518016 | elapsed time per iteration (ms): 83433.0 | throughput per GPU (TFLOP/s/GPU): 43.7 | MFU 13.99% | FLOPs estimator: workload-aware | logical tokens/s: 201085.7 | valid tokens/s: 201085.7 | physical tokens/s: 201549.6 | loss tokens/s: 147807.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.596600E-01 | lm-global-supervised-tokens: 1.233205E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.481496E-02 | lm-token-buffer-mib: 5.803278E+01 | global_load_balancing_loss: 9.988661E-01 | loss scale: 1.0 | grad norm: 0.062 | params norm: 11381.232 | params rms: 0.061626 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 17:04:39.896381 | finish at 2026-10-10 14:32:14 + [2026-10-07 21:29:02] iteration 4048/ 6855 | consumed samples: 518144 | elapsed time per iteration (ms): 87911.9 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.41% | FLOPs estimator: workload-aware | logical tokens/s: 190840.7 | valid tokens/s: 190840.7 | physical tokens/s: 191358.9 | loss tokens/s: 146948.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.298117E-01 | lm-global-supervised-tokens: 1.291853E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.437765E-02 | lm-token-buffer-mib: 5.809412E+01 | global_load_balancing_loss: 9.996055E-01 | loss scale: 1.0 | grad norm: 0.061 | params norm: 11381.174 | params rms: 0.061625 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 20:32:48.830578 | finish at 2026-10-10 18:01:51 + [2026-10-07 21:30:28] iteration 4049/ 6855 | consumed samples: 518272 | elapsed time per iteration (ms): 85923.8 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.63% | FLOPs estimator: workload-aware | logical tokens/s: 195256.7 | valid tokens/s: 195256.7 | physical tokens/s: 195852.8 | loss tokens/s: 145780.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.945498E-01 | lm-global-supervised-tokens: 1.252597E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.220604E-02 | lm-token-buffer-mib: 5.827534E+01 | global_load_balancing_loss: 9.990505E-01 | loss scale: 1.0 | grad norm: 0.042 | params norm: 11381.115 | params rms: 0.061625 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 18:58:22.149995 | finish at 2026-10-10 16:28:51 + [2026-10-07 21:32:18] iteration 4050/ 6855 | consumed samples: 518400 | elapsed time per iteration (ms): 109335.7 | throughput per GPU (TFLOP/s/GPU): 33.4 | MFU 10.71% | FLOPs estimator: workload-aware | logical tokens/s: 153446.7 | valid tokens/s: 153446.7 | physical tokens/s: 153965.5 | loss tokens/s: 113466.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.764678E-01 | lm-global-supervised-tokens: 1.240592E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.746855E-02 | lm-token-buffer-mib: 5.817550E+01 | global_load_balancing_loss: 1.000821E+00 | loss scale: 1.0 | grad norm: 0.081 | params norm: 11381.053 | params rms: 0.061625 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 13:11:26.536278 | finish at 2026-10-11 10:43:44 + [2026-10-07 21:33:47] iteration 4051/ 6855 | consumed samples: 518528 | elapsed time per iteration (ms): 89192.7 | throughput per GPU (TFLOP/s/GPU): 41.1 | MFU 13.19% | FLOPs estimator: workload-aware | logical tokens/s: 188100.2 | valid tokens/s: 188100.2 | physical tokens/s: 188539.3 | loss tokens/s: 142605.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.641538E-01 | lm-global-supervised-tokens: 1.271937E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.217920E-02 | lm-token-buffer-mib: 5.809006E+01 | global_load_balancing_loss: 1.000326E+00 | loss scale: 1.0 | grad norm: 0.042 | params norm: 11380.992 | params rms: 0.061624 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 21:28:16.204863 | finish at 2026-10-10 19:02:03 + [2026-10-07 21:35:17] iteration 4052/ 6855 | consumed samples: 518656 | elapsed time per iteration (ms): 90532.0 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.54% | FLOPs estimator: workload-aware | logical tokens/s: 185317.6 | valid tokens/s: 185317.6 | physical tokens/s: 185679.9 | loss tokens/s: 134401.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.093164E-01 | lm-global-supervised-tokens: 1.216760E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.231838E-02 | lm-token-buffer-mib: 5.821616E+01 | global_load_balancing_loss: 1.002364E+00 | loss scale: 1.0 | grad norm: 0.063 | params norm: 11380.930 | params rms: 0.061624 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 22:29:21.093934 | finish at 2026-10-10 20:04:39 + [2026-10-07 21:36:47] iteration 4053/ 6855 | consumed samples: 518784 | elapsed time per iteration (ms): 89488.8 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.45% | FLOPs estimator: workload-aware | logical tokens/s: 187477.6 | valid tokens/s: 187477.6 | physical tokens/s: 188098.4 | loss tokens/s: 138363.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.629179E-01 | lm-global-supervised-tokens: 1.238197E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.420652E-02 | lm-token-buffer-mib: 5.806615E+01 | global_load_balancing_loss: 1.001021E+00 | loss scale: 1.0 | grad norm: 0.086 | params norm: 11380.868 | params rms: 0.061624 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 21:39:07.657152 | finish at 2026-10-10 19:15:55 + [2026-10-07 21:38:16] iteration 4054/ 6855 | consumed samples: 518912 | elapsed time per iteration (ms): 88573.4 | throughput per GPU (TFLOP/s/GPU): 41.3 | MFU 13.24% | FLOPs estimator: workload-aware | logical tokens/s: 189415.4 | valid tokens/s: 189415.4 | physical tokens/s: 189817.6 | loss tokens/s: 140899.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.724780E-01 | lm-global-supervised-tokens: 1.247993E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.512516E-02 | lm-token-buffer-mib: 5.819589E+01 | global_load_balancing_loss: 1.000375E+00 | loss scale: 1.0 | grad norm: 0.135 | params norm: 11380.805 | params rms: 0.061623 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 20:54:54.044708 | finish at 2026-10-10 18:33:10 + [2026-10-07 21:39:46] iteration 4055/ 6855 | consumed samples: 519040 | elapsed time per iteration (ms): 90687.7 | throughput per GPU (TFLOP/s/GPU): 41.1 | MFU 13.18% | FLOPs estimator: workload-aware | logical tokens/s: 184999.9 | valid tokens/s: 184999.9 | physical tokens/s: 185347.4 | loss tokens/s: 137680.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.067302E-01 | lm-global-supervised-tokens: 1.248589E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.539637E-02 | lm-token-buffer-mib: 5.813337E+01 | global_load_balancing_loss: 9.996483E-01 | loss scale: 1.0 | grad norm: 0.198 | params norm: 11380.742 | params rms: 0.061623 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 22:32:05.479984 | finish at 2026-10-10 20:11:52 + [2026-10-07 21:41:18] iteration 4056/ 6855 | consumed samples: 519168 | elapsed time per iteration (ms): 91967.0 | throughput per GPU (TFLOP/s/GPU): 41.6 | MFU 13.33% | FLOPs estimator: workload-aware | logical tokens/s: 182425.7 | valid tokens/s: 182425.7 | physical tokens/s: 182647.0 | loss tokens/s: 125953.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.612088E-01 | lm-global-supervised-tokens: 1.158358E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.162416E-02 | lm-token-buffer-mib: 5.803372E+01 | global_load_balancing_loss: 9.989692E-01 | loss scale: 1.0 | grad norm: 0.045 | params norm: 11380.681 | params rms: 0.061623 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 23:30:15.502891 | finish at 2026-10-10 21:11:34 + [2026-10-07 21:42:49] iteration 4057/ 6855 | consumed samples: 519296 | elapsed time per iteration (ms): 90440.4 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.76% | FLOPs estimator: workload-aware | logical tokens/s: 185505.6 | valid tokens/s: 185505.5 | physical tokens/s: 185787.9 | loss tokens/s: 131972.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.163787E-01 | lm-global-supervised-tokens: 1.193563E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.139785E-02 | lm-token-buffer-mib: 5.808700E+01 | global_load_balancing_loss: 9.997563E-01 | loss scale: 1.0 | grad norm: 0.129 | params norm: 11380.619 | params rms: 0.061622 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 22:17:32.290912 | finish at 2026-10-10 20:00:21 + [2026-10-07 21:44:19] iteration 4058/ 6855 | consumed samples: 519424 | elapsed time per iteration (ms): 90518.5 | throughput per GPU (TFLOP/s/GPU): 41.3 | MFU 13.24% | FLOPs estimator: workload-aware | logical tokens/s: 185345.7 | valid tokens/s: 185345.7 | physical tokens/s: 185785.8 | loss tokens/s: 136687.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.793501E-01 | lm-global-supervised-tokens: 1.237273E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.262630E-02 | lm-token-buffer-mib: 5.807929E+01 | global_load_balancing_loss: 9.997069E-01 | loss scale: 1.0 | grad norm: 0.152 | params norm: 11380.557 | params rms: 0.061622 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 22:19:40.123383 | finish at 2026-10-10 20:03:59 + [2026-10-07 21:45:46] iteration 4059/ 6855 | consumed samples: 519552 | elapsed time per iteration (ms): 87144.5 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.67% | FLOPs estimator: workload-aware | logical tokens/s: 192521.7 | valid tokens/s: 192521.7 | physical tokens/s: 192950.9 | loss tokens/s: 135349.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.506397E-01 | lm-global-supervised-tokens: 1.179495E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.113283E-02 | lm-token-buffer-mib: 5.805233E+01 | global_load_balancing_loss: 1.000769E+00 | loss scale: 1.0 | grad norm: 0.142 | params norm: 11380.495 | params rms: 0.061622 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 19:40:56.016048 | finish at 2026-10-10 17:26:42 + [2026-10-07 21:47:33] iteration 4060/ 6855 | consumed samples: 519680 | elapsed time per iteration (ms): 107096.8 | throughput per GPU (TFLOP/s/GPU): 34.3 | MFU 10.99% | FLOPs estimator: workload-aware | logical tokens/s: 156654.7 | valid tokens/s: 156654.7 | physical tokens/s: 157027.7 | loss tokens/s: 120282.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.872182E-01 | lm-global-supervised-tokens: 1.288186E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.374159E-02 | lm-token-buffer-mib: 5.806710E+01 | global_load_balancing_loss: 9.995280E-01 | loss scale: 1.0 | grad norm: 0.108 | params norm: 11380.436 | params rms: 0.061621 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 11:08:55.598230 | finish at 2026-10-11 08:56:29 + [2026-10-07 21:49:00] iteration 4061/ 6855 | consumed samples: 519808 | elapsed time per iteration (ms): 86358.7 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.68% | FLOPs estimator: workload-aware | logical tokens/s: 194273.2 | valid tokens/s: 194273.2 | physical tokens/s: 194738.8 | loss tokens/s: 142567.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.023622E-01 | lm-global-supervised-tokens: 1.231195E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.362731E-02 | lm-token-buffer-mib: 5.806766E+01 | global_load_balancing_loss: 1.000066E+00 | loss scale: 1.0 | grad norm: 0.079 | params norm: 11380.378 | params rms: 0.061621 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 19:01:26.214565 | finish at 2026-10-10 16:50:26 + [2026-10-07 21:50:28] iteration 4062/ 6855 | consumed samples: 519936 | elapsed time per iteration (ms): 88063.7 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.43% | FLOPs estimator: workload-aware | logical tokens/s: 190511.6 | valid tokens/s: 190511.6 | physical tokens/s: 190840.3 | loss tokens/s: 140184.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.426418E-01 | lm-global-supervised-tokens: 1.234519E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.253092E-02 | lm-token-buffer-mib: 5.804570E+01 | global_load_balancing_loss: 1.000564E+00 | loss scale: 1.0 | grad norm: 0.040 | params norm: 11380.319 | params rms: 0.061621 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 20:19:21.993899 | finish at 2026-10-10 18:09:50 + [2026-10-07 21:51:54] iteration 4063/ 6855 | consumed samples: 520064 | elapsed time per iteration (ms): 86346.5 | throughput per GPU (TFLOP/s/GPU): 41.5 | MFU 13.30% | FLOPs estimator: workload-aware | logical tokens/s: 194300.6 | valid tokens/s: 194300.6 | physical tokens/s: 194903.6 | loss tokens/s: 144635.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.678158E-01 | lm-global-supervised-tokens: 1.248875E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.163899E-02 | lm-token-buffer-mib: 5.809914E+01 | global_load_balancing_loss: 1.000510E+00 | loss scale: 1.0 | grad norm: 0.167 | params norm: 11380.261 | params rms: 0.061620 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 18:57:59.443752 | finish at 2026-10-10 16:49:54 + [2026-10-07 21:53:21] iteration 4064/ 6855 | consumed samples: 520192 | elapsed time per iteration (ms): 87146.8 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.52% | FLOPs estimator: workload-aware | logical tokens/s: 192515.9 | valid tokens/s: 192515.9 | physical tokens/s: 192960.6 | loss tokens/s: 145692.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.824149E-01 | lm-global-supervised-tokens: 1.269660E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.675780E-02 | lm-token-buffer-mib: 5.830567E+01 | global_load_balancing_loss: 1.000172E+00 | loss scale: 1.0 | grad norm: 0.140 | params norm: 11380.205 | params rms: 0.061620 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 19:33:46.850669 | finish at 2026-10-10 17:27:08 + [2026-10-07 21:54:48] iteration 4065/ 6855 | consumed samples: 520320 | elapsed time per iteration (ms): 87056.2 | throughput per GPU (TFLOP/s/GPU): 41.6 | MFU 13.32% | FLOPs estimator: workload-aware | logical tokens/s: 192716.7 | valid tokens/s: 192716.7 | physical tokens/s: 193259.2 | loss tokens/s: 142049.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.080433E-01 | lm-global-supervised-tokens: 1.236631E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.127384E-02 | lm-token-buffer-mib: 5.842999E+01 | global_load_balancing_loss: 1.000778E+00 | loss scale: 1.0 | grad norm: 0.167 | params norm: 11380.149 | params rms: 0.061620 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 19:28:06.916480 | finish at 2026-10-10 17:22:55 + [2026-10-07 21:56:17] iteration 4066/ 6855 | consumed samples: 520448 | elapsed time per iteration (ms): 88300.6 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.53% | FLOPs estimator: workload-aware | logical tokens/s: 190000.6 | valid tokens/s: 190000.6 | physical tokens/s: 190304.0 | loss tokens/s: 142358.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.630682E-01 | lm-global-supervised-tokens: 1.257030E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.470773E-02 | lm-token-buffer-mib: 5.814603E+01 | global_load_balancing_loss: 1.001668E+00 | loss scale: 1.0 | grad norm: 0.152 | params norm: 11380.093 | params rms: 0.061620 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 20:24:30.292421 | finish at 2026-10-10 18:20:47 + [2026-10-07 21:57:44] iteration 4067/ 6855 | consumed samples: 520576 | elapsed time per iteration (ms): 87075.0 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.70% | FLOPs estimator: workload-aware | logical tokens/s: 192675.5 | valid tokens/s: 192675.5 | physical tokens/s: 193169.3 | loss tokens/s: 142466.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.980826E-01 | lm-global-supervised-tokens: 1.240529E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.382107E-02 | lm-token-buffer-mib: 5.811711E+01 | global_load_balancing_loss: 9.997367E-01 | loss scale: 1.0 | grad norm: 0.144 | params norm: 11380.037 | params rms: 0.061619 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 19:26:05.001091 | finish at 2026-10-10 17:23:49 + [2026-10-07 21:59:13] iteration 4068/ 6855 | consumed samples: 520704 | elapsed time per iteration (ms): 89047.3 | throughput per GPU (TFLOP/s/GPU): 40.9 | MFU 13.12% | FLOPs estimator: workload-aware | logical tokens/s: 188407.4 | valid tokens/s: 188407.4 | physical tokens/s: 188783.2 | loss tokens/s: 143542.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.190994E-01 | lm-global-supervised-tokens: 1.278208E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.162446E-02 | lm-token-buffer-mib: 5.815140E+01 | global_load_balancing_loss: 1.000277E+00 | loss scale: 1.0 | grad norm: 0.053 | params norm: 11379.979 | params rms: 0.061619 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 20:56:14.881860 | finish at 2026-10-10 18:55:28 + [2026-10-07 22:00:40] iteration 4069/ 6855 | consumed samples: 520832 | elapsed time per iteration (ms): 87468.2 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.44% | FLOPs estimator: workload-aware | logical tokens/s: 191808.2 | valid tokens/s: 191808.2 | physical tokens/s: 192321.1 | loss tokens/s: 143195.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.290358E-01 | lm-global-supervised-tokens: 1.252502E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.185228E-02 | lm-token-buffer-mib: 5.825288E+01 | global_load_balancing_loss: 9.999480E-01 | loss scale: 1.0 | grad norm: 0.220 | params norm: 11379.921 | params rms: 0.061619 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 19:41:26.433010 | finish at 2026-10-10 17:42:07 + [2026-10-07 22:02:32] iteration 4070/ 6855 | consumed samples: 520960 | elapsed time per iteration (ms): 112062.9 | throughput per GPU (TFLOP/s/GPU): 34.1 | MFU 10.92% | FLOPs estimator: workload-aware | logical tokens/s: 149711.9 | valid tokens/s: 149711.9 | physical tokens/s: 149947.5 | loss tokens/s: 106123.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.104039E-01 | lm-global-supervised-tokens: 1.189245E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.171665E-02 | lm-token-buffer-mib: 5.807419E+01 | global_load_balancing_loss: 9.998350E-01 | loss scale: 1.0 | grad norm: 0.072 | params norm: 11379.864 | params rms: 0.061618 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 14:41:35.150125 | finish at 2026-10-11 12:44:07 + [2026-10-07 22:04:00] iteration 4071/ 6855 | consumed samples: 521088 | elapsed time per iteration (ms): 87224.8 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.51% | FLOPs estimator: workload-aware | logical tokens/s: 192344.3 | valid tokens/s: 192344.3 | physical tokens/s: 192912.9 | loss tokens/s: 140982.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.222686E-01 | lm-global-supervised-tokens: 1.229718E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.262398E-02 | lm-token-buffer-mib: 5.813734E+01 | global_load_balancing_loss: 1.000205E+00 | loss scale: 1.0 | grad norm: 0.271 | params norm: 11379.804 | params rms: 0.061618 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 19:27:13.784431 | finish at 2026-10-10 17:31:13 + [2026-10-07 22:05:27] iteration 4072/ 6855 | consumed samples: 521216 | elapsed time per iteration (ms): 87429.9 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.51% | FLOPs estimator: workload-aware | logical tokens/s: 191892.8 | valid tokens/s: 191892.8 | physical tokens/s: 192283.6 | loss tokens/s: 138720.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.354406E-01 | lm-global-supervised-tokens: 1.212834E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.146514E-02 | lm-token-buffer-mib: 5.819686E+01 | global_load_balancing_loss: 1.000247E+00 | loss scale: 1.0 | grad norm: 0.127 | params norm: 11379.744 | params rms: 0.061618 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 19:35:17.516344 | finish at 2026-10-10 17:40:44 + [2026-10-07 22:06:53] iteration 4073/ 6855 | consumed samples: 521344 | elapsed time per iteration (ms): 86085.0 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.61% | FLOPs estimator: workload-aware | logical tokens/s: 194890.9 | valid tokens/s: 194890.8 | physical tokens/s: 195420.4 | loss tokens/s: 145130.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.426339E-01 | lm-global-supervised-tokens: 1.249360E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.144239E-02 | lm-token-buffer-mib: 5.806836E+01 | global_load_balancing_loss: 9.993440E-01 | loss scale: 1.0 | grad norm: 0.259 | params norm: 11379.684 | params rms: 0.061617 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 18:31:28.478065 | finish at 2026-10-10 16:38:22 + [2026-10-07 22:08:19] iteration 4074/ 6855 | consumed samples: 521472 | elapsed time per iteration (ms): 86441.1 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.40% | FLOPs estimator: workload-aware | logical tokens/s: 194088.1 | valid tokens/s: 194088.0 | physical tokens/s: 194633.9 | loss tokens/s: 146659.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.384639E-01 | lm-global-supervised-tokens: 1.267740E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.253412E-02 | lm-token-buffer-mib: 5.820264E+01 | global_load_balancing_loss: 9.997967E-01 | loss scale: 1.0 | grad norm: 0.087 | params norm: 11379.628 | params rms: 0.061617 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 18:46:32.655674 | finish at 2026-10-10 16:54:52 + [2026-10-07 22:09:47] iteration 4075/ 6855 | consumed samples: 521600 | elapsed time per iteration (ms): 87067.3 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.53% | FLOPs estimator: workload-aware | logical tokens/s: 192691.2 | valid tokens/s: 192691.2 | physical tokens/s: 193043.1 | loss tokens/s: 141222.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.690695E-01 | lm-global-supervised-tokens: 1.229587E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.842435E-02 | lm-token-buffer-mib: 5.815218E+01 | global_load_balancing_loss: 9.994240E-01 | loss scale: 1.0 | grad norm: 0.159 | params norm: 11379.570 | params rms: 0.061617 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 19:14:07.139297 | finish at 2026-10-10 17:23:54 + [2026-10-07 22:11:12] iteration 4076/ 6855 | consumed samples: 521728 | elapsed time per iteration (ms): 85808.2 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.60% | FLOPs estimator: workload-aware | logical tokens/s: 195519.0 | valid tokens/s: 195518.9 | physical tokens/s: 195972.8 | loss tokens/s: 144533.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.909903E-01 | lm-global-supervised-tokens: 1.240218E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.185138E-02 | lm-token-buffer-mib: 5.804879E+01 | global_load_balancing_loss: 1.000157E+00 | loss scale: 1.0 | grad norm: 0.311 | params norm: 11379.510 | params rms: 0.061616 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 18:14:21.068969 | finish at 2026-10-10 16:25:33 + [2026-10-07 22:12:40] iteration 4077/ 6855 | consumed samples: 521856 | elapsed time per iteration (ms): 87350.0 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.67% | FLOPs estimator: workload-aware | logical tokens/s: 192068.1 | valid tokens/s: 192068.1 | physical tokens/s: 192479.0 | loss tokens/s: 135863.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.454799E-01 | lm-global-supervised-tokens: 1.186764E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.334699E-02 | lm-token-buffer-mib: 5.817962E+01 | global_load_balancing_loss: 1.000377E+00 | loss scale: 1.0 | grad norm: 0.126 | params norm: 11379.449 | params rms: 0.061616 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 19:24:18.352721 | finish at 2026-10-10 17:36:58 + [2026-10-07 22:14:07] iteration 4078/ 6855 | consumed samples: 521984 | elapsed time per iteration (ms): 87011.5 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.39% | FLOPs estimator: workload-aware | logical tokens/s: 192816.0 | valid tokens/s: 192816.0 | physical tokens/s: 193353.7 | loss tokens/s: 150193.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.153243E-01 | lm-global-supervised-tokens: 1.306859E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.305662E-02 | lm-token-buffer-mib: 5.807046E+01 | global_load_balancing_loss: 1.000007E+00 | loss scale: 1.0 | grad norm: 0.179 | params norm: 11379.390 | params rms: 0.061616 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 19:07:10.981518 | finish at 2026-10-10 17:21:18 + [2026-10-07 22:15:37] iteration 4079/ 6855 | consumed samples: 522112 | elapsed time per iteration (ms): 90133.4 | throughput per GPU (TFLOP/s/GPU): 41.4 | MFU 13.28% | FLOPs estimator: workload-aware | logical tokens/s: 186137.0 | valid tokens/s: 186137.0 | physical tokens/s: 186625.9 | loss tokens/s: 134846.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.554298E-01 | lm-global-supervised-tokens: 1.215416E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.222964E-02 | lm-token-buffer-mib: 5.809914E+01 | global_load_balancing_loss: 1.001678E+00 | loss scale: 1.0 | grad norm: 0.247 | params norm: 11379.330 | params rms: 0.061615 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 21:30:10.386469 | finish at 2026-10-10 19:45:47 + [2026-10-07 22:17:26] iteration 4080/ 6855 | consumed samples: 522240 | elapsed time per iteration (ms): 108856.4 | throughput per GPU (TFLOP/s/GPU): 34.7 | MFU 11.11% | FLOPs estimator: workload-aware | logical tokens/s: 154122.5 | valid tokens/s: 154122.5 | physical tokens/s: 154554.0 | loss tokens/s: 114582.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.620821E-01 | lm-global-supervised-tokens: 1.247305E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.443344E-02 | lm-token-buffer-mib: 5.814400E+01 | global_load_balancing_loss: 1.000945E+00 | loss scale: 1.0 | grad norm: 0.171 | params norm: 11379.270 | params rms: 0.061615 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 11:54:36.434612 | finish at 2026-10-11 10:12:02 +times across ranks (ms): + save-checkpoint: + rank 0: 24930.80 + rank 1: 24930.86 + rank 2: 24930.80 + rank 3: 24930.80 + rank 4: 24930.80 + rank 5: 24930.80 + rank 6: 24930.79 + rank 7: 24930.84 + rank 8: 24930.88 + rank 9: 24930.85 + rank 10: 24930.85 + rank 11: 24930.85 + rank 12: 24930.86 + rank 13: 24930.87 + rank 14: 24930.86 + rank 15: 24930.85 + rank 16: 24930.49 + rank 17: 24930.42 + rank 18: 24930.40 + rank 19: 24930.44 + rank 20: 24930.40 + rank 21: 24930.51 + rank 22: 24930.40 + rank 23: 24930.43 + rank 24: 24931.05 + rank 25: 24931.02 + rank 26: 24931.06 + rank 27: 24931.05 + rank 28: 24931.04 + rank 29: 24931.14 + rank 30: 24931.03 + rank 31: 24931.13 + rank 32: 24932.11 + rank 33: 24931.07 + rank 34: 24930.98 + rank 35: 24931.20 + rank 36: 24930.98 + rank 37: 24930.93 + rank 38: 24930.95 + rank 39: 24930.94 + rank 40: 24932.17 + rank 41: 24932.12 + rank 42: 24932.13 + rank 43: 24932.22 + rank 44: 24932.12 + rank 45: 24932.12 + rank 46: 24932.13 + rank 47: 24932.14 + rank 48: 24931.75 + rank 49: 24931.71 + rank 50: 24931.70 + rank 51: 24931.71 + rank 52: 24931.70 + rank 53: 24931.73 + rank 54: 24931.76 + rank 55: 24931.71 + rank 56: 24931.23 + rank 57: 24931.23 + rank 58: 24931.21 + rank 59: 24931.23 + rank 60: 24931.21 + rank 61: 24931.22 + rank 62: 24931.21 + rank 63: 24931.22 + [2026-10-07 22:19:20] iteration 4081/ 6855 | consumed samples: 522368 | elapsed time per iteration (ms): 89309.6 | throughput per GPU (TFLOP/s/GPU): 43.1 | MFU 13.80% | FLOPs estimator: workload-aware | logical tokens/s: 187854.1 | valid tokens/s: 187854.1 | physical tokens/s: 188253.6 | loss tokens/s: 135417.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.695628E-01 | lm-global-supervised-tokens: 1.209408E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.256397E-02 | lm-token-buffer-mib: 5.810808E+01 | global_load_balancing_loss: 9.999360E-01 | loss scale: 1.0 | grad norm: 0.090 | params norm: 11379.210 | params rms: 0.061615 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 20:49:04.952411 | finish at 2026-10-10 19:08:25 + [2026-10-07 22:20:52] iteration 4082/ 6855 | consumed samples: 522496 | elapsed time per iteration (ms): 91386.5 | throughput per GPU (TFLOP/s/GPU): 40.7 | MFU 13.05% | FLOPs estimator: workload-aware | logical tokens/s: 183584.9 | valid tokens/s: 183584.8 | physical tokens/s: 183876.7 | loss tokens/s: 136757.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.848940E-01 | lm-global-supervised-tokens: 1.249783E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.564425E-02 | lm-token-buffer-mib: 5.805443E+01 | global_load_balancing_loss: 9.995590E-01 | loss scale: 1.0 | grad norm: 0.066 | params norm: 11379.151 | params rms: 0.061614 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 22:23:34.900366 | finish at 2026-10-10 20:44:27 + [2026-10-07 22:22:17] iteration 4083/ 6855 | consumed samples: 522624 | elapsed time per iteration (ms): 84871.5 | throughput per GPU (TFLOP/s/GPU): 43.7 | MFU 13.99% | FLOPs estimator: workload-aware | logical tokens/s: 197677.9 | valid tokens/s: 197677.9 | physical tokens/s: 198085.3 | loss tokens/s: 143311.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.929264E-01 | lm-global-supervised-tokens: 1.216306E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.151705E-02 | lm-token-buffer-mib: 5.808177E+01 | global_load_balancing_loss: 9.999035E-01 | loss scale: 1.0 | grad norm: 0.115 | params norm: 11379.093 | params rms: 0.061614 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 17:21:03.734596 | finish at 2026-10-10 15:43:20 + [2026-10-07 22:23:43] iteration 4084/ 6855 | consumed samples: 522752 | elapsed time per iteration (ms): 86415.4 | throughput per GPU (TFLOP/s/GPU): 43.1 | MFU 13.81% | FLOPs estimator: workload-aware | logical tokens/s: 194145.5 | valid tokens/s: 194145.5 | physical tokens/s: 194702.4 | loss tokens/s: 143817.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.862370E-01 | lm-global-supervised-tokens: 1.242800E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.442325E-02 | lm-token-buffer-mib: 5.812486E+01 | global_load_balancing_loss: 9.994739E-01 | loss scale: 1.0 | grad norm: 0.074 | params norm: 11379.035 | params rms: 0.061614 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 18:30:56.983629 | finish at 2026-10-10 16:54:40 + [2026-10-07 22:25:11] iteration 4085/ 6855 | consumed samples: 522880 | elapsed time per iteration (ms): 87598.6 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.70% | FLOPs estimator: workload-aware | logical tokens/s: 191523.6 | valid tokens/s: 191523.6 | physical tokens/s: 192197.4 | loss tokens/s: 143336.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.872024E-01 | lm-global-supervised-tokens: 1.255606E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.390211E-02 | lm-token-buffer-mib: 5.834935E+01 | global_load_balancing_loss: 1.001557E+00 | loss scale: 1.0 | grad norm: 0.125 | params norm: 11378.975 | params rms: 0.061613 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 19:24:08.142886 | finish at 2026-10-10 17:49:19 + [2026-10-07 22:26:41] iteration 4086/ 6855 | consumed samples: 523008 | elapsed time per iteration (ms): 90345.9 | throughput per GPU (TFLOP/s/GPU): 41.4 | MFU 13.28% | FLOPs estimator: workload-aware | logical tokens/s: 185699.2 | valid tokens/s: 185699.2 | physical tokens/s: 186280.3 | loss tokens/s: 131439.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.954364E-01 | lm-global-supervised-tokens: 1.187502E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.318413E-02 | lm-token-buffer-mib: 5.812389E+01 | global_load_balancing_loss: 1.000188E+00 | loss scale: 1.0 | grad norm: 0.268 | params norm: 11378.916 | params rms: 0.061613 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 21:29:27.789340 | finish at 2026-10-10 19:56:09 + [2026-10-07 22:28:10] iteration 4087/ 6855 | consumed samples: 523136 | elapsed time per iteration (ms): 89457.6 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.53% | FLOPs estimator: workload-aware | logical tokens/s: 187543.2 | valid tokens/s: 187543.2 | physical tokens/s: 187899.3 | loss tokens/s: 135493.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.482454E-01 | lm-global-supervised-tokens: 1.212093E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.277219E-02 | lm-token-buffer-mib: 5.804514E+01 | global_load_balancing_loss: 9.998924E-01 | loss scale: 1.0 | grad norm: 0.248 | params norm: 11378.858 | params rms: 0.061613 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 20:46:58.504475 | finish at 2026-10-10 19:15:09 + [2026-10-07 22:29:39] iteration 4088/ 6855 | consumed samples: 523264 | elapsed time per iteration (ms): 88715.5 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.38% | FLOPs estimator: workload-aware | logical tokens/s: 189112.1 | valid tokens/s: 189112.1 | physical tokens/s: 189494.7 | loss tokens/s: 138703.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.370940E-01 | lm-global-supervised-tokens: 1.230519E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.469311E-02 | lm-token-buffer-mib: 5.808936E+01 | global_load_balancing_loss: 1.000716E+00 | loss scale: 1.0 | grad norm: 0.228 | params norm: 11378.800 | params rms: 0.061613 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 20:11:15.881181 | finish at 2026-10-10 18:40:55 + [2026-10-07 22:31:07] iteration 4089/ 6855 | consumed samples: 523392 | elapsed time per iteration (ms): 87883.8 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.39% | FLOPs estimator: workload-aware | logical tokens/s: 190901.9 | valid tokens/s: 190901.9 | physical tokens/s: 191258.7 | loss tokens/s: 141820.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.882315E-01 | lm-global-supervised-tokens: 1.246369E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.300720E-02 | lm-token-buffer-mib: 5.812457E+01 | global_load_balancing_loss: 1.000038E+00 | loss scale: 1.0 | grad norm: 0.205 | params norm: 11378.735 | params rms: 0.061612 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 19:31:26.641002 | finish at 2026-10-10 18:02:34 + [2026-10-07 22:32:53] iteration 4090/ 6855 | consumed samples: 523520 | elapsed time per iteration (ms): 105950.9 | throughput per GPU (TFLOP/s/GPU): 33.9 | MFU 10.86% | FLOPs estimator: workload-aware | logical tokens/s: 158348.7 | valid tokens/s: 158348.7 | physical tokens/s: 158798.1 | loss tokens/s: 124825.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.645466E-01 | lm-global-supervised-tokens: 1.322541E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.184569E-02 | lm-token-buffer-mib: 5.827563E+01 | global_load_balancing_loss: 1.001609E+00 | loss scale: 1.0 | grad norm: 0.416 | params norm: 11378.675 | params rms: 0.061612 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 9:22:34.254537 | finish at 2026-10-11 07:55:27 + [2026-10-07 22:34:23] iteration 4091/ 6855 | consumed samples: 523648 | elapsed time per iteration (ms): 89593.9 | throughput per GPU (TFLOP/s/GPU): 41.4 | MFU 13.27% | FLOPs estimator: workload-aware | logical tokens/s: 187258.1 | valid tokens/s: 187258.1 | physical tokens/s: 187715.3 | loss tokens/s: 138566.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.504048E-01 | lm-global-supervised-tokens: 1.241474E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.568436E-02 | lm-token-buffer-mib: 5.827216E+01 | global_load_balancing_loss: 1.000562E+00 | loss scale: 1.0 | grad norm: 0.449 | params norm: 11378.619 | params rms: 0.061612 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 20:47:17.472946 | finish at 2026-10-10 19:21:40 + [2026-10-07 22:35:53] iteration 4092/ 6855 | consumed samples: 523776 | elapsed time per iteration (ms): 90777.2 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.50% | FLOPs estimator: workload-aware | logical tokens/s: 184817.4 | valid tokens/s: 184817.4 | physical tokens/s: 185110.0 | loss tokens/s: 135125.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.518883E-01 | lm-global-supervised-tokens: 1.226629E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.489255E-02 | lm-token-buffer-mib: 5.811459E+01 | global_load_balancing_loss: 1.000094E+00 | loss scale: 1.0 | grad norm: 0.153 | params norm: 11378.566 | params rms: 0.061611 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 21:40:17.290908 | finish at 2026-10-10 20:16:11 + [2026-10-07 22:37:23] iteration 4093/ 6855 | consumed samples: 523904 | elapsed time per iteration (ms): 89200.3 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.48% | FLOPs estimator: workload-aware | logical tokens/s: 188084.1 | valid tokens/s: 188084.1 | physical tokens/s: 188544.8 | loss tokens/s: 135637.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.136295E-01 | lm-global-supervised-tokens: 1.209893E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.780394E-02 | lm-token-buffer-mib: 5.815150E+01 | global_load_balancing_loss: 9.993388E-01 | loss scale: 1.0 | grad norm: 0.088 | params norm: 11378.513 | params rms: 0.061611 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 20:26:11.237101 | finish at 2026-10-10 19:03:34 + [2026-10-07 22:38:50] iteration 4094/ 6855 | consumed samples: 524032 | elapsed time per iteration (ms): 87587.2 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.69% | FLOPs estimator: workload-aware | logical tokens/s: 191548.2 | valid tokens/s: 191548.2 | physical tokens/s: 191999.5 | loss tokens/s: 135679.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.739748E-01 | lm-global-supervised-tokens: 1.188383E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.517824E-02 | lm-token-buffer-mib: 5.807739E+01 | global_load_balancing_loss: 9.998189E-01 | loss scale: 1.0 | grad norm: 0.186 | params norm: 11378.462 | params rms: 0.061611 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 19:10:28.375511 | finish at 2026-10-10 17:49:19 + [2026-10-07 22:40:19] iteration 4095/ 6855 | consumed samples: 524160 | elapsed time per iteration (ms): 88885.3 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.54% | FLOPs estimator: workload-aware | logical tokens/s: 188750.8 | valid tokens/s: 188750.8 | physical tokens/s: 189176.6 | loss tokens/s: 137865.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.409072E-01 | lm-global-supervised-tokens: 1.225419E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.109212E-02 | lm-token-buffer-mib: 5.811819E+01 | global_load_balancing_loss: 9.998412E-01 | loss scale: 1.0 | grad norm: 0.136 | params norm: 11378.410 | params rms: 0.061610 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 20:08:43.527803 | finish at 2026-10-10 18:49:03 + [2026-10-07 22:41:50] iteration 4096/ 6855 | consumed samples: 524288 | elapsed time per iteration (ms): 91144.2 | throughput per GPU (TFLOP/s/GPU): 41.6 | MFU 13.33% | FLOPs estimator: workload-aware | logical tokens/s: 184072.8 | valid tokens/s: 184072.8 | physical tokens/s: 184545.1 | loss tokens/s: 132639.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.319291E-01 | lm-global-supervised-tokens: 1.208933E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.208283E-02 | lm-token-buffer-mib: 5.813491E+01 | global_load_balancing_loss: 1.000364E+00 | loss scale: 1.0 | grad norm: 0.069 | params norm: 11378.360 | params rms: 0.061610 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 21:51:06.940130 | finish at 2026-10-10 20:32:57 + [2026-10-07 22:43:17] iteration 4097/ 6855 | consumed samples: 524416 | elapsed time per iteration (ms): 86630.5 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.59% | FLOPs estimator: workload-aware | logical tokens/s: 193663.4 | valid tokens/s: 193663.4 | physical tokens/s: 194126.4 | loss tokens/s: 143072.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.398456E-01 | lm-global-supervised-tokens: 1.239442E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.243934E-02 | lm-token-buffer-mib: 5.820291E+01 | global_load_balancing_loss: 1.000304E+00 | loss scale: 1.0 | grad norm: 0.140 | params norm: 11378.312 | params rms: 0.061610 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 18:22:06.920531 | finish at 2026-10-10 17:05:24 + [2026-10-07 22:44:45] iteration 4098/ 6855 | consumed samples: 524544 | elapsed time per iteration (ms): 88555.7 | throughput per GPU (TFLOP/s/GPU): 41.7 | MFU 13.37% | FLOPs estimator: workload-aware | logical tokens/s: 189453.8 | valid tokens/s: 189453.8 | physical tokens/s: 189986.7 | loss tokens/s: 136937.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 5.968300E-01 | lm-global-supervised-tokens: 1.212659E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.201378E-02 | lm-token-buffer-mib: 5.827903E+01 | global_load_balancing_loss: 1.002217E+00 | loss scale: 1.0 | grad norm: 0.091 | params norm: 11378.263 | params rms: 0.061610 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 19:49:07.966478 | finish at 2026-10-10 18:33:53 + [2026-10-07 22:46:16] iteration 4099/ 6855 | consumed samples: 524672 | elapsed time per iteration (ms): 90619.6 | throughput per GPU (TFLOP/s/GPU): 40.8 | MFU 13.06% | FLOPs estimator: workload-aware | logical tokens/s: 185138.4 | valid tokens/s: 185138.4 | physical tokens/s: 185608.1 | loss tokens/s: 135590.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.121988E-01 | lm-global-supervised-tokens: 1.228719E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.473268E-02 | lm-token-buffer-mib: 5.808493E+01 | global_load_balancing_loss: 1.000401E+00 | loss scale: 1.0 | grad norm: 0.042 | params norm: 11378.213 | params rms: 0.061609 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 21:22:27.587533 | finish at 2026-10-10 20:08:44 + [2026-10-07 22:48:05] iteration 4100/ 6855 | consumed samples: 524800 | elapsed time per iteration (ms): 109048.9 | throughput per GPU (TFLOP/s/GPU): 33.6 | MFU 10.76% | FLOPs estimator: workload-aware | logical tokens/s: 153850.1 | valid tokens/s: 153850.1 | physical tokens/s: 154418.0 | loss tokens/s: 110292.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.301448E-01 | lm-global-supervised-tokens: 1.202726E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.289200E-02 | lm-token-buffer-mib: 5.828320E+01 | global_load_balancing_loss: 1.000122E+00 | loss scale: 1.0 | grad norm: 0.114 | params norm: 11378.162 | params rms: 0.061609 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 11:27:09.597678 | finish at 2026-10-11 10:15:15 + [2026-10-07 22:49:34] iteration 4101/ 6855 | consumed samples: 524928 | elapsed time per iteration (ms): 88582.9 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.51% | FLOPs estimator: workload-aware | logical tokens/s: 189395.5 | valid tokens/s: 189395.5 | physical tokens/s: 189950.4 | loss tokens/s: 134648.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.681339E-01 | lm-global-supervised-tokens: 1.192758E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.415159E-02 | lm-token-buffer-mib: 5.811051E+01 | global_load_balancing_loss: 9.993954E-01 | loss scale: 1.0 | grad norm: 0.045 | params norm: 11378.111 | params rms: 0.061609 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 19:45:57.202330 | finish at 2026-10-10 18:35:31 + [2026-10-07 22:51:03] iteration 4102/ 6855 | consumed samples: 525056 | elapsed time per iteration (ms): 89237.0 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.40% | FLOPs estimator: workload-aware | logical tokens/s: 188006.9 | valid tokens/s: 188006.9 | physical tokens/s: 188520.3 | loss tokens/s: 135242.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.212919E-01 | lm-global-supervised-tokens: 1.206868E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.353545E-02 | lm-token-buffer-mib: 5.814265E+01 | global_load_balancing_loss: 9.986644E-01 | loss scale: 1.0 | grad norm: 0.066 | params norm: 11378.059 | params rms: 0.061608 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 20:14:29.571894 | finish at 2026-10-10 19:05:32 + [2026-10-07 22:52:28] iteration 4103/ 6855 | consumed samples: 525184 | elapsed time per iteration (ms): 85432.5 | throughput per GPU (TFLOP/s/GPU): 43.8 | MFU 14.03% | FLOPs estimator: workload-aware | logical tokens/s: 196379.0 | valid tokens/s: 196379.0 | physical tokens/s: 196886.0 | loss tokens/s: 140007.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.565419E-01 | lm-global-supervised-tokens: 1.196116E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.106407E-02 | lm-token-buffer-mib: 5.806870E+01 | global_load_balancing_loss: 9.991938E-01 | loss scale: 1.0 | grad norm: 0.055 | params norm: 11378.007 | params rms: 0.061608 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 17:18:30.298737 | finish at 2026-10-10 16:10:59 + [2026-10-07 22:53:57] iteration 4104/ 6855 | consumed samples: 525312 | elapsed time per iteration (ms): 88465.9 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.56% | FLOPs estimator: workload-aware | logical tokens/s: 189646.0 | valid tokens/s: 189646.0 | physical tokens/s: 190031.3 | loss tokens/s: 140152.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.107318E-01 | lm-global-supervised-tokens: 1.239874E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.135162E-02 | lm-token-buffer-mib: 5.806110E+01 | global_load_balancing_loss: 9.998029E-01 | loss scale: 1.0 | grad norm: 0.051 | params norm: 11377.954 | params rms: 0.061608 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 19:36:09.559569 | finish at 2026-10-10 18:30:06 + [2026-10-07 22:55:19] iteration 4105/ 6855 | consumed samples: 525440 | elapsed time per iteration (ms): 81698.5 | throughput per GPU (TFLOP/s/GPU): 43.9 | MFU 14.08% | FLOPs estimator: workload-aware | logical tokens/s: 205354.9 | valid tokens/s: 205354.8 | physical tokens/s: 205935.9 | loss tokens/s: 154342.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.377122E-01 | lm-global-supervised-tokens: 1.260952E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.155813E-02 | lm-token-buffer-mib: 5.812657E+01 | global_load_balancing_loss: 1.001221E+00 | loss scale: 1.0 | grad norm: 0.080 | params norm: 11377.901 | params rms: 0.061608 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 14:24:30.867562 | finish at 2026-10-10 13:19:49 + [2026-10-07 22:56:49] iteration 4106/ 6855 | consumed samples: 525568 | elapsed time per iteration (ms): 90743.8 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.41% | FLOPs estimator: workload-aware | logical tokens/s: 184885.4 | valid tokens/s: 184885.4 | physical tokens/s: 185170.7 | loss tokens/s: 136197.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.172568E-01 | lm-global-supervised-tokens: 1.235908E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.840117E-02 | lm-token-buffer-mib: 5.818577E+01 | global_load_balancing_loss: 1.001802E+00 | loss scale: 1.0 | grad norm: 0.037 | params norm: 11377.848 | params rms: 0.061607 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 21:17:34.827900 | finish at 2026-10-10 20:14:24 + [2026-10-07 22:58:17] iteration 4107/ 6855 | consumed samples: 525696 | elapsed time per iteration (ms): 87282.8 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.48% | FLOPs estimator: workload-aware | logical tokens/s: 192216.4 | valid tokens/s: 192216.4 | physical tokens/s: 192768.1 | loss tokens/s: 145057.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.476116E-01 | lm-global-supervised-tokens: 1.266098E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.727067E-02 | lm-token-buffer-mib: 5.825710E+01 | global_load_balancing_loss: 1.000588E+00 | loss scale: 1.0 | grad norm: 0.077 | params norm: 11377.793 | params rms: 0.061607 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 18:37:33.216185 | finish at 2026-10-10 17:35:50 + [2026-10-07 22:59:44] iteration 4108/ 6855 | consumed samples: 525824 | elapsed time per iteration (ms): 87733.3 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.66% | FLOPs estimator: workload-aware | logical tokens/s: 191228.7 | valid tokens/s: 191228.7 | physical tokens/s: 191610.3 | loss tokens/s: 142979.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.711904E-01 | lm-global-supervised-tokens: 1.254408E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.245172E-02 | lm-token-buffer-mib: 5.804208E+01 | global_load_balancing_loss: 1.001531E+00 | loss scale: 1.0 | grad norm: 0.042 | params norm: 11377.738 | params rms: 0.061607 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 18:56:43.264990 | finish at 2026-10-10 17:56:28 + [2026-10-07 23:01:12] iteration 4109/ 6855 | consumed samples: 525952 | elapsed time per iteration (ms): 87731.2 | throughput per GPU (TFLOP/s/GPU): 41.0 | MFU 13.14% | FLOPs estimator: workload-aware | logical tokens/s: 191233.5 | valid tokens/s: 191233.5 | physical tokens/s: 191681.6 | loss tokens/s: 148257.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.766902E-01 | lm-global-supervised-tokens: 1.300684E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.165422E-02 | lm-token-buffer-mib: 5.814691E+01 | global_load_balancing_loss: 9.996842E-01 | loss scale: 1.0 | grad norm: 0.116 | params norm: 11377.682 | params rms: 0.061606 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 18:55:09.981860 | finish at 2026-10-10 17:56:22 + [2026-10-07 23:03:05] iteration 4110/ 6855 | consumed samples: 526080 | elapsed time per iteration (ms): 113457.6 | throughput per GPU (TFLOP/s/GPU): 33.1 | MFU 10.61% | FLOPs estimator: workload-aware | logical tokens/s: 147872.1 | valid tokens/s: 147872.1 | physical tokens/s: 148277.1 | loss tokens/s: 110597.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.673196E-01 | lm-global-supervised-tokens: 1.254815E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.242131E-02 | lm-token-buffer-mib: 5.829906E+01 | global_load_balancing_loss: 9.996218E-01 | loss scale: 1.0 | grad norm: 0.049 | params norm: 11377.625 | params rms: 0.061606 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 14:30:41.140462 | finish at 2026-10-11 13:33:47 + [2026-10-07 23:04:31] iteration 4111/ 6855 | consumed samples: 526208 | elapsed time per iteration (ms): 86006.0 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.59% | FLOPs estimator: workload-aware | logical tokens/s: 195069.6 | valid tokens/s: 195069.6 | physical tokens/s: 195444.7 | loss tokens/s: 148513.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.788555E-01 | lm-global-supervised-tokens: 1.277308E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.145775E-02 | lm-token-buffer-mib: 5.806259E+01 | global_load_balancing_loss: 9.990886E-01 | loss scale: 1.0 | grad norm: 0.154 | params norm: 11377.566 | params rms: 0.061606 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 17:33:20.362711 | finish at 2026-10-10 16:37:52 + [2026-10-07 23:05:58] iteration 4112/ 6855 | consumed samples: 526336 | elapsed time per iteration (ms): 86464.2 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.56% | FLOPs estimator: workload-aware | logical tokens/s: 194036.6 | valid tokens/s: 194036.6 | physical tokens/s: 194498.1 | loss tokens/s: 141960.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.700364E-01 | lm-global-supervised-tokens: 1.227445E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.176302E-02 | lm-token-buffer-mib: 5.808304E+01 | global_load_balancing_loss: 1.000119E+00 | loss scale: 1.0 | grad norm: 0.071 | params norm: 11377.508 | params rms: 0.061606 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 17:52:51.236564 | finish at 2026-10-10 16:58:49 + [2026-10-07 23:07:26] iteration 4113/ 6855 | consumed samples: 526464 | elapsed time per iteration (ms): 88221.8 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.45% | FLOPs estimator: workload-aware | logical tokens/s: 190170.5 | valid tokens/s: 190170.5 | physical tokens/s: 190717.5 | loss tokens/s: 137574.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.338111E-01 | lm-global-supervised-tokens: 1.213709E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.150152E-02 | lm-token-buffer-mib: 5.812366E+01 | global_load_balancing_loss: 1.000223E+00 | loss scale: 1.0 | grad norm: 0.163 | params norm: 11377.450 | params rms: 0.061605 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 19:11:44.149694 | finish at 2026-10-10 18:19:10 + [2026-10-07 23:08:54] iteration 4114/ 6855 | consumed samples: 526592 | elapsed time per iteration (ms): 87368.5 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.62% | FLOPs estimator: workload-aware | logical tokens/s: 192027.9 | valid tokens/s: 192027.9 | physical tokens/s: 192513.2 | loss tokens/s: 137115.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.539989E-01 | lm-global-supervised-tokens: 1.197960E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 8.408806E-02 | lm-token-buffer-mib: 5.809286E+01 | global_load_balancing_loss: 1.000558E+00 | loss scale: 1.0 | grad norm: 0.128 | params norm: 11377.391 | params rms: 0.061605 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 18:31:16.993787 | finish at 2026-10-10 17:40:11 + [2026-10-07 23:10:20] iteration 4115/ 6855 | consumed samples: 526720 | elapsed time per iteration (ms): 86886.3 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.64% | FLOPs estimator: workload-aware | logical tokens/s: 193093.7 | valid tokens/s: 193093.7 | physical tokens/s: 193436.5 | loss tokens/s: 138909.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.068825E-01 | lm-global-supervised-tokens: 1.206932E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.955940E-02 | lm-token-buffer-mib: 5.807489E+01 | global_load_balancing_loss: 1.001683E+00 | loss scale: 1.0 | grad norm: 0.062 | params norm: 11377.330 | params rms: 0.061605 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 18:07:48.430882 | finish at 2026-10-10 17:18:09 + [2026-10-07 23:11:47] iteration 4116/ 6855 | consumed samples: 526848 | elapsed time per iteration (ms): 86493.6 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.65% | FLOPs estimator: workload-aware | logical tokens/s: 193969.7 | valid tokens/s: 193969.6 | physical tokens/s: 194454.7 | loss tokens/s: 142715.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.796138E-01 | lm-global-supervised-tokens: 1.234400E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.159457E-02 | lm-token-buffer-mib: 5.809439E+01 | global_load_balancing_loss: 1.001031E+00 | loss scale: 1.0 | grad norm: 0.049 | params norm: 11377.269 | params rms: 0.061604 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 17:48:26.008632 | finish at 2026-10-10 17:00:13 + [2026-10-07 23:13:16] iteration 4117/ 6855 | consumed samples: 526976 | elapsed time per iteration (ms): 88690.3 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.48% | FLOPs estimator: workload-aware | logical tokens/s: 189166.1 | valid tokens/s: 189166.1 | physical tokens/s: 189594.9 | loss tokens/s: 140228.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.537743E-01 | lm-global-supervised-tokens: 1.243693E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.727265E-02 | lm-token-buffer-mib: 5.808745E+01 | global_load_balancing_loss: 9.983792E-01 | loss scale: 1.0 | grad norm: 0.068 | params norm: 11377.209 | params rms: 0.061604 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 19:27:14.057034 | finish at 2026-10-10 18:40:30 + [2026-10-07 23:14:46] iteration 4118/ 6855 | consumed samples: 527104 | elapsed time per iteration (ms): 90419.8 | throughput per GPU (TFLOP/s/GPU): 41.5 | MFU 13.30% | FLOPs estimator: workload-aware | logical tokens/s: 185547.1 | valid tokens/s: 185547.1 | physical tokens/s: 186033.1 | loss tokens/s: 132908.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.522773E-01 | lm-global-supervised-tokens: 1.201753E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.248706E-02 | lm-token-buffer-mib: 5.812268E+01 | global_load_balancing_loss: 9.990655E-01 | loss scale: 1.0 | grad norm: 0.068 | params norm: 11377.149 | params rms: 0.061604 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 20:44:38.998591 | finish at 2026-10-10 19:59:25 + [2026-10-07 23:16:10] iteration 4119/ 6855 | consumed samples: 527232 | elapsed time per iteration (ms): 84322.0 | throughput per GPU (TFLOP/s/GPU): 44.2 | MFU 14.16% | FLOPs estimator: workload-aware | logical tokens/s: 198966.0 | valid tokens/s: 198966.0 | physical tokens/s: 199615.2 | loss tokens/s: 151067.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.446241E-01 | lm-global-supervised-tokens: 1.273827E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.767904E-02 | lm-token-buffer-mib: 5.818857E+01 | global_load_balancing_loss: 9.992549E-01 | loss scale: 1.0 | grad norm: 0.129 | params norm: 11377.088 | params rms: 0.061603 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 16:05:04.977070 | finish at 2026-10-10 15:21:15 + [2026-10-07 23:17:58] iteration 4120/ 6855 | consumed samples: 527360 | elapsed time per iteration (ms): 107344.1 | throughput per GPU (TFLOP/s/GPU): 34.4 | MFU 11.04% | FLOPs estimator: workload-aware | logical tokens/s: 156293.5 | valid tokens/s: 156293.5 | physical tokens/s: 156638.7 | loss tokens/s: 122306.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.023828E-01 | lm-global-supervised-tokens: 1.312892E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.319963E-02 | lm-token-buffer-mib: 5.809331E+01 | global_load_balancing_loss: 9.993976E-01 | loss scale: 1.0 | grad norm: 0.053 | params norm: 11377.026 | params rms: 0.061603 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 9:33:06.097194 | finish at 2026-10-11 08:51:04 +times across ranks (ms): + save-checkpoint: + rank 0: 26571.42 + rank 1: 26571.50 + rank 2: 26571.46 + rank 3: 26571.43 + rank 4: 26571.43 + rank 5: 26571.41 + rank 6: 26571.44 + rank 7: 26571.47 + rank 8: 26570.20 + rank 9: 26570.15 + rank 10: 26570.14 + rank 11: 26570.19 + rank 12: 26570.18 + rank 13: 26570.16 + rank 14: 26570.23 + rank 15: 26570.13 + rank 16: 26570.77 + rank 17: 26570.75 + rank 18: 26570.78 + rank 19: 26570.79 + rank 20: 26570.82 + rank 21: 26570.80 + rank 22: 26570.74 + rank 23: 26570.74 + rank 24: 26570.82 + rank 25: 26570.85 + rank 26: 26570.86 + rank 27: 26570.54 + rank 28: 26570.80 + rank 29: 26571.57 + rank 30: 26570.78 + rank 31: 26570.83 + rank 32: 26570.66 + rank 33: 26570.75 + rank 34: 26570.73 + rank 35: 26570.67 + rank 36: 26570.53 + rank 37: 26570.55 + rank 38: 26570.54 + rank 39: 26570.54 + rank 40: 26570.34 + rank 41: 26570.29 + rank 42: 26570.31 + rank 43: 26570.32 + rank 44: 26570.31 + rank 45: 26570.30 + rank 46: 26570.31 + rank 47: 26570.29 + rank 48: 26569.33 + rank 49: 26569.39 + rank 50: 26569.37 + rank 51: 26569.25 + rank 52: 26569.47 + rank 53: 26569.37 + rank 54: 26569.41 + rank 55: 26569.38 + rank 56: 26570.57 + rank 57: 26570.60 + rank 58: 26570.52 + rank 59: 26570.56 + rank 60: 26570.52 + rank 61: 26570.52 + rank 62: 26570.53 + rank 63: 26570.55 + [2026-10-07 23:19:58] iteration 4121/ 6855 | consumed samples: 527488 | elapsed time per iteration (ms): 93701.4 | throughput per GPU (TFLOP/s/GPU): 39.4 | MFU 12.61% | FLOPs estimator: workload-aware | logical tokens/s: 179048.7 | valid tokens/s: 179048.7 | physical tokens/s: 179473.7 | loss tokens/s: 127837.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.588081E-01 | lm-global-supervised-tokens: 1.197856E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 8.172773E-01 | lm-token-buffer-mib: 5.811741E+01 | global_load_balancing_loss: 1.000879E+00 | loss scale: 1.0 | grad norm: 0.073 | params norm: 11376.964 | params rms: 0.061603 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 23:09:39.690942 | finish at 2026-10-10 22:29:38 + [2026-10-07 23:21:33] iteration 4122/ 6855 | consumed samples: 527616 | elapsed time per iteration (ms): 95428.6 | throughput per GPU (TFLOP/s/GPU): 39.0 | MFU 12.50% | FLOPs estimator: workload-aware | logical tokens/s: 175807.9 | valid tokens/s: 175807.9 | physical tokens/s: 176241.7 | loss tokens/s: 123007.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 5.722805E-01 | lm-global-supervised-tokens: 1.173844E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.862133E-02 | lm-token-buffer-mib: 5.803313E+01 | global_load_balancing_loss: 1.000797E+00 | loss scale: 1.0 | grad norm: 0.066 | params norm: 11376.904 | params rms: 0.061602 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 0:26:46.387457 | finish at 2026-10-10 23:48:20 + [2026-10-07 23:22:59] iteration 4123/ 6855 | consumed samples: 527744 | elapsed time per iteration (ms): 85888.6 | throughput per GPU (TFLOP/s/GPU): 42.8 | MFU 13.72% | FLOPs estimator: workload-aware | logical tokens/s: 195336.8 | valid tokens/s: 195336.8 | physical tokens/s: 195622.8 | loss tokens/s: 146753.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.088284E-01 | lm-global-supervised-tokens: 1.260447E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.192736E-02 | lm-token-buffer-mib: 5.807967E+01 | global_load_balancing_loss: 1.000097E+00 | loss scale: 1.0 | grad norm: 0.037 | params norm: 11376.843 | params rms: 0.061602 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 17:10:47.712172 | finish at 2026-10-10 16:33:47 + [2026-10-07 23:24:26] iteration 4124/ 6855 | consumed samples: 527872 | elapsed time per iteration (ms): 86421.4 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.60% | FLOPs estimator: workload-aware | logical tokens/s: 194132.7 | valid tokens/s: 194132.7 | physical tokens/s: 194547.3 | loss tokens/s: 147168.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.485779E-01 | lm-global-supervised-tokens: 1.271850E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.177475E-02 | lm-token-buffer-mib: 5.802780E+01 | global_load_balancing_loss: 1.001628E+00 | loss scale: 1.0 | grad norm: 0.063 | params norm: 11376.782 | params rms: 0.061602 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 17:33:36.753737 | finish at 2026-10-10 16:58:03 + [2026-10-07 23:25:56] iteration 4125/ 6855 | consumed samples: 528000 | elapsed time per iteration (ms): 89995.6 | throughput per GPU (TFLOP/s/GPU): 42.8 | MFU 13.71% | FLOPs estimator: workload-aware | logical tokens/s: 186422.4 | valid tokens/s: 186422.4 | physical tokens/s: 186729.0 | loss tokens/s: 134394.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.714382E-01 | lm-global-supervised-tokens: 1.209494E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.163988E-02 | lm-token-buffer-mib: 5.807471E+01 | global_load_balancing_loss: 1.002003E+00 | loss scale: 1.0 | grad norm: 0.056 | params norm: 11376.721 | params rms: 0.061601 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 20:14:48.105118 | finish at 2026-10-10 19:40:44 + [2026-10-07 23:27:25] iteration 4126/ 6855 | consumed samples: 528128 | elapsed time per iteration (ms): 88950.8 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.41% | FLOPs estimator: workload-aware | logical tokens/s: 188611.9 | valid tokens/s: 188611.9 | physical tokens/s: 188972.5 | loss tokens/s: 136305.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.621335E-01 | lm-global-supervised-tokens: 1.212450E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.005874E-02 | lm-token-buffer-mib: 5.811075E+01 | global_load_balancing_loss: 9.996937E-01 | loss scale: 1.0 | grad norm: 0.043 | params norm: 11376.658 | params rms: 0.061601 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 19:25:46.803311 | finish at 2026-10-10 18:53:12 + [2026-10-07 23:28:50] iteration 4127/ 6855 | consumed samples: 528256 | elapsed time per iteration (ms): 85087.4 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.79% | FLOPs estimator: workload-aware | logical tokens/s: 197176.2 | valid tokens/s: 197176.2 | physical tokens/s: 197666.7 | loss tokens/s: 140944.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.651711E-01 | lm-global-supervised-tokens: 1.199262E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.291950E-02 | lm-token-buffer-mib: 5.815730E+01 | global_load_balancing_loss: 9.996225E-01 | loss scale: 1.0 | grad norm: 0.071 | params norm: 11376.595 | params rms: 0.061601 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 16:28:38.455708 | finish at 2026-10-10 15:57:28 + [2026-10-07 23:30:17] iteration 4128/ 6855 | consumed samples: 528384 | elapsed time per iteration (ms): 87121.0 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.57% | FLOPs estimator: workload-aware | logical tokens/s: 192573.8 | valid tokens/s: 192573.8 | physical tokens/s: 193227.5 | loss tokens/s: 142613.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.900219E-01 | lm-global-supervised-tokens: 1.242462E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.185420E-02 | lm-token-buffer-mib: 5.838714E+01 | global_load_balancing_loss: 9.986199E-01 | loss scale: 1.0 | grad norm: 0.051 | params norm: 11376.532 | params rms: 0.061600 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 17:59:38.841008 | finish at 2026-10-10 17:29:56 + [2026-10-07 23:31:42] iteration 4129/ 6855 | consumed samples: 528512 | elapsed time per iteration (ms): 85301.5 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.56% | FLOPs estimator: workload-aware | logical tokens/s: 196680.9 | valid tokens/s: 196680.9 | physical tokens/s: 197090.8 | loss tokens/s: 154108.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.122305E-01 | lm-global-supervised-tokens: 1.314566E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.780949E-02 | lm-token-buffer-mib: 5.805091E+01 | global_load_balancing_loss: 9.991207E-01 | loss scale: 1.0 | grad norm: 0.038 | params norm: 11376.469 | params rms: 0.061600 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 16:35:31.954216 | finish at 2026-10-10 16:07:14 + [2026-10-07 23:33:34] iteration 4130/ 6855 | consumed samples: 528640 | elapsed time per iteration (ms): 111936.8 | throughput per GPU (TFLOP/s/GPU): 33.0 | MFU 10.59% | FLOPs estimator: workload-aware | logical tokens/s: 149880.8 | valid tokens/s: 149880.8 | physical tokens/s: 150273.4 | loss tokens/s: 110049.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.430290E-01 | lm-global-supervised-tokens: 1.231862E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.861568E-02 | lm-token-buffer-mib: 5.812510E+01 | global_load_balancing_loss: 1.001131E+00 | loss scale: 1.0 | grad norm: 0.088 | params norm: 11376.407 | params rms: 0.061600 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 12:43:47.682555 | finish at 2026-10-11 12:17:22 + [2026-10-07 23:35:05] iteration 4131/ 6855 | consumed samples: 528768 | elapsed time per iteration (ms): 90890.3 | throughput per GPU (TFLOP/s/GPU): 41.3 | MFU 13.22% | FLOPs estimator: workload-aware | logical tokens/s: 184586.6 | valid tokens/s: 184586.6 | physical tokens/s: 184962.2 | loss tokens/s: 134800.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.309033E-01 | lm-global-supervised-tokens: 1.225209E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.460997E-02 | lm-token-buffer-mib: 5.805629E+01 | global_load_balancing_loss: 9.997418E-01 | loss scale: 1.0 | grad norm: 0.053 | params norm: 11376.345 | params rms: 0.061599 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 20:46:25.306538 | finish at 2026-10-10 20:21:30 + [2026-10-07 23:36:30] iteration 4132/ 6855 | consumed samples: 528896 | elapsed time per iteration (ms): 85313.9 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.69% | FLOPs estimator: workload-aware | logical tokens/s: 196652.9 | valid tokens/s: 196652.9 | physical tokens/s: 197284.9 | loss tokens/s: 153464.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.487300E-01 | lm-global-supervised-tokens: 1.309266E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.171662E-02 | lm-token-buffer-mib: 5.807614E+01 | global_load_balancing_loss: 9.996013E-01 | loss scale: 1.0 | grad norm: 0.060 | params norm: 11376.284 | params rms: 0.061599 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 16:31:49.617244 | finish at 2026-10-10 16:08:20 + [2026-10-07 23:37:59] iteration 4133/ 6855 | consumed samples: 529024 | elapsed time per iteration (ms): 88658.7 | throughput per GPU (TFLOP/s/GPU): 41.0 | MFU 13.13% | FLOPs estimator: workload-aware | logical tokens/s: 189233.4 | valid tokens/s: 189233.4 | physical tokens/s: 189660.0 | loss tokens/s: 143304.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.266516E-01 | lm-global-supervised-tokens: 1.270515E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.174559E-02 | lm-token-buffer-mib: 5.828069E+01 | global_load_balancing_loss: 1.000900E+00 | loss scale: 1.0 | grad norm: 0.047 | params norm: 11376.222 | params rms: 0.061599 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 19:02:08.994350 | finish at 2026-10-10 18:40:08 + [2026-10-07 23:39:26] iteration 4134/ 6855 | consumed samples: 529152 | elapsed time per iteration (ms): 87350.4 | throughput per GPU (TFLOP/s/GPU): 42.8 | MFU 13.72% | FLOPs estimator: workload-aware | logical tokens/s: 192068.0 | valid tokens/s: 192068.0 | physical tokens/s: 192647.2 | loss tokens/s: 135508.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.688316E-01 | lm-global-supervised-tokens: 1.183670E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.172151E-02 | lm-token-buffer-mib: 5.856397E+01 | global_load_balancing_loss: 1.000672E+00 | loss scale: 1.0 | grad norm: 0.094 | params norm: 11376.160 | params rms: 0.061598 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 18:01:20.397451 | finish at 2026-10-10 17:40:47 + [2026-10-07 23:40:53] iteration 4135/ 6855 | consumed samples: 529280 | elapsed time per iteration (ms): 86967.9 | throughput per GPU (TFLOP/s/GPU): 43.7 | MFU 14.01% | FLOPs estimator: workload-aware | logical tokens/s: 192912.2 | valid tokens/s: 192912.2 | physical tokens/s: 193353.5 | loss tokens/s: 140082.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.952267E-01 | lm-global-supervised-tokens: 1.218267E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.155739E-02 | lm-token-buffer-mib: 5.810170E+01 | global_load_balancing_loss: 1.000376E+00 | loss scale: 1.0 | grad norm: 0.088 | params norm: 11376.099 | params rms: 0.061598 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 17:42:32.719879 | finish at 2026-10-10 17:23:26 + [2026-10-07 23:42:22] iteration 4136/ 6855 | consumed samples: 529408 | elapsed time per iteration (ms): 88277.2 | throughput per GPU (TFLOP/s/GPU): 41.6 | MFU 13.33% | FLOPs estimator: workload-aware | logical tokens/s: 190051.3 | valid tokens/s: 190051.3 | physical tokens/s: 190590.4 | loss tokens/s: 145137.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.575834E-01 | lm-global-supervised-tokens: 1.281231E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.303096E-02 | lm-token-buffer-mib: 5.812325E+01 | global_load_balancing_loss: 1.000841E+00 | loss scale: 1.0 | grad norm: 0.108 | params norm: 11376.037 | params rms: 0.061598 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 18:40:25.842242 | finish at 2026-10-10 18:22:47 + [2026-10-07 23:43:50] iteration 4137/ 6855 | consumed samples: 529536 | elapsed time per iteration (ms): 87926.8 | throughput per GPU (TFLOP/s/GPU): 40.7 | MFU 13.05% | FLOPs estimator: workload-aware | logical tokens/s: 190807.9 | valid tokens/s: 190807.9 | physical tokens/s: 191324.5 | loss tokens/s: 143371.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.276381E-01 | lm-global-supervised-tokens: 1.260616E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.339260E-02 | lm-token-buffer-mib: 5.812269E+01 | global_load_balancing_loss: 1.000162E+00 | loss scale: 1.0 | grad norm: 0.204 | params norm: 11375.977 | params rms: 0.061597 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 18:23:04.965968 | finish at 2026-10-10 18:06:55 + [2026-10-07 23:45:20] iteration 4138/ 6855 | consumed samples: 529664 | elapsed time per iteration (ms): 90692.4 | throughput per GPU (TFLOP/s/GPU): 41.4 | MFU 13.26% | FLOPs estimator: workload-aware | logical tokens/s: 184990.3 | valid tokens/s: 184990.2 | physical tokens/s: 185357.4 | loss tokens/s: 135207.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.619644E-01 | lm-global-supervised-tokens: 1.226234E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.239101E-02 | lm-token-buffer-mib: 5.812273E+01 | global_load_balancing_loss: 1.000179E+00 | loss scale: 1.0 | grad norm: 0.210 | params norm: 11375.915 | params rms: 0.061597 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 20:26:51.304508 | finish at 2026-10-10 20:12:12 + [2026-10-07 23:46:51] iteration 4139/ 6855 | consumed samples: 529792 | elapsed time per iteration (ms): 90778.0 | throughput per GPU (TFLOP/s/GPU): 41.3 | MFU 13.25% | FLOPs estimator: workload-aware | logical tokens/s: 184814.7 | valid tokens/s: 184814.7 | physical tokens/s: 185313.3 | loss tokens/s: 133286.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.517280E-01 | lm-global-supervised-tokens: 1.209948E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.148372E-02 | lm-token-buffer-mib: 5.825514E+01 | global_load_balancing_loss: 1.000468E+00 | loss scale: 1.0 | grad norm: 0.105 | params norm: 11375.851 | params rms: 0.061597 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 20:29:13.051554 | finish at 2026-10-10 20:16:04 + [2026-10-07 23:48:38] iteration 4140/ 6855 | consumed samples: 529920 | elapsed time per iteration (ms): 106466.5 | throughput per GPU (TFLOP/s/GPU): 34.9 | MFU 11.19% | FLOPs estimator: workload-aware | logical tokens/s: 157581.3 | valid tokens/s: 157581.3 | physical tokens/s: 158016.8 | loss tokens/s: 118113.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.706162E-01 | lm-global-supervised-tokens: 1.257510E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.355376E-02 | lm-token-buffer-mib: 5.809282E+01 | global_load_balancing_loss: 9.997646E-01 | loss scale: 1.0 | grad norm: 0.063 | params norm: 11375.787 | params rms: 0.061596 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 8:17:36.428515 | finish at 2026-10-11 08:06:14 + [2026-10-07 23:50:05] iteration 4141/ 6855 | consumed samples: 530048 | elapsed time per iteration (ms): 87403.2 | throughput per GPU (TFLOP/s/GPU): 43.1 | MFU 13.82% | FLOPs estimator: workload-aware | logical tokens/s: 191951.4 | valid tokens/s: 191951.4 | physical tokens/s: 192258.1 | loss tokens/s: 146342.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.493073E-01 | lm-global-supervised-tokens: 1.279076E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.154268E-02 | lm-token-buffer-mib: 5.803682E+01 | global_load_balancing_loss: 9.984034E-01 | loss scale: 1.0 | grad norm: 0.039 | params norm: 11375.725 | params rms: 0.061596 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 17:53:32.203675 | finish at 2026-10-10 17:43:37 + [2026-10-07 23:51:29] iteration 4142/ 6855 | consumed samples: 530176 | elapsed time per iteration (ms): 84495.8 | throughput per GPU (TFLOP/s/GPU): 43.2 | MFU 13.83% | FLOPs estimator: workload-aware | logical tokens/s: 198556.1 | valid tokens/s: 198556.1 | physical tokens/s: 199038.1 | loss tokens/s: 143559.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.072893E-01 | lm-global-supervised-tokens: 1.213013E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.499899E-02 | lm-token-buffer-mib: 5.817755E+01 | global_load_balancing_loss: 1.000316E+00 | loss scale: 1.0 | grad norm: 0.032 | params norm: 11375.664 | params rms: 0.061596 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 15:40:37.148140 | finish at 2026-10-10 15:32:07 + [2026-10-07 23:52:57] iteration 4143/ 6855 | consumed samples: 530304 | elapsed time per iteration (ms): 87267.0 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.57% | FLOPs estimator: workload-aware | logical tokens/s: 192251.5 | valid tokens/s: 192251.4 | physical tokens/s: 192736.2 | loss tokens/s: 143067.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.903986E-01 | lm-global-supervised-tokens: 1.248509E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.512938E-02 | lm-token-buffer-mib: 5.805804E+01 | global_load_balancing_loss: 1.000868E+00 | loss scale: 1.0 | grad norm: 0.055 | params norm: 11375.603 | params rms: 0.061595 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 17:44:28.197647 | finish at 2026-10-10 17:37:25 + [2026-10-07 23:54:27] iteration 4144/ 6855 | consumed samples: 530432 | elapsed time per iteration (ms): 90277.3 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.51% | FLOPs estimator: workload-aware | logical tokens/s: 185840.3 | valid tokens/s: 185840.3 | physical tokens/s: 186186.0 | loss tokens/s: 129521.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.825475E-01 | lm-global-supervised-tokens: 1.169289E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.626560E-02 | lm-token-buffer-mib: 5.817429E+01 | global_load_balancing_loss: 1.001167E+00 | loss scale: 1.0 | grad norm: 0.053 | params norm: 11375.542 | params rms: 0.061595 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 19:59:01.732184 | finish at 2026-10-10 19:53:29 + [2026-10-07 23:55:57] iteration 4145/ 6855 | consumed samples: 530560 | elapsed time per iteration (ms): 89836.0 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.45% | FLOPs estimator: workload-aware | logical tokens/s: 186753.6 | valid tokens/s: 186753.6 | physical tokens/s: 187274.8 | loss tokens/s: 128748.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.403283E-01 | lm-global-supervised-tokens: 1.156622E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.198082E-02 | lm-token-buffer-mib: 5.811871E+01 | global_load_balancing_loss: 1.001494E+00 | loss scale: 1.0 | grad norm: 0.052 | params norm: 11375.481 | params rms: 0.061595 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 19:37:35.693653 | finish at 2026-10-10 19:33:32 + [2026-10-07 23:57:26] iteration 4146/ 6855 | consumed samples: 530688 | elapsed time per iteration (ms): 88904.5 | throughput per GPU (TFLOP/s/GPU): 41.1 | MFU 13.18% | FLOPs estimator: workload-aware | logical tokens/s: 188709.9 | valid tokens/s: 188709.9 | physical tokens/s: 189225.1 | loss tokens/s: 139201.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.098023E-01 | lm-global-supervised-tokens: 1.237562E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.137739E-02 | lm-token-buffer-mib: 5.813204E+01 | global_load_balancing_loss: 9.995597E-01 | loss scale: 1.0 | grad norm: 0.041 | params norm: 11375.419 | params rms: 0.061594 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 18:54:02.233038 | finish at 2026-10-10 18:51:28 + [2026-10-07 23:58:55] iteration 4147/ 6855 | consumed samples: 530816 | elapsed time per iteration (ms): 88981.0 | throughput per GPU (TFLOP/s/GPU): 41.0 | MFU 13.14% | FLOPs estimator: workload-aware | logical tokens/s: 188548.0 | valid tokens/s: 188548.0 | physical tokens/s: 189089.2 | loss tokens/s: 146475.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.805230E-01 | lm-global-supervised-tokens: 1.303350E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.138443E-02 | lm-token-buffer-mib: 5.808704E+01 | global_load_balancing_loss: 9.999522E-01 | loss scale: 1.0 | grad norm: 0.066 | params norm: 11375.358 | params rms: 0.061594 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 18:56:00.492976 | finish at 2026-10-10 18:54:55 + [2026-10-08 00:00:22] iteration 4148/ 6855 | consumed samples: 530944 | elapsed time per iteration (ms): 87739.9 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.40% | FLOPs estimator: workload-aware | logical tokens/s: 191214.6 | valid tokens/s: 191214.6 | physical tokens/s: 191775.3 | loss tokens/s: 140214.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.639491E-01 | lm-global-supervised-tokens: 1.230239E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.803148E-02 | lm-token-buffer-mib: 5.832597E+01 | global_load_balancing_loss: 9.990654E-01 | loss scale: 1.0 | grad norm: 0.038 | params norm: 11375.298 | params rms: 0.061594 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 17:58:32.001896 | finish at 2026-10-10 17:58:54 + [2026-10-08 00:01:46] iteration 4149/ 6855 | consumed samples: 531072 | elapsed time per iteration (ms): 83838.2 | throughput per GPU (TFLOP/s/GPU): 43.7 | MFU 14.00% | FLOPs estimator: workload-aware | logical tokens/s: 200114.1 | valid tokens/s: 200114.1 | physical tokens/s: 200456.9 | loss tokens/s: 148118.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.849650E-01 | lm-global-supervised-tokens: 1.241800E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.807688E-02 | lm-token-buffer-mib: 5.806274E+01 | global_load_balancing_loss: 9.998993E-01 | loss scale: 1.0 | grad norm: 0.056 | params norm: 11375.238 | params rms: 0.061593 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 15:01:06.042353 | finish at 2026-10-10 15:02:52 + [2026-10-08 00:03:34] iteration 4150/ 6855 | consumed samples: 531200 | elapsed time per iteration (ms): 107490.7 | throughput per GPU (TFLOP/s/GPU): 34.5 | MFU 11.07% | FLOPs estimator: workload-aware | logical tokens/s: 156080.2 | valid tokens/s: 156080.2 | physical tokens/s: 156402.2 | loss tokens/s: 114828.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.989514E-01 | lm-global-supervised-tokens: 1.234294E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.232374E-02 | lm-token-buffer-mib: 5.826781E+01 | global_load_balancing_loss: 9.993294E-01 | loss scale: 1.0 | grad norm: 0.047 | params norm: 11375.181 | params rms: 0.061593 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 8:46:02.260323 | finish at 2026-10-11 08:49:36 + [2026-10-08 00:05:01] iteration 4151/ 6855 | consumed samples: 531328 | elapsed time per iteration (ms): 87283.5 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.46% | FLOPs estimator: workload-aware | logical tokens/s: 192214.3 | valid tokens/s: 192214.3 | physical tokens/s: 192699.1 | loss tokens/s: 144163.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.810661E-01 | lm-global-supervised-tokens: 1.258307E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.192054E-02 | lm-token-buffer-mib: 5.817075E+01 | global_load_balancing_loss: 9.999681E-01 | loss scale: 1.0 | grad norm: 0.073 | params norm: 11375.123 | params rms: 0.061593 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 17:33:34.610958 | finish at 2026-10-10 17:38:36 + [2026-10-08 00:06:27] iteration 4152/ 6855 | consumed samples: 531456 | elapsed time per iteration (ms): 86371.9 | throughput per GPU (TFLOP/s/GPU): 41.5 | MFU 13.31% | FLOPs estimator: workload-aware | logical tokens/s: 194243.9 | valid tokens/s: 194243.9 | physical tokens/s: 194754.5 | loss tokens/s: 145409.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.319304E-01 | lm-global-supervised-tokens: 1.255930E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.146107E-02 | lm-token-buffer-mib: 5.810562E+01 | global_load_balancing_loss: 1.001605E+00 | loss scale: 1.0 | grad norm: 0.044 | params norm: 11375.063 | params rms: 0.061592 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 16:51:03.216276 | finish at 2026-10-10 16:57:31 + [2026-10-08 00:07:55] iteration 4153/ 6855 | consumed samples: 531584 | elapsed time per iteration (ms): 87411.5 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.53% | FLOPs estimator: workload-aware | logical tokens/s: 191933.6 | valid tokens/s: 191933.6 | physical tokens/s: 192470.0 | loss tokens/s: 148459.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.563023E-01 | lm-global-supervised-tokens: 1.297707E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.435445E-02 | lm-token-buffer-mib: 5.809296E+01 | global_load_balancing_loss: 1.001484E+00 | loss scale: 1.0 | grad norm: 0.050 | params norm: 11375.005 | params rms: 0.061592 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 17:36:25.887755 | finish at 2026-10-10 17:44:21 + [2026-10-08 00:09:25] iteration 4154/ 6855 | consumed samples: 531712 | elapsed time per iteration (ms): 90513.7 | throughput per GPU (TFLOP/s/GPU): 41.7 | MFU 13.36% | FLOPs estimator: workload-aware | logical tokens/s: 185354.8 | valid tokens/s: 185354.8 | physical tokens/s: 185929.2 | loss tokens/s: 133994.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.067317E-01 | lm-global-supervised-tokens: 1.212830E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.143012E-02 | lm-token-buffer-mib: 5.831190E+01 | global_load_balancing_loss: 1.000588E+00 | loss scale: 1.0 | grad norm: 0.055 | params norm: 11374.949 | params rms: 0.061592 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 19:54:37.506299 | finish at 2026-10-10 20:04:03 + [2026-10-08 00:10:50] iteration 4155/ 6855 | consumed samples: 531840 | elapsed time per iteration (ms): 84493.8 | throughput per GPU (TFLOP/s/GPU): 43.1 | MFU 13.81% | FLOPs estimator: workload-aware | logical tokens/s: 198561.1 | valid tokens/s: 198561.0 | physical tokens/s: 198990.7 | loss tokens/s: 152448.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.542807E-01 | lm-global-supervised-tokens: 1.288093E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.144559E-02 | lm-token-buffer-mib: 5.818878E+01 | global_load_balancing_loss: 1.001088E+00 | loss scale: 1.0 | grad norm: 0.077 | params norm: 11374.893 | params rms: 0.061591 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 15:22:13.344769 | finish at 2026-10-10 15:33:03 + [2026-10-08 00:12:16] iteration 4156/ 6855 | consumed samples: 531968 | elapsed time per iteration (ms): 86246.8 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.49% | FLOPs estimator: workload-aware | logical tokens/s: 194525.4 | valid tokens/s: 194525.4 | physical tokens/s: 194923.0 | loss tokens/s: 146711.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.760042E-01 | lm-global-supervised-tokens: 1.265337E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.180869E-02 | lm-token-buffer-mib: 5.819079E+01 | global_load_balancing_loss: 9.994700E-01 | loss scale: 1.0 | grad norm: 0.054 | params norm: 11374.836 | params rms: 0.061591 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 16:39:40.120776 | finish at 2026-10-10 16:51:56 + [2026-10-08 00:13:44] iteration 4157/ 6855 | consumed samples: 532096 | elapsed time per iteration (ms): 87512.5 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.57% | FLOPs estimator: workload-aware | logical tokens/s: 191711.9 | valid tokens/s: 191711.9 | physical tokens/s: 192174.0 | loss tokens/s: 138010.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.713203E-01 | lm-global-supervised-tokens: 1.207762E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.091146E-02 | lm-token-buffer-mib: 5.811639E+01 | global_load_balancing_loss: 1.000090E+00 | loss scale: 1.0 | grad norm: 0.061 | params norm: 11374.777 | params rms: 0.061591 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 17:35:08.602267 | finish at 2026-10-10 17:48:52 + [2026-10-08 00:15:10] iteration 4158/ 6855 | consumed samples: 532224 | elapsed time per iteration (ms): 86660.7 | throughput per GPU (TFLOP/s/GPU): 42.8 | MFU 13.73% | FLOPs estimator: workload-aware | logical tokens/s: 193596.4 | valid tokens/s: 193596.3 | physical tokens/s: 194081.3 | loss tokens/s: 138076.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.275020E-01 | lm-global-supervised-tokens: 1.196583E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.768351E-02 | lm-token-buffer-mib: 5.817654E+01 | global_load_balancing_loss: 9.999061E-01 | loss scale: 1.0 | grad norm: 0.090 | params norm: 11374.720 | params rms: 0.061590 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 16:55:24.008434 | finish at 2026-10-10 17:10:34 + [2026-10-08 00:16:37] iteration 4159/ 6855 | consumed samples: 532352 | elapsed time per iteration (ms): 86867.2 | throughput per GPU (TFLOP/s/GPU): 41.7 | MFU 13.36% | FLOPs estimator: workload-aware | logical tokens/s: 193135.9 | valid tokens/s: 193135.9 | physical tokens/s: 193712.3 | loss tokens/s: 146035.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.524412E-01 | lm-global-supervised-tokens: 1.268569E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.276569E-02 | lm-token-buffer-mib: 5.804080E+01 | global_load_balancing_loss: 9.998969E-01 | loss scale: 1.0 | grad norm: 0.078 | params norm: 11374.664 | params rms: 0.061590 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 17:03:14.037130 | finish at 2026-10-10 17:19:51 + [2026-10-08 00:18:27] iteration 4160/ 6855 | consumed samples: 532480 | elapsed time per iteration (ms): 110292.8 | throughput per GPU (TFLOP/s/GPU): 33.7 | MFU 10.80% | FLOPs estimator: workload-aware | logical tokens/s: 152114.9 | valid tokens/s: 152114.9 | physical tokens/s: 152413.1 | loss tokens/s: 111463.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.416765E-01 | lm-global-supervised-tokens: 1.229365E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.319761E-02 | lm-token-buffer-mib: 5.807272E+01 | global_load_balancing_loss: 9.999524E-01 | loss scale: 1.0 | grad norm: 0.037 | params norm: 11374.608 | params rms: 0.061590 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 10:33:58.989203 | finish at 2026-10-11 10:52:26 +times across ranks (ms): + save-checkpoint: + rank 0: 25529.29 + rank 1: 25529.33 + rank 2: 25529.31 + rank 3: 25529.32 + rank 4: 25529.29 + rank 5: 25529.24 + rank 6: 25529.24 + rank 7: 25529.27 + rank 8: 25529.89 + rank 9: 25529.88 + rank 10: 25529.88 + rank 11: 25529.86 + rank 12: 25529.86 + rank 13: 25529.87 + rank 14: 25529.87 + rank 15: 25529.85 + rank 16: 25529.59 + rank 17: 25529.54 + rank 18: 25529.55 + rank 19: 25529.56 + rank 20: 25529.52 + rank 21: 25529.56 + rank 22: 25529.53 + rank 23: 25529.51 + rank 24: 25529.63 + rank 25: 25529.56 + rank 26: 25529.60 + rank 27: 25529.47 + rank 28: 25529.53 + rank 29: 25530.02 + rank 30: 25529.59 + rank 31: 25529.63 + rank 32: 25528.68 + rank 33: 25528.64 + rank 34: 25528.71 + rank 35: 25528.65 + rank 36: 25528.58 + rank 37: 25528.54 + rank 38: 25528.67 + rank 39: 25528.59 + rank 40: 25530.31 + rank 41: 25530.28 + rank 42: 25530.29 + rank 43: 25530.32 + rank 44: 25530.28 + rank 45: 25530.29 + rank 46: 25530.36 + rank 47: 25530.28 + rank 48: 25529.72 + rank 49: 25529.88 + rank 50: 25529.81 + rank 51: 25529.66 + rank 52: 25529.84 + rank 53: 25529.82 + rank 54: 25529.82 + rank 55: 25529.87 + rank 56: 25529.60 + rank 57: 25529.55 + rank 58: 25529.51 + rank 59: 25529.52 + rank 60: 25529.52 + rank 61: 25529.52 + rank 62: 25529.56 + rank 63: 25529.38 + [2026-10-08 00:20:19] iteration 4161/ 6855 | consumed samples: 532608 | elapsed time per iteration (ms): 85806.7 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.70% | FLOPs estimator: workload-aware | logical tokens/s: 195522.2 | valid tokens/s: 195522.2 | physical tokens/s: 195957.5 | loss tokens/s: 149095.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.701186E-01 | lm-global-supervised-tokens: 1.279335E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.727365E-01 | lm-token-buffer-mib: 5.810736E+01 | global_load_balancing_loss: 1.000386E+00 | loss scale: 1.0 | grad norm: 0.076 | params norm: 11374.553 | params rms: 0.061590 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 16:12:43.251703 | finish at 2026-10-10 16:33:02 + [2026-10-08 00:21:50] iteration 4162/ 6855 | consumed samples: 532736 | elapsed time per iteration (ms): 90593.5 | throughput per GPU (TFLOP/s/GPU): 40.8 | MFU 13.06% | FLOPs estimator: workload-aware | logical tokens/s: 185192.2 | valid tokens/s: 185192.1 | physical tokens/s: 185537.3 | loss tokens/s: 136280.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.941686E-01 | lm-global-supervised-tokens: 1.234616E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.686959E-02 | lm-token-buffer-mib: 5.806249E+01 | global_load_balancing_loss: 9.999095E-01 | loss scale: 1.0 | grad norm: 0.068 | params norm: 11374.498 | params rms: 0.061589 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 19:46:08.398600 | finish at 2026-10-10 20:07:58 + [2026-10-08 00:23:16] iteration 4163/ 6855 | consumed samples: 532864 | elapsed time per iteration (ms): 86248.6 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.75% | FLOPs estimator: workload-aware | logical tokens/s: 194521.4 | valid tokens/s: 194521.3 | physical tokens/s: 194941.5 | loss tokens/s: 144480.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.004972E-01 | lm-global-supervised-tokens: 1.246128E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.217404E-02 | lm-token-buffer-mib: 5.816913E+01 | global_load_balancing_loss: 9.997473E-01 | loss scale: 1.0 | grad norm: 0.053 | params norm: 11374.442 | params rms: 0.061589 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 16:29:41.318504 | finish at 2026-10-10 16:52:57 + [2026-10-08 00:24:42] iteration 4164/ 6855 | consumed samples: 532992 | elapsed time per iteration (ms): 86266.9 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.55% | FLOPs estimator: workload-aware | logical tokens/s: 194480.3 | valid tokens/s: 194480.3 | physical tokens/s: 194890.2 | loss tokens/s: 142930.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.499181E-01 | lm-global-supervised-tokens: 1.233014E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.241582E-02 | lm-token-buffer-mib: 5.807410E+01 | global_load_balancing_loss: 1.000148E+00 | loss scale: 1.0 | grad norm: 0.128 | params norm: 11374.384 | params rms: 0.061589 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 16:29:04.305058 | finish at 2026-10-10 16:53:47 + [2026-10-08 00:26:09] iteration 4165/ 6855 | consumed samples: 533120 | elapsed time per iteration (ms): 87057.6 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.63% | FLOPs estimator: workload-aware | logical tokens/s: 192713.7 | valid tokens/s: 192713.7 | physical tokens/s: 193139.4 | loss tokens/s: 136354.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.008495E-01 | lm-global-supervised-tokens: 1.187069E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.262332E-02 | lm-token-buffer-mib: 5.820586E+01 | global_load_balancing_loss: 1.001157E+00 | loss scale: 1.0 | grad norm: 0.154 | params norm: 11374.327 | params rms: 0.061588 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 17:03:04.810658 | finish at 2026-10-10 17:29:14 + [2026-10-08 00:27:37] iteration 4166/ 6855 | consumed samples: 533248 | elapsed time per iteration (ms): 87715.5 | throughput per GPU (TFLOP/s/GPU): 43.2 | MFU 13.85% | FLOPs estimator: workload-aware | logical tokens/s: 191268.3 | valid tokens/s: 191268.3 | physical tokens/s: 191648.3 | loss tokens/s: 140249.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.552780E-01 | lm-global-supervised-tokens: 1.230209E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.621630E-02 | lm-token-buffer-mib: 5.819030E+01 | global_load_balancing_loss: 1.000029E+00 | loss scale: 1.0 | grad norm: 0.083 | params norm: 11374.268 | params rms: 0.061588 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 17:31:06.945834 | finish at 2026-10-10 17:58:44 + [2026-10-08 00:29:05] iteration 4167/ 6855 | consumed samples: 533376 | elapsed time per iteration (ms): 87600.2 | throughput per GPU (TFLOP/s/GPU): 43.2 | MFU 13.84% | FLOPs estimator: workload-aware | logical tokens/s: 191519.7 | valid tokens/s: 191519.7 | physical tokens/s: 192003.0 | loss tokens/s: 135912.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.691642E-01 | lm-global-supervised-tokens: 1.190592E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.304434E-02 | lm-token-buffer-mib: 5.822179E+01 | global_load_balancing_loss: 1.000315E+00 | loss scale: 1.0 | grad norm: 0.065 | params norm: 11374.208 | params rms: 0.061588 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 17:24:29.424591 | finish at 2026-10-10 17:53:34 + [2026-10-08 00:30:35] iteration 4168/ 6855 | consumed samples: 533504 | elapsed time per iteration (ms): 90444.0 | throughput per GPU (TFLOP/s/GPU): 41.5 | MFU 13.31% | FLOPs estimator: workload-aware | logical tokens/s: 185497.2 | valid tokens/s: 185497.2 | physical tokens/s: 185933.3 | loss tokens/s: 134027.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.673796E-01 | lm-global-supervised-tokens: 1.212196E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.164447E-02 | lm-token-buffer-mib: 5.807686E+01 | global_load_balancing_loss: 1.001095E+00 | loss scale: 1.0 | grad norm: 0.144 | params norm: 11374.149 | params rms: 0.061587 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 19:30:22.921671 | finish at 2026-10-10 20:00:58 + [2026-10-08 00:32:03] iteration 4169/ 6855 | consumed samples: 533632 | elapsed time per iteration (ms): 87543.4 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.45% | FLOPs estimator: workload-aware | logical tokens/s: 191644.2 | valid tokens/s: 191644.2 | physical tokens/s: 192014.0 | loss tokens/s: 143089.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.662572E-01 | lm-global-supervised-tokens: 1.252651E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.116618E-02 | lm-token-buffer-mib: 5.820206E+01 | global_load_balancing_loss: 9.999636E-01 | loss scale: 1.0 | grad norm: 0.119 | params norm: 11374.091 | params rms: 0.061587 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 17:19:01.520660 | finish at 2026-10-10 17:51:04 + [2026-10-08 00:33:55] iteration 4170/ 6855 | consumed samples: 533760 | elapsed time per iteration (ms): 112352.6 | throughput per GPU (TFLOP/s/GPU): 32.8 | MFU 10.51% | FLOPs estimator: workload-aware | logical tokens/s: 149325.9 | valid tokens/s: 149325.9 | physical tokens/s: 149667.6 | loss tokens/s: 110927.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.705552E-01 | lm-global-supervised-tokens: 1.246296E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.243054E-02 | lm-token-buffer-mib: 5.825159E+01 | global_load_balancing_loss: 9.998755E-01 | loss scale: 1.0 | grad norm: 0.050 | params norm: 11374.033 | params rms: 0.061587 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 11:47:46.648682 | finish at 2026-10-11 12:21:42 + [2026-10-08 00:35:21] iteration 4171/ 6855 | consumed samples: 533888 | elapsed time per iteration (ms): 86088.0 | throughput per GPU (TFLOP/s/GPU): 42.8 | MFU 13.71% | FLOPs estimator: workload-aware | logical tokens/s: 194884.1 | valid tokens/s: 194884.1 | physical tokens/s: 195334.0 | loss tokens/s: 149666.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.685837E-01 | lm-global-supervised-tokens: 1.288447E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.319849E-02 | lm-token-buffer-mib: 5.811054E+01 | global_load_balancing_loss: 9.993440E-01 | loss scale: 1.0 | grad norm: 0.081 | params norm: 11373.976 | params rms: 0.061586 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 16:11:00.082733 | finish at 2026-10-10 16:46:21 + [2026-10-08 00:36:51] iteration 4172/ 6855 | consumed samples: 534016 | elapsed time per iteration (ms): 89699.6 | throughput per GPU (TFLOP/s/GPU): 41.3 | MFU 13.23% | FLOPs estimator: workload-aware | logical tokens/s: 187037.4 | valid tokens/s: 187037.4 | physical tokens/s: 187507.3 | loss tokens/s: 140052.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.749068E-01 | lm-global-supervised-tokens: 1.256266E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.857200E-02 | lm-token-buffer-mib: 5.807662E+01 | global_load_balancing_loss: 9.993101E-01 | loss scale: 1.0 | grad norm: 0.046 | params norm: 11373.918 | params rms: 0.061586 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 18:51:03.941673 | finish at 2026-10-10 19:27:55 + [2026-10-08 00:38:15] iteration 4173/ 6855 | consumed samples: 534144 | elapsed time per iteration (ms): 83806.3 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.51% | FLOPs estimator: workload-aware | logical tokens/s: 200190.1 | valid tokens/s: 200190.1 | physical tokens/s: 200812.5 | loss tokens/s: 154184.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.198511E-01 | lm-global-supervised-tokens: 1.292165E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.307758E-02 | lm-token-buffer-mib: 5.809727E+01 | global_load_balancing_loss: 1.000357E+00 | loss scale: 1.0 | grad norm: 0.098 | params norm: 11373.859 | params rms: 0.061586 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 14:26:08.566737 | finish at 2026-10-10 15:04:23 + [2026-10-08 00:39:42] iteration 4174/ 6855 | consumed samples: 534272 | elapsed time per iteration (ms): 87203.9 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.62% | FLOPs estimator: workload-aware | logical tokens/s: 192390.0 | valid tokens/s: 192390.0 | physical tokens/s: 192962.9 | loss tokens/s: 141256.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.948219E-01 | lm-global-supervised-tokens: 1.231807E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.752734E-02 | lm-token-buffer-mib: 5.816789E+01 | global_load_balancing_loss: 1.000019E+00 | loss scale: 1.0 | grad norm: 0.058 | params norm: 11373.802 | params rms: 0.061585 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 16:56:33.640185 | finish at 2026-10-10 17:36:15 + [2026-10-08 00:41:07] iteration 4175/ 6855 | consumed samples: 534400 | elapsed time per iteration (ms): 84876.8 | throughput per GPU (TFLOP/s/GPU): 43.4 | MFU 13.91% | FLOPs estimator: workload-aware | logical tokens/s: 197665.4 | valid tokens/s: 197665.4 | physical tokens/s: 198093.6 | loss tokens/s: 146175.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.371835E-01 | lm-global-supervised-tokens: 1.240687E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.730416E-02 | lm-token-buffer-mib: 5.812325E+01 | global_load_balancing_loss: 1.000316E+00 | loss scale: 1.0 | grad norm: 0.060 | params norm: 11373.744 | params rms: 0.061585 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 15:11:09.812021 | finish at 2026-10-10 15:52:17 + [2026-10-08 00:42:32] iteration 4176/ 6855 | consumed samples: 534528 | elapsed time per iteration (ms): 85417.3 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.57% | FLOPs estimator: workload-aware | logical tokens/s: 196414.7 | valid tokens/s: 196414.7 | physical tokens/s: 196868.4 | loss tokens/s: 145294.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.762269E-01 | lm-global-supervised-tokens: 1.241067E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.269666E-02 | lm-token-buffer-mib: 5.812952E+01 | global_load_balancing_loss: 1.001440E+00 | loss scale: 1.0 | grad norm: 0.077 | params norm: 11373.685 | params rms: 0.061585 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 15:33:53.000315 | finish at 2026-10-10 16:16:25 + [2026-10-08 00:43:58] iteration 4177/ 6855 | consumed samples: 534656 | elapsed time per iteration (ms): 85865.2 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.53% | FLOPs estimator: workload-aware | logical tokens/s: 195389.6 | valid tokens/s: 195389.6 | physical tokens/s: 195867.1 | loss tokens/s: 148292.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.517324E-01 | lm-global-supervised-tokens: 1.273314E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.917602E-02 | lm-token-buffer-mib: 5.813358E+01 | global_load_balancing_loss: 1.001451E+00 | loss scale: 1.0 | grad norm: 0.055 | params norm: 11373.626 | params rms: 0.061584 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 15:52:26.929735 | finish at 2026-10-10 16:36:25 + [2026-10-08 00:45:23] iteration 4178/ 6855 | consumed samples: 534784 | elapsed time per iteration (ms): 84831.9 | throughput per GPU (TFLOP/s/GPU): 41.7 | MFU 13.38% | FLOPs estimator: workload-aware | logical tokens/s: 197769.9 | valid tokens/s: 197769.9 | physical tokens/s: 198200.8 | loss tokens/s: 151811.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.358635E-01 | lm-global-supervised-tokens: 1.287846E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.418981E-02 | lm-token-buffer-mib: 5.805162E+01 | global_load_balancing_loss: 1.000414E+00 | loss scale: 1.0 | grad norm: 0.049 | params norm: 11373.567 | params rms: 0.061584 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 15:04:54.924499 | finish at 2026-10-10 15:50:18 + [2026-10-08 00:46:48] iteration 4179/ 6855 | consumed samples: 534912 | elapsed time per iteration (ms): 85434.2 | throughput per GPU (TFLOP/s/GPU): 43.8 | MFU 14.03% | FLOPs estimator: workload-aware | logical tokens/s: 196375.0 | valid tokens/s: 196375.0 | physical tokens/s: 196713.4 | loss tokens/s: 153876.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.525594E-01 | lm-global-supervised-tokens: 1.314632E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.864824E-02 | lm-token-buffer-mib: 5.829267E+01 | global_load_balancing_loss: 9.990135E-01 | loss scale: 1.0 | grad norm: 0.045 | params norm: 11373.508 | params rms: 0.061584 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 15:30:21.841493 | finish at 2026-10-10 16:17:10 + [2026-10-08 00:48:35] iteration 4180/ 6855 | consumed samples: 535040 | elapsed time per iteration (ms): 106457.3 | throughput per GPU (TFLOP/s/GPU): 34.6 | MFU 11.10% | FLOPs estimator: workload-aware | logical tokens/s: 157595.7 | valid tokens/s: 157595.6 | physical tokens/s: 158063.7 | loss tokens/s: 119387.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.220026E-01 | lm-global-supervised-tokens: 1.270972E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.160485E-02 | lm-token-buffer-mib: 5.821397E+01 | global_load_balancing_loss: 1.000847E+00 | loss scale: 1.0 | grad norm: 0.079 | params norm: 11373.446 | params rms: 0.061584 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 7:06:13.278636 | finish at 2026-10-11 07:54:48 + [2026-10-08 00:50:01] iteration 4181/ 6855 | consumed samples: 535168 | elapsed time per iteration (ms): 86650.1 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.78% | FLOPs estimator: workload-aware | logical tokens/s: 193619.8 | valid tokens/s: 193619.8 | physical tokens/s: 194192.3 | loss tokens/s: 147403.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.515163E-01 | lm-global-supervised-tokens: 1.277256E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.276883E-02 | lm-token-buffer-mib: 5.836481E+01 | global_load_balancing_loss: 1.000989E+00 | loss scale: 1.0 | grad norm: 0.138 | params norm: 11373.385 | params rms: 0.061583 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 16:21:42.406908 | finish at 2026-10-10 17:11:44 + [2026-10-08 00:51:30] iteration 4182/ 6855 | consumed samples: 535296 | elapsed time per iteration (ms): 88369.6 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.79% | FLOPs estimator: workload-aware | logical tokens/s: 189852.6 | valid tokens/s: 189852.6 | physical tokens/s: 190555.6 | loss tokens/s: 131947.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.520234E-01 | lm-global-supervised-tokens: 1.166016E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.455804E-02 | lm-token-buffer-mib: 5.807276E+01 | global_load_balancing_loss: 1.000844E+00 | loss scale: 1.0 | grad norm: 0.053 | params norm: 11373.325 | params rms: 0.061583 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 17:36:51.809034 | finish at 2026-10-10 18:28:22 + [2026-10-08 00:52:55] iteration 4183/ 6855 | consumed samples: 535424 | elapsed time per iteration (ms): 85173.2 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.69% | FLOPs estimator: workload-aware | logical tokens/s: 196977.6 | valid tokens/s: 196977.6 | physical tokens/s: 197413.8 | loss tokens/s: 143241.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.293387E-01 | lm-global-supervised-tokens: 1.220036E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.513570E-02 | lm-token-buffer-mib: 5.817166E+01 | global_load_balancing_loss: 9.996667E-01 | loss scale: 1.0 | grad norm: 0.140 | params norm: 11373.264 | params rms: 0.061583 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 15:13:02.811134 | finish at 2026-10-10 16:05:58 + [2026-10-08 00:54:25] iteration 4184/ 6855 | consumed samples: 535552 | elapsed time per iteration (ms): 89715.0 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.51% | FLOPs estimator: workload-aware | logical tokens/s: 187005.7 | valid tokens/s: 187005.7 | physical tokens/s: 187258.9 | loss tokens/s: 137888.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.930493E-01 | lm-global-supervised-tokens: 1.237070E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.766011E-02 | lm-token-buffer-mib: 5.809752E+01 | global_load_balancing_loss: 9.997367E-01 | loss scale: 1.0 | grad norm: 0.062 | params norm: 11373.203 | params rms: 0.061582 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 18:33:48.805527 | finish at 2026-10-10 19:28:13 + [2026-10-08 00:55:54] iteration 4185/ 6855 | consumed samples: 535680 | elapsed time per iteration (ms): 88973.4 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.66% | FLOPs estimator: workload-aware | logical tokens/s: 188564.1 | valid tokens/s: 188564.1 | physical tokens/s: 189241.5 | loss tokens/s: 133309.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.796744E-01 | lm-global-supervised-tokens: 1.186103E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.279446E-02 | lm-token-buffer-mib: 5.811292E+01 | global_load_balancing_loss: 9.998966E-01 | loss scale: 1.0 | grad norm: 0.174 | params norm: 11373.143 | params rms: 0.061582 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 17:59:18.940115 | finish at 2026-10-10 18:55:13 + [2026-10-08 00:57:21] iteration 4186/ 6855 | consumed samples: 535808 | elapsed time per iteration (ms): 87739.4 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.47% | FLOPs estimator: workload-aware | logical tokens/s: 191215.8 | valid tokens/s: 191215.8 | physical tokens/s: 191614.4 | loss tokens/s: 137750.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.393639E-01 | lm-global-supervised-tokens: 1.208614E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.599615E-02 | lm-token-buffer-mib: 5.813900E+01 | global_load_balancing_loss: 9.996337E-01 | loss scale: 1.0 | grad norm: 0.055 | params norm: 11373.082 | params rms: 0.061582 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 17:02:56.582446 | finish at 2026-10-10 18:00:18 + [2026-10-08 00:58:49] iteration 4187/ 6855 | consumed samples: 535936 | elapsed time per iteration (ms): 88136.9 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.45% | FLOPs estimator: workload-aware | logical tokens/s: 190353.3 | valid tokens/s: 190353.3 | physical tokens/s: 190904.4 | loss tokens/s: 147136.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.016394E-01 | lm-global-supervised-tokens: 1.296817E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.175939E-02 | lm-token-buffer-mib: 5.802612E+01 | global_load_balancing_loss: 9.993168E-01 | loss scale: 1.0 | grad norm: 0.386 | params norm: 11373.020 | params rms: 0.061581 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 17:19:09.271961 | finish at 2026-10-10 18:17:59 + [2026-10-08 01:00:16] iteration 4188/ 6855 | consumed samples: 536064 | elapsed time per iteration (ms): 86273.5 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.50% | FLOPs estimator: workload-aware | logical tokens/s: 194465.4 | valid tokens/s: 194465.4 | physical tokens/s: 194943.5 | loss tokens/s: 149086.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.982238E-01 | lm-global-supervised-tokens: 1.286220E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.238159E-02 | lm-token-buffer-mib: 5.813878E+01 | global_load_balancing_loss: 1.000397E+00 | loss scale: 1.0 | grad norm: 0.325 | params norm: 11372.957 | params rms: 0.061581 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 15:54:51.340475 | finish at 2026-10-10 16:55:07 + [2026-10-08 01:01:41] iteration 4189/ 6855 | consumed samples: 536192 | elapsed time per iteration (ms): 85717.9 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.62% | FLOPs estimator: workload-aware | logical tokens/s: 195725.6 | valid tokens/s: 195725.6 | physical tokens/s: 196249.2 | loss tokens/s: 146869.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.246060E-01 | lm-global-supervised-tokens: 1.258938E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.250052E-02 | lm-token-buffer-mib: 5.821167E+01 | global_load_balancing_loss: 1.001156E+00 | loss scale: 1.0 | grad norm: 0.205 | params norm: 11372.896 | params rms: 0.061581 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 15:28:43.849675 | finish at 2026-10-10 16:30:25 + [2026-10-08 01:03:27] iteration 4190/ 6855 | consumed samples: 536320 | elapsed time per iteration (ms): 105765.5 | throughput per GPU (TFLOP/s/GPU): 33.9 | MFU 10.88% | FLOPs estimator: workload-aware | logical tokens/s: 158626.2 | valid tokens/s: 158626.2 | physical tokens/s: 158996.4 | loss tokens/s: 121812.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.162685E-01 | lm-global-supervised-tokens: 1.288350E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.267145E-02 | lm-token-buffer-mib: 5.839675E+01 | global_load_balancing_loss: 1.001131E+00 | loss scale: 1.0 | grad norm: 0.052 | params norm: 11372.837 | params rms: 0.061580 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 6:17:45.105972 | finish at 2026-10-11 07:21:12 + [2026-10-08 01:04:55] iteration 4191/ 6855 | consumed samples: 536448 | elapsed time per iteration (ms): 87706.4 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.59% | FLOPs estimator: workload-aware | logical tokens/s: 191288.0 | valid tokens/s: 191288.0 | physical tokens/s: 191642.4 | loss tokens/s: 137174.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 5.847415E-01 | lm-global-supervised-tokens: 1.203103E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.093309E-02 | lm-token-buffer-mib: 5.815701E+01 | global_load_balancing_loss: 1.000465E+00 | loss scale: 1.0 | grad norm: 0.143 | params norm: 11372.778 | params rms: 0.061580 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 16:54:09.763000 | finish at 2026-10-10 17:59:05 + [2026-10-08 01:06:25] iteration 4192/ 6855 | consumed samples: 536576 | elapsed time per iteration (ms): 89814.3 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.48% | FLOPs estimator: workload-aware | logical tokens/s: 186798.9 | valid tokens/s: 186798.9 | physical tokens/s: 187169.6 | loss tokens/s: 132865.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.393069E-01 | lm-global-supervised-tokens: 1.193321E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.357816E-02 | lm-token-buffer-mib: 5.811729E+01 | global_load_balancing_loss: 1.000662E+00 | loss scale: 1.0 | grad norm: 0.186 | params norm: 11372.718 | params rms: 0.061580 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 18:26:15.528679 | finish at 2026-10-10 19:32:40 + [2026-10-08 01:07:52] iteration 4193/ 6855 | consumed samples: 536704 | elapsed time per iteration (ms): 87179.8 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.68% | FLOPs estimator: workload-aware | logical tokens/s: 192443.8 | valid tokens/s: 192443.8 | physical tokens/s: 192770.8 | loss tokens/s: 143257.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.706846E-01 | lm-global-supervised-tokens: 1.248915E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.381762E-02 | lm-token-buffer-mib: 5.808804E+01 | global_load_balancing_loss: 1.000328E+00 | loss scale: 1.0 | grad norm: 0.042 | params norm: 11372.655 | params rms: 0.061579 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 16:27:52.553433 | finish at 2026-10-10 17:35:44 + [2026-10-08 01:09:20] iteration 4194/ 6855 | consumed samples: 536832 | elapsed time per iteration (ms): 88132.2 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.48% | FLOPs estimator: workload-aware | logical tokens/s: 190363.9 | valid tokens/s: 190363.9 | physical tokens/s: 190685.9 | loss tokens/s: 143590.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.505402E-01 | lm-global-supervised-tokens: 1.265495E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.145471E-02 | lm-token-buffer-mib: 5.814063E+01 | global_load_balancing_loss: 1.000513E+00 | loss scale: 1.0 | grad norm: 0.071 | params norm: 11372.592 | params rms: 0.061579 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 17:08:39.746776 | finish at 2026-10-10 18:18:00 + [2026-10-08 01:10:47] iteration 4195/ 6855 | consumed samples: 536960 | elapsed time per iteration (ms): 86495.4 | throughput per GPU (TFLOP/s/GPU): 43.1 | MFU 13.81% | FLOPs estimator: workload-aware | logical tokens/s: 193965.9 | valid tokens/s: 193965.9 | physical tokens/s: 194236.3 | loss tokens/s: 142742.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.488847E-01 | lm-global-supervised-tokens: 1.234658E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.152103E-02 | lm-token-buffer-mib: 5.805192E+01 | global_load_balancing_loss: 1.000899E+00 | loss scale: 1.0 | grad norm: 0.113 | params norm: 11372.530 | params rms: 0.061579 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 15:54:37.687769 | finish at 2026-10-10 17:05:24 + [2026-10-08 01:12:12] iteration 4196/ 6855 | consumed samples: 537088 | elapsed time per iteration (ms): 85729.4 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.74% | FLOPs estimator: workload-aware | logical tokens/s: 195699.3 | valid tokens/s: 195699.3 | physical tokens/s: 196293.6 | loss tokens/s: 147047.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.270057E-01 | lm-global-supervised-tokens: 1.260629E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.147591E-02 | lm-token-buffer-mib: 5.819987E+01 | global_load_balancing_loss: 1.000028E+00 | loss scale: 1.0 | grad norm: 0.108 | params norm: 11372.467 | params rms: 0.061578 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 15:19:14.402115 | finish at 2026-10-10 16:31:27 + [2026-10-08 01:13:39] iteration 4197/ 6855 | consumed samples: 537216 | elapsed time per iteration (ms): 86721.5 | throughput per GPU (TFLOP/s/GPU): 43.6 | MFU 13.96% | FLOPs estimator: workload-aware | logical tokens/s: 193460.4 | valid tokens/s: 193460.3 | physical tokens/s: 193952.0 | loss tokens/s: 143060.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.578773E-01 | lm-global-supervised-tokens: 1.240645E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.217716E-02 | lm-token-buffer-mib: 5.815251E+01 | global_load_balancing_loss: 9.994288E-01 | loss scale: 1.0 | grad norm: 0.069 | params norm: 11372.408 | params rms: 0.061578 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 16:01:45.770868 | finish at 2026-10-10 17:15:25 + [2026-10-08 01:15:07] iteration 4198/ 6855 | consumed samples: 537344 | elapsed time per iteration (ms): 88224.4 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.45% | FLOPs estimator: workload-aware | logical tokens/s: 190164.8 | valid tokens/s: 190164.8 | physical tokens/s: 190568.8 | loss tokens/s: 140984.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.375427E-01 | lm-global-supervised-tokens: 1.243830E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.442081E-02 | lm-token-buffer-mib: 5.805303E+01 | global_load_balancing_loss: 9.989466E-01 | loss scale: 1.0 | grad norm: 0.099 | params norm: 11372.348 | params rms: 0.061578 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 17:06:52.173903 | finish at 2026-10-10 18:21:59 + [2026-10-08 01:16:33] iteration 4199/ 6855 | consumed samples: 537472 | elapsed time per iteration (ms): 85539.6 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.62% | FLOPs estimator: workload-aware | logical tokens/s: 196132.1 | valid tokens/s: 196132.1 | physical tokens/s: 196560.7 | loss tokens/s: 147328.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.590097E-01 | lm-global-supervised-tokens: 1.260242E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.418693E-02 | lm-token-buffer-mib: 5.822100E+01 | global_load_balancing_loss: 9.988230E-01 | loss scale: 1.0 | grad norm: 0.179 | params norm: 11372.288 | params rms: 0.061577 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 15:06:33.282440 | finish at 2026-10-10 16:23:06 + [2026-10-08 01:18:21] iteration 4200/ 6855 | consumed samples: 537600 | elapsed time per iteration (ms): 108183.4 | throughput per GPU (TFLOP/s/GPU): 34.0 | MFU 10.89% | FLOPs estimator: workload-aware | logical tokens/s: 155081.0 | valid tokens/s: 155081.0 | physical tokens/s: 155402.5 | loss tokens/s: 116285.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.371759E-01 | lm-global-supervised-tokens: 1.258012E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.256109E-02 | lm-token-buffer-mib: 5.841765E+01 | global_load_balancing_loss: 1.000942E+00 | loss scale: 1.0 | grad norm: 0.146 | params norm: 11372.227 | params rms: 0.061577 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 7:47:06.834991 | finish at 2026-10-11 09:05:28 +times across ranks (ms): + save-checkpoint: + rank 0: 26020.51 + rank 1: 26020.58 + rank 2: 26020.54 + rank 3: 26020.51 + rank 4: 26020.48 + rank 5: 26020.45 + rank 6: 26020.45 + rank 7: 26020.46 + rank 8: 26019.19 + rank 9: 26019.16 + rank 10: 26019.16 + rank 11: 26019.22 + rank 12: 26019.16 + rank 13: 26019.20 + rank 14: 26019.17 + rank 15: 26019.21 + rank 16: 26020.35 + rank 17: 26020.34 + rank 18: 26020.29 + rank 19: 26020.34 + rank 20: 26020.31 + rank 21: 26020.34 + rank 22: 26020.30 + rank 23: 26020.28 + rank 24: 26020.92 + rank 25: 26021.19 + rank 26: 26020.94 + rank 27: 26020.98 + rank 28: 26020.88 + rank 29: 26020.93 + rank 30: 26020.93 + rank 31: 26020.91 + rank 32: 26020.82 + rank 33: 26020.78 + rank 34: 26020.66 + rank 35: 26020.71 + rank 36: 26020.79 + rank 37: 26020.67 + rank 38: 26020.65 + rank 39: 26020.76 + rank 40: 26020.32 + rank 41: 26020.29 + rank 42: 26020.31 + rank 43: 26020.32 + rank 44: 26020.29 + rank 45: 26020.29 + rank 46: 26020.31 + rank 47: 26020.29 + rank 48: 26020.09 + rank 49: 26020.25 + rank 50: 26020.18 + rank 51: 26020.17 + rank 52: 26020.26 + rank 53: 26020.23 + rank 54: 26020.24 + rank 55: 26020.25 + rank 56: 26019.74 + rank 57: 26019.71 + rank 58: 26019.71 + rank 59: 26019.69 + rank 60: 26019.71 + rank 61: 26019.71 + rank 62: 26019.71 + rank 63: 26019.55 + [2026-10-08 01:20:11] iteration 4201/ 6855 | consumed samples: 537728 | elapsed time per iteration (ms): 83705.2 | throughput per GPU (TFLOP/s/GPU): 44.1 | MFU 14.13% | FLOPs estimator: workload-aware | logical tokens/s: 200431.7 | valid tokens/s: 200431.7 | physical tokens/s: 200886.3 | loss tokens/s: 150164.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.285137E-01 | lm-global-supervised-tokens: 1.256951E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.294717E+00 | lm-token-buffer-mib: 5.820929E+01 | global_load_balancing_loss: 1.000960E+00 | loss scale: 1.0 | grad norm: 0.095 | params norm: 11372.168 | params rms: 0.061577 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 13:42:33.708888 | finish at 2026-10-10 15:02:45 + [2026-10-08 01:21:42] iteration 4202/ 6855 | consumed samples: 537856 | elapsed time per iteration (ms): 90854.4 | throughput per GPU (TFLOP/s/GPU): 40.4 | MFU 12.95% | FLOPs estimator: workload-aware | logical tokens/s: 184660.3 | valid tokens/s: 184660.3 | physical tokens/s: 185082.6 | loss tokens/s: 141174.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.442933E-01 | lm-global-supervised-tokens: 1.282630E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.362759E-02 | lm-token-buffer-mib: 5.807417E+01 | global_load_balancing_loss: 1.000860E+00 | loss scale: 1.0 | grad norm: 0.128 | params norm: 11372.108 | params rms: 0.061576 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 18:57:16.768528 | finish at 2026-10-10 20:18:58 + [2026-10-08 01:23:07] iteration 4203/ 6855 | consumed samples: 537984 | elapsed time per iteration (ms): 85224.2 | throughput per GPU (TFLOP/s/GPU): 44.6 | MFU 14.28% | FLOPs estimator: workload-aware | logical tokens/s: 196859.7 | valid tokens/s: 196859.7 | physical tokens/s: 197365.3 | loss tokens/s: 143627.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.814066E-01 | lm-global-supervised-tokens: 1.224055E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.323398E-02 | lm-token-buffer-mib: 5.804829E+01 | global_load_balancing_loss: 1.001365E+00 | loss scale: 1.0 | grad norm: 0.077 | params norm: 11372.047 | params rms: 0.061576 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 14:46:54.637862 | finish at 2026-10-10 16:10:02 + [2026-10-08 01:24:38] iteration 4204/ 6855 | consumed samples: 538112 | elapsed time per iteration (ms): 90806.2 | throughput per GPU (TFLOP/s/GPU): 41.7 | MFU 13.38% | FLOPs estimator: workload-aware | logical tokens/s: 184758.1 | valid tokens/s: 184758.1 | physical tokens/s: 185232.4 | loss tokens/s: 135848.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.069427E-01 | lm-global-supervised-tokens: 1.233586E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.187569E-02 | lm-token-buffer-mib: 5.810435E+01 | global_load_balancing_loss: 1.000560E+00 | loss scale: 1.0 | grad norm: 0.064 | params norm: 11371.984 | params rms: 0.061576 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 18:52:07.110495 | finish at 2026-10-10 20:16:45 + [2026-10-08 01:26:07] iteration 4205/ 6855 | consumed samples: 538240 | elapsed time per iteration (ms): 88958.4 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.49% | FLOPs estimator: workload-aware | logical tokens/s: 188596.2 | valid tokens/s: 188596.2 | physical tokens/s: 189089.9 | loss tokens/s: 135073.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.595814E-01 | lm-global-supervised-tokens: 1.201590E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.511361E-02 | lm-token-buffer-mib: 5.822651E+01 | global_load_balancing_loss: 9.997053E-01 | loss scale: 1.0 | grad norm: 0.140 | params norm: 11371.921 | params rms: 0.061575 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 17:28:59.782143 | finish at 2026-10-10 18:55:06 + [2026-10-08 01:27:36] iteration 4206/ 6855 | consumed samples: 538368 | elapsed time per iteration (ms): 89768.7 | throughput per GPU (TFLOP/s/GPU): 41.5 | MFU 13.31% | FLOPs estimator: workload-aware | logical tokens/s: 186893.4 | valid tokens/s: 186893.4 | physical tokens/s: 187200.1 | loss tokens/s: 142666.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.574345E-01 | lm-global-supervised-tokens: 1.280702E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.179728E-02 | lm-token-buffer-mib: 5.816510E+01 | global_load_balancing_loss: 9.999653E-01 | loss scale: 1.0 | grad norm: 0.089 | params norm: 11371.859 | params rms: 0.061575 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 18:03:17.404098 | finish at 2026-10-10 19:30:54 + [2026-10-08 01:29:05] iteration 4207/ 6855 | consumed samples: 538496 | elapsed time per iteration (ms): 88144.2 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.54% | FLOPs estimator: workload-aware | logical tokens/s: 190338.2 | valid tokens/s: 190338.2 | physical tokens/s: 190789.2 | loss tokens/s: 137004.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.324561E-01 | lm-global-supervised-tokens: 1.207612E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.315999E-02 | lm-token-buffer-mib: 5.807968E+01 | global_load_balancing_loss: 9.999650E-01 | loss scale: 1.0 | grad norm: 0.178 | params norm: 11371.798 | params rms: 0.061575 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 16:50:05.779327 | finish at 2026-10-10 18:19:10 + [2026-10-08 01:30:33] iteration 4208/ 6855 | consumed samples: 538624 | elapsed time per iteration (ms): 88279.9 | throughput per GPU (TFLOP/s/GPU): 41.0 | MFU 13.14% | FLOPs estimator: workload-aware | logical tokens/s: 190045.6 | valid tokens/s: 190045.5 | physical tokens/s: 190554.5 | loss tokens/s: 150057.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.527961E-01 | lm-global-supervised-tokens: 1.324708E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.437216E-02 | lm-token-buffer-mib: 5.810504E+01 | global_load_balancing_loss: 1.000698E+00 | loss scale: 1.0 | grad norm: 0.114 | params norm: 11371.735 | params rms: 0.061574 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 16:54:36.795152 | finish at 2026-10-10 18:25:10 + [2026-10-08 01:32:01] iteration 4209/ 6855 | consumed samples: 538752 | elapsed time per iteration (ms): 88251.1 | throughput per GPU (TFLOP/s/GPU): 41.3 | MFU 13.25% | FLOPs estimator: workload-aware | logical tokens/s: 190107.4 | valid tokens/s: 190107.4 | physical tokens/s: 190498.7 | loss tokens/s: 142166.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.668155E-01 | lm-global-supervised-tokens: 1.254634E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.465973E-02 | lm-token-buffer-mib: 5.804157E+01 | global_load_balancing_loss: 9.997662E-01 | loss scale: 1.0 | grad norm: 0.142 | params norm: 11371.673 | params rms: 0.061574 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 16:51:52.330018 | finish at 2026-10-10 18:23:53 + [2026-10-08 01:33:49] iteration 4210/ 6855 | consumed samples: 538880 | elapsed time per iteration (ms): 107782.2 | throughput per GPU (TFLOP/s/GPU): 34.0 | MFU 10.90% | FLOPs estimator: workload-aware | logical tokens/s: 155658.0 | valid tokens/s: 155658.0 | physical tokens/s: 156080.2 | loss tokens/s: 117431.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.543677E-01 | lm-global-supervised-tokens: 1.265703E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.382777E-02 | lm-token-buffer-mib: 5.812270E+01 | global_load_balancing_loss: 1.000178E+00 | loss scale: 1.0 | grad norm: 0.055 | params norm: 11371.611 | params rms: 0.061574 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 7:11:23.975384 | finish at 2026-10-11 08:45:13 + [2026-10-08 01:35:16] iteration 4211/ 6855 | consumed samples: 539008 | elapsed time per iteration (ms): 87114.6 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.75% | FLOPs estimator: workload-aware | logical tokens/s: 192587.2 | valid tokens/s: 192587.2 | physical tokens/s: 193055.5 | loss tokens/s: 144071.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.564777E-01 | lm-global-supervised-tokens: 1.255071E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.201890E-02 | lm-token-buffer-mib: 5.818511E+01 | global_load_balancing_loss: 9.997483E-01 | loss scale: 1.0 | grad norm: 0.224 | params norm: 11371.550 | params rms: 0.061573 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 15:58:50.951819 | finish at 2026-10-10 17:34:07 + [2026-10-08 01:36:44] iteration 4212/ 6855 | consumed samples: 539136 | elapsed time per iteration (ms): 87964.6 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.44% | FLOPs estimator: workload-aware | logical tokens/s: 190726.7 | valid tokens/s: 190726.7 | physical tokens/s: 191184.3 | loss tokens/s: 134543.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.172360E-01 | lm-global-supervised-tokens: 1.183508E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.285517E-02 | lm-token-buffer-mib: 5.822118E+01 | global_load_balancing_loss: 1.001311E+00 | loss scale: 1.0 | grad norm: 0.108 | params norm: 11371.485 | params rms: 0.061573 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 16:34:50.533809 | finish at 2026-10-10 18:11:35 + [2026-10-08 01:38:11] iteration 4213/ 6855 | consumed samples: 539264 | elapsed time per iteration (ms): 87227.7 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.64% | FLOPs estimator: workload-aware | logical tokens/s: 192337.5 | valid tokens/s: 192337.5 | physical tokens/s: 192694.6 | loss tokens/s: 141362.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.346895E-01 | lm-global-supervised-tokens: 1.233068E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.275445E-02 | lm-token-buffer-mib: 5.816966E+01 | global_load_balancing_loss: 9.996525E-01 | loss scale: 1.0 | grad norm: 0.097 | params norm: 11371.420 | params rms: 0.061573 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 16:00:55.562600 | finish at 2026-10-10 17:39:07 + [2026-10-08 01:39:43] iteration 4214/ 6855 | consumed samples: 539392 | elapsed time per iteration (ms): 91785.9 | throughput per GPU (TFLOP/s/GPU): 40.4 | MFU 12.96% | FLOPs estimator: workload-aware | logical tokens/s: 182786.3 | valid tokens/s: 182786.3 | physical tokens/s: 183247.6 | loss tokens/s: 136063.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.519916E-01 | lm-global-supervised-tokens: 1.248875E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.705233E-02 | lm-token-buffer-mib: 5.808852E+01 | global_load_balancing_loss: 1.000046E+00 | loss scale: 1.0 | grad norm: 0.083 | params norm: 11371.357 | params rms: 0.061572 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 19:20:06.669879 | finish at 2026-10-10 20:59:50 + [2026-10-08 01:41:14] iteration 4215/ 6855 | consumed samples: 539520 | elapsed time per iteration (ms): 91215.1 | throughput per GPU (TFLOP/s/GPU): 40.9 | MFU 13.11% | FLOPs estimator: workload-aware | logical tokens/s: 183929.6 | valid tokens/s: 183929.6 | physical tokens/s: 184423.8 | loss tokens/s: 138365.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.689170E-01 | lm-global-supervised-tokens: 1.262101E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.315041E-02 | lm-token-buffer-mib: 5.803989E+01 | global_load_balancing_loss: 9.996161E-01 | loss scale: 1.0 | grad norm: 0.097 | params norm: 11371.296 | params rms: 0.061572 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 18:53:27.897491 | finish at 2026-10-10 20:34:42 + [2026-10-08 01:42:43] iteration 4216/ 6855 | consumed samples: 539648 | elapsed time per iteration (ms): 88307.8 | throughput per GPU (TFLOP/s/GPU): 41.4 | MFU 13.28% | FLOPs estimator: workload-aware | logical tokens/s: 189985.3 | valid tokens/s: 189985.3 | physical tokens/s: 190704.4 | loss tokens/s: 146165.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.951940E-01 | lm-global-supervised-tokens: 1.290757E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.159627E-02 | lm-token-buffer-mib: 5.809687E+01 | global_load_balancing_loss: 1.000164E+00 | loss scale: 1.0 | grad norm: 0.145 | params norm: 11371.235 | params rms: 0.061572 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 16:44:04.252255 | finish at 2026-10-10 18:26:47 + [2026-10-08 01:44:11] iteration 4217/ 6855 | consumed samples: 539776 | elapsed time per iteration (ms): 88831.1 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.60% | FLOPs estimator: workload-aware | logical tokens/s: 188866.2 | valid tokens/s: 188866.2 | physical tokens/s: 189258.7 | loss tokens/s: 131293.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.988775E-01 | lm-global-supervised-tokens: 1.166297E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.844914E-02 | lm-token-buffer-mib: 5.805230E+01 | global_load_balancing_loss: 1.000205E+00 | loss scale: 1.0 | grad norm: 0.112 | params norm: 11371.173 | params rms: 0.061571 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 17:05:36.324152 | finish at 2026-10-10 18:49:48 + [2026-10-08 01:45:36] iteration 4218/ 6855 | consumed samples: 539904 | elapsed time per iteration (ms): 84484.8 | throughput per GPU (TFLOP/s/GPU): 43.2 | MFU 13.83% | FLOPs estimator: workload-aware | logical tokens/s: 198582.2 | valid tokens/s: 198582.2 | physical tokens/s: 199051.4 | loss tokens/s: 143602.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.388214E-01 | lm-global-supervised-tokens: 1.213219E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.312081E-02 | lm-token-buffer-mib: 5.815394E+01 | global_load_balancing_loss: 1.000161E+00 | loss scale: 1.0 | grad norm: 0.104 | params norm: 11371.110 | params rms: 0.061571 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 13:53:06.391459 | finish at 2026-10-10 15:38:42 + [2026-10-08 01:47:03] iteration 4219/ 6855 | consumed samples: 540032 | elapsed time per iteration (ms): 87354.3 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.57% | FLOPs estimator: workload-aware | logical tokens/s: 192059.3 | valid tokens/s: 192059.3 | physical tokens/s: 192522.1 | loss tokens/s: 146793.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.509394E-01 | lm-global-supervised-tokens: 1.282304E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.053489E-02 | lm-token-buffer-mib: 5.824077E+01 | global_load_balancing_loss: 9.996112E-01 | loss scale: 1.0 | grad norm: 0.053 | params norm: 11371.049 | params rms: 0.061571 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 15:57:45.878296 | finish at 2026-10-10 17:44:49 + [2026-10-08 01:48:53] iteration 4220/ 6855 | consumed samples: 540160 | elapsed time per iteration (ms): 109439.5 | throughput per GPU (TFLOP/s/GPU): 33.9 | MFU 10.87% | FLOPs estimator: workload-aware | logical tokens/s: 153301.0 | valid tokens/s: 153301.0 | physical tokens/s: 153790.3 | loss tokens/s: 116208.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.094411E-01 | lm-global-supervised-tokens: 1.271784E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.281258E-02 | lm-token-buffer-mib: 5.804259E+01 | global_load_balancing_loss: 9.997542E-01 | loss scale: 1.0 | grad norm: 0.167 | params norm: 11370.989 | params rms: 0.061570 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 8:06:13.148711 | finish at 2026-10-11 09:55:06 + [2026-10-08 01:50:20] iteration 4221/ 6855 | consumed samples: 540288 | elapsed time per iteration (ms): 87362.7 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.48% | FLOPs estimator: workload-aware | logical tokens/s: 192040.9 | valid tokens/s: 192040.9 | physical tokens/s: 192471.6 | loss tokens/s: 143254.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.590130E-01 | lm-global-supervised-tokens: 1.251510E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.193974E-02 | lm-token-buffer-mib: 5.813934E+01 | global_load_balancing_loss: 1.002072E+00 | loss scale: 1.0 | grad norm: 0.100 | params norm: 11370.929 | params rms: 0.061570 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 15:55:13.275146 | finish at 2026-10-10 17:45:33 + [2026-10-08 01:51:49] iteration 4222/ 6855 | consumed samples: 540416 | elapsed time per iteration (ms): 88742.8 | throughput per GPU (TFLOP/s/GPU): 41.0 | MFU 13.14% | FLOPs estimator: workload-aware | logical tokens/s: 189054.0 | valid tokens/s: 189054.0 | physical tokens/s: 189516.0 | loss tokens/s: 145019.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.432654E-01 | lm-global-supervised-tokens: 1.286939E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.694651E-02 | lm-token-buffer-mib: 5.808218E+01 | global_load_balancing_loss: 1.000238E+00 | loss scale: 1.0 | grad norm: 0.163 | params norm: 11370.868 | params rms: 0.061570 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 16:54:19.733384 | finish at 2026-10-10 18:46:09 + [2026-10-08 01:53:19] iteration 4223/ 6855 | consumed samples: 540544 | elapsed time per iteration (ms): 90168.4 | throughput per GPU (TFLOP/s/GPU): 41.1 | MFU 13.18% | FLOPs estimator: workload-aware | logical tokens/s: 186064.8 | valid tokens/s: 186064.8 | physical tokens/s: 186515.6 | loss tokens/s: 138481.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.347623E-01 | lm-global-supervised-tokens: 1.248661E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.291478E-02 | lm-token-buffer-mib: 5.807681E+01 | global_load_balancing_loss: 1.002309E+00 | loss scale: 1.0 | grad norm: 0.362 | params norm: 11370.809 | params rms: 0.061569 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 17:55:23.174963 | finish at 2026-10-10 19:48:42 + [2026-10-08 01:54:51] iteration 4224/ 6855 | consumed samples: 540672 | elapsed time per iteration (ms): 92307.6 | throughput per GPU (TFLOP/s/GPU): 41.4 | MFU 13.26% | FLOPs estimator: workload-aware | logical tokens/s: 181752.9 | valid tokens/s: 181752.9 | physical tokens/s: 182144.4 | loss tokens/s: 130705.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.612774E-01 | lm-global-supervised-tokens: 1.206507E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.742319E-02 | lm-token-buffer-mib: 5.817149E+01 | global_load_balancing_loss: 1.000556E+00 | loss scale: 1.0 | grad norm: 0.184 | params norm: 11370.750 | params rms: 0.061569 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 19:27:41.342075 | finish at 2026-10-10 21:22:33 + [2026-10-08 01:56:17] iteration 4225/ 6855 | consumed samples: 540800 | elapsed time per iteration (ms): 86075.2 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.59% | FLOPs estimator: workload-aware | logical tokens/s: 194912.7 | valid tokens/s: 194912.7 | physical tokens/s: 195461.0 | loss tokens/s: 148360.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.617249E-01 | lm-global-supervised-tokens: 1.277011E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.521234E-02 | lm-token-buffer-mib: 5.809175E+01 | global_load_balancing_loss: 9.999717E-01 | loss scale: 1.0 | grad norm: 0.211 | params norm: 11370.691 | params rms: 0.061569 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 14:52:57.675259 | finish at 2026-10-10 16:49:15 + [2026-10-08 01:57:47] iteration 4226/ 6855 | consumed samples: 540928 | elapsed time per iteration (ms): 89370.1 | throughput per GPU (TFLOP/s/GPU): 41.7 | MFU 13.37% | FLOPs estimator: workload-aware | logical tokens/s: 187726.4 | valid tokens/s: 187726.4 | physical tokens/s: 188114.0 | loss tokens/s: 136736.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.644878E-01 | lm-global-supervised-tokens: 1.222011E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.183880E-02 | lm-token-buffer-mib: 5.816920E+01 | global_load_balancing_loss: 9.993277E-01 | loss scale: 1.0 | grad norm: 0.055 | params norm: 11370.632 | params rms: 0.061568 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 17:15:53.894548 | finish at 2026-10-10 19:13:41 + [2026-10-08 01:59:17] iteration 4227/ 6855 | consumed samples: 541056 | elapsed time per iteration (ms): 90323.1 | throughput per GPU (TFLOP/s/GPU): 41.3 | MFU 13.25% | FLOPs estimator: workload-aware | logical tokens/s: 185746.4 | valid tokens/s: 185746.4 | physical tokens/s: 186162.2 | loss tokens/s: 140799.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.499432E-01 | lm-global-supervised-tokens: 1.271742E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.317254E-02 | lm-token-buffer-mib: 5.826345E+01 | global_load_balancing_loss: 9.981770E-01 | loss scale: 1.0 | grad norm: 0.175 | params norm: 11370.572 | params rms: 0.061568 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 17:56:09.218565 | finish at 2026-10-10 19:55:26 + [2026-10-08 02:00:43] iteration 4228/ 6855 | consumed samples: 541184 | elapsed time per iteration (ms): 86124.2 | throughput per GPU (TFLOP/s/GPU): 43.1 | MFU 13.82% | FLOPs estimator: workload-aware | logical tokens/s: 194802.2 | valid tokens/s: 194802.2 | physical tokens/s: 195231.3 | loss tokens/s: 146283.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.959634E-01 | lm-global-supervised-tokens: 1.259852E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.265813E-02 | lm-token-buffer-mib: 5.807256E+01 | global_load_balancing_loss: 9.986553E-01 | loss scale: 1.0 | grad norm: 0.132 | params norm: 11370.512 | params rms: 0.061568 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 14:50:48.351968 | finish at 2026-10-10 16:51:31 + [2026-10-08 02:02:13] iteration 4229/ 6855 | consumed samples: 541312 | elapsed time per iteration (ms): 90325.9 | throughput per GPU (TFLOP/s/GPU): 42.8 | MFU 13.72% | FLOPs estimator: workload-aware | logical tokens/s: 185740.5 | valid tokens/s: 185740.4 | physical tokens/s: 186253.6 | loss tokens/s: 129130.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.657402E-01 | lm-global-supervised-tokens: 1.166382E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.208501E-02 | lm-token-buffer-mib: 5.814638E+01 | global_load_balancing_loss: 1.000118E+00 | loss scale: 1.0 | grad norm: 0.083 | params norm: 11370.453 | params rms: 0.061567 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 17:53:15.779170 | finish at 2026-10-10 19:55:29 + [2026-10-08 02:04:04] iteration 4230/ 6855 | consumed samples: 541440 | elapsed time per iteration (ms): 110839.7 | throughput per GPU (TFLOP/s/GPU): 33.6 | MFU 10.77% | FLOPs estimator: workload-aware | logical tokens/s: 151364.7 | valid tokens/s: 151364.7 | physical tokens/s: 151816.4 | loss tokens/s: 111489.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.572681E-01 | lm-global-supervised-tokens: 1.235750E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.296096E-02 | lm-token-buffer-mib: 5.843720E+01 | global_load_balancing_loss: 1.001009E+00 | loss scale: 1.0 | grad norm: 0.121 | params norm: 11370.392 | params rms: 0.061567 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 8:49:14.210579 | finish at 2026-10-11 10:53:19 + [2026-10-08 02:05:34] iteration 4231/ 6855 | consumed samples: 541568 | elapsed time per iteration (ms): 89532.6 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.38% | FLOPs estimator: workload-aware | logical tokens/s: 187386.6 | valid tokens/s: 187386.6 | physical tokens/s: 187738.0 | loss tokens/s: 138873.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.037151E-01 | lm-global-supervised-tokens: 1.243369E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.358002E-02 | lm-token-buffer-mib: 5.808974E+01 | global_load_balancing_loss: 1.000585E+00 | loss scale: 1.0 | grad norm: 0.132 | params norm: 11370.333 | params rms: 0.061567 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 17:15:33.579742 | finish at 2026-10-10 19:21:07 + [2026-10-08 02:07:02] iteration 4232/ 6855 | consumed samples: 541696 | elapsed time per iteration (ms): 87705.1 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.57% | FLOPs estimator: workload-aware | logical tokens/s: 191290.9 | valid tokens/s: 191290.9 | physical tokens/s: 191814.2 | loss tokens/s: 140584.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.475616E-01 | lm-global-supervised-tokens: 1.232997E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.145752E-02 | lm-token-buffer-mib: 5.804757E+01 | global_load_balancing_loss: 1.002190E+00 | loss scale: 1.0 | grad norm: 0.077 | params norm: 11370.274 | params rms: 0.061566 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 15:54:10.498093 | finish at 2026-10-10 18:01:12 + [2026-10-08 02:08:27] iteration 4233/ 6855 | consumed samples: 541824 | elapsed time per iteration (ms): 85301.5 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.69% | FLOPs estimator: workload-aware | logical tokens/s: 196681.0 | valid tokens/s: 196681.0 | physical tokens/s: 197195.1 | loss tokens/s: 155457.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.375676E-01 | lm-global-supervised-tokens: 1.326074E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.194902E-02 | lm-token-buffer-mib: 5.822957E+01 | global_load_balancing_loss: 1.000483E+00 | loss scale: 1.0 | grad norm: 0.174 | params norm: 11370.215 | params rms: 0.061566 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 14:07:40.591352 | finish at 2026-10-10 16:16:07 + [2026-10-08 02:09:55] iteration 4234/ 6855 | consumed samples: 541952 | elapsed time per iteration (ms): 88511.8 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.51% | FLOPs estimator: workload-aware | logical tokens/s: 189546.9 | valid tokens/s: 189546.9 | physical tokens/s: 189953.2 | loss tokens/s: 132357.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.650854E-01 | lm-global-supervised-tokens: 1.171523E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.284935E-02 | lm-token-buffer-mib: 5.814151E+01 | global_load_balancing_loss: 1.001090E+00 | loss scale: 1.0 | grad norm: 0.160 | params norm: 11370.156 | params rms: 0.061566 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 16:26:29.478549 | finish at 2026-10-10 18:36:25 + [2026-10-08 02:11:22] iteration 4235/ 6855 | consumed samples: 542080 | elapsed time per iteration (ms): 86305.3 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.44% | FLOPs estimator: workload-aware | logical tokens/s: 194392.8 | valid tokens/s: 194392.8 | physical tokens/s: 194928.9 | loss tokens/s: 146072.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.589864E-01 | lm-global-supervised-tokens: 1.260678E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.236395E-02 | lm-token-buffer-mib: 5.810814E+01 | global_load_balancing_loss: 9.991367E-01 | loss scale: 1.0 | grad norm: 0.046 | params norm: 11370.097 | params rms: 0.061565 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 14:48:39.804788 | finish at 2026-10-10 17:00:01 + [2026-10-08 02:12:48] iteration 4236/ 6855 | consumed samples: 542208 | elapsed time per iteration (ms): 85944.5 | throughput per GPU (TFLOP/s/GPU): 43.1 | MFU 13.81% | FLOPs estimator: workload-aware | logical tokens/s: 195209.4 | valid tokens/s: 195209.4 | physical tokens/s: 195662.5 | loss tokens/s: 152656.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.605403E-01 | lm-global-supervised-tokens: 1.311998E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.243563E-02 | lm-token-buffer-mib: 5.807875E+01 | global_load_balancing_loss: 9.996122E-01 | loss scale: 1.0 | grad norm: 0.118 | params norm: 11370.040 | params rms: 0.061565 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 14:31:28.607955 | finish at 2026-10-10 16:44:16 + [2026-10-08 02:14:15] iteration 4237/ 6855 | consumed samples: 542336 | elapsed time per iteration (ms): 87438.9 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.77% | FLOPs estimator: workload-aware | logical tokens/s: 191872.6 | valid tokens/s: 191872.6 | physical tokens/s: 192208.0 | loss tokens/s: 140646.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.949304E-01 | lm-global-supervised-tokens: 1.229802E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.159908E-02 | lm-token-buffer-mib: 5.818607E+01 | global_load_balancing_loss: 9.999534E-01 | loss scale: 1.0 | grad norm: 0.058 | params norm: 11369.984 | params rms: 0.061565 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 15:35:15.125072 | finish at 2026-10-10 17:49:30 + [2026-10-08 02:15:42] iteration 4238/ 6855 | consumed samples: 542464 | elapsed time per iteration (ms): 87303.1 | throughput per GPU (TFLOP/s/GPU): 41.6 | MFU 13.33% | FLOPs estimator: workload-aware | logical tokens/s: 192171.2 | valid tokens/s: 192171.2 | physical tokens/s: 192668.4 | loss tokens/s: 151317.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.412677E-01 | lm-global-supervised-tokens: 1.321049E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.203531E-02 | lm-token-buffer-mib: 5.819533E+01 | global_load_balancing_loss: 1.000221E+00 | loss scale: 1.0 | grad norm: 0.068 | params norm: 11369.926 | params rms: 0.061564 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 15:27:52.138736 | finish at 2026-10-10 17:43:34 + [2026-10-08 02:17:11] iteration 4239/ 6855 | consumed samples: 542592 | elapsed time per iteration (ms): 88848.3 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.60% | FLOPs estimator: workload-aware | logical tokens/s: 188828.1 | valid tokens/s: 188828.1 | physical tokens/s: 189277.3 | loss tokens/s: 142215.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.916795E-01 | lm-global-supervised-tokens: 1.263560E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.620226E-02 | lm-token-buffer-mib: 5.807961E+01 | global_load_balancing_loss: 1.000039E+00 | loss scale: 1.0 | grad norm: 0.051 | params norm: 11369.870 | params rms: 0.061564 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 16:33:47.258778 | finish at 2026-10-10 18:50:58 + [2026-10-08 02:19:00] iteration 4240/ 6855 | consumed samples: 542720 | elapsed time per iteration (ms): 109153.0 | throughput per GPU (TFLOP/s/GPU): 34.1 | MFU 10.93% | FLOPs estimator: workload-aware | logical tokens/s: 153703.4 | valid tokens/s: 153703.4 | physical tokens/s: 154019.7 | loss tokens/s: 109242.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.482337E-01 | lm-global-supervised-tokens: 1.192414E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.138975E-02 | lm-token-buffer-mib: 5.809175E+01 | global_load_balancing_loss: 1.000863E+00 | loss scale: 1.0 | grad norm: 0.036 | params norm: 11369.814 | params rms: 0.061564 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 7:17:15.016748 | finish at 2026-10-11 09:36:15 +times across ranks (ms): + save-checkpoint: + rank 0: 24856.41 + rank 1: 24856.46 + rank 2: 24856.44 + rank 3: 24856.43 + rank 4: 24856.28 + rank 5: 24856.39 + rank 6: 24856.41 + rank 7: 24856.43 + rank 8: 24856.75 + rank 9: 24856.70 + rank 10: 24856.72 + rank 11: 24856.71 + rank 12: 24856.71 + rank 13: 24856.70 + rank 14: 24856.71 + rank 15: 24856.70 + rank 16: 24856.28 + rank 17: 24856.26 + rank 18: 24856.26 + rank 19: 24856.31 + rank 20: 24856.23 + rank 21: 24856.29 + rank 22: 24856.24 + rank 23: 24856.22 + rank 24: 24856.89 + rank 25: 24856.76 + rank 26: 24856.85 + rank 27: 24856.79 + rank 28: 24856.82 + rank 29: 24856.89 + rank 30: 24856.79 + rank 31: 24856.74 + rank 32: 24856.43 + rank 33: 24856.65 + rank 34: 24856.62 + rank 35: 24856.58 + rank 36: 24856.49 + rank 37: 24856.51 + rank 38: 24856.56 + rank 39: 24856.55 + rank 40: 24856.35 + rank 41: 24856.33 + rank 42: 24856.34 + rank 43: 24856.35 + rank 44: 24856.32 + rank 45: 24856.33 + rank 46: 24856.34 + rank 47: 24856.32 + rank 48: 24856.45 + rank 49: 24856.42 + rank 50: 24856.42 + rank 51: 24856.33 + rank 52: 24856.45 + rank 53: 24856.41 + rank 54: 24856.38 + rank 55: 24856.46 + rank 56: 24858.72 + rank 57: 24858.67 + rank 58: 24858.65 + rank 59: 24858.66 + rank 60: 24858.66 + rank 61: 24858.68 + rank 62: 24858.68 + rank 63: 24858.68 + [2026-10-08 02:20:54] iteration 4241/ 6855 | consumed samples: 542848 | elapsed time per iteration (ms): 88519.6 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.38% | FLOPs estimator: workload-aware | logical tokens/s: 189530.4 | valid tokens/s: 189530.4 | physical tokens/s: 190034.3 | loss tokens/s: 140418.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.701170E-01 | lm-global-supervised-tokens: 1.242976E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 9.080147E-01 | lm-token-buffer-mib: 5.820377E+01 | global_load_balancing_loss: 1.000677E+00 | loss scale: 1.0 | grad norm: 0.049 | params norm: 11369.757 | params rms: 0.061564 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 16:16:30.279049 | finish at 2026-10-10 18:37:24 + [2026-10-08 02:22:25] iteration 4242/ 6855 | consumed samples: 542976 | elapsed time per iteration (ms): 91041.5 | throughput per GPU (TFLOP/s/GPU): 40.2 | MFU 12.89% | FLOPs estimator: workload-aware | logical tokens/s: 184280.6 | valid tokens/s: 184280.5 | physical tokens/s: 184777.9 | loss tokens/s: 137281.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.460472E-01 | lm-global-supervised-tokens: 1.249828E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.412607E-02 | lm-token-buffer-mib: 5.816341E+01 | global_load_balancing_loss: 1.001112E+00 | loss scale: 1.0 | grad norm: 0.104 | params norm: 11369.700 | params rms: 0.061563 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 18:04:51.517613 | finish at 2026-10-10 20:27:16 + [2026-10-08 02:23:51] iteration 4243/ 6855 | consumed samples: 543104 | elapsed time per iteration (ms): 85688.5 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.54% | FLOPs estimator: workload-aware | logical tokens/s: 195792.2 | valid tokens/s: 195792.2 | physical tokens/s: 196342.4 | loss tokens/s: 145205.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.980581E-01 | lm-global-supervised-tokens: 1.244246E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.506251E-02 | lm-token-buffer-mib: 5.819588E+01 | global_load_balancing_loss: 1.001103E+00 | loss scale: 1.0 | grad norm: 0.042 | params norm: 11369.642 | params rms: 0.061563 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 14:10:18.378625 | finish at 2026-10-10 16:34:09 + [2026-10-08 02:25:17] iteration 4244/ 6855 | consumed samples: 543232 | elapsed time per iteration (ms): 86631.2 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.63% | FLOPs estimator: workload-aware | logical tokens/s: 193662.2 | valid tokens/s: 193662.1 | physical tokens/s: 194066.5 | loss tokens/s: 143758.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.319459E-01 | lm-global-supervised-tokens: 1.245399E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.187745E-02 | lm-token-buffer-mib: 5.811776E+01 | global_load_balancing_loss: 1.000260E+00 | loss scale: 1.0 | grad norm: 0.314 | params norm: 11369.584 | params rms: 0.061563 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 14:49:54.034138 | finish at 2026-10-10 17:15:11 + [2026-10-08 02:26:46] iteration 4245/ 6855 | consumed samples: 543360 | elapsed time per iteration (ms): 88385.2 | throughput per GPU (TFLOP/s/GPU): 41.7 | MFU 13.36% | FLOPs estimator: workload-aware | logical tokens/s: 189819.0 | valid tokens/s: 189819.0 | physical tokens/s: 190260.2 | loss tokens/s: 140217.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.707894E-01 | lm-global-supervised-tokens: 1.239312E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.158097E-02 | lm-token-buffer-mib: 5.816172E+01 | global_load_balancing_loss: 9.989770E-01 | loss scale: 1.0 | grad norm: 0.057 | params norm: 11369.523 | params rms: 0.061562 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 16:04:45.256319 | finish at 2026-10-10 18:31:31 + [2026-10-08 02:28:11] iteration 4246/ 6855 | consumed samples: 543488 | elapsed time per iteration (ms): 85735.0 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.40% | FLOPs estimator: workload-aware | logical tokens/s: 195686.6 | valid tokens/s: 195686.6 | physical tokens/s: 196096.9 | loss tokens/s: 142775.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.047285E-01 | lm-global-supervised-tokens: 1.224086E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.433409E-02 | lm-token-buffer-mib: 5.810745E+01 | global_load_balancing_loss: 9.985043E-01 | loss scale: 1.0 | grad norm: 0.120 | params norm: 11369.470 | params rms: 0.061562 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 14:08:02.576160 | finish at 2026-10-10 16:36:14 + [2026-10-08 02:29:39] iteration 4247/ 6855 | consumed samples: 543616 | elapsed time per iteration (ms): 87361.5 | throughput per GPU (TFLOP/s/GPU): 41.5 | MFU 13.30% | FLOPs estimator: workload-aware | logical tokens/s: 192043.0 | valid tokens/s: 192043.0 | physical tokens/s: 192603.6 | loss tokens/s: 141522.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.805823E-01 | lm-global-supervised-tokens: 1.236363E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.318160E-02 | lm-token-buffer-mib: 5.834630E+01 | global_load_balancing_loss: 9.996212E-01 | loss scale: 1.0 | grad norm: 0.135 | params norm: 11369.421 | params rms: 0.061562 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 15:17:18.814449 | finish at 2026-10-10 17:46:58 + [2026-10-08 02:31:03] iteration 4248/ 6855 | consumed samples: 543744 | elapsed time per iteration (ms): 84208.3 | throughput per GPU (TFLOP/s/GPU): 43.1 | MFU 13.82% | FLOPs estimator: workload-aware | logical tokens/s: 199234.5 | valid tokens/s: 199234.5 | physical tokens/s: 199689.6 | loss tokens/s: 153339.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.089959E-01 | lm-global-supervised-tokens: 1.291244E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.639463E-02 | lm-token-buffer-mib: 5.819119E+01 | global_load_balancing_loss: 1.000134E+00 | loss scale: 1.0 | grad norm: 0.052 | params norm: 11369.375 | params rms: 0.061561 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 12:58:50.967539 | finish at 2026-10-10 15:29:54 + [2026-10-08 02:32:28] iteration 4249/ 6855 | consumed samples: 543872 | elapsed time per iteration (ms): 85219.9 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.55% | FLOPs estimator: workload-aware | logical tokens/s: 196869.5 | valid tokens/s: 196869.5 | physical tokens/s: 197407.3 | loss tokens/s: 151850.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.166163E-01 | lm-global-supervised-tokens: 1.294069E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.250630E-02 | lm-token-buffer-mib: 5.817431E+01 | global_load_balancing_loss: 1.000756E+00 | loss scale: 1.0 | grad norm: 0.068 | params norm: 11369.329 | params rms: 0.061561 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 13:41:22.978349 | finish at 2026-10-10 16:13:51 + [2026-10-08 02:34:17] iteration 4250/ 6855 | consumed samples: 544000 | elapsed time per iteration (ms): 109207.4 | throughput per GPU (TFLOP/s/GPU): 33.5 | MFU 10.74% | FLOPs estimator: workload-aware | logical tokens/s: 153627.1 | valid tokens/s: 153627.0 | physical tokens/s: 153888.2 | loss tokens/s: 113638.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.477274E-01 | lm-global-supervised-tokens: 1.241019E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.353827E-02 | lm-token-buffer-mib: 5.818320E+01 | global_load_balancing_loss: 1.001649E+00 | loss scale: 1.0 | grad norm: 0.040 | params norm: 11369.285 | params rms: 0.061561 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 7:01:25.323799 | finish at 2026-10-11 09:35:43 + [2026-10-08 02:35:51] iteration 4251/ 6855 | consumed samples: 544128 | elapsed time per iteration (ms): 94065.1 | throughput per GPU (TFLOP/s/GPU): 40.8 | MFU 13.07% | FLOPs estimator: workload-aware | logical tokens/s: 178357.4 | valid tokens/s: 178357.4 | physical tokens/s: 178862.6 | loss tokens/s: 125317.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.148078E-01 | lm-global-supervised-tokens: 1.178800E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.326991E-02 | lm-token-buffer-mib: 5.809425E+01 | global_load_balancing_loss: 1.001353E+00 | loss scale: 1.0 | grad norm: 0.064 | params norm: 11369.242 | params rms: 0.061561 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 20:02:25.578641 | finish at 2026-10-10 22:38:17 + [2026-10-08 02:37:28] iteration 4252/ 6855 | consumed samples: 544256 | elapsed time per iteration (ms): 96780.8 | throughput per GPU (TFLOP/s/GPU): 39.6 | MFU 12.69% | FLOPs estimator: workload-aware | logical tokens/s: 173352.5 | valid tokens/s: 173352.5 | physical tokens/s: 173749.9 | loss tokens/s: 129016.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.697235E-01 | lm-global-supervised-tokens: 1.248637E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.160484E-02 | lm-token-buffer-mib: 5.827535E+01 | global_load_balancing_loss: 1.000203E+00 | loss scale: 1.0 | grad norm: 0.081 | params norm: 11369.196 | params rms: 0.061560 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 21:58:40.550515 | finish at 2026-10-11 00:36:09 + [2026-10-08 02:38:55] iteration 4253/ 6855 | consumed samples: 544384 | elapsed time per iteration (ms): 87121.0 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.57% | FLOPs estimator: workload-aware | logical tokens/s: 192571.9 | valid tokens/s: 192571.9 | physical tokens/s: 193150.8 | loss tokens/s: 146324.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.382691E-01 | lm-global-supervised-tokens: 1.274796E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.388800E-02 | lm-token-buffer-mib: 5.823177E+01 | global_load_balancing_loss: 9.995733E-01 | loss scale: 1.0 | grad norm: 0.063 | params norm: 11369.147 | params rms: 0.061560 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 14:58:08.969929 | finish at 2026-10-10 17:37:04 + [2026-10-08 02:40:28] iteration 4254/ 6855 | consumed samples: 544512 | elapsed time per iteration (ms): 93013.9 | throughput per GPU (TFLOP/s/GPU): 40.0 | MFU 12.84% | FLOPs estimator: workload-aware | logical tokens/s: 180372.6 | valid tokens/s: 180372.6 | physical tokens/s: 180820.7 | loss tokens/s: 127312.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.453165E-01 | lm-global-supervised-tokens: 1.184178E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.262088E-02 | lm-token-buffer-mib: 5.805226E+01 | global_load_balancing_loss: 9.998075E-01 | loss scale: 1.0 | grad norm: 0.043 | params norm: 11369.101 | params rms: 0.061560 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 19:12:09.047346 | finish at 2026-10-10 21:52:37 + [2026-10-08 02:41:55] iteration 4255/ 6855 | consumed samples: 544640 | elapsed time per iteration (ms): 86322.5 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.65% | FLOPs estimator: workload-aware | logical tokens/s: 194354.5 | valid tokens/s: 194354.5 | physical tokens/s: 194832.6 | loss tokens/s: 144797.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.724837E-01 | lm-global-supervised-tokens: 1.249928E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.412469E-02 | lm-token-buffer-mib: 5.804653E+01 | global_load_balancing_loss: 1.000887E+00 | loss scale: 1.0 | grad norm: 0.031 | params norm: 11369.057 | params rms: 0.061560 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 14:20:38.615894 | finish at 2026-10-10 17:02:33 + [2026-10-08 02:43:18] iteration 4256/ 6855 | consumed samples: 544768 | elapsed time per iteration (ms): 82996.2 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.80% | FLOPs estimator: workload-aware | logical tokens/s: 202143.5 | valid tokens/s: 202143.5 | physical tokens/s: 202536.7 | loss tokens/s: 159001.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.315352E-01 | lm-global-supervised-tokens: 1.319653E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.144161E-02 | lm-token-buffer-mib: 5.802312E+01 | global_load_balancing_loss: 9.997784E-01 | loss scale: 1.0 | grad norm: 0.083 | params norm: 11369.014 | params rms: 0.061560 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 11:55:07.091179 | finish at 2026-10-10 14:38:25 + [2026-10-08 02:44:45] iteration 4257/ 6855 | consumed samples: 544896 | elapsed time per iteration (ms): 87142.4 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.67% | FLOPs estimator: workload-aware | logical tokens/s: 192526.1 | valid tokens/s: 192526.1 | physical tokens/s: 192923.3 | loss tokens/s: 140085.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.496823E-01 | lm-global-supervised-tokens: 1.220736E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.313869E-02 | lm-token-buffer-mib: 5.816264E+01 | global_load_balancing_loss: 9.999932E-01 | loss scale: 1.0 | grad norm: 0.073 | params norm: 11368.968 | params rms: 0.061559 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 14:53:15.858021 | finish at 2026-10-10 17:38:01 + [2026-10-08 02:46:12] iteration 4258/ 6855 | consumed samples: 545024 | elapsed time per iteration (ms): 87196.5 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.54% | FLOPs estimator: workload-aware | logical tokens/s: 192406.0 | valid tokens/s: 192406.0 | physical tokens/s: 192999.7 | loss tokens/s: 139391.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.785749E-01 | lm-global-supervised-tokens: 1.215445E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.162682E-02 | lm-token-buffer-mib: 5.806084E+01 | global_load_balancing_loss: 9.993300E-01 | loss scale: 1.0 | grad norm: 0.086 | params norm: 11368.922 | params rms: 0.061559 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 14:54:09.275371 | finish at 2026-10-10 17:40:21 + [2026-10-08 02:47:38] iteration 4259/ 6855 | consumed samples: 545152 | elapsed time per iteration (ms): 85876.5 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.49% | FLOPs estimator: workload-aware | logical tokens/s: 195364.1 | valid tokens/s: 195364.1 | physical tokens/s: 195862.0 | loss tokens/s: 151053.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.718903E-01 | lm-global-supervised-tokens: 1.297191E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.155701E-02 | lm-token-buffer-mib: 5.809795E+01 | global_load_balancing_loss: 9.989505E-01 | loss scale: 1.0 | grad norm: 0.119 | params norm: 11368.874 | params rms: 0.061559 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 13:55:35.516886 | finish at 2026-10-10 16:43:13 + [2026-10-08 02:49:28] iteration 4260/ 6855 | consumed samples: 545280 | elapsed time per iteration (ms): 110237.1 | throughput per GPU (TFLOP/s/GPU): 34.8 | MFU 11.16% | FLOPs estimator: workload-aware | logical tokens/s: 152191.3 | valid tokens/s: 152191.3 | physical tokens/s: 152494.3 | loss tokens/s: 108944.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.584911E-01 | lm-global-supervised-tokens: 1.200969E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.617508E-02 | lm-token-buffer-mib: 5.806391E+01 | global_load_balancing_loss: 1.000346E+00 | loss scale: 1.0 | grad norm: 0.061 | params norm: 11368.827 | params rms: 0.061558 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 7:27:45.188824 | finish at 2026-10-11 10:17:13 + [2026-10-08 02:50:56] iteration 4261/ 6855 | consumed samples: 545408 | elapsed time per iteration (ms): 88323.9 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.46% | FLOPs estimator: workload-aware | logical tokens/s: 189950.2 | valid tokens/s: 189950.2 | physical tokens/s: 190439.4 | loss tokens/s: 142206.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.752548E-01 | lm-global-supervised-tokens: 1.256021E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.520041E-02 | lm-token-buffer-mib: 5.813744E+01 | global_load_balancing_loss: 1.000799E+00 | loss scale: 1.0 | grad norm: 0.072 | params norm: 11368.776 | params rms: 0.061558 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 15:38:32.268301 | finish at 2026-10-10 18:29:29 + [2026-10-08 02:52:24] iteration 4262/ 6855 | consumed samples: 545536 | elapsed time per iteration (ms): 87721.8 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.58% | FLOPs estimator: workload-aware | logical tokens/s: 191254.8 | valid tokens/s: 191254.8 | physical tokens/s: 191739.3 | loss tokens/s: 146135.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.897845E-01 | lm-global-supervised-tokens: 1.281923E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.148228E-02 | lm-token-buffer-mib: 5.810561E+01 | global_load_balancing_loss: 1.001185E+00 | loss scale: 1.0 | grad norm: 0.071 | params norm: 11368.724 | params rms: 0.061558 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 15:11:02.582500 | finish at 2026-10-10 18:03:27 + [2026-10-08 02:53:50] iteration 4263/ 6855 | consumed samples: 545664 | elapsed time per iteration (ms): 85377.0 | throughput per GPU (TFLOP/s/GPU): 43.4 | MFU 13.92% | FLOPs estimator: workload-aware | logical tokens/s: 196507.0 | valid tokens/s: 196507.0 | physical tokens/s: 197026.5 | loss tokens/s: 150829.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.513548E-01 | lm-global-supervised-tokens: 1.287741E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.166638E-02 | lm-token-buffer-mib: 5.808917E+01 | global_load_balancing_loss: 1.000430E+00 | loss scale: 1.0 | grad norm: 0.132 | params norm: 11368.674 | params rms: 0.061558 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 13:28:17.194748 | finish at 2026-10-10 16:22:07 + [2026-10-08 02:55:14] iteration 4264/ 6855 | consumed samples: 545792 | elapsed time per iteration (ms): 84331.8 | throughput per GPU (TFLOP/s/GPU): 43.1 | MFU 13.80% | FLOPs estimator: workload-aware | logical tokens/s: 198942.8 | valid tokens/s: 198942.8 | physical tokens/s: 199440.9 | loss tokens/s: 144829.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.151211E-01 | lm-global-supervised-tokens: 1.221369E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.163616E-02 | lm-token-buffer-mib: 5.820966E+01 | global_load_balancing_loss: 1.002081E+00 | loss scale: 1.0 | grad norm: 0.104 | params norm: 11368.624 | params rms: 0.061557 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 12:41:43.569592 | finish at 2026-10-10 15:36:57 + [2026-10-08 02:56:42] iteration 4265/ 6855 | consumed samples: 545920 | elapsed time per iteration (ms): 88422.3 | throughput per GPU (TFLOP/s/GPU): 41.0 | MFU 13.15% | FLOPs estimator: workload-aware | logical tokens/s: 189739.0 | valid tokens/s: 189739.0 | physical tokens/s: 190240.2 | loss tokens/s: 139588.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.616483E-01 | lm-global-supervised-tokens: 1.234277E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.168015E-02 | lm-token-buffer-mib: 5.820891E+01 | global_load_balancing_loss: 9.999176E-01 | loss scale: 1.0 | grad norm: 0.038 | params norm: 11368.573 | params rms: 0.061557 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 15:36:53.699214 | finish at 2026-10-10 18:33:36 + [2026-10-08 02:58:07] iteration 4266/ 6855 | consumed samples: 546048 | elapsed time per iteration (ms): 84405.0 | throughput per GPU (TFLOP/s/GPU): 43.7 | MFU 14.00% | FLOPs estimator: workload-aware | logical tokens/s: 198769.7 | valid tokens/s: 198769.7 | physical tokens/s: 199133.1 | loss tokens/s: 149363.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.879101E-01 | lm-global-supervised-tokens: 1.260701E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.156167E-02 | lm-token-buffer-mib: 5.803653E+01 | global_load_balancing_loss: 9.998498E-01 | loss scale: 1.0 | grad norm: 0.046 | params norm: 11368.520 | params rms: 0.061557 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 12:42:04.631343 | finish at 2026-10-10 15:40:11 + [2026-10-08 02:59:32] iteration 4267/ 6855 | consumed samples: 546176 | elapsed time per iteration (ms): 85546.2 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.64% | FLOPs estimator: workload-aware | logical tokens/s: 196116.7 | valid tokens/s: 196116.7 | physical tokens/s: 196677.4 | loss tokens/s: 149092.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.441296E-01 | lm-global-supervised-tokens: 1.275428E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.362631E-02 | lm-token-buffer-mib: 5.841846E+01 | global_load_balancing_loss: 9.998370E-01 | loss scale: 1.0 | grad norm: 0.083 | params norm: 11368.466 | params rms: 0.061557 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 13:29:53.447227 | finish at 2026-10-10 16:29:26 + [2026-10-08 03:00:57] iteration 4268/ 6855 | consumed samples: 546304 | elapsed time per iteration (ms): 84312.3 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.63% | FLOPs estimator: workload-aware | logical tokens/s: 198988.7 | valid tokens/s: 198988.7 | physical tokens/s: 199477.0 | loss tokens/s: 151820.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.388826E-01 | lm-global-supervised-tokens: 1.280032E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.526393E-02 | lm-token-buffer-mib: 5.809955E+01 | global_load_balancing_loss: 9.995896E-01 | loss scale: 1.0 | grad norm: 0.140 | params norm: 11368.412 | params rms: 0.061556 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 12:35:15.969357 | finish at 2026-10-10 15:36:13 + [2026-10-08 03:02:25] iteration 4269/ 6855 | consumed samples: 546432 | elapsed time per iteration (ms): 88000.4 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.51% | FLOPs estimator: workload-aware | logical tokens/s: 190649.1 | valid tokens/s: 190649.1 | physical tokens/s: 191002.3 | loss tokens/s: 140371.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.753606E-01 | lm-global-supervised-tokens: 1.235277E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.464344E-02 | lm-token-buffer-mib: 5.806842E+01 | global_load_balancing_loss: 9.992087E-01 | loss scale: 1.0 | grad norm: 0.114 | params norm: 11368.357 | params rms: 0.061556 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 15:12:49.110407 | finish at 2026-10-10 18:15:14 + [2026-10-08 03:04:13] iteration 4270/ 6855 | consumed samples: 546560 | elapsed time per iteration (ms): 108732.8 | throughput per GPU (TFLOP/s/GPU): 33.2 | MFU 10.65% | FLOPs estimator: workload-aware | logical tokens/s: 154297.5 | valid tokens/s: 154297.5 | physical tokens/s: 154690.2 | loss tokens/s: 117004.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.240656E-01 | lm-global-supervised-tokens: 1.272219E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.202822E-02 | lm-token-buffer-mib: 5.814059E+01 | global_load_balancing_loss: 1.000504E+00 | loss scale: 1.0 | grad norm: 0.094 | params norm: 11368.300 | params rms: 0.061556 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 6:04:34.198036 | finish at 2026-10-11 09:08:47 + [2026-10-08 03:05:44] iteration 4271/ 6855 | consumed samples: 546688 | elapsed time per iteration (ms): 90700.8 | throughput per GPU (TFLOP/s/GPU): 41.0 | MFU 13.14% | FLOPs estimator: workload-aware | logical tokens/s: 184972.8 | valid tokens/s: 184972.8 | physical tokens/s: 185408.3 | loss tokens/s: 138151.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.582503E-01 | lm-global-supervised-tokens: 1.253049E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.442125E-02 | lm-token-buffer-mib: 5.831016E+01 | global_load_balancing_loss: 1.001208E+00 | loss scale: 1.0 | grad norm: 0.095 | params norm: 11368.243 | params rms: 0.061555 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 17:06:10.927425 | finish at 2026-10-10 20:11:55 + [2026-10-08 03:07:12] iteration 4272/ 6855 | consumed samples: 546816 | elapsed time per iteration (ms): 88080.7 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.56% | FLOPs estimator: workload-aware | logical tokens/s: 190475.3 | valid tokens/s: 190475.3 | physical tokens/s: 190864.2 | loss tokens/s: 140106.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 5.976402E-01 | lm-global-supervised-tokens: 1.234065E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.351839E-02 | lm-token-buffer-mib: 5.803500E+01 | global_load_balancing_loss: 1.001194E+00 | loss scale: 1.0 | grad norm: 0.083 | params norm: 11368.185 | params rms: 0.061555 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 15:11:52.438042 | finish at 2026-10-10 18:19:05 + [2026-10-08 03:08:34] iteration 4273/ 6855 | consumed samples: 546944 | elapsed time per iteration (ms): 81817.6 | throughput per GPU (TFLOP/s/GPU): 44.7 | MFU 14.34% | FLOPs estimator: workload-aware | logical tokens/s: 205055.6 | valid tokens/s: 205055.6 | physical tokens/s: 205581.7 | loss tokens/s: 148584.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.714004E-01 | lm-global-supervised-tokens: 1.215686E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.169490E-02 | lm-token-buffer-mib: 5.808250E+01 | global_load_balancing_loss: 1.000031E+00 | loss scale: 1.0 | grad norm: 0.074 | params norm: 11368.123 | params rms: 0.061555 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 10:40:53.156500 | finish at 2026-10-10 13:49:27 + [2026-10-08 03:10:01] iteration 4274/ 6855 | consumed samples: 547072 | elapsed time per iteration (ms): 86837.6 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.50% | FLOPs estimator: workload-aware | logical tokens/s: 193201.9 | valid tokens/s: 193201.9 | physical tokens/s: 193711.5 | loss tokens/s: 149067.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.334804E-01 | lm-global-supervised-tokens: 1.294470E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.507308E-02 | lm-token-buffer-mib: 5.828326E+01 | global_load_balancing_loss: 9.996260E-01 | loss scale: 1.0 | grad norm: 0.197 | params norm: 11368.061 | params rms: 0.061554 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 14:15:27.833274 | finish at 2026-10-10 17:25:29 + [2026-10-08 03:11:25] iteration 4275/ 6855 | consumed samples: 547200 | elapsed time per iteration (ms): 83857.2 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.77% | FLOPs estimator: workload-aware | logical tokens/s: 200068.0 | valid tokens/s: 200067.9 | physical tokens/s: 200466.2 | loss tokens/s: 146505.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.355772E-01 | lm-global-supervised-tokens: 1.228550E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.664275E-02 | lm-token-buffer-mib: 5.813335E+01 | global_load_balancing_loss: 9.992179E-01 | loss scale: 1.0 | grad norm: 0.063 | params norm: 11367.996 | params rms: 0.061554 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 12:05:51.677871 | finish at 2026-10-10 15:17:16 + [2026-10-08 03:12:53] iteration 4276/ 6855 | consumed samples: 547328 | elapsed time per iteration (ms): 88595.5 | throughput per GPU (TFLOP/s/GPU): 41.4 | MFU 13.27% | FLOPs estimator: workload-aware | logical tokens/s: 189368.6 | valid tokens/s: 189368.6 | physical tokens/s: 189733.3 | loss tokens/s: 143699.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.662494E-01 | lm-global-supervised-tokens: 1.273110E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.474240E-02 | lm-token-buffer-mib: 5.808714E+01 | global_load_balancing_loss: 9.994071E-01 | loss scale: 1.0 | grad norm: 0.076 | params norm: 11367.934 | params rms: 0.061554 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 15:28:07.868881 | finish at 2026-10-10 18:41:01 + [2026-10-08 03:14:21] iteration 4277/ 6855 | consumed samples: 547456 | elapsed time per iteration (ms): 87383.7 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.45% | FLOPs estimator: workload-aware | logical tokens/s: 191993.1 | valid tokens/s: 191993.1 | physical tokens/s: 192336.3 | loss tokens/s: 146095.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.119937E-01 | lm-global-supervised-tokens: 1.276640E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.291028E-02 | lm-token-buffer-mib: 5.804720E+01 | global_load_balancing_loss: 1.000565E+00 | loss scale: 1.0 | grad norm: 0.071 | params norm: 11367.874 | params rms: 0.061553 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 14:34:35.113174 | finish at 2026-10-10 17:48:56 + [2026-10-08 03:15:46] iteration 4278/ 6855 | consumed samples: 547584 | elapsed time per iteration (ms): 84952.6 | throughput per GPU (TFLOP/s/GPU): 43.5 | MFU 13.93% | FLOPs estimator: workload-aware | logical tokens/s: 197488.5 | valid tokens/s: 197488.5 | physical tokens/s: 197889.8 | loss tokens/s: 148964.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.892381E-01 | lm-global-supervised-tokens: 1.265495E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.139505E-02 | lm-token-buffer-mib: 5.817493E+01 | global_load_balancing_loss: 1.000823E+00 | loss scale: 1.0 | grad norm: 0.074 | params norm: 11367.816 | params rms: 0.061553 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 12:48:42.783850 | finish at 2026-10-10 16:04:28 + [2026-10-08 03:17:10] iteration 4279/ 6855 | consumed samples: 547712 | elapsed time per iteration (ms): 84646.3 | throughput per GPU (TFLOP/s/GPU): 44.1 | MFU 14.12% | FLOPs estimator: workload-aware | logical tokens/s: 198203.7 | valid tokens/s: 198203.7 | physical tokens/s: 198575.1 | loss tokens/s: 147156.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.893417E-01 | lm-global-supervised-tokens: 1.245621E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.155775E-02 | lm-token-buffer-mib: 5.807130E+01 | global_load_balancing_loss: 1.001133E+00 | loss scale: 1.0 | grad norm: 0.085 | params norm: 11367.757 | params rms: 0.061553 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 12:34:08.951912 | finish at 2026-10-10 15:51:19 + [2026-10-08 03:19:04] iteration 4280/ 6855 | consumed samples: 547840 | elapsed time per iteration (ms): 114247.5 | throughput per GPU (TFLOP/s/GPU): 32.7 | MFU 10.49% | FLOPs estimator: workload-aware | logical tokens/s: 146849.5 | valid tokens/s: 146849.5 | physical tokens/s: 147130.9 | loss tokens/s: 106505.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.141494E-01 | lm-global-supervised-tokens: 1.216798E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.157111E-02 | lm-token-buffer-mib: 5.815247E+01 | global_load_balancing_loss: 9.997467E-01 | loss scale: 1.0 | grad norm: 0.034 | params norm: 11367.700 | params rms: 0.061552 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 9:43:07.328315 | finish at 2026-10-11 13:02:12 +times across ranks (ms): + save-checkpoint: + rank 0: 23818.94 + rank 1: 23818.99 + rank 2: 23818.96 + rank 3: 23818.94 + rank 4: 23818.96 + rank 5: 23818.94 + rank 6: 23818.92 + rank 7: 23818.95 + rank 8: 23818.05 + rank 9: 23818.02 + rank 10: 23818.03 + rank 11: 23818.02 + rank 12: 23818.05 + rank 13: 23818.06 + rank 14: 23818.05 + rank 15: 23818.04 + rank 16: 23818.24 + rank 17: 23818.19 + rank 18: 23818.20 + rank 19: 23818.22 + rank 20: 23818.19 + rank 21: 23818.19 + rank 22: 23818.19 + rank 23: 23818.19 + rank 24: 23819.43 + rank 25: 23819.45 + rank 26: 23819.46 + rank 27: 23819.44 + rank 28: 23819.44 + rank 29: 23819.43 + rank 30: 23819.44 + rank 31: 23819.42 + rank 32: 23817.97 + rank 33: 23818.09 + rank 34: 23817.95 + rank 35: 23817.93 + rank 36: 23817.93 + rank 37: 23817.96 + rank 38: 23817.94 + rank 39: 23817.93 + rank 40: 23818.09 + rank 41: 23818.08 + rank 42: 23818.08 + rank 43: 23818.11 + rank 44: 23818.08 + rank 45: 23818.08 + rank 46: 23818.08 + rank 47: 23818.10 + rank 48: 23818.44 + rank 49: 23818.42 + rank 50: 23818.37 + rank 51: 23818.48 + rank 52: 23818.38 + rank 53: 23818.37 + rank 54: 23818.37 + rank 55: 23818.35 + rank 56: 23818.18 + rank 57: 23818.17 + rank 58: 23818.13 + rank 59: 23818.13 + rank 60: 23818.14 + rank 61: 23818.14 + rank 62: 23818.13 + rank 63: 23818.13 + [2026-10-08 03:20:58] iteration 4281/ 6855 | consumed samples: 547968 | elapsed time per iteration (ms): 89840.2 | throughput per GPU (TFLOP/s/GPU): 41.4 | MFU 13.25% | FLOPs estimator: workload-aware | logical tokens/s: 186744.7 | valid tokens/s: 186744.7 | physical tokens/s: 187186.0 | loss tokens/s: 135352.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.688309E-01 | lm-global-supervised-tokens: 1.216005E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.359408E+00 | lm-token-buffer-mib: 5.805578E+01 | global_load_balancing_loss: 1.000211E+00 | loss scale: 1.0 | grad norm: 0.052 | params norm: 11367.643 | params rms: 0.061552 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 16:14:08.723760 | finish at 2026-10-10 19:35:07 + [2026-10-08 03:22:32] iteration 4282/ 6855 | consumed samples: 548096 | elapsed time per iteration (ms): 93727.0 | throughput per GPU (TFLOP/s/GPU): 39.1 | MFU 12.53% | FLOPs estimator: workload-aware | logical tokens/s: 179000.7 | valid tokens/s: 179000.7 | physical tokens/s: 179550.6 | loss tokens/s: 134200.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.351269E-01 | lm-global-supervised-tokens: 1.257818E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.517208E-02 | lm-token-buffer-mib: 5.811209E+01 | global_load_balancing_loss: 1.001402E+00 | loss scale: 1.0 | grad norm: 0.119 | params norm: 11367.585 | params rms: 0.061552 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 18:59:19.578970 | finish at 2026-10-10 22:21:51 + [2026-10-08 03:23:57] iteration 4283/ 6855 | consumed samples: 548224 | elapsed time per iteration (ms): 84863.2 | throughput per GPU (TFLOP/s/GPU): 43.6 | MFU 13.97% | FLOPs estimator: workload-aware | logical tokens/s: 197695.9 | valid tokens/s: 197695.9 | physical tokens/s: 198073.3 | loss tokens/s: 143809.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.893005E-01 | lm-global-supervised-tokens: 1.220410E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.879845E-02 | lm-token-buffer-mib: 5.815292E+01 | global_load_balancing_loss: 1.000329E+00 | loss scale: 1.0 | grad norm: 0.086 | params norm: 11367.528 | params rms: 0.061551 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 12:37:48.141685 | finish at 2026-10-10 16:01:45 + [2026-10-08 03:25:24] iteration 4284/ 6855 | consumed samples: 548352 | elapsed time per iteration (ms): 87374.5 | throughput per GPU (TFLOP/s/GPU): 43.1 | MFU 13.83% | FLOPs estimator: workload-aware | logical tokens/s: 192014.7 | valid tokens/s: 192014.6 | physical tokens/s: 192420.5 | loss tokens/s: 137278.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.718284E-01 | lm-global-supervised-tokens: 1.199463E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.144644E-02 | lm-token-buffer-mib: 5.808659E+01 | global_load_balancing_loss: 9.987838E-01 | loss scale: 1.0 | grad norm: 0.055 | params norm: 11367.470 | params rms: 0.061551 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 14:23:59.808331 | finish at 2026-10-10 17:49:24 + [2026-10-08 03:26:49] iteration 4285/ 6855 | consumed samples: 548480 | elapsed time per iteration (ms): 85272.0 | throughput per GPU (TFLOP/s/GPU): 42.8 | MFU 13.73% | FLOPs estimator: workload-aware | logical tokens/s: 196749.3 | valid tokens/s: 196749.3 | physical tokens/s: 197191.6 | loss tokens/s: 143510.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.180370E-01 | lm-global-supervised-tokens: 1.223746E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.172418E-02 | lm-token-buffer-mib: 5.816280E+01 | global_load_balancing_loss: 9.992418E-01 | loss scale: 1.0 | grad norm: 0.148 | params norm: 11367.413 | params rms: 0.061551 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 12:52:28.913968 | finish at 2026-10-10 16:19:18 + [2026-10-08 03:28:13] iteration 4286/ 6855 | consumed samples: 548608 | elapsed time per iteration (ms): 83335.4 | throughput per GPU (TFLOP/s/GPU): 43.5 | MFU 13.95% | FLOPs estimator: workload-aware | logical tokens/s: 201321.1 | valid tokens/s: 201321.1 | physical tokens/s: 201837.8 | loss tokens/s: 150765.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.601840E-01 | lm-global-supervised-tokens: 1.256408E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.242651E-02 | lm-token-buffer-mib: 5.808916E+01 | global_load_balancing_loss: 1.000618E+00 | loss scale: 1.0 | grad norm: 0.136 | params norm: 11367.356 | params rms: 0.061551 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 11:28:08.542419 | finish at 2026-10-10 14:56:21 + [2026-10-08 03:29:41] iteration 4287/ 6855 | consumed samples: 548736 | elapsed time per iteration (ms): 88372.5 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.45% | FLOPs estimator: workload-aware | logical tokens/s: 189846.2 | valid tokens/s: 189846.2 | physical tokens/s: 190345.0 | loss tokens/s: 136465.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.716505E-01 | lm-global-supervised-tokens: 1.205977E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.152848E-02 | lm-token-buffer-mib: 5.814497E+01 | global_load_balancing_loss: 1.001158E+00 | loss scale: 1.0 | grad norm: 0.401 | params norm: 11367.300 | params rms: 0.061550 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 15:02:20.590874 | finish at 2026-10-10 18:32:02 + [2026-10-08 03:31:09] iteration 4288/ 6855 | consumed samples: 548864 | elapsed time per iteration (ms): 88322.6 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.45% | FLOPs estimator: workload-aware | logical tokens/s: 189953.8 | valid tokens/s: 189953.8 | physical tokens/s: 190490.4 | loss tokens/s: 142862.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.847596E-01 | lm-global-supervised-tokens: 1.261798E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.104189E-02 | lm-token-buffer-mib: 5.815894E+01 | global_load_balancing_loss: 1.000463E+00 | loss scale: 1.0 | grad norm: 0.313 | params norm: 11367.242 | params rms: 0.061550 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 14:58:44.080863 | finish at 2026-10-10 18:29:54 + [2026-10-08 03:32:34] iteration 4289/ 6855 | consumed samples: 548992 | elapsed time per iteration (ms): 84626.3 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.62% | FLOPs estimator: workload-aware | logical tokens/s: 198249.8 | valid tokens/s: 198249.8 | physical tokens/s: 198724.3 | loss tokens/s: 152918.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.372945E-01 | lm-global-supervised-tokens: 1.294094E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.478557E-02 | lm-token-buffer-mib: 5.809827E+01 | global_load_balancing_loss: 9.993342E-01 | loss scale: 1.0 | grad norm: 0.219 | params norm: 11367.183 | params rms: 0.061550 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 12:19:11.032823 | finish at 2026-10-10 15:51:45 + [2026-10-08 03:34:24] iteration 4290/ 6855 | consumed samples: 549120 | elapsed time per iteration (ms): 109954.2 | throughput per GPU (TFLOP/s/GPU): 33.5 | MFU 10.74% | FLOPs estimator: workload-aware | logical tokens/s: 152583.1 | valid tokens/s: 152583.1 | physical tokens/s: 153162.0 | loss tokens/s: 108875.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.574028E-01 | lm-global-supervised-tokens: 1.197127E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.302626E-02 | lm-token-buffer-mib: 5.814175E+01 | global_load_balancing_loss: 1.000686E+00 | loss scale: 1.0 | grad norm: 0.064 | params norm: 11367.123 | params rms: 0.061549 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 6:20:32.627038 | finish at 2026-10-11 09:54:57 + [2026-10-08 03:35:50] iteration 4291/ 6855 | consumed samples: 549248 | elapsed time per iteration (ms): 86258.2 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.52% | FLOPs estimator: workload-aware | logical tokens/s: 194499.6 | valid tokens/s: 194499.6 | physical tokens/s: 194923.2 | loss tokens/s: 138633.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.523800E-01 | lm-global-supervised-tokens: 1.195825E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.513403E-02 | lm-token-buffer-mib: 5.818111E+01 | global_load_balancing_loss: 1.000836E+00 | loss scale: 1.0 | grad norm: 0.159 | params norm: 11367.061 | params rms: 0.061549 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 13:26:06.126098 | finish at 2026-10-10 17:01:56 + [2026-10-08 03:37:15] iteration 4292/ 6855 | consumed samples: 549376 | elapsed time per iteration (ms): 84258.9 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.42% | FLOPs estimator: workload-aware | logical tokens/s: 199113.9 | valid tokens/s: 199113.9 | physical tokens/s: 199609.1 | loss tokens/s: 149831.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.656872E-01 | lm-global-supervised-tokens: 1.262460E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.414109E-02 | lm-token-buffer-mib: 5.805313E+01 | global_load_balancing_loss: 9.996104E-01 | loss scale: 1.0 | grad norm: 0.208 | params norm: 11366.997 | params rms: 0.061549 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 11:59:15.593511 | finish at 2026-10-10 15:36:30 + [2026-10-08 03:38:40] iteration 4293/ 6855 | consumed samples: 549504 | elapsed time per iteration (ms): 85082.8 | throughput per GPU (TFLOP/s/GPU): 43.5 | MFU 13.94% | FLOPs estimator: workload-aware | logical tokens/s: 197186.3 | valid tokens/s: 197186.3 | physical tokens/s: 197536.8 | loss tokens/s: 150543.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.796049E-01 | lm-global-supervised-tokens: 1.280864E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.899671E-02 | lm-token-buffer-mib: 5.806119E+01 | global_load_balancing_loss: 9.996761E-01 | loss scale: 1.0 | grad norm: 0.041 | params norm: 11366.937 | params rms: 0.061548 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 12:33:02.217057 | finish at 2026-10-10 16:11:42 + [2026-10-08 03:40:05] iteration 4294/ 6855 | consumed samples: 549632 | elapsed time per iteration (ms): 85472.9 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.79% | FLOPs estimator: workload-aware | logical tokens/s: 196286.9 | valid tokens/s: 196286.9 | physical tokens/s: 196627.8 | loss tokens/s: 142824.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.617504E-01 | lm-global-supervised-tokens: 1.220766E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.273825E-02 | lm-token-buffer-mib: 5.814183E+01 | global_load_balancing_loss: 1.001408E+00 | loss scale: 1.0 | grad norm: 0.084 | params norm: 11366.877 | params rms: 0.061548 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 12:48:16.162623 | finish at 2026-10-10 16:28:21 + [2026-10-08 03:41:34] iteration 4295/ 6855 | consumed samples: 549760 | elapsed time per iteration (ms): 88566.9 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.58% | FLOPs estimator: workload-aware | logical tokens/s: 189429.3 | valid tokens/s: 189429.3 | physical tokens/s: 190001.3 | loss tokens/s: 134425.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.727523E-01 | lm-global-supervised-tokens: 1.190564E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.153789E-02 | lm-token-buffer-mib: 5.803738E+01 | global_load_balancing_loss: 1.000620E+00 | loss scale: 1.0 | grad norm: 0.121 | params norm: 11366.816 | params rms: 0.061548 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 14:58:51.179199 | finish at 2026-10-10 18:40:25 + [2026-10-08 03:43:00] iteration 4296/ 6855 | consumed samples: 549888 | elapsed time per iteration (ms): 86335.9 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.62% | FLOPs estimator: workload-aware | logical tokens/s: 194323.9 | valid tokens/s: 194323.9 | physical tokens/s: 194929.5 | loss tokens/s: 143068.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.756474E-01 | lm-global-supervised-tokens: 1.235192E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.023804E-02 | lm-token-buffer-mib: 5.825296E+01 | global_load_balancing_loss: 1.000523E+00 | loss scale: 1.0 | grad norm: 0.053 | params norm: 11366.755 | params rms: 0.061547 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 13:22:13.663452 | finish at 2026-10-10 17:05:14 + [2026-10-08 03:44:25] iteration 4297/ 6855 | consumed samples: 550016 | elapsed time per iteration (ms): 85456.8 | throughput per GPU (TFLOP/s/GPU): 42.8 | MFU 13.70% | FLOPs estimator: workload-aware | logical tokens/s: 196323.8 | valid tokens/s: 196323.8 | physical tokens/s: 196851.8 | loss tokens/s: 144151.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.539882E-01 | lm-global-supervised-tokens: 1.231876E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.167620E-02 | lm-token-buffer-mib: 5.813192E+01 | global_load_balancing_loss: 9.999923E-01 | loss scale: 1.0 | grad norm: 0.102 | params norm: 11366.694 | params rms: 0.061547 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 12:43:18.461426 | finish at 2026-10-10 16:27:44 + [2026-10-08 03:45:51] iteration 4298/ 6855 | consumed samples: 550144 | elapsed time per iteration (ms): 86011.4 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.74% | FLOPs estimator: workload-aware | logical tokens/s: 195057.6 | valid tokens/s: 195057.6 | physical tokens/s: 195411.8 | loss tokens/s: 144548.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.529399E-01 | lm-global-supervised-tokens: 1.243281E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.140803E-02 | lm-token-buffer-mib: 5.810249E+01 | global_load_balancing_loss: 9.995071E-01 | loss scale: 1.0 | grad norm: 0.069 | params norm: 11366.631 | params rms: 0.061547 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 13:05:31.267420 | finish at 2026-10-10 16:51:23 + [2026-10-08 03:47:20] iteration 4299/ 6855 | consumed samples: 550272 | elapsed time per iteration (ms): 88818.0 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.60% | FLOPs estimator: workload-aware | logical tokens/s: 188893.9 | valid tokens/s: 188893.8 | physical tokens/s: 189251.0 | loss tokens/s: 140079.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.898885E-01 | lm-global-supervised-tokens: 1.244156E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.363545E-02 | lm-token-buffer-mib: 5.810743E+01 | global_load_balancing_loss: 9.998272E-01 | loss scale: 1.0 | grad norm: 0.041 | params norm: 11366.569 | params rms: 0.061546 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 15:03:38.821607 | finish at 2026-10-10 18:50:59 + [2026-10-08 03:49:07] iteration 4300/ 6855 | consumed samples: 550400 | elapsed time per iteration (ms): 107155.8 | throughput per GPU (TFLOP/s/GPU): 34.1 | MFU 10.93% | FLOPs estimator: workload-aware | logical tokens/s: 156568.5 | valid tokens/s: 156568.4 | physical tokens/s: 156991.4 | loss tokens/s: 115625.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.907790E-01 | lm-global-supervised-tokens: 1.238990E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.693594E-02 | lm-token-buffer-mib: 5.817696E+01 | global_load_balancing_loss: 9.997818E-01 | loss scale: 1.0 | grad norm: 0.084 | params norm: 11366.508 | params rms: 0.061546 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 4:03:02.951698 | finish at 2026-10-11 07:52:10 + [2026-10-08 03:50:35] iteration 4301/ 6855 | consumed samples: 550528 | elapsed time per iteration (ms): 87158.6 | throughput per GPU (TFLOP/s/GPU): 43.2 | MFU 13.85% | FLOPs estimator: workload-aware | logical tokens/s: 192490.2 | valid tokens/s: 192490.2 | physical tokens/s: 193014.7 | loss tokens/s: 141569.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.998158E-01 | lm-global-supervised-tokens: 1.233899E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.521876E-02 | lm-token-buffer-mib: 5.808709E+01 | global_load_balancing_loss: 1.000699E+00 | loss scale: 1.0 | grad norm: 0.098 | params norm: 11366.448 | params rms: 0.061546 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 13:50:03.152928 | finish at 2026-10-10 17:40:38 + [2026-10-08 03:52:01] iteration 4302/ 6855 | consumed samples: 550656 | elapsed time per iteration (ms): 86743.9 | throughput per GPU (TFLOP/s/GPU): 42.8 | MFU 13.73% | FLOPs estimator: workload-aware | logical tokens/s: 193409.8 | valid tokens/s: 193409.8 | physical tokens/s: 193894.1 | loss tokens/s: 143162.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.718438E-01 | lm-global-supervised-tokens: 1.241850E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.145633E-02 | lm-token-buffer-mib: 5.824274E+01 | global_load_balancing_loss: 1.000449E+00 | loss scale: 1.0 | grad norm: 0.037 | params norm: 11366.385 | params rms: 0.061545 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 13:30:57.224941 | finish at 2026-10-10 17:22:59 + [2026-10-08 03:53:28] iteration 4303/ 6855 | consumed samples: 550784 | elapsed time per iteration (ms): 86466.8 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.75% | FLOPs estimator: workload-aware | logical tokens/s: 194030.2 | valid tokens/s: 194030.2 | physical tokens/s: 194402.0 | loss tokens/s: 142732.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.542794E-01 | lm-global-supervised-tokens: 1.234162E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.407990E-02 | lm-token-buffer-mib: 5.818059E+01 | global_load_balancing_loss: 1.000194E+00 | loss scale: 1.0 | grad norm: 0.069 | params norm: 11366.324 | params rms: 0.061545 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 13:17:43.223034 | finish at 2026-10-10 17:11:11 + [2026-10-08 03:54:55] iteration 4304/ 6855 | consumed samples: 550912 | elapsed time per iteration (ms): 86771.2 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.46% | FLOPs estimator: workload-aware | logical tokens/s: 193348.8 | valid tokens/s: 193348.8 | physical tokens/s: 193739.3 | loss tokens/s: 143288.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.289882E-01 | lm-global-supervised-tokens: 1.243335E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.138451E-02 | lm-token-buffer-mib: 5.813362E+01 | global_load_balancing_loss: 9.998132E-01 | loss scale: 1.0 | grad norm: 0.069 | params norm: 11366.262 | params rms: 0.061545 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 13:29:13.316454 | finish at 2026-10-10 17:24:08 + [2026-10-08 03:56:22] iteration 4305/ 6855 | consumed samples: 551040 | elapsed time per iteration (ms): 87407.0 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.47% | FLOPs estimator: workload-aware | logical tokens/s: 191943.4 | valid tokens/s: 191943.4 | physical tokens/s: 192466.9 | loss tokens/s: 146456.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.833197E-01 | lm-global-supervised-tokens: 1.280134E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.149056E-02 | lm-token-buffer-mib: 5.806757E+01 | global_load_balancing_loss: 9.989386E-01 | loss scale: 1.0 | grad norm: 0.036 | params norm: 11366.200 | params rms: 0.061544 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 13:54:47.967503 | finish at 2026-10-10 17:51:10 + [2026-10-08 03:57:50] iteration 4306/ 6855 | consumed samples: 551168 | elapsed time per iteration (ms): 88148.0 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.46% | FLOPs estimator: workload-aware | logical tokens/s: 190329.6 | valid tokens/s: 190329.6 | physical tokens/s: 190842.6 | loss tokens/s: 140501.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.752620E-01 | lm-global-supervised-tokens: 1.238491E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.758524E-02 | lm-token-buffer-mib: 5.810269E+01 | global_load_balancing_loss: 1.000276E+00 | loss scale: 1.0 | grad norm: 0.039 | params norm: 11366.138 | params rms: 0.061544 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 14:24:49.363219 | finish at 2026-10-10 18:22:39 + [2026-10-08 03:59:19] iteration 4307/ 6855 | consumed samples: 551296 | elapsed time per iteration (ms): 88932.3 | throughput per GPU (TFLOP/s/GPU): 41.4 | MFU 13.28% | FLOPs estimator: workload-aware | logical tokens/s: 188651.2 | valid tokens/s: 188651.2 | physical tokens/s: 189168.5 | loss tokens/s: 146470.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.806787E-01 | lm-global-supervised-tokens: 1.302595E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.203653E-02 | lm-token-buffer-mib: 5.811024E+01 | global_load_balancing_loss: 9.994774E-01 | loss scale: 1.0 | grad norm: 0.047 | params norm: 11366.078 | params rms: 0.061544 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 14:56:39.470864 | finish at 2026-10-10 18:55:59 + [2026-10-08 04:00:48] iteration 4308/ 6855 | consumed samples: 551424 | elapsed time per iteration (ms): 88975.5 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.53% | FLOPs estimator: workload-aware | logical tokens/s: 188559.7 | valid tokens/s: 188559.7 | physical tokens/s: 189145.5 | loss tokens/s: 130908.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.601382E-01 | lm-global-supervised-tokens: 1.164762E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.173559E-02 | lm-token-buffer-mib: 5.809534E+01 | global_load_balancing_loss: 1.001893E+00 | loss scale: 1.0 | grad norm: 0.052 | params norm: 11366.017 | params rms: 0.061543 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 14:57:00.725080 | finish at 2026-10-10 18:57:49 + [2026-10-08 04:02:18] iteration 4309/ 6855 | consumed samples: 551552 | elapsed time per iteration (ms): 89635.1 | throughput per GPU (TFLOP/s/GPU): 41.6 | MFU 13.32% | FLOPs estimator: workload-aware | logical tokens/s: 187172.1 | valid tokens/s: 187172.1 | physical tokens/s: 187532.4 | loss tokens/s: 136856.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.304283E-01 | lm-global-supervised-tokens: 1.226711E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.705333E-02 | lm-token-buffer-mib: 5.811630E+01 | global_load_balancing_loss: 1.001370E+00 | loss scale: 1.0 | grad norm: 0.080 | params norm: 11365.954 | params rms: 0.061543 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 15:23:30.987381 | finish at 2026-10-10 19:25:49 + [2026-10-08 04:04:05] iteration 4310/ 6855 | consumed samples: 551680 | elapsed time per iteration (ms): 107589.7 | throughput per GPU (TFLOP/s/GPU): 34.1 | MFU 10.92% | FLOPs estimator: workload-aware | logical tokens/s: 155937.1 | valid tokens/s: 155937.1 | physical tokens/s: 156351.4 | loss tokens/s: 113597.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.198237E-01 | lm-global-supervised-tokens: 1.222196E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.313398E-02 | lm-token-buffer-mib: 5.815707E+01 | global_load_balancing_loss: 9.998074E-01 | loss scale: 1.0 | grad norm: 0.043 | params norm: 11365.891 | params rms: 0.061543 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 4:03:35.689374 | finish at 2026-10-11 08:07:41 + [2026-10-08 04:05:33] iteration 4311/ 6855 | consumed samples: 551808 | elapsed time per iteration (ms): 87769.1 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.58% | FLOPs estimator: workload-aware | logical tokens/s: 191151.0 | valid tokens/s: 191151.0 | physical tokens/s: 191788.0 | loss tokens/s: 139937.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.623544E-01 | lm-global-supervised-tokens: 1.228219E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.347001E-02 | lm-token-buffer-mib: 5.802629E+01 | global_load_balancing_loss: 1.000070E+00 | loss scale: 1.0 | grad norm: 0.072 | params norm: 11365.827 | params rms: 0.061542 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 14:01:24.545998 | finish at 2026-10-10 18:06:58 + [2026-10-08 04:07:00] iteration 4312/ 6855 | consumed samples: 551936 | elapsed time per iteration (ms): 87206.0 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.54% | FLOPs estimator: workload-aware | logical tokens/s: 192386.0 | valid tokens/s: 192386.0 | physical tokens/s: 193135.0 | loss tokens/s: 145733.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.465521E-01 | lm-global-supervised-tokens: 1.270886E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.290906E-02 | lm-token-buffer-mib: 5.805307E+01 | global_load_balancing_loss: 9.996558E-01 | loss scale: 1.0 | grad norm: 0.056 | params norm: 11365.762 | params rms: 0.061542 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 13:36:04.851559 | finish at 2026-10-10 17:43:05 + [2026-10-08 04:08:26] iteration 4313/ 6855 | consumed samples: 552064 | elapsed time per iteration (ms): 86158.3 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.64% | FLOPs estimator: workload-aware | logical tokens/s: 194725.5 | valid tokens/s: 194725.5 | physical tokens/s: 195107.1 | loss tokens/s: 145833.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.233634E-01 | lm-global-supervised-tokens: 1.256472E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.202608E-02 | lm-token-buffer-mib: 5.805755E+01 | global_load_balancing_loss: 9.995621E-01 | loss scale: 1.0 | grad norm: 0.045 | params norm: 11365.699 | params rms: 0.061542 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 12:50:14.372344 | finish at 2026-10-10 16:58:41 + [2026-10-08 04:09:51] iteration 4314/ 6855 | consumed samples: 552192 | elapsed time per iteration (ms): 85029.9 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.62% | FLOPs estimator: workload-aware | logical tokens/s: 197309.3 | valid tokens/s: 197309.3 | physical tokens/s: 197714.1 | loss tokens/s: 152060.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.509442E-01 | lm-global-supervised-tokens: 1.292971E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.164640E-02 | lm-token-buffer-mib: 5.819086E+01 | global_load_balancing_loss: 1.000086E+00 | loss scale: 1.0 | grad norm: 0.040 | params norm: 11365.636 | params rms: 0.061541 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 12:01:01.049998 | finish at 2026-10-10 16:10:52 + [2026-10-08 04:11:18] iteration 4315/ 6855 | consumed samples: 552320 | elapsed time per iteration (ms): 86178.6 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.64% | FLOPs estimator: workload-aware | logical tokens/s: 194678.9 | valid tokens/s: 194678.9 | physical tokens/s: 195389.4 | loss tokens/s: 141692.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.586892E-01 | lm-global-supervised-tokens: 1.221089E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.140678E-02 | lm-token-buffer-mib: 5.825686E+01 | global_load_balancing_loss: 1.000677E+00 | loss scale: 1.0 | grad norm: 0.043 | params norm: 11365.571 | params rms: 0.061541 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 12:48:13.575754 | finish at 2026-10-10 16:59:31 + [2026-10-08 04:12:45] iteration 4316/ 6855 | consumed samples: 552448 | elapsed time per iteration (ms): 87199.8 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.74% | FLOPs estimator: workload-aware | logical tokens/s: 192399.7 | valid tokens/s: 192399.7 | physical tokens/s: 193011.2 | loss tokens/s: 140721.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.279564E-01 | lm-global-supervised-tokens: 1.227091E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.608848E-02 | lm-token-buffer-mib: 5.828098E+01 | global_load_balancing_loss: 1.001178E+00 | loss scale: 1.0 | grad norm: 0.061 | params norm: 11365.507 | params rms: 0.061541 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 13:30:00.210515 | finish at 2026-10-10 17:42:45 + [2026-10-08 04:14:14] iteration 4317/ 6855 | consumed samples: 552576 | elapsed time per iteration (ms): 89009.3 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.43% | FLOPs estimator: workload-aware | logical tokens/s: 188488.3 | valid tokens/s: 188488.3 | physical tokens/s: 188921.9 | loss tokens/s: 135503.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.110908E-01 | lm-global-supervised-tokens: 1.206107E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.328132E-02 | lm-token-buffer-mib: 5.824844E+01 | global_load_balancing_loss: 9.994681E-01 | loss scale: 1.0 | grad norm: 0.048 | params norm: 11365.443 | params rms: 0.061540 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 14:45:05.632429 | finish at 2026-10-10 18:59:19 + [2026-10-08 04:15:37] iteration 4318/ 6855 | consumed samples: 552704 | elapsed time per iteration (ms): 83321.4 | throughput per GPU (TFLOP/s/GPU): 43.3 | MFU 13.88% | FLOPs estimator: workload-aware | logical tokens/s: 201354.7 | valid tokens/s: 201354.7 | physical tokens/s: 201859.1 | loss tokens/s: 155552.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.610010E-01 | lm-global-supervised-tokens: 1.296085E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.921220E-02 | lm-token-buffer-mib: 5.810178E+01 | global_load_balancing_loss: 9.997264E-01 | loss scale: 1.0 | grad norm: 0.051 | params norm: 11365.379 | params rms: 0.061540 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 10:43:06.430932 | finish at 2026-10-10 14:58:44 + [2026-10-08 04:17:03] iteration 4319/ 6855 | consumed samples: 552832 | elapsed time per iteration (ms): 86305.8 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.63% | FLOPs estimator: workload-aware | logical tokens/s: 194392.7 | valid tokens/s: 194392.7 | physical tokens/s: 194960.4 | loss tokens/s: 141264.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.581413E-01 | lm-global-supervised-tokens: 1.219193E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.375879E-02 | lm-token-buffer-mib: 5.813908E+01 | global_load_balancing_loss: 1.000702E+00 | loss scale: 1.0 | grad norm: 0.031 | params norm: 11365.315 | params rms: 0.061539 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 12:47:51.481493 | finish at 2026-10-10 17:04:55 + [2026-10-08 04:18:57] iteration 4320/ 6855 | consumed samples: 552960 | elapsed time per iteration (ms): 113195.8 | throughput per GPU (TFLOP/s/GPU): 32.9 | MFU 10.53% | FLOPs estimator: workload-aware | logical tokens/s: 148214.1 | valid tokens/s: 148214.1 | physical tokens/s: 148518.6 | loss tokens/s: 110355.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.961374E-01 | lm-global-supervised-tokens: 1.249173E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.190527E-02 | lm-token-buffer-mib: 5.823267E+01 | global_load_balancing_loss: 1.000647E+00 | loss scale: 1.0 | grad norm: 0.040 | params norm: 11365.251 | params rms: 0.061539 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 7:42:31.313882 | finish at 2026-10-11 12:01:28 +times across ranks (ms): + save-checkpoint: + rank 0: 24420.42 + rank 1: 24420.94 + rank 2: 24420.35 + rank 3: 24420.54 + rank 4: 24421.45 + rank 5: 24420.43 + rank 6: 24421.20 + rank 7: 24420.66 + rank 8: 24420.69 + rank 9: 24420.65 + rank 10: 24420.65 + rank 11: 24420.66 + rank 12: 24420.68 + rank 13: 24420.66 + rank 14: 24420.52 + rank 15: 24420.64 + rank 16: 24420.52 + rank 17: 24420.46 + rank 18: 24420.45 + rank 19: 24420.47 + rank 20: 24420.44 + rank 21: 24420.47 + rank 22: 24420.44 + rank 23: 24420.45 + rank 24: 24420.77 + rank 25: 24420.80 + rank 26: 24420.86 + rank 27: 24420.94 + rank 28: 24420.76 + rank 29: 24420.82 + rank 30: 24420.79 + rank 31: 24420.93 + rank 32: 24420.79 + rank 33: 24420.82 + rank 34: 24420.74 + rank 35: 24420.72 + rank 36: 24420.66 + rank 37: 24420.66 + rank 38: 24420.70 + rank 39: 24420.66 + rank 40: 24420.81 + rank 41: 24420.77 + rank 42: 24420.79 + rank 43: 24420.77 + rank 44: 24420.77 + rank 45: 24420.77 + rank 46: 24420.81 + rank 47: 24420.76 + rank 48: 24421.02 + rank 49: 24421.12 + rank 50: 24421.04 + rank 51: 24420.90 + rank 52: 24421.13 + rank 53: 24421.09 + rank 54: 24421.10 + rank 55: 24421.08 + rank 56: 24420.65 + rank 57: 24420.64 + rank 58: 24420.63 + rank 59: 24420.63 + rank 60: 24420.63 + rank 61: 24420.65 + rank 62: 24420.65 + rank 63: 24420.64 + [2026-10-08 04:21:27] iteration 4321/ 6855 | consumed samples: 553088 | elapsed time per iteration (ms): 125532.1 | throughput per GPU (TFLOP/s/GPU): 30.0 | MFU 9.61% | FLOPs estimator: workload-aware | logical tokens/s: 133648.6 | valid tokens/s: 133648.6 | physical tokens/s: 133921.5 | loss tokens/s: 98848.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.319439E-01 | lm-global-supervised-tokens: 1.240868E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.345805E+00 | lm-token-buffer-mib: 5.809985E+01 | global_load_balancing_loss: 9.998150E-01 | loss scale: 1.0 | grad norm: 0.060 | params norm: 11365.187 | params rms: 0.061539 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 16:21:38.253702 | finish at 2026-10-11 20:43:05 + [2026-10-08 04:22:58] iteration 4322/ 6855 | consumed samples: 553216 | elapsed time per iteration (ms): 90904.9 | throughput per GPU (TFLOP/s/GPU): 40.2 | MFU 12.87% | FLOPs estimator: workload-aware | logical tokens/s: 184556.9 | valid tokens/s: 184556.9 | physical tokens/s: 184969.0 | loss tokens/s: 137774.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.759096E-01 | lm-global-supervised-tokens: 1.252433E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.563794E-02 | lm-token-buffer-mib: 5.803405E+01 | global_load_balancing_loss: 9.998404E-01 | loss scale: 1.0 | grad norm: 0.030 | params norm: 11365.124 | params rms: 0.061538 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 15:57:42.162616 | finish at 2026-10-10 20:20:40 + [2026-10-08 04:24:23] iteration 4323/ 6855 | consumed samples: 553344 | elapsed time per iteration (ms): 85046.0 | throughput per GPU (TFLOP/s/GPU): 43.9 | MFU 14.05% | FLOPs estimator: workload-aware | logical tokens/s: 197272.1 | valid tokens/s: 197272.1 | physical tokens/s: 197718.2 | loss tokens/s: 153854.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.943317E-01 | lm-global-supervised-tokens: 1.308470E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.143536E-02 | lm-token-buffer-mib: 5.822823E+01 | global_load_balancing_loss: 9.995776E-01 | loss scale: 1.0 | grad norm: 0.066 | params norm: 11365.060 | params rms: 0.061538 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 11:48:56.389910 | finish at 2026-10-10 16:13:19 + [2026-10-08 04:25:47] iteration 4324/ 6855 | consumed samples: 553472 | elapsed time per iteration (ms): 84504.2 | throughput per GPU (TFLOP/s/GPU): 43.2 | MFU 13.84% | FLOPs estimator: workload-aware | logical tokens/s: 198536.2 | valid tokens/s: 198536.2 | physical tokens/s: 198972.5 | loss tokens/s: 149162.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.690544E-01 | lm-global-supervised-tokens: 1.260485E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.154323E-02 | lm-token-buffer-mib: 5.810921E+01 | global_load_balancing_loss: 1.000625E+00 | loss scale: 1.0 | grad norm: 0.051 | params norm: 11365.000 | params rms: 0.061538 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 11:24:40.161532 | finish at 2026-10-10 15:50:27 + [2026-10-08 04:27:12] iteration 4325/ 6855 | consumed samples: 553600 | elapsed time per iteration (ms): 84999.7 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.79% | FLOPs estimator: workload-aware | logical tokens/s: 197379.6 | valid tokens/s: 197379.6 | physical tokens/s: 197781.1 | loss tokens/s: 154653.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.826224E-01 | lm-global-supervised-tokens: 1.314550E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.418331E-02 | lm-token-buffer-mib: 5.802271E+01 | global_load_balancing_loss: 1.000337E+00 | loss scale: 1.0 | grad norm: 0.044 | params norm: 11364.940 | params rms: 0.061537 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 11:44:09.338894 | finish at 2026-10-10 16:11:22 + [2026-10-08 04:28:37] iteration 4326/ 6855 | consumed samples: 553728 | elapsed time per iteration (ms): 84927.4 | throughput per GPU (TFLOP/s/GPU): 42.8 | MFU 13.70% | FLOPs estimator: workload-aware | logical tokens/s: 197547.2 | valid tokens/s: 197547.2 | physical tokens/s: 197987.9 | loss tokens/s: 151022.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.711037E-01 | lm-global-supervised-tokens: 1.282594E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.486300E-02 | lm-token-buffer-mib: 5.824454E+01 | global_load_balancing_loss: 9.991658E-01 | loss scale: 1.0 | grad norm: 0.033 | params norm: 11364.878 | params rms: 0.061537 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 11:39:41.503417 | finish at 2026-10-10 16:08:19 + [2026-10-08 04:30:07] iteration 4327/ 6855 | consumed samples: 553856 | elapsed time per iteration (ms): 90138.4 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.49% | FLOPs estimator: workload-aware | logical tokens/s: 186126.9 | valid tokens/s: 186126.9 | physical tokens/s: 186561.9 | loss tokens/s: 132384.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.433215E-01 | lm-global-supervised-tokens: 1.193293E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.580334E-02 | lm-token-buffer-mib: 5.808433E+01 | global_load_balancing_loss: 1.000784E+00 | loss scale: 1.0 | grad norm: 0.058 | params norm: 11364.818 | params rms: 0.061537 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 15:17:49.800056 | finish at 2026-10-10 19:47:57 + [2026-10-08 04:31:36] iteration 4328/ 6855 | consumed samples: 553984 | elapsed time per iteration (ms): 88227.7 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.52% | FLOPs estimator: workload-aware | logical tokens/s: 190158.1 | valid tokens/s: 190158.1 | physical tokens/s: 190607.5 | loss tokens/s: 140056.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.239575E-01 | lm-global-supervised-tokens: 1.235685E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.227863E-02 | lm-token-buffer-mib: 5.802538E+01 | global_load_balancing_loss: 1.001124E+00 | loss scale: 1.0 | grad norm: 0.083 | params norm: 11364.759 | params rms: 0.061536 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 13:55:51.390658 | finish at 2026-10-10 18:27:27 + [2026-10-08 04:33:04] iteration 4329/ 6855 | consumed samples: 554112 | elapsed time per iteration (ms): 88766.7 | throughput per GPU (TFLOP/s/GPU): 41.2 | MFU 13.20% | FLOPs estimator: workload-aware | logical tokens/s: 189003.1 | valid tokens/s: 189003.1 | physical tokens/s: 189386.4 | loss tokens/s: 141976.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.146960E-01 | lm-global-supervised-tokens: 1.260282E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.335556E-02 | lm-token-buffer-mib: 5.808858E+01 | global_load_balancing_loss: 1.001301E+00 | loss scale: 1.0 | grad norm: 0.132 | params norm: 11364.701 | params rms: 0.061536 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 14:17:04.773407 | finish at 2026-10-10 18:50:09 + [2026-10-08 04:34:53] iteration 4330/ 6855 | consumed samples: 554240 | elapsed time per iteration (ms): 109020.1 | throughput per GPU (TFLOP/s/GPU): 33.1 | MFU 10.61% | FLOPs estimator: workload-aware | logical tokens/s: 153889.8 | valid tokens/s: 153889.8 | physical tokens/s: 154379.3 | loss tokens/s: 118758.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.132924E-01 | lm-global-supervised-tokens: 1.294708E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.248954E-02 | lm-token-buffer-mib: 5.816354E+01 | global_load_balancing_loss: 9.990918E-01 | loss scale: 1.0 | grad norm: 0.327 | params norm: 11364.642 | params rms: 0.061536 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 4:27:55.743163 | finish at 2026-10-11 09:02:49 + [2026-10-08 04:36:24] iteration 4331/ 6855 | consumed samples: 554368 | elapsed time per iteration (ms): 90721.3 | throughput per GPU (TFLOP/s/GPU): 40.5 | MFU 13.00% | FLOPs estimator: workload-aware | logical tokens/s: 184931.2 | valid tokens/s: 184931.2 | physical tokens/s: 185399.8 | loss tokens/s: 134160.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.817711E-01 | lm-global-supervised-tokens: 1.217117E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.135814E-02 | lm-token-buffer-mib: 5.813613E+01 | global_load_balancing_loss: 1.000415E+00 | loss scale: 1.0 | grad norm: 0.480 | params norm: 11364.581 | params rms: 0.061536 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 15:36:20.545870 | finish at 2026-10-10 20:12:45 + [2026-10-08 04:37:53] iteration 4332/ 6855 | consumed samples: 554496 | elapsed time per iteration (ms): 89141.9 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.49% | FLOPs estimator: workload-aware | logical tokens/s: 188207.9 | valid tokens/s: 188207.9 | physical tokens/s: 188580.2 | loss tokens/s: 143808.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.115291E-01 | lm-global-supervised-tokens: 1.281933E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.217211E-02 | lm-token-buffer-mib: 5.819011E+01 | global_load_balancing_loss: 1.000123E+00 | loss scale: 1.0 | grad norm: 0.099 | params norm: 11364.521 | params rms: 0.061535 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 14:28:25.036114 | finish at 2026-10-10 19:06:18 + [2026-10-08 04:39:21] iteration 4333/ 6855 | consumed samples: 554624 | elapsed time per iteration (ms): 87432.0 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.64% | FLOPs estimator: workload-aware | logical tokens/s: 191888.2 | valid tokens/s: 191888.2 | physical tokens/s: 192324.5 | loss tokens/s: 142281.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.751379E-01 | lm-global-supervised-tokens: 1.243995E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.522500E-02 | lm-token-buffer-mib: 5.818635E+01 | global_load_balancing_loss: 1.000090E+00 | loss scale: 1.0 | grad norm: 0.468 | params norm: 11364.461 | params rms: 0.061535 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 13:15:03.615042 | finish at 2026-10-10 17:54:24 + [2026-10-08 04:40:48] iteration 4334/ 6855 | consumed samples: 554752 | elapsed time per iteration (ms): 87241.6 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.66% | FLOPs estimator: workload-aware | logical tokens/s: 192307.1 | valid tokens/s: 192306.9 | physical tokens/s: 192698.2 | loss tokens/s: 146546.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.867911E-01 | lm-global-supervised-tokens: 1.278494E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.523034E-02 | lm-token-buffer-mib: 5.816911E+01 | global_load_balancing_loss: 9.990699E-01 | loss scale: 1.0 | grad norm: 0.090 | params norm: 11364.401 | params rms: 0.061535 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 13:05:36.075495 | finish at 2026-10-10 17:46:24 + [2026-10-08 04:42:13] iteration 4335/ 6855 | consumed samples: 554880 | elapsed time per iteration (ms): 85061.1 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.78% | FLOPs estimator: workload-aware | logical tokens/s: 197236.9 | valid tokens/s: 197236.9 | physical tokens/s: 197685.3 | loss tokens/s: 153418.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.701518E-01 | lm-global-supervised-tokens: 1.304995E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.151576E-02 | lm-token-buffer-mib: 5.830488E+01 | global_load_balancing_loss: 9.991591E-01 | loss scale: 1.0 | grad norm: 0.315 | params norm: 11364.340 | params rms: 0.061534 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 11:32:33.950987 | finish at 2026-10-10 16:14:47 + [2026-10-08 04:43:38] iteration 4336/ 6855 | consumed samples: 555008 | elapsed time per iteration (ms): 85529.3 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.60% | FLOPs estimator: workload-aware | logical tokens/s: 196156.6 | valid tokens/s: 196156.6 | physical tokens/s: 196725.8 | loss tokens/s: 145148.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.465725E-01 | lm-global-supervised-tokens: 1.241445E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.136966E-02 | lm-token-buffer-mib: 5.818397E+01 | global_load_balancing_loss: 1.001492E+00 | loss scale: 1.0 | grad norm: 0.137 | params norm: 11364.281 | params rms: 0.061534 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 11:50:48.370897 | finish at 2026-10-10 16:34:27 + [2026-10-08 04:45:05] iteration 4337/ 6855 | consumed samples: 555136 | elapsed time per iteration (ms): 86637.7 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.44% | FLOPs estimator: workload-aware | logical tokens/s: 193647.1 | valid tokens/s: 193647.1 | physical tokens/s: 194097.1 | loss tokens/s: 145804.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.301441E-01 | lm-global-supervised-tokens: 1.263217E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.335943E-02 | lm-token-buffer-mib: 5.812316E+01 | global_load_balancing_loss: 1.000680E+00 | loss scale: 1.0 | grad norm: 0.053 | params norm: 11364.222 | params rms: 0.061534 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 12:35:53.698787 | finish at 2026-10-10 17:20:59 + [2026-10-08 04:46:31] iteration 4338/ 6855 | consumed samples: 555264 | elapsed time per iteration (ms): 86248.6 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.77% | FLOPs estimator: workload-aware | logical tokens/s: 194521.3 | valid tokens/s: 194521.3 | physical tokens/s: 195126.8 | loss tokens/s: 144927.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.553259E-01 | lm-global-supervised-tokens: 1.249976E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.098095E-02 | lm-token-buffer-mib: 5.808008E+01 | global_load_balancing_loss: 1.000250E+00 | loss scale: 1.0 | grad norm: 0.089 | params norm: 11364.163 | params rms: 0.061533 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 12:18:07.699811 | finish at 2026-10-10 17:04:39 + [2026-10-08 04:48:00] iteration 4339/ 6855 | consumed samples: 555392 | elapsed time per iteration (ms): 88233.0 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.48% | FLOPs estimator: workload-aware | logical tokens/s: 190145.7 | valid tokens/s: 190145.7 | physical tokens/s: 190521.3 | loss tokens/s: 146899.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.006382E-01 | lm-global-supervised-tokens: 1.296136E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.400885E-02 | lm-token-buffer-mib: 5.804889E+01 | global_load_balancing_loss: 1.000649E+00 | loss scale: 1.0 | grad norm: 0.156 | params norm: 11364.102 | params rms: 0.061533 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 13:39:54.104529 | finish at 2026-10-10 18:27:54 + [2026-10-08 04:49:46] iteration 4340/ 6855 | consumed samples: 555520 | elapsed time per iteration (ms): 106200.2 | throughput per GPU (TFLOP/s/GPU): 34.4 | MFU 11.02% | FLOPs estimator: workload-aware | logical tokens/s: 157977.0 | valid tokens/s: 157977.0 | physical tokens/s: 158273.1 | loss tokens/s: 117272.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.201030E-01 | lm-global-supervised-tokens: 1.245431E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.383153E-02 | lm-token-buffer-mib: 5.810976E+01 | global_load_balancing_loss: 1.000955E+00 | loss scale: 1.0 | grad norm: 0.052 | params norm: 11364.039 | params rms: 0.061533 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 2:11:33.619530 | finish at 2026-10-11 07:01:19 + [2026-10-08 04:51:12] iteration 4341/ 6855 | consumed samples: 555648 | elapsed time per iteration (ms): 86236.0 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.78% | FLOPs estimator: workload-aware | logical tokens/s: 194549.8 | valid tokens/s: 194549.8 | physical tokens/s: 194987.2 | loss tokens/s: 140768.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.570019E-01 | lm-global-supervised-tokens: 1.213933E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.311224E-02 | lm-token-buffer-mib: 5.814015E+01 | global_load_balancing_loss: 1.000576E+00 | loss scale: 1.0 | grad norm: 0.103 | params norm: 11363.978 | params rms: 0.061532 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 12:13:17.248411 | finish at 2026-10-10 17:04:29 + [2026-10-08 04:52:41] iteration 4342/ 6855 | consumed samples: 555776 | elapsed time per iteration (ms): 89047.4 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.60% | FLOPs estimator: workload-aware | logical tokens/s: 188407.5 | valid tokens/s: 188407.5 | physical tokens/s: 188720.4 | loss tokens/s: 136633.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.031590E-01 | lm-global-supervised-tokens: 1.216689E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.056764E-02 | lm-token-buffer-mib: 5.821515E+01 | global_load_balancing_loss: 9.995004E-01 | loss scale: 1.0 | grad norm: 0.049 | params norm: 11363.918 | params rms: 0.061532 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 14:09:36.103612 | finish at 2026-10-10 19:02:17 + [2026-10-08 04:54:11] iteration 4343/ 6855 | consumed samples: 555904 | elapsed time per iteration (ms): 89935.9 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.67% | FLOPs estimator: workload-aware | logical tokens/s: 186546.1 | valid tokens/s: 186546.1 | physical tokens/s: 186847.9 | loss tokens/s: 132839.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.919454E-01 | lm-global-supervised-tokens: 1.194705E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.207349E-02 | lm-token-buffer-mib: 5.806425E+01 | global_load_balancing_loss: 9.994176E-01 | loss scale: 1.0 | grad norm: 0.060 | params norm: 11363.857 | params rms: 0.061532 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 14:45:19.050804 | finish at 2026-10-10 19:39:30 + [2026-10-08 04:55:37] iteration 4344/ 6855 | consumed samples: 556032 | elapsed time per iteration (ms): 85654.1 | throughput per GPU (TFLOP/s/GPU): 42.8 | MFU 13.73% | FLOPs estimator: workload-aware | logical tokens/s: 195871.4 | valid tokens/s: 195871.4 | physical tokens/s: 196301.9 | loss tokens/s: 147138.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.447737E-01 | lm-global-supervised-tokens: 1.260302E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.231673E-02 | lm-token-buffer-mib: 5.809832E+01 | global_load_balancing_loss: 9.999159E-01 | loss scale: 1.0 | grad norm: 0.126 | params norm: 11363.797 | params rms: 0.061531 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 11:44:37.434176 | finish at 2026-10-10 16:40:14 + [2026-10-08 04:57:03] iteration 4345/ 6855 | consumed samples: 556160 | elapsed time per iteration (ms): 86385.7 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.50% | FLOPs estimator: workload-aware | logical tokens/s: 194212.2 | valid tokens/s: 194212.2 | physical tokens/s: 194632.8 | loss tokens/s: 143464.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.294274E-01 | lm-global-supervised-tokens: 1.239326E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.714774E-02 | lm-token-buffer-mib: 5.811586E+01 | global_load_balancing_loss: 1.000508E+00 | loss scale: 1.0 | grad norm: 0.100 | params norm: 11363.738 | params rms: 0.061531 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 12:13:48.077047 | finish at 2026-10-10 17:10:51 + [2026-10-08 04:58:32] iteration 4346/ 6855 | consumed samples: 556288 | elapsed time per iteration (ms): 89083.3 | throughput per GPU (TFLOP/s/GPU): 41.4 | MFU 13.28% | FLOPs estimator: workload-aware | logical tokens/s: 188331.6 | valid tokens/s: 188331.6 | physical tokens/s: 188694.8 | loss tokens/s: 138825.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.366183E-01 | lm-global-supervised-tokens: 1.236700E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.137687E-02 | lm-token-buffer-mib: 5.837523E+01 | global_load_balancing_loss: 1.000042E+00 | loss scale: 1.0 | grad norm: 0.048 | params norm: 11363.678 | params rms: 0.061531 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 14:05:10.002378 | finish at 2026-10-10 19:03:42 + [2026-10-08 04:59:59] iteration 4347/ 6855 | consumed samples: 556416 | elapsed time per iteration (ms): 86981.9 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.41% | FLOPs estimator: workload-aware | logical tokens/s: 192881.4 | valid tokens/s: 192881.4 | physical tokens/s: 193355.5 | loss tokens/s: 142755.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.462010E-01 | lm-global-supervised-tokens: 1.241716E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.224204E-02 | lm-token-buffer-mib: 5.803616E+01 | global_load_balancing_loss: 1.000419E+00 | loss scale: 1.0 | grad norm: 0.075 | params norm: 11363.617 | params rms: 0.061530 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 12:35:50.641617 | finish at 2026-10-10 17:35:50 + [2026-10-08 05:01:27] iteration 4348/ 6855 | consumed samples: 556544 | elapsed time per iteration (ms): 88133.9 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.56% | FLOPs estimator: workload-aware | logical tokens/s: 190359.8 | valid tokens/s: 190359.8 | physical tokens/s: 190896.9 | loss tokens/s: 137665.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.247497E-01 | lm-global-supervised-tokens: 1.213302E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.235857E-02 | lm-token-buffer-mib: 5.809758E+01 | global_load_balancing_loss: 1.000622E+00 | loss scale: 1.0 | grad norm: 0.069 | params norm: 11363.559 | params rms: 0.061530 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 13:22:31.732544 | finish at 2026-10-10 18:23:59 + [2026-10-08 05:02:53] iteration 4349/ 6855 | consumed samples: 556672 | elapsed time per iteration (ms): 85956.4 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.66% | FLOPs estimator: workload-aware | logical tokens/s: 195182.9 | valid tokens/s: 195182.9 | physical tokens/s: 195668.1 | loss tokens/s: 146234.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.680934E-01 | lm-global-supervised-tokens: 1.256982E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.668484E-02 | lm-token-buffer-mib: 5.822280E+01 | global_load_balancing_loss: 1.000360E+00 | loss scale: 1.0 | grad norm: 0.045 | params norm: 11363.499 | params rms: 0.061530 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 11:50:06.720867 | finish at 2026-10-10 16:53:00 + [2026-10-08 05:04:42] iteration 4350/ 6855 | consumed samples: 556800 | elapsed time per iteration (ms): 108571.2 | throughput per GPU (TFLOP/s/GPU): 34.2 | MFU 10.95% | FLOPs estimator: workload-aware | logical tokens/s: 154527.3 | valid tokens/s: 154527.3 | physical tokens/s: 154767.2 | loss tokens/s: 112169.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.778095E-01 | lm-global-supervised-tokens: 1.217834E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.347355E-02 | lm-token-buffer-mib: 5.811127E+01 | global_load_balancing_loss: 1.000417E+00 | loss scale: 1.0 | grad norm: 0.042 | params norm: 11363.440 | params rms: 0.061529 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 3:32:50.904111 | finish at 2026-10-11 08:37:33 + [2026-10-08 05:06:10] iteration 4351/ 6855 | consumed samples: 556928 | elapsed time per iteration (ms): 88099.8 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.44% | FLOPs estimator: workload-aware | logical tokens/s: 190434.1 | valid tokens/s: 190434.1 | physical tokens/s: 190844.9 | loss tokens/s: 145744.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.693432E-01 | lm-global-supervised-tokens: 1.284001E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.107075E-02 | lm-token-buffer-mib: 5.828220E+01 | global_load_balancing_loss: 1.000654E+00 | loss scale: 1.0 | grad norm: 0.047 | params norm: 11363.379 | params rms: 0.061529 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 13:16:41.788433 | finish at 2026-10-10 18:22:52 + [2026-10-08 05:07:38] iteration 4352/ 6855 | consumed samples: 557056 | elapsed time per iteration (ms): 87810.5 | throughput per GPU (TFLOP/s/GPU): 40.9 | MFU 13.12% | FLOPs estimator: workload-aware | logical tokens/s: 191060.4 | valid tokens/s: 191060.4 | physical tokens/s: 191572.0 | loss tokens/s: 144564.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 5.898435E-01 | lm-global-supervised-tokens: 1.269430E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.020429E-02 | lm-token-buffer-mib: 5.810003E+01 | global_load_balancing_loss: 9.989539E-01 | loss scale: 1.0 | grad norm: 0.042 | params norm: 11363.318 | params rms: 0.061529 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 13:03:09.697975 | finish at 2026-10-10 18:10:47 + [2026-10-08 05:09:03] iteration 4353/ 6855 | consumed samples: 557184 | elapsed time per iteration (ms): 85455.9 | throughput per GPU (TFLOP/s/GPU): 44.0 | MFU 14.10% | FLOPs estimator: workload-aware | logical tokens/s: 196326.0 | valid tokens/s: 196326.0 | physical tokens/s: 196995.2 | loss tokens/s: 137257.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.644883E-01 | lm-global-supervised-tokens: 1.172950E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.296888E-02 | lm-token-buffer-mib: 5.808920E+01 | global_load_balancing_loss: 1.000504E+00 | loss scale: 1.0 | grad norm: 0.036 | params norm: 11363.258 | params rms: 0.061528 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 11:23:30.608594 | finish at 2026-10-10 16:32:34 + [2026-10-08 05:10:29] iteration 4354/ 6855 | consumed samples: 557312 | elapsed time per iteration (ms): 86086.8 | throughput per GPU (TFLOP/s/GPU): 43.1 | MFU 13.80% | FLOPs estimator: workload-aware | logical tokens/s: 194886.8 | valid tokens/s: 194886.8 | physical tokens/s: 195235.5 | loss tokens/s: 134004.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.756264E-01 | lm-global-supervised-tokens: 1.153605E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.139686E-02 | lm-token-buffer-mib: 5.806833E+01 | global_load_balancing_loss: 9.996898E-01 | loss scale: 1.0 | grad norm: 0.040 | params norm: 11363.197 | params rms: 0.061528 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 11:48:23.141308 | finish at 2026-10-10 16:58:52 + [2026-10-08 05:11:57] iteration 4355/ 6855 | consumed samples: 557440 | elapsed time per iteration (ms): 88210.2 | throughput per GPU (TFLOP/s/GPU): 41.4 | MFU 13.26% | FLOPs estimator: workload-aware | logical tokens/s: 190195.7 | valid tokens/s: 190195.7 | physical tokens/s: 190665.6 | loss tokens/s: 140716.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.945933E-01 | lm-global-supervised-tokens: 1.241264E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.135384E-02 | lm-token-buffer-mib: 5.816040E+01 | global_load_balancing_loss: 1.000348E+00 | loss scale: 1.0 | grad norm: 0.050 | params norm: 11363.136 | params rms: 0.061528 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 13:15:25.599122 | finish at 2026-10-10 18:27:23 + [2026-10-08 05:13:22] iteration 4356/ 6855 | consumed samples: 557568 | elapsed time per iteration (ms): 84824.0 | throughput per GPU (TFLOP/s/GPU): 42.8 | MFU 13.70% | FLOPs estimator: workload-aware | logical tokens/s: 197787.8 | valid tokens/s: 197787.8 | physical tokens/s: 198350.6 | loss tokens/s: 147730.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.070611E-01 | lm-global-supervised-tokens: 1.253105E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.304840E-02 | lm-token-buffer-mib: 5.822029E+01 | global_load_balancing_loss: 1.000974E+00 | loss scale: 1.0 | grad norm: 0.043 | params norm: 11363.075 | params rms: 0.061527 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 10:52:55.073821 | finish at 2026-10-10 16:06:17 + [2026-10-08 05:14:53] iteration 4357/ 6855 | consumed samples: 557696 | elapsed time per iteration (ms): 90534.7 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.41% | FLOPs estimator: workload-aware | logical tokens/s: 185311.5 | valid tokens/s: 185311.5 | physical tokens/s: 185704.3 | loss tokens/s: 133092.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.757299E-01 | lm-global-supervised-tokens: 1.204948E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.470658E-02 | lm-token-buffer-mib: 5.822880E+01 | global_load_balancing_loss: 1.000837E+00 | loss scale: 1.0 | grad norm: 0.049 | params norm: 11363.014 | params rms: 0.061527 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 14:49:15.801293 | finish at 2026-10-10 20:04:09 + [2026-10-08 05:16:24] iteration 4358/ 6855 | consumed samples: 557824 | elapsed time per iteration (ms): 90854.7 | throughput per GPU (TFLOP/s/GPU): 41.6 | MFU 13.33% | FLOPs estimator: workload-aware | logical tokens/s: 184659.6 | valid tokens/s: 184659.6 | physical tokens/s: 185102.2 | loss tokens/s: 128821.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.847547E-01 | lm-global-supervised-tokens: 1.170405E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.134302E-02 | lm-token-buffer-mib: 5.822701E+01 | global_load_balancing_loss: 1.000212E+00 | loss scale: 1.0 | grad norm: 0.061 | params norm: 11362.954 | params rms: 0.061527 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 15:01:04.262919 | finish at 2026-10-10 20:17:28 + [2026-10-08 05:17:52] iteration 4359/ 6855 | consumed samples: 557952 | elapsed time per iteration (ms): 88649.2 | throughput per GPU (TFLOP/s/GPU): 41.7 | MFU 13.38% | FLOPs estimator: workload-aware | logical tokens/s: 189253.6 | valid tokens/s: 189253.6 | physical tokens/s: 189729.0 | loss tokens/s: 141736.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.667202E-01 | lm-global-supervised-tokens: 1.256480E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.133615E-02 | lm-token-buffer-mib: 5.816184E+01 | global_load_balancing_loss: 9.987449E-01 | loss scale: 1.0 | grad norm: 0.045 | params norm: 11362.892 | params rms: 0.061526 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 13:27:48.451309 | finish at 2026-10-10 18:45:41 + [2026-10-08 05:19:42] iteration 4360/ 6855 | consumed samples: 558080 | elapsed time per iteration (ms): 110209.0 | throughput per GPU (TFLOP/s/GPU): 33.4 | MFU 10.70% | FLOPs estimator: workload-aware | logical tokens/s: 152230.6 | valid tokens/s: 152230.6 | physical tokens/s: 152571.9 | loss tokens/s: 114004.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.922123E-01 | lm-global-supervised-tokens: 1.256429E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.168441E-02 | lm-token-buffer-mib: 5.811134E+01 | global_load_balancing_loss: 9.994636E-01 | loss scale: 1.0 | grad norm: 0.068 | params norm: 11362.831 | params rms: 0.061526 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 4:22:51.448733 | finish at 2026-10-11 09:42:34 +times across ranks (ms): + save-checkpoint: + rank 0: 24322.76 + rank 1: 24322.82 + rank 2: 24322.78 + rank 3: 24322.80 + rank 4: 24322.82 + rank 5: 24322.77 + rank 6: 24322.76 + rank 7: 24322.80 + rank 8: 24322.61 + rank 9: 24322.60 + rank 10: 24322.58 + rank 11: 24322.58 + rank 12: 24322.59 + rank 13: 24322.57 + rank 14: 24322.58 + rank 15: 24322.57 + rank 16: 24323.12 + rank 17: 24323.10 + rank 18: 24323.08 + rank 19: 24323.10 + rank 20: 24323.09 + rank 21: 24323.10 + rank 22: 24323.08 + rank 23: 24323.08 + rank 24: 24322.86 + rank 25: 24322.85 + rank 26: 24322.86 + rank 27: 24322.86 + rank 28: 24322.83 + rank 29: 24322.84 + rank 30: 24322.84 + rank 31: 24322.87 + rank 32: 24324.50 + rank 33: 24324.52 + rank 34: 24324.49 + rank 35: 24324.45 + rank 36: 24324.46 + rank 37: 24324.46 + rank 38: 24324.46 + rank 39: 24324.44 + rank 40: 24322.45 + rank 41: 24322.43 + rank 42: 24322.44 + rank 43: 24322.45 + rank 44: 24322.42 + rank 45: 24322.50 + rank 46: 24322.45 + rank 47: 24322.42 + rank 48: 24322.72 + rank 49: 24322.69 + rank 50: 24322.68 + rank 51: 24322.69 + rank 52: 24322.70 + rank 53: 24322.70 + rank 54: 24322.70 + rank 55: 24322.69 + rank 56: 24322.47 + rank 57: 24322.46 + rank 58: 24322.46 + rank 59: 24322.48 + rank 60: 24322.47 + rank 61: 24322.46 + rank 62: 24322.48 + rank 63: 24322.46 + [2026-10-08 05:21:50] iteration 4361/ 6855 | consumed samples: 558208 | elapsed time per iteration (ms): 103458.7 | throughput per GPU (TFLOP/s/GPU): 36.0 | MFU 11.55% | FLOPs estimator: workload-aware | logical tokens/s: 162163.1 | valid tokens/s: 162163.1 | physical tokens/s: 162654.5 | loss tokens/s: 119862.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.611981E-01 | lm-global-supervised-tokens: 1.240082E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.161991E-02 | lm-token-buffer-mib: 5.806034E+01 | global_load_balancing_loss: 1.000739E+00 | loss scale: 1.0 | grad norm: 0.096 | params norm: 11362.771 | params rms: 0.061526 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 23:40:26.096361 | finish at 2026-10-11 05:02:17 + [2026-10-08 05:23:23] iteration 4362/ 6855 | consumed samples: 558336 | elapsed time per iteration (ms): 92705.1 | throughput per GPU (TFLOP/s/GPU): 39.7 | MFU 12.73% | FLOPs estimator: workload-aware | logical tokens/s: 180974.1 | valid tokens/s: 180974.0 | physical tokens/s: 181298.0 | loss tokens/s: 128446.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.401381E-01 | lm-global-supervised-tokens: 1.190766E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.766652E-02 | lm-token-buffer-mib: 5.802360E+01 | global_load_balancing_loss: 1.002151E+00 | loss scale: 1.0 | grad norm: 0.094 | params norm: 11362.711 | params rms: 0.061525 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 16:11:53.766898 | finish at 2026-10-10 21:35:17 + [2026-10-08 05:24:49] iteration 4363/ 6855 | consumed samples: 558464 | elapsed time per iteration (ms): 85764.7 | throughput per GPU (TFLOP/s/GPU): 43.2 | MFU 13.85% | FLOPs estimator: workload-aware | logical tokens/s: 195618.5 | valid tokens/s: 195618.5 | physical tokens/s: 196000.1 | loss tokens/s: 151769.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.857953E-01 | lm-global-supervised-tokens: 1.301650E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.605206E-02 | lm-token-buffer-mib: 5.809624E+01 | global_load_balancing_loss: 9.999668E-01 | loss scale: 1.0 | grad norm: 0.075 | params norm: 11362.643 | params rms: 0.061525 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 11:22:05.613822 | finish at 2026-10-10 16:46:55 + [2026-10-08 05:26:12] iteration 4364/ 6855 | consumed samples: 558592 | elapsed time per iteration (ms): 82618.2 | throughput per GPU (TFLOP/s/GPU): 43.8 | MFU 14.03% | FLOPs estimator: workload-aware | logical tokens/s: 203068.5 | valid tokens/s: 203068.5 | physical tokens/s: 203470.2 | loss tokens/s: 147700.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.461728E-01 | lm-global-supervised-tokens: 1.220272E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.276397E-02 | lm-token-buffer-mib: 5.805165E+01 | global_load_balancing_loss: 1.002252E+00 | loss scale: 1.0 | grad norm: 0.077 | params norm: 11362.579 | params rms: 0.061525 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 9:10:01.841928 | finish at 2026-10-10 14:36:13 + [2026-10-08 05:27:39] iteration 4365/ 6855 | consumed samples: 558720 | elapsed time per iteration (ms): 87366.7 | throughput per GPU (TFLOP/s/GPU): 42.8 | MFU 13.71% | FLOPs estimator: workload-aware | logical tokens/s: 192032.0 | valid tokens/s: 192032.0 | physical tokens/s: 192487.9 | loss tokens/s: 136818.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.129788E-01 | lm-global-supervised-tokens: 1.195339E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.107378E-02 | lm-token-buffer-mib: 5.820542E+01 | global_load_balancing_loss: 1.000750E+00 | loss scale: 1.0 | grad norm: 0.073 | params norm: 11362.520 | params rms: 0.061524 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 12:25:43.202162 | finish at 2026-10-10 17:53:22 + [2026-10-08 05:29:03] iteration 4366/ 6855 | consumed samples: 558848 | elapsed time per iteration (ms): 84562.9 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.75% | FLOPs estimator: workload-aware | logical tokens/s: 198398.8 | valid tokens/s: 198398.8 | physical tokens/s: 198780.1 | loss tokens/s: 148747.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.918983E-01 | lm-global-supervised-tokens: 1.257856E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.161716E-02 | lm-token-buffer-mib: 5.810809E+01 | global_load_balancing_loss: 9.996241E-01 | loss scale: 1.0 | grad norm: 0.047 | params norm: 11362.466 | params rms: 0.061524 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 10:27:57.067760 | finish at 2026-10-10 15:57:01 + [2026-10-08 05:30:27] iteration 4367/ 6855 | consumed samples: 558976 | elapsed time per iteration (ms): 83418.4 | throughput per GPU (TFLOP/s/GPU): 43.8 | MFU 14.03% | FLOPs estimator: workload-aware | logical tokens/s: 201121.2 | valid tokens/s: 201121.2 | physical tokens/s: 201837.3 | loss tokens/s: 149811.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.722819E-01 | lm-global-supervised-tokens: 1.249699E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.149168E-02 | lm-token-buffer-mib: 5.809356E+01 | global_load_balancing_loss: 9.994439E-01 | loss scale: 1.0 | grad norm: 0.167 | params norm: 11362.413 | params rms: 0.061524 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 9:39:04.933647 | finish at 2026-10-10 15:09:32 + [2026-10-08 05:31:54] iteration 4368/ 6855 | consumed samples: 559104 | elapsed time per iteration (ms): 86724.3 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.59% | FLOPs estimator: workload-aware | logical tokens/s: 193454.5 | valid tokens/s: 193454.5 | physical tokens/s: 194017.3 | loss tokens/s: 148813.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.952968E-01 | lm-global-supervised-tokens: 1.290573E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.470991E-02 | lm-token-buffer-mib: 5.833535E+01 | global_load_balancing_loss: 9.980286E-01 | loss scale: 1.0 | grad norm: 0.177 | params norm: 11362.361 | params rms: 0.061523 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 11:54:43.380120 | finish at 2026-10-10 17:26:37 + [2026-10-08 05:33:21] iteration 4369/ 6855 | consumed samples: 559232 | elapsed time per iteration (ms): 87627.7 | throughput per GPU (TFLOP/s/GPU): 43.1 | MFU 13.83% | FLOPs estimator: workload-aware | logical tokens/s: 191460.0 | valid tokens/s: 191460.0 | physical tokens/s: 192161.3 | loss tokens/s: 141534.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.852739E-01 | lm-global-supervised-tokens: 1.240231E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.584220E-02 | lm-token-buffer-mib: 5.833353E+01 | global_load_balancing_loss: 9.988897E-01 | loss scale: 1.0 | grad norm: 0.041 | params norm: 11362.310 | params rms: 0.061523 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 12:30:42.497395 | finish at 2026-10-10 18:04:04 + [2026-10-08 05:35:12] iteration 4370/ 6855 | consumed samples: 559360 | elapsed time per iteration (ms): 111205.3 | throughput per GPU (TFLOP/s/GPU): 33.9 | MFU 10.87% | FLOPs estimator: workload-aware | logical tokens/s: 150866.7 | valid tokens/s: 150866.7 | physical tokens/s: 151273.5 | loss tokens/s: 103310.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.047832E-01 | lm-global-supervised-tokens: 1.148863E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.460612E-02 | lm-token-buffer-mib: 5.822250E+01 | global_load_balancing_loss: 1.000596E+00 | loss scale: 1.0 | grad norm: 0.210 | params norm: 11362.260 | params rms: 0.061523 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 4:45:45.142884 | finish at 2026-10-11 10:20:58 + [2026-10-08 05:36:42] iteration 4371/ 6855 | consumed samples: 559488 | elapsed time per iteration (ms): 89785.0 | throughput per GPU (TFLOP/s/GPU): 41.2 | MFU 13.22% | FLOPs estimator: workload-aware | logical tokens/s: 186859.5 | valid tokens/s: 186859.5 | physical tokens/s: 187390.5 | loss tokens/s: 135993.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.949011E-01 | lm-global-supervised-tokens: 1.221015E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.370768E-02 | lm-token-buffer-mib: 5.823773E+01 | global_load_balancing_loss: 1.000125E+00 | loss scale: 1.0 | grad norm: 0.219 | params norm: 11362.209 | params rms: 0.061523 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 13:57:05.885728 | finish at 2026-10-10 19:33:48 + [2026-10-08 05:38:12] iteration 4372/ 6855 | consumed samples: 559616 | elapsed time per iteration (ms): 89304.4 | throughput per GPU (TFLOP/s/GPU): 41.4 | MFU 13.27% | FLOPs estimator: workload-aware | logical tokens/s: 187863.1 | valid tokens/s: 187863.1 | physical tokens/s: 188327.9 | loss tokens/s: 139366.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.594261E-01 | lm-global-supervised-tokens: 1.244606E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.165163E-02 | lm-token-buffer-mib: 5.820159E+01 | global_load_balancing_loss: 1.001186E+00 | loss scale: 1.0 | grad norm: 0.113 | params norm: 11362.155 | params rms: 0.061522 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 13:35:42.824527 | finish at 2026-10-10 19:13:54 + [2026-10-08 05:39:43] iteration 4373/ 6855 | consumed samples: 559744 | elapsed time per iteration (ms): 91778.9 | throughput per GPU (TFLOP/s/GPU): 41.5 | MFU 13.30% | FLOPs estimator: workload-aware | logical tokens/s: 182799.9 | valid tokens/s: 182799.8 | physical tokens/s: 183137.2 | loss tokens/s: 129585.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.671943E-01 | lm-global-supervised-tokens: 1.189322E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.856458E-02 | lm-token-buffer-mib: 5.805068E+01 | global_load_balancing_loss: 1.000451E+00 | loss scale: 1.0 | grad norm: 0.071 | params norm: 11362.099 | params rms: 0.061522 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 15:16:35.172763 | finish at 2026-10-10 20:56:18 + [2026-10-08 05:41:12] iteration 4374/ 6855 | consumed samples: 559872 | elapsed time per iteration (ms): 89026.2 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.48% | FLOPs estimator: workload-aware | logical tokens/s: 188451.9 | valid tokens/s: 188451.9 | physical tokens/s: 188919.0 | loss tokens/s: 144542.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.835663E-01 | lm-global-supervised-tokens: 1.286804E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.151488E-02 | lm-token-buffer-mib: 5.811405E+01 | global_load_balancing_loss: 1.000200E+00 | loss scale: 1.0 | grad norm: 0.090 | params norm: 11362.047 | params rms: 0.061522 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 13:21:13.959158 | finish at 2026-10-10 19:02:26 + [2026-10-08 05:42:41] iteration 4375/ 6855 | consumed samples: 560000 | elapsed time per iteration (ms): 88642.6 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.55% | FLOPs estimator: workload-aware | logical tokens/s: 189267.6 | valid tokens/s: 189267.6 | physical tokens/s: 189659.1 | loss tokens/s: 142199.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.310165E-01 | lm-global-supervised-tokens: 1.260494E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.157073E-02 | lm-token-buffer-mib: 5.803677E+01 | global_load_balancing_loss: 1.000686E+00 | loss scale: 1.0 | grad norm: 0.082 | params norm: 11361.995 | params rms: 0.061522 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 13:03:53.650513 | finish at 2026-10-10 18:46:35 + [2026-10-08 05:44:11] iteration 4376/ 6855 | consumed samples: 560128 | elapsed time per iteration (ms): 89686.7 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.60% | FLOPs estimator: workload-aware | logical tokens/s: 187064.8 | valid tokens/s: 187064.7 | physical tokens/s: 187325.1 | loss tokens/s: 127922.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.688724E-01 | lm-global-supervised-tokens: 1.147297E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.160127E-02 | lm-token-buffer-mib: 5.804811E+01 | global_load_balancing_loss: 1.000269E+00 | loss scale: 1.0 | grad norm: 0.073 | params norm: 11361.943 | params rms: 0.061521 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 13:45:33.247998 | finish at 2026-10-10 19:29:44 + [2026-10-08 05:45:36] iteration 4377/ 6855 | consumed samples: 560256 | elapsed time per iteration (ms): 85804.4 | throughput per GPU (TFLOP/s/GPU): 43.4 | MFU 13.92% | FLOPs estimator: workload-aware | logical tokens/s: 195528.0 | valid tokens/s: 195528.0 | physical tokens/s: 195977.0 | loss tokens/s: 144580.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.841476E-01 | lm-global-supervised-tokens: 1.240563E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.268968E-02 | lm-token-buffer-mib: 5.826239E+01 | global_load_balancing_loss: 1.000416E+00 | loss scale: 1.0 | grad norm: 0.046 | params norm: 11361.889 | params rms: 0.061521 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 11:03:43.400601 | finish at 2026-10-10 16:49:20 + [2026-10-08 05:47:07] iteration 4378/ 6855 | consumed samples: 560384 | elapsed time per iteration (ms): 90287.5 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.51% | FLOPs estimator: workload-aware | logical tokens/s: 185819.5 | valid tokens/s: 185819.5 | physical tokens/s: 186192.1 | loss tokens/s: 132399.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.890494E-01 | lm-global-supervised-tokens: 1.195404E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.462658E-02 | lm-token-buffer-mib: 5.814441E+01 | global_load_balancing_loss: 1.000129E+00 | loss scale: 1.0 | grad norm: 0.083 | params norm: 11361.835 | params rms: 0.061521 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 14:07:22.085294 | finish at 2026-10-10 19:54:29 + [2026-10-08 05:48:32] iteration 4379/ 6855 | consumed samples: 560512 | elapsed time per iteration (ms): 85078.8 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.63% | FLOPs estimator: workload-aware | logical tokens/s: 197195.3 | valid tokens/s: 197195.3 | physical tokens/s: 197800.1 | loss tokens/s: 146496.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.395198E-01 | lm-global-supervised-tokens: 1.246370E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.025968E-02 | lm-token-buffer-mib: 5.815060E+01 | global_load_balancing_loss: 1.000762E+00 | loss scale: 1.0 | grad norm: 0.043 | params norm: 11361.782 | params rms: 0.061520 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 10:30:55.089818 | finish at 2026-10-10 16:19:27 + [2026-10-08 05:50:21] iteration 4380/ 6855 | consumed samples: 560640 | elapsed time per iteration (ms): 109074.5 | throughput per GPU (TFLOP/s/GPU): 34.3 | MFU 10.99% | FLOPs estimator: workload-aware | logical tokens/s: 153814.2 | valid tokens/s: 153814.2 | physical tokens/s: 154294.8 | loss tokens/s: 110206.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.149154E-01 | lm-global-supervised-tokens: 1.202069E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.522513E-02 | lm-token-buffer-mib: 5.817345E+01 | global_load_balancing_loss: 1.000818E+00 | loss scale: 1.0 | grad norm: 0.115 | params norm: 11361.729 | params rms: 0.061520 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 2:59:19.375906 | finish at 2026-10-11 08:49:40 + [2026-10-08 05:51:45] iteration 4381/ 6855 | consumed samples: 560768 | elapsed time per iteration (ms): 84601.4 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.77% | FLOPs estimator: workload-aware | logical tokens/s: 198308.9 | valid tokens/s: 198308.9 | physical tokens/s: 198754.1 | loss tokens/s: 150746.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.715336E-01 | lm-global-supervised-tokens: 1.275338E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.213560E-02 | lm-token-buffer-mib: 5.821589E+01 | global_load_balancing_loss: 9.999284E-01 | loss scale: 1.0 | grad norm: 0.047 | params norm: 11361.676 | params rms: 0.061520 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 10:08:23.879865 | finish at 2026-10-10 16:00:09 + [2026-10-08 05:53:12] iteration 4382/ 6855 | consumed samples: 560896 | elapsed time per iteration (ms): 86185.3 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.51% | FLOPs estimator: workload-aware | logical tokens/s: 194663.2 | valid tokens/s: 194663.2 | physical tokens/s: 195017.0 | loss tokens/s: 143333.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.671832E-01 | lm-global-supervised-tokens: 1.235328E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.455021E-02 | lm-token-buffer-mib: 5.803986E+01 | global_load_balancing_loss: 1.000046E+00 | loss scale: 1.0 | grad norm: 0.074 | params norm: 11361.622 | params rms: 0.061519 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 11:12:16.314981 | finish at 2026-10-10 17:05:28 + [2026-10-08 05:54:41] iteration 4383/ 6855 | consumed samples: 561024 | elapsed time per iteration (ms): 88872.8 | throughput per GPU (TFLOP/s/GPU): 41.7 | MFU 13.36% | FLOPs estimator: workload-aware | logical tokens/s: 188777.5 | valid tokens/s: 188777.4 | physical tokens/s: 189118.6 | loss tokens/s: 139988.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.581618E-01 | lm-global-supervised-tokens: 1.244117E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.311298E-02 | lm-token-buffer-mib: 5.808422E+01 | global_load_balancing_loss: 9.991665E-01 | loss scale: 1.0 | grad norm: 0.050 | params norm: 11361.567 | params rms: 0.061519 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 13:01:33.587809 | finish at 2026-10-10 18:56:14 + [2026-10-08 05:56:08] iteration 4384/ 6855 | consumed samples: 561152 | elapsed time per iteration (ms): 87550.1 | throughput per GPU (TFLOP/s/GPU): 41.7 | MFU 13.36% | FLOPs estimator: workload-aware | logical tokens/s: 191629.6 | valid tokens/s: 191629.6 | physical tokens/s: 192226.6 | loss tokens/s: 144021.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.114949E-01 | lm-global-supervised-tokens: 1.260908E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.377498E-02 | lm-token-buffer-mib: 5.812579E+01 | global_load_balancing_loss: 9.990801E-01 | loss scale: 1.0 | grad norm: 0.047 | params norm: 11361.513 | params rms: 0.061519 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 12:05:36.262559 | finish at 2026-10-10 18:01:44 + [2026-10-08 05:57:39] iteration 4385/ 6855 | consumed samples: 561280 | elapsed time per iteration (ms): 91249.9 | throughput per GPU (TFLOP/s/GPU): 41.6 | MFU 13.34% | FLOPs estimator: workload-aware | logical tokens/s: 183859.4 | valid tokens/s: 183859.4 | physical tokens/s: 184272.6 | loss tokens/s: 134435.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.783838E-01 | lm-global-supervised-tokens: 1.226720E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.287234E-02 | lm-token-buffer-mib: 5.807027E+01 | global_load_balancing_loss: 9.996265E-01 | loss scale: 1.0 | grad norm: 0.065 | params norm: 11361.458 | params rms: 0.061519 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 14:36:27.315087 | finish at 2026-10-10 20:34:07 + [2026-10-08 05:59:09] iteration 4386/ 6855 | consumed samples: 561408 | elapsed time per iteration (ms): 90016.2 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.54% | FLOPs estimator: workload-aware | logical tokens/s: 186379.7 | valid tokens/s: 186379.7 | physical tokens/s: 186663.9 | loss tokens/s: 134476.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.964104E-01 | lm-global-supervised-tokens: 1.210510E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.148430E-02 | lm-token-buffer-mib: 5.810838E+01 | global_load_balancing_loss: 1.000873E+00 | loss scale: 1.0 | grad norm: 0.038 | params norm: 11361.399 | params rms: 0.061518 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 13:44:10.116282 | finish at 2026-10-10 19:43:19 + [2026-10-08 06:00:40] iteration 4387/ 6855 | consumed samples: 561536 | elapsed time per iteration (ms): 90166.8 | throughput per GPU (TFLOP/s/GPU): 41.6 | MFU 13.34% | FLOPs estimator: workload-aware | logical tokens/s: 186068.1 | valid tokens/s: 186068.1 | physical tokens/s: 186402.3 | loss tokens/s: 134553.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.773071E-01 | lm-global-supervised-tokens: 1.213229E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.151637E-02 | lm-token-buffer-mib: 5.807553E+01 | global_load_balancing_loss: 1.001380E+00 | loss scale: 1.0 | grad norm: 0.086 | params norm: 11361.340 | params rms: 0.061518 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 13:48:51.678342 | finish at 2026-10-10 19:49:31 + [2026-10-08 06:02:06] iteration 4388/ 6855 | consumed samples: 561664 | elapsed time per iteration (ms): 86921.0 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.77% | FLOPs estimator: workload-aware | logical tokens/s: 193016.5 | valid tokens/s: 193016.5 | physical tokens/s: 193445.3 | loss tokens/s: 139033.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.525602E-01 | lm-global-supervised-tokens: 1.208491E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.373877E-02 | lm-token-buffer-mib: 5.815549E+01 | global_load_balancing_loss: 9.996382E-01 | loss scale: 1.0 | grad norm: 0.084 | params norm: 11361.280 | params rms: 0.061518 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 11:33:54.208764 | finish at 2026-10-10 17:36:01 + [2026-10-08 06:03:35] iteration 4389/ 6855 | consumed samples: 561792 | elapsed time per iteration (ms): 88754.6 | throughput per GPU (TFLOP/s/GPU): 41.3 | MFU 13.24% | FLOPs estimator: workload-aware | logical tokens/s: 189028.3 | valid tokens/s: 189028.3 | physical tokens/s: 189485.1 | loss tokens/s: 140511.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.148978E-01 | lm-global-supervised-tokens: 1.247103E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.493906E-02 | lm-token-buffer-mib: 5.810403E+01 | global_load_balancing_loss: 9.999603E-01 | loss scale: 1.0 | grad norm: 0.036 | params norm: 11361.219 | params rms: 0.061517 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 12:47:48.915447 | finish at 2026-10-10 18:51:24 + [2026-10-08 06:05:26] iteration 4390/ 6855 | consumed samples: 561920 | elapsed time per iteration (ms): 110390.4 | throughput per GPU (TFLOP/s/GPU): 34.2 | MFU 10.97% | FLOPs estimator: workload-aware | logical tokens/s: 151980.7 | valid tokens/s: 151980.7 | physical tokens/s: 152358.9 | loss tokens/s: 109219.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.595960E-01 | lm-global-supervised-tokens: 1.205675E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.146237E-02 | lm-token-buffer-mib: 5.805603E+01 | global_load_balancing_loss: 1.001002E+00 | loss scale: 1.0 | grad norm: 0.055 | params norm: 11361.158 | params rms: 0.061517 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 3:35:12.360518 | finish at 2026-10-11 09:40:38 + [2026-10-08 06:06:53] iteration 4391/ 6855 | consumed samples: 562048 | elapsed time per iteration (ms): 87529.8 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.63% | FLOPs estimator: workload-aware | logical tokens/s: 191674.0 | valid tokens/s: 191674.0 | physical tokens/s: 192115.6 | loss tokens/s: 142805.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.536776E-01 | lm-global-supervised-tokens: 1.249977E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.565668E-02 | lm-token-buffer-mib: 5.807099E+01 | global_load_balancing_loss: 1.000643E+00 | loss scale: 1.0 | grad norm: 0.112 | params norm: 11361.097 | params rms: 0.061517 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 11:54:33.461121 | finish at 2026-10-10 18:01:27 + [2026-10-08 06:08:21] iteration 4392/ 6855 | consumed samples: 562176 | elapsed time per iteration (ms): 87892.5 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.62% | FLOPs estimator: workload-aware | logical tokens/s: 190883.2 | valid tokens/s: 190883.2 | physical tokens/s: 191417.5 | loss tokens/s: 140729.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.988454E-01 | lm-global-supervised-tokens: 1.236904E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.423384E-02 | lm-token-buffer-mib: 5.827229E+01 | global_load_balancing_loss: 9.998981E-01 | loss scale: 1.0 | grad norm: 0.192 | params norm: 11361.037 | params rms: 0.061516 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 12:07:59.162130 | finish at 2026-10-10 18:16:20 + [2026-10-08 06:09:48] iteration 4393/ 6855 | consumed samples: 562304 | elapsed time per iteration (ms): 87027.9 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.47% | FLOPs estimator: workload-aware | logical tokens/s: 192779.6 | valid tokens/s: 192779.6 | physical tokens/s: 193186.6 | loss tokens/s: 141957.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.275775E-01 | lm-global-supervised-tokens: 1.235429E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.289361E-02 | lm-token-buffer-mib: 5.816589E+01 | global_load_balancing_loss: 9.998776E-01 | loss scale: 1.0 | grad norm: 0.109 | params norm: 11360.977 | params rms: 0.061516 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 11:31:02.705601 | finish at 2026-10-10 17:40:51 + [2026-10-08 06:11:14] iteration 4394/ 6855 | consumed samples: 562432 | elapsed time per iteration (ms): 85991.9 | throughput per GPU (TFLOP/s/GPU): 43.1 | MFU 13.82% | FLOPs estimator: workload-aware | logical tokens/s: 195101.7 | valid tokens/s: 195101.7 | physical tokens/s: 195540.7 | loss tokens/s: 145755.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.464561E-01 | lm-global-supervised-tokens: 1.253379E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.897921E-02 | lm-token-buffer-mib: 5.819749E+01 | global_load_balancing_loss: 9.994946E-01 | loss scale: 1.0 | grad norm: 0.117 | params norm: 11360.918 | params rms: 0.061516 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 10:47:06.018293 | finish at 2026-10-10 16:58:20 + [2026-10-08 06:12:44] iteration 4395/ 6855 | consumed samples: 562560 | elapsed time per iteration (ms): 89638.1 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.53% | FLOPs estimator: workload-aware | logical tokens/s: 187165.6 | valid tokens/s: 187165.6 | physical tokens/s: 187645.3 | loss tokens/s: 128593.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.459355E-01 | lm-global-supervised-tokens: 1.152687E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.820281E-02 | lm-token-buffer-mib: 5.813206E+01 | global_load_balancing_loss: 1.000649E+00 | loss scale: 1.0 | grad norm: 0.113 | params norm: 11360.858 | params rms: 0.061515 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 13:15:09.802608 | finish at 2026-10-10 19:27:53 + [2026-10-08 06:14:13] iteration 4396/ 6855 | consumed samples: 562688 | elapsed time per iteration (ms): 89191.6 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.45% | FLOPs estimator: workload-aware | logical tokens/s: 188102.0 | valid tokens/s: 188102.0 | physical tokens/s: 188490.0 | loss tokens/s: 132584.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.146832E-01 | lm-global-supervised-tokens: 1.182540E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.282055E-02 | lm-token-buffer-mib: 5.820577E+01 | global_load_balancing_loss: 1.000045E+00 | loss scale: 1.0 | grad norm: 0.067 | params norm: 11360.797 | params rms: 0.061515 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 12:55:22.141090 | finish at 2026-10-10 19:09:35 + [2026-10-08 06:15:40] iteration 4397/ 6855 | consumed samples: 562816 | elapsed time per iteration (ms): 86718.6 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.56% | FLOPs estimator: workload-aware | logical tokens/s: 193466.7 | valid tokens/s: 193466.7 | physical tokens/s: 193983.8 | loss tokens/s: 150016.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.476179E-01 | lm-global-supervised-tokens: 1.300917E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.143624E-02 | lm-token-buffer-mib: 5.809933E+01 | global_load_balancing_loss: 1.001410E+00 | loss scale: 1.0 | grad norm: 0.086 | params norm: 11360.734 | params rms: 0.061515 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 11:12:34.250319 | finish at 2026-10-10 17:28:14 + [2026-10-08 06:17:08] iteration 4398/ 6855 | consumed samples: 562944 | elapsed time per iteration (ms): 88770.1 | throughput per GPU (TFLOP/s/GPU): 41.5 | MFU 13.29% | FLOPs estimator: workload-aware | logical tokens/s: 188995.5 | valid tokens/s: 188995.4 | physical tokens/s: 189388.8 | loss tokens/s: 140456.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.232655E-01 | lm-global-supervised-tokens: 1.246836E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.233566E-02 | lm-token-buffer-mib: 5.825386E+01 | global_load_balancing_loss: 9.997958E-01 | loss scale: 1.0 | grad norm: 0.138 | params norm: 11360.671 | params rms: 0.061514 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 12:35:08.097910 | finish at 2026-10-10 18:52:16 + [2026-10-08 06:18:32] iteration 4399/ 6855 | consumed samples: 563072 | elapsed time per iteration (ms): 83544.8 | throughput per GPU (TFLOP/s/GPU): 43.6 | MFU 13.97% | FLOPs estimator: workload-aware | logical tokens/s: 200816.7 | valid tokens/s: 200816.7 | physical tokens/s: 201193.2 | loss tokens/s: 148050.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.633143E-01 | lm-global-supervised-tokens: 1.236885E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.304263E-02 | lm-token-buffer-mib: 5.806687E+01 | global_load_balancing_loss: 1.000254E+00 | loss scale: 1.0 | grad norm: 0.065 | params norm: 11360.609 | params rms: 0.061514 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 8:59:46.147774 | finish at 2026-10-10 15:18:18 + [2026-10-08 06:20:24] iteration 4400/ 6855 | consumed samples: 563200 | elapsed time per iteration (ms): 111767.5 | throughput per GPU (TFLOP/s/GPU): 33.1 | MFU 10.62% | FLOPs estimator: workload-aware | logical tokens/s: 150107.9 | valid tokens/s: 150107.9 | physical tokens/s: 150352.0 | loss tokens/s: 108426.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.376415E-01 | lm-global-supervised-tokens: 1.211851E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.137350E-02 | lm-token-buffer-mib: 5.814668E+01 | global_load_balancing_loss: 1.000725E+00 | loss scale: 1.0 | grad norm: 0.065 | params norm: 11360.549 | params rms: 0.061514 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 4:13:09.194753 | finish at 2026-10-11 10:33:33 +times across ranks (ms): + save-checkpoint: + rank 0: 24369.57 + rank 1: 24369.58 + rank 2: 24369.61 + rank 3: 24369.57 + rank 4: 24369.58 + rank 5: 24369.56 + rank 6: 24369.57 + rank 7: 24369.58 + rank 8: 24370.12 + rank 9: 24370.09 + rank 10: 24370.08 + rank 11: 24370.08 + rank 12: 24370.09 + rank 13: 24370.08 + rank 14: 24370.10 + rank 15: 24370.07 + rank 16: 24370.12 + rank 17: 24370.12 + rank 18: 24370.16 + rank 19: 24370.14 + rank 20: 24370.10 + rank 21: 24370.13 + rank 22: 24370.12 + rank 23: 24370.14 + rank 24: 24369.83 + rank 25: 24369.80 + rank 26: 24369.83 + rank 27: 24369.86 + rank 28: 24369.81 + rank 29: 24369.82 + rank 30: 24369.83 + rank 31: 24369.81 + rank 32: 24369.84 + rank 33: 24369.84 + rank 34: 24369.81 + rank 35: 24369.79 + rank 36: 24369.80 + rank 37: 24369.80 + rank 38: 24369.82 + rank 39: 24369.82 + rank 40: 24369.95 + rank 41: 24369.94 + rank 42: 24369.95 + rank 43: 24369.95 + rank 44: 24369.97 + rank 45: 24369.95 + rank 46: 24369.95 + rank 47: 24369.93 + rank 48: 24370.19 + rank 49: 24370.19 + rank 50: 24370.32 + rank 51: 24370.18 + rank 52: 24370.21 + rank 53: 24370.17 + rank 54: 24370.17 + rank 55: 24370.18 + rank 56: 24371.10 + rank 57: 24371.09 + rank 58: 24371.06 + rank 59: 24371.06 + rank 60: 24371.07 + rank 61: 24371.06 + rank 62: 24371.16 + rank 63: 24371.06 + [2026-10-08 06:22:13] iteration 4401/ 6855 | consumed samples: 563328 | elapsed time per iteration (ms): 84684.1 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.74% | FLOPs estimator: workload-aware | logical tokens/s: 198114.3 | valid tokens/s: 198114.3 | physical tokens/s: 198673.0 | loss tokens/s: 151421.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.996380E-01 | lm-global-supervised-tokens: 1.282299E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.180060E+00 | lm-token-buffer-mib: 5.805492E+01 | global_load_balancing_loss: 9.992130E-01 | loss scale: 1.0 | grad norm: 0.093 | params norm: 11360.488 | params rms: 0.061513 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 9:43:34.847885 | finish at 2026-10-10 16:05:48 + [2026-10-08 06:23:45] iteration 4402/ 6855 | consumed samples: 563456 | elapsed time per iteration (ms): 92127.9 | throughput per GPU (TFLOP/s/GPU): 41.3 | MFU 13.25% | FLOPs estimator: workload-aware | logical tokens/s: 182107.9 | valid tokens/s: 182107.9 | physical tokens/s: 182531.5 | loss tokens/s: 133843.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.132879E-01 | lm-global-supervised-tokens: 1.233073E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.195067E-02 | lm-token-buffer-mib: 5.833449E+01 | global_load_balancing_loss: 9.990034E-01 | loss scale: 1.0 | grad norm: 0.098 | params norm: 11360.425 | params rms: 0.061513 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 14:46:29.661804 | finish at 2026-10-10 21:10:15 + [2026-10-08 06:25:11] iteration 4403/ 6855 | consumed samples: 563584 | elapsed time per iteration (ms): 85922.6 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.66% | FLOPs estimator: workload-aware | logical tokens/s: 195259.5 | valid tokens/s: 195259.5 | physical tokens/s: 195712.8 | loss tokens/s: 145329.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.725013E-01 | lm-global-supervised-tokens: 1.248708E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.228946E-02 | lm-token-buffer-mib: 5.818181E+01 | global_load_balancing_loss: 9.991717E-01 | loss scale: 1.0 | grad norm: 0.130 | params norm: 11360.362 | params rms: 0.061513 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 10:31:22.266136 | finish at 2026-10-10 16:56:33 + [2026-10-08 06:26:40] iteration 4404/ 6855 | consumed samples: 563712 | elapsed time per iteration (ms): 89462.8 | throughput per GPU (TFLOP/s/GPU): 41.7 | MFU 13.35% | FLOPs estimator: workload-aware | logical tokens/s: 187532.7 | valid tokens/s: 187532.7 | physical tokens/s: 187949.3 | loss tokens/s: 139537.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.883033E-01 | lm-global-supervised-tokens: 1.248340E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.401218E-02 | lm-token-buffer-mib: 5.827040E+01 | global_load_balancing_loss: 1.001355E+00 | loss scale: 1.0 | grad norm: 0.067 | params norm: 11360.301 | params rms: 0.061512 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 12:54:33.444996 | finish at 2026-10-10 19:21:14 + [2026-10-08 06:28:07] iteration 4405/ 6855 | consumed samples: 563840 | elapsed time per iteration (ms): 86736.0 | throughput per GPU (TFLOP/s/GPU): 43.1 | MFU 13.82% | FLOPs estimator: workload-aware | logical tokens/s: 193427.8 | valid tokens/s: 193427.7 | physical tokens/s: 193853.9 | loss tokens/s: 137580.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.739776E-01 | lm-global-supervised-tokens: 1.193314E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.928846E-02 | lm-token-buffer-mib: 5.818518E+01 | global_load_balancing_loss: 1.001262E+00 | loss scale: 1.0 | grad norm: 0.146 | params norm: 11360.240 | params rms: 0.061512 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 11:01:43.182626 | finish at 2026-10-10 17:29:50 + [2026-10-08 06:29:36] iteration 4406/ 6855 | consumed samples: 563968 | elapsed time per iteration (ms): 88987.3 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.69% | FLOPs estimator: workload-aware | logical tokens/s: 188534.4 | valid tokens/s: 188534.4 | physical tokens/s: 189160.8 | loss tokens/s: 134251.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.801221E-01 | lm-global-supervised-tokens: 1.194663E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.237005E-02 | lm-token-buffer-mib: 5.820184E+01 | global_load_balancing_loss: 1.001388E+00 | loss scale: 1.0 | grad norm: 0.150 | params norm: 11360.177 | params rms: 0.061512 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 12:32:09.923193 | finish at 2026-10-10 19:01:46 + [2026-10-08 06:31:06] iteration 4407/ 6855 | consumed samples: 564096 | elapsed time per iteration (ms): 89961.5 | throughput per GPU (TFLOP/s/GPU): 40.9 | MFU 13.10% | FLOPs estimator: workload-aware | logical tokens/s: 186492.6 | valid tokens/s: 186492.6 | physical tokens/s: 187208.9 | loss tokens/s: 135601.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.197398E-01 | lm-global-supervised-tokens: 1.219889E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.135213E-02 | lm-token-buffer-mib: 5.811290E+01 | global_load_balancing_loss: 1.001168E+00 | loss scale: 1.0 | grad norm: 0.154 | params norm: 11360.117 | params rms: 0.061511 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 13:10:25.631012 | finish at 2026-10-10 19:41:32 + [2026-10-08 06:32:33] iteration 4408/ 6855 | consumed samples: 564224 | elapsed time per iteration (ms): 86857.4 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.54% | FLOPs estimator: workload-aware | logical tokens/s: 193157.6 | valid tokens/s: 193157.6 | physical tokens/s: 193474.2 | loss tokens/s: 142985.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.698192E-01 | lm-global-supervised-tokens: 1.241937E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.274294E-02 | lm-token-buffer-mib: 5.816840E+01 | global_load_balancing_loss: 9.986325E-01 | loss scale: 1.0 | grad norm: 0.064 | params norm: 11360.059 | params rms: 0.061511 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 11:02:20.124747 | finish at 2026-10-10 17:34:53 + [2026-10-08 06:34:04] iteration 4409/ 6855 | consumed samples: 564352 | elapsed time per iteration (ms): 90716.9 | throughput per GPU (TFLOP/s/GPU): 40.8 | MFU 13.08% | FLOPs estimator: workload-aware | logical tokens/s: 184939.8 | valid tokens/s: 184939.8 | physical tokens/s: 185351.6 | loss tokens/s: 141064.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.493599E-01 | lm-global-supervised-tokens: 1.279690E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.127105E-02 | lm-token-buffer-mib: 5.811293E+01 | global_load_balancing_loss: 9.993884E-01 | loss scale: 1.0 | grad norm: 0.044 | params norm: 11359.998 | params rms: 0.061511 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 13:38:13.418703 | finish at 2026-10-10 20:12:17 + [2026-10-08 06:35:51] iteration 4410/ 6855 | consumed samples: 564480 | elapsed time per iteration (ms): 107166.6 | throughput per GPU (TFLOP/s/GPU): 34.9 | MFU 11.18% | FLOPs estimator: workload-aware | logical tokens/s: 156552.3 | valid tokens/s: 156552.3 | physical tokens/s: 156851.3 | loss tokens/s: 114542.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.427478E-01 | lm-global-supervised-tokens: 1.227515E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.202211E-02 | lm-token-buffer-mib: 5.810620E+01 | global_load_balancing_loss: 9.990627E-01 | loss scale: 1.0 | grad norm: 0.080 | params norm: 11359.939 | params rms: 0.061510 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 0:47:02.349797 | finish at 2026-10-11 07:22:53 + [2026-10-08 06:37:18] iteration 4411/ 6855 | consumed samples: 564608 | elapsed time per iteration (ms): 87393.2 | throughput per GPU (TFLOP/s/GPU): 41.7 | MFU 13.38% | FLOPs estimator: workload-aware | logical tokens/s: 191972.3 | valid tokens/s: 191972.3 | physical tokens/s: 192309.0 | loss tokens/s: 141040.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.634727E-01 | lm-global-supervised-tokens: 1.232602E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.496255E-02 | lm-token-buffer-mib: 5.805486E+01 | global_load_balancing_loss: 1.001055E+00 | loss scale: 1.0 | grad norm: 0.035 | params norm: 11359.881 | params rms: 0.061510 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 11:19:48.989347 | finish at 2026-10-10 17:57:07 + [2026-10-08 06:38:46] iteration 4412/ 6855 | consumed samples: 564736 | elapsed time per iteration (ms): 87677.8 | throughput per GPU (TFLOP/s/GPU): 41.4 | MFU 13.27% | FLOPs estimator: workload-aware | logical tokens/s: 191350.2 | valid tokens/s: 191350.2 | physical tokens/s: 191936.5 | loss tokens/s: 146146.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.626222E-01 | lm-global-supervised-tokens: 1.281383E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.446551E-02 | lm-token-buffer-mib: 5.806731E+01 | global_load_balancing_loss: 1.000291E+00 | loss scale: 1.0 | grad norm: 0.076 | params norm: 11359.825 | params rms: 0.061510 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 11:29:56.905443 | finish at 2026-10-10 18:08:43 + [2026-10-08 06:40:16] iteration 4413/ 6855 | consumed samples: 564864 | elapsed time per iteration (ms): 89689.2 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.80% | FLOPs estimator: workload-aware | logical tokens/s: 187059.5 | valid tokens/s: 187059.5 | physical tokens/s: 187413.8 | loss tokens/s: 132406.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.284364E-01 | lm-global-supervised-tokens: 1.187541E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.164001E-02 | lm-token-buffer-mib: 5.813244E+01 | global_load_balancing_loss: 1.000330E+00 | loss scale: 1.0 | grad norm: 0.038 | params norm: 11359.768 | params rms: 0.061509 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 12:50:21.045429 | finish at 2026-10-10 19:30:37 + [2026-10-08 06:41:38] iteration 4414/ 6855 | consumed samples: 564992 | elapsed time per iteration (ms): 82755.1 | throughput per GPU (TFLOP/s/GPU): 43.4 | MFU 13.90% | FLOPs estimator: workload-aware | logical tokens/s: 202733.2 | valid tokens/s: 202733.2 | physical tokens/s: 203098.6 | loss tokens/s: 156689.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.487582E-01 | lm-global-supervised-tokens: 1.296689E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.307336E-02 | lm-token-buffer-mib: 5.815154E+01 | global_load_balancing_loss: 9.999696E-01 | loss scale: 1.0 | grad norm: 0.063 | params norm: 11359.711 | params rms: 0.061509 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 8:06:45.164792 | finish at 2026-10-10 14:48:24 + [2026-10-08 06:43:07] iteration 4415/ 6855 | consumed samples: 565120 | elapsed time per iteration (ms): 88986.8 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.39% | FLOPs estimator: workload-aware | logical tokens/s: 188535.9 | valid tokens/s: 188535.9 | physical tokens/s: 188865.4 | loss tokens/s: 141733.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.521638E-01 | lm-global-supervised-tokens: 1.261244E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.409460E-02 | lm-token-buffer-mib: 5.814651E+01 | global_load_balancing_loss: 1.000793E+00 | loss scale: 1.0 | grad norm: 0.048 | params norm: 11359.655 | params rms: 0.061509 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 12:18:47.794218 | finish at 2026-10-10 19:01:55 + [2026-10-08 06:44:37] iteration 4416/ 6855 | consumed samples: 565248 | elapsed time per iteration (ms): 89702.7 | throughput per GPU (TFLOP/s/GPU): 41.6 | MFU 13.34% | FLOPs estimator: workload-aware | logical tokens/s: 187030.8 | valid tokens/s: 187030.8 | physical tokens/s: 187436.9 | loss tokens/s: 134727.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.890398E-01 | lm-global-supervised-tokens: 1.208546E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.300309E-02 | lm-token-buffer-mib: 5.807879E+01 | global_load_balancing_loss: 1.001807E+00 | loss scale: 1.0 | grad norm: 0.036 | params norm: 11359.600 | params rms: 0.061509 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 12:46:24.992633 | finish at 2026-10-10 19:31:02 + [2026-10-08 06:46:06] iteration 4417/ 6855 | consumed samples: 565376 | elapsed time per iteration (ms): 88966.7 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.41% | FLOPs estimator: workload-aware | logical tokens/s: 188578.6 | valid tokens/s: 188578.6 | physical tokens/s: 189091.0 | loss tokens/s: 136004.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.439864E-01 | lm-global-supervised-tokens: 1.209983E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.196945E-02 | lm-token-buffer-mib: 5.814607E+01 | global_load_balancing_loss: 9.993529E-01 | loss scale: 1.0 | grad norm: 0.098 | params norm: 11359.544 | params rms: 0.061508 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 12:15:00.835132 | finish at 2026-10-10 19:01:07 + [2026-10-08 06:47:34] iteration 4418/ 6855 | consumed samples: 565504 | elapsed time per iteration (ms): 88495.9 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.49% | FLOPs estimator: workload-aware | logical tokens/s: 189581.3 | valid tokens/s: 189581.3 | physical tokens/s: 190016.1 | loss tokens/s: 144870.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.612970E-01 | lm-global-supervised-tokens: 1.282047E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.234464E-02 | lm-token-buffer-mib: 5.807732E+01 | global_load_balancing_loss: 1.000458E+00 | loss scale: 1.0 | grad norm: 0.157 | params norm: 11359.487 | params rms: 0.061508 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 11:54:24.557482 | finish at 2026-10-10 18:41:59 + [2026-10-08 06:49:02] iteration 4419/ 6855 | consumed samples: 565632 | elapsed time per iteration (ms): 87203.5 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.76% | FLOPs estimator: workload-aware | logical tokens/s: 192391.0 | valid tokens/s: 192391.0 | physical tokens/s: 192769.2 | loss tokens/s: 144715.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.564600E-01 | lm-global-supervised-tokens: 1.261971E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.415405E-02 | lm-token-buffer-mib: 5.815634E+01 | global_load_balancing_loss: 9.999748E-01 | loss scale: 1.0 | grad norm: 0.092 | params norm: 11359.430 | params rms: 0.061508 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 11:00:27.655804 | finish at 2026-10-10 17:49:29 + [2026-10-08 06:50:52] iteration 4420/ 6855 | consumed samples: 565760 | elapsed time per iteration (ms): 109808.6 | throughput per GPU (TFLOP/s/GPU): 33.7 | MFU 10.79% | FLOPs estimator: workload-aware | logical tokens/s: 152785.8 | valid tokens/s: 152785.8 | physical tokens/s: 153162.5 | loss tokens/s: 109068.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.577840E-01 | lm-global-supervised-tokens: 1.197668E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.126119E-02 | lm-token-buffer-mib: 5.817832E+01 | global_load_balancing_loss: 9.991364E-01 | loss scale: 1.0 | grad norm: 0.058 | params norm: 11359.373 | params rms: 0.061507 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 2:16:23.977450 | finish at 2026-10-11 09:07:15 + [2026-10-08 06:52:21] iteration 4421/ 6855 | consumed samples: 565888 | elapsed time per iteration (ms): 89368.5 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.60% | FLOPs estimator: workload-aware | logical tokens/s: 187586.9 | valid tokens/s: 187586.9 | physical tokens/s: 188251.0 | loss tokens/s: 133257.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.821319E-01 | lm-global-supervised-tokens: 1.190906E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.444546E-02 | lm-token-buffer-mib: 5.803342E+01 | global_load_balancing_loss: 1.001325E+00 | loss scale: 1.0 | grad norm: 0.088 | params norm: 11359.315 | params rms: 0.061507 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 12:25:22.811183 | finish at 2026-10-10 19:17:44 + [2026-10-08 06:53:47] iteration 4422/ 6855 | consumed samples: 566016 | elapsed time per iteration (ms): 86026.4 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.50% | FLOPs estimator: workload-aware | logical tokens/s: 195024.0 | valid tokens/s: 195024.0 | physical tokens/s: 195471.4 | loss tokens/s: 140142.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.305295E-01 | lm-global-supervised-tokens: 1.205593E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.321379E-02 | lm-token-buffer-mib: 5.807228E+01 | global_load_balancing_loss: 9.995778E-01 | loss scale: 1.0 | grad norm: 0.075 | params norm: 11359.257 | params rms: 0.061507 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 10:08:22.190812 | finish at 2026-10-10 17:02:09 + [2026-10-08 06:55:13] iteration 4423/ 6855 | consumed samples: 566144 | elapsed time per iteration (ms): 86311.8 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.65% | FLOPs estimator: workload-aware | logical tokens/s: 194378.8 | valid tokens/s: 194378.8 | physical tokens/s: 194828.4 | loss tokens/s: 146530.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.848623E-01 | lm-global-supervised-tokens: 1.264735E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.446615E-02 | lm-token-buffer-mib: 5.826212E+01 | global_load_balancing_loss: 9.996795E-01 | loss scale: 1.0 | grad norm: 0.032 | params norm: 11359.199 | params rms: 0.061506 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 10:18:30.251801 | finish at 2026-10-10 17:13:43 + [2026-10-08 06:56:42] iteration 4424/ 6855 | consumed samples: 566272 | elapsed time per iteration (ms): 88802.4 | throughput per GPU (TFLOP/s/GPU): 41.7 | MFU 13.35% | FLOPs estimator: workload-aware | logical tokens/s: 188926.9 | valid tokens/s: 188926.9 | physical tokens/s: 189465.1 | loss tokens/s: 136067.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.391834E-01 | lm-global-supervised-tokens: 1.208309E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.181017E-02 | lm-token-buffer-mib: 5.808131E+01 | global_load_balancing_loss: 1.000042E+00 | loss scale: 1.0 | grad norm: 0.081 | params norm: 11359.141 | params rms: 0.061506 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 11:57:58.711179 | finish at 2026-10-10 18:54:41 + [2026-10-08 06:58:15] iteration 4425/ 6855 | consumed samples: 566400 | elapsed time per iteration (ms): 92886.5 | throughput per GPU (TFLOP/s/GPU): 41.7 | MFU 13.37% | FLOPs estimator: workload-aware | logical tokens/s: 180620.6 | valid tokens/s: 180620.6 | physical tokens/s: 181009.3 | loss tokens/s: 128134.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.024046E-01 | lm-global-supervised-tokens: 1.190198E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.253034E-02 | lm-token-buffer-mib: 5.807585E+01 | global_load_balancing_loss: 1.000295E+00 | loss scale: 1.0 | grad norm: 0.055 | params norm: 11359.079 | params rms: 0.061506 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 14:41:54.091008 | finish at 2026-10-10 21:40:09 + [2026-10-08 06:59:44] iteration 4426/ 6855 | consumed samples: 566528 | elapsed time per iteration (ms): 88951.6 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.52% | FLOPs estimator: workload-aware | logical tokens/s: 188609.8 | valid tokens/s: 188609.8 | physical tokens/s: 188920.6 | loss tokens/s: 134820.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.211201E-01 | lm-global-supervised-tokens: 1.199247E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.606921E-02 | lm-token-buffer-mib: 5.809830E+01 | global_load_balancing_loss: 1.000613E+00 | loss scale: 1.0 | grad norm: 0.044 | params norm: 11359.019 | params rms: 0.061505 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 12:01:03.406467 | finish at 2026-10-10 19:00:47 + [2026-10-08 07:01:11] iteration 4427/ 6855 | consumed samples: 566656 | elapsed time per iteration (ms): 87531.4 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.78% | FLOPs estimator: workload-aware | logical tokens/s: 191670.5 | valid tokens/s: 191670.5 | physical tokens/s: 192185.6 | loss tokens/s: 135145.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.674129E-01 | lm-global-supervised-tokens: 1.182949E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.578374E-02 | lm-token-buffer-mib: 5.814090E+01 | global_load_balancing_loss: 1.000172E+00 | loss scale: 1.0 | grad norm: 0.040 | params norm: 11358.960 | params rms: 0.061505 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 11:02:06.238537 | finish at 2026-10-10 18:03:18 + [2026-10-08 07:02:39] iteration 4428/ 6855 | consumed samples: 566784 | elapsed time per iteration (ms): 87820.3 | throughput per GPU (TFLOP/s/GPU): 43.1 | MFU 13.82% | FLOPs estimator: workload-aware | logical tokens/s: 191040.1 | valid tokens/s: 191040.1 | physical tokens/s: 191245.1 | loss tokens/s: 133704.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.352326E-01 | lm-global-supervised-tokens: 1.174198E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.151853E-02 | lm-token-buffer-mib: 5.811614E+01 | global_load_balancing_loss: 1.000549E+00 | loss scale: 1.0 | grad norm: 0.038 | params norm: 11358.903 | params rms: 0.061505 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 11:12:19.852725 | finish at 2026-10-10 18:14:59 + [2026-10-08 07:04:04] iteration 4429/ 6855 | consumed samples: 566912 | elapsed time per iteration (ms): 84818.3 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.60% | FLOPs estimator: workload-aware | logical tokens/s: 197801.9 | valid tokens/s: 197801.8 | physical tokens/s: 198327.4 | loss tokens/s: 151613.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.397809E-01 | lm-global-supervised-tokens: 1.285955E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.288519E-02 | lm-token-buffer-mib: 5.823998E+01 | global_load_balancing_loss: 1.000296E+00 | loss scale: 1.0 | grad norm: 0.037 | params norm: 11358.845 | params rms: 0.061504 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 9:09:29.146657 | finish at 2026-10-10 16:13:33 + [2026-10-08 07:05:52] iteration 4430/ 6855 | consumed samples: 567040 | elapsed time per iteration (ms): 108400.3 | throughput per GPU (TFLOP/s/GPU): 34.0 | MFU 10.91% | FLOPs estimator: workload-aware | logical tokens/s: 154770.6 | valid tokens/s: 154770.6 | physical tokens/s: 155099.7 | loss tokens/s: 115936.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.539109E-01 | lm-global-supervised-tokens: 1.256758E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.469291E-02 | lm-token-buffer-mib: 5.811550E+01 | global_load_balancing_loss: 1.000714E+00 | loss scale: 1.0 | grad norm: 0.040 | params norm: 11358.789 | params rms: 0.061504 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 1:01:10.669746 | finish at 2026-10-11 08:07:03 + [2026-10-08 07:07:25] iteration 4431/ 6855 | consumed samples: 567168 | elapsed time per iteration (ms): 92776.4 | throughput per GPU (TFLOP/s/GPU): 40.6 | MFU 13.02% | FLOPs estimator: workload-aware | logical tokens/s: 180834.8 | valid tokens/s: 180834.8 | physical tokens/s: 181261.1 | loss tokens/s: 126188.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.038823E-01 | lm-global-supervised-tokens: 1.170730E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.339847E-02 | lm-token-buffer-mib: 5.830205E+01 | global_load_balancing_loss: 9.999907E-01 | loss scale: 1.0 | grad norm: 0.042 | params norm: 11358.734 | params rms: 0.061504 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 14:28:10.003063 | finish at 2026-10-10 21:35:35 + [2026-10-08 07:08:51] iteration 4432/ 6855 | consumed samples: 567296 | elapsed time per iteration (ms): 85837.3 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.61% | FLOPs estimator: workload-aware | logical tokens/s: 195453.2 | valid tokens/s: 195453.2 | physical tokens/s: 195948.7 | loss tokens/s: 144208.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.660589E-01 | lm-global-supervised-tokens: 1.237846E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.602214E-02 | lm-token-buffer-mib: 5.824825E+01 | global_load_balancing_loss: 1.000102E+00 | loss scale: 1.0 | grad norm: 0.049 | params norm: 11358.679 | params rms: 0.061504 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 9:46:23.794226 | finish at 2026-10-10 16:55:15 + [2026-10-08 07:10:18] iteration 4433/ 6855 | consumed samples: 567424 | elapsed time per iteration (ms): 87309.5 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.78% | FLOPs estimator: workload-aware | logical tokens/s: 192157.0 | valid tokens/s: 192157.0 | physical tokens/s: 192688.6 | loss tokens/s: 140997.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.905402E-01 | lm-global-supervised-tokens: 1.231042E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.155647E-02 | lm-token-buffer-mib: 5.830535E+01 | global_load_balancing_loss: 1.000158E+00 | loss scale: 1.0 | grad norm: 0.031 | params norm: 11358.624 | params rms: 0.061503 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 10:44:23.688637 | finish at 2026-10-10 17:54:42 + [2026-10-08 07:11:51] iteration 4434/ 6855 | consumed samples: 567552 | elapsed time per iteration (ms): 92478.1 | throughput per GPU (TFLOP/s/GPU): 41.4 | MFU 13.28% | FLOPs estimator: workload-aware | logical tokens/s: 181418.1 | valid tokens/s: 181418.1 | physical tokens/s: 181800.2 | loss tokens/s: 129093.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.264072E-01 | lm-global-supervised-tokens: 1.193831E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.170336E-02 | lm-token-buffer-mib: 5.810437E+01 | global_load_balancing_loss: 9.998788E-01 | loss scale: 1.0 | grad norm: 0.034 | params norm: 11358.568 | params rms: 0.061503 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 14:11:29.562790 | finish at 2026-10-10 21:23:20 + [2026-10-08 07:13:18] iteration 4435/ 6855 | consumed samples: 567680 | elapsed time per iteration (ms): 87004.1 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.64% | FLOPs estimator: workload-aware | logical tokens/s: 192832.0 | valid tokens/s: 192832.0 | physical tokens/s: 193323.1 | loss tokens/s: 142770.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.003047E-01 | lm-global-supervised-tokens: 1.242162E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.154794E-02 | lm-token-buffer-mib: 5.805948E+01 | global_load_balancing_loss: 1.000588E+00 | loss scale: 1.0 | grad norm: 0.056 | params norm: 11358.513 | params rms: 0.061503 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 10:29:09.914703 | finish at 2026-10-10 17:42:28 + [2026-10-08 07:14:44] iteration 4436/ 6855 | consumed samples: 567808 | elapsed time per iteration (ms): 86652.4 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.65% | FLOPs estimator: workload-aware | logical tokens/s: 193615.1 | valid tokens/s: 193615.1 | physical tokens/s: 194135.9 | loss tokens/s: 145545.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.442961E-01 | lm-global-supervised-tokens: 1.261189E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.430862E-02 | lm-token-buffer-mib: 5.817090E+01 | global_load_balancing_loss: 9.996150E-01 | loss scale: 1.0 | grad norm: 0.100 | params norm: 11358.459 | params rms: 0.061502 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 10:13:32.138339 | finish at 2026-10-10 17:28:17 + [2026-10-08 07:16:13] iteration 4437/ 6855 | consumed samples: 567936 | elapsed time per iteration (ms): 88631.0 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.45% | FLOPs estimator: workload-aware | logical tokens/s: 189292.8 | valid tokens/s: 189292.8 | physical tokens/s: 189743.0 | loss tokens/s: 145559.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.165459E-01 | lm-global-supervised-tokens: 1.290108E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.241653E-02 | lm-token-buffer-mib: 5.811664E+01 | global_load_balancing_loss: 9.995459E-01 | loss scale: 1.0 | grad norm: 0.230 | params norm: 11358.404 | params rms: 0.061502 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 11:31:49.819787 | finish at 2026-10-10 18:48:03 + [2026-10-08 07:17:40] iteration 4438/ 6855 | consumed samples: 568064 | elapsed time per iteration (ms): 87346.8 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.40% | FLOPs estimator: workload-aware | logical tokens/s: 192075.3 | valid tokens/s: 192075.3 | physical tokens/s: 192640.0 | loss tokens/s: 139023.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.326612E-01 | lm-global-supervised-tokens: 1.214327E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.187826E-02 | lm-token-buffer-mib: 5.809726E+01 | global_load_balancing_loss: 9.997184E-01 | loss scale: 1.0 | grad norm: 0.462 | params norm: 11358.348 | params rms: 0.061502 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 10:38:37.293033 | finish at 2026-10-10 17:56:18 + [2026-10-08 07:19:11] iteration 4439/ 6855 | consumed samples: 568192 | elapsed time per iteration (ms): 90726.8 | throughput per GPU (TFLOP/s/GPU): 41.3 | MFU 13.25% | FLOPs estimator: workload-aware | logical tokens/s: 184920.2 | valid tokens/s: 184920.2 | physical tokens/s: 185323.2 | loss tokens/s: 129381.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.371814E-01 | lm-global-supervised-tokens: 1.173833E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.608743E-02 | lm-token-buffer-mib: 5.807343E+01 | global_load_balancing_loss: 1.001106E+00 | loss scale: 1.0 | grad norm: 0.174 | params norm: 11358.292 | params rms: 0.061501 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 12:53:15.897449 | finish at 2026-10-10 20:12:27 + [2026-10-08 07:21:02] iteration 4440/ 6855 | consumed samples: 568320 | elapsed time per iteration (ms): 110624.7 | throughput per GPU (TFLOP/s/GPU): 34.0 | MFU 10.91% | FLOPs estimator: workload-aware | logical tokens/s: 151658.2 | valid tokens/s: 151658.2 | physical tokens/s: 151938.6 | loss tokens/s: 109068.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.554812E-01 | lm-global-supervised-tokens: 1.206572E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.547682E-02 | lm-token-buffer-mib: 5.809315E+01 | global_load_balancing_loss: 1.000491E+00 | loss scale: 1.0 | grad norm: 0.444 | params norm: 11358.237 | params rms: 0.061501 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 2:12:38.624182 | finish at 2026-10-11 09:33:40 +times across ranks (ms): + save-checkpoint: + rank 0: 24032.33 + rank 1: 24032.36 + rank 2: 24032.37 + rank 3: 24032.32 + rank 4: 24032.36 + rank 5: 24032.34 + rank 6: 24032.35 + rank 7: 24032.41 + rank 8: 24032.27 + rank 9: 24032.26 + rank 10: 24032.26 + rank 11: 24032.26 + rank 12: 24032.27 + rank 13: 24032.26 + rank 14: 24032.27 + rank 15: 24032.25 + rank 16: 24032.05 + rank 17: 24032.17 + rank 18: 24032.07 + rank 19: 24032.05 + rank 20: 24032.06 + rank 21: 24032.04 + rank 22: 24032.05 + rank 23: 24032.01 + rank 24: 24031.32 + rank 25: 24031.29 + rank 26: 24031.32 + rank 27: 24031.32 + rank 28: 24031.29 + rank 29: 24031.28 + rank 30: 24031.37 + rank 31: 24031.30 + rank 32: 24031.82 + rank 33: 24031.79 + rank 34: 24031.80 + rank 35: 24031.78 + rank 36: 24031.79 + rank 37: 24031.78 + rank 38: 24031.80 + rank 39: 24031.82 + rank 40: 24032.11 + rank 41: 24032.09 + rank 42: 24032.12 + rank 43: 24032.11 + rank 44: 24032.09 + rank 45: 24032.10 + rank 46: 24032.11 + rank 47: 24032.12 + rank 48: 24032.06 + rank 49: 24032.04 + rank 50: 24032.03 + rank 51: 24032.03 + rank 52: 24032.02 + rank 53: 24032.03 + rank 54: 24032.03 + rank 55: 24032.02 + rank 56: 24032.06 + rank 57: 24032.06 + rank 58: 24032.02 + rank 59: 24032.03 + rank 60: 24032.03 + rank 61: 24032.04 + rank 62: 24032.03 + rank 63: 24032.04 + [2026-10-08 07:23:05] iteration 4441/ 6855 | consumed samples: 568448 | elapsed time per iteration (ms): 99339.7 | throughput per GPU (TFLOP/s/GPU): 36.8 | MFU 11.80% | FLOPs estimator: workload-aware | logical tokens/s: 168887.3 | valid tokens/s: 168887.3 | physical tokens/s: 169346.4 | loss tokens/s: 127348.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.366623E-01 | lm-global-supervised-tokens: 1.265072E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.318454E-02 | lm-token-buffer-mib: 5.813426E+01 | global_load_balancing_loss: 1.001306E+00 | loss scale: 1.0 | grad norm: 0.074 | params norm: 11358.182 | params rms: 0.061501 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 18:36:46.011588 | finish at 2026-10-11 01:59:51 + [2026-10-08 07:24:39] iteration 4442/ 6855 | consumed samples: 568576 | elapsed time per iteration (ms): 93479.0 | throughput per GPU (TFLOP/s/GPU): 40.0 | MFU 12.83% | FLOPs estimator: workload-aware | logical tokens/s: 179475.8 | valid tokens/s: 179475.8 | physical tokens/s: 179769.0 | loss tokens/s: 130530.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.926942E-01 | lm-global-supervised-tokens: 1.220184E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.570146E-02 | lm-token-buffer-mib: 5.806557E+01 | global_load_balancing_loss: 9.993140E-01 | loss scale: 1.0 | grad norm: 0.257 | params norm: 11358.127 | params rms: 0.061501 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 14:39:24.752431 | finish at 2026-10-10 22:04:04 + [2026-10-08 07:26:07] iteration 4443/ 6855 | consumed samples: 568704 | elapsed time per iteration (ms): 87848.5 | throughput per GPU (TFLOP/s/GPU): 41.3 | MFU 13.24% | FLOPs estimator: workload-aware | logical tokens/s: 190978.8 | valid tokens/s: 190978.8 | physical tokens/s: 191618.9 | loss tokens/s: 141542.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.633994E-01 | lm-global-supervised-tokens: 1.243425E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.173718E-02 | lm-token-buffer-mib: 5.804793E+01 | global_load_balancing_loss: 1.000140E+00 | loss scale: 1.0 | grad norm: 0.297 | params norm: 11358.071 | params rms: 0.061500 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 10:51:30.499798 | finish at 2026-10-10 18:17:37 + [2026-10-08 07:27:32] iteration 4444/ 6855 | consumed samples: 568832 | elapsed time per iteration (ms): 85573.3 | throughput per GPU (TFLOP/s/GPU): 43.1 | MFU 13.80% | FLOPs estimator: workload-aware | logical tokens/s: 196055.9 | valid tokens/s: 196055.9 | physical tokens/s: 196479.6 | loss tokens/s: 142046.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.385275E-01 | lm-global-supervised-tokens: 1.215539E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.056985E-02 | lm-token-buffer-mib: 5.804000E+01 | global_load_balancing_loss: 9.998786E-01 | loss scale: 1.0 | grad norm: 0.179 | params norm: 11358.016 | params rms: 0.061500 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 9:18:37.273158 | finish at 2026-10-10 16:46:10 + [2026-10-08 07:29:00] iteration 4445/ 6855 | consumed samples: 568960 | elapsed time per iteration (ms): 87782.5 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.62% | FLOPs estimator: workload-aware | logical tokens/s: 191122.0 | valid tokens/s: 191122.0 | physical tokens/s: 191449.0 | loss tokens/s: 139677.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.448368E-01 | lm-global-supervised-tokens: 1.226127E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.347308E-02 | lm-token-buffer-mib: 5.808622E+01 | global_load_balancing_loss: 1.000459E+00 | loss scale: 1.0 | grad norm: 0.216 | params norm: 11357.961 | params rms: 0.061500 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 10:45:55.867014 | finish at 2026-10-10 18:14:56 + [2026-10-08 07:30:28] iteration 4446/ 6855 | consumed samples: 569088 | elapsed time per iteration (ms): 87595.9 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.55% | FLOPs estimator: workload-aware | logical tokens/s: 191529.6 | valid tokens/s: 191529.6 | physical tokens/s: 191957.5 | loss tokens/s: 141214.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.574029E-01 | lm-global-supervised-tokens: 1.236981E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.187696E-02 | lm-token-buffer-mib: 5.812988E+01 | global_load_balancing_loss: 1.000048E+00 | loss scale: 1.0 | grad norm: 0.217 | params norm: 11357.906 | params rms: 0.061499 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 10:36:58.533580 | finish at 2026-10-10 18:07:26 + [2026-10-08 07:31:55] iteration 4447/ 6855 | consumed samples: 569216 | elapsed time per iteration (ms): 87384.9 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.38% | FLOPs estimator: workload-aware | logical tokens/s: 191991.9 | valid tokens/s: 191991.9 | physical tokens/s: 192479.8 | loss tokens/s: 141625.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.745043E-01 | lm-global-supervised-tokens: 1.237597E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.177529E-02 | lm-token-buffer-mib: 5.808301E+01 | global_load_balancing_loss: 9.998940E-01 | loss scale: 1.0 | grad norm: 0.067 | params norm: 11357.851 | params rms: 0.061499 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 10:27:02.834257 | finish at 2026-10-10 17:58:58 + [2026-10-08 07:33:24] iteration 4448/ 6855 | consumed samples: 569344 | elapsed time per iteration (ms): 89242.8 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.54% | FLOPs estimator: workload-aware | logical tokens/s: 187994.8 | valid tokens/s: 187994.8 | physical tokens/s: 188417.6 | loss tokens/s: 139878.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.420454E-01 | lm-global-supervised-tokens: 1.248315E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.800686E-02 | lm-token-buffer-mib: 5.815941E+01 | global_load_balancing_loss: 1.000270E+00 | loss scale: 1.0 | grad norm: 0.185 | params norm: 11357.798 | params rms: 0.061499 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 11:40:07.373684 | finish at 2026-10-10 19:13:32 + [2026-10-08 07:34:53] iteration 4449/ 6855 | consumed samples: 569472 | elapsed time per iteration (ms): 89070.1 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.57% | FLOPs estimator: workload-aware | logical tokens/s: 188359.0 | valid tokens/s: 188359.0 | physical tokens/s: 188833.4 | loss tokens/s: 140071.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.469990E-01 | lm-global-supervised-tokens: 1.247619E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.803477E-02 | lm-token-buffer-mib: 5.819199E+01 | global_load_balancing_loss: 9.999770E-01 | loss scale: 1.0 | grad norm: 0.183 | params norm: 11357.744 | params rms: 0.061498 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 11:31:42.597093 | finish at 2026-10-10 19:06:36 + [2026-10-08 07:36:40] iteration 4450/ 6855 | consumed samples: 569600 | elapsed time per iteration (ms): 106726.9 | throughput per GPU (TFLOP/s/GPU): 34.2 | MFU 10.96% | FLOPs estimator: workload-aware | logical tokens/s: 157197.6 | valid tokens/s: 157197.6 | physical tokens/s: 157519.4 | loss tokens/s: 119322.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.700106E-01 | lm-global-supervised-tokens: 1.273486E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.124558E-02 | lm-token-buffer-mib: 5.811440E+01 | global_load_balancing_loss: 1.000333E+00 | loss scale: 1.0 | grad norm: 0.055 | params norm: 11357.690 | params rms: 0.061498 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 23:17:58.167219 | finish at 2026-10-11 06:54:38 + [2026-10-08 07:38:09] iteration 4451/ 6855 | consumed samples: 569728 | elapsed time per iteration (ms): 88924.5 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.49% | FLOPs estimator: workload-aware | logical tokens/s: 188667.8 | valid tokens/s: 188667.8 | physical tokens/s: 189088.6 | loss tokens/s: 135860.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.503143E-01 | lm-global-supervised-tokens: 1.208132E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.139661E-02 | lm-token-buffer-mib: 5.812434E+01 | global_load_balancing_loss: 1.001945E+00 | loss scale: 1.0 | grad norm: 0.078 | params norm: 11357.637 | params rms: 0.061498 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 11:22:54.440657 | finish at 2026-10-10 19:01:03 + [2026-10-08 07:39:37] iteration 4452/ 6855 | consumed samples: 569856 | elapsed time per iteration (ms): 88436.0 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.52% | FLOPs estimator: workload-aware | logical tokens/s: 189709.2 | valid tokens/s: 189709.2 | physical tokens/s: 190147.4 | loss tokens/s: 143825.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.870463E-01 | lm-global-supervised-tokens: 1.271940E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.191692E-02 | lm-token-buffer-mib: 5.804187E+01 | global_load_balancing_loss: 9.997473E-01 | loss scale: 1.0 | grad norm: 0.153 | params norm: 11357.584 | params rms: 0.061498 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 11:01:51.822272 | finish at 2026-10-10 18:41:29 + [2026-10-08 07:41:06] iteration 4453/ 6855 | consumed samples: 569984 | elapsed time per iteration (ms): 88601.5 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.44% | FLOPs estimator: workload-aware | logical tokens/s: 189355.2 | valid tokens/s: 189355.2 | physical tokens/s: 189790.6 | loss tokens/s: 142224.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.570785E-01 | lm-global-supervised-tokens: 1.260130E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.151617E-02 | lm-token-buffer-mib: 5.810876E+01 | global_load_balancing_loss: 9.990092E-01 | loss scale: 1.0 | grad norm: 0.067 | params norm: 11357.530 | params rms: 0.061497 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 11:07:00.919909 | finish at 2026-10-10 18:48:07 + [2026-10-08 07:42:32] iteration 4454/ 6855 | consumed samples: 570112 | elapsed time per iteration (ms): 86279.7 | throughput per GPU (TFLOP/s/GPU): 42.8 | MFU 13.73% | FLOPs estimator: workload-aware | logical tokens/s: 194450.5 | valid tokens/s: 194450.5 | physical tokens/s: 194818.2 | loss tokens/s: 142245.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.961561E-01 | lm-global-supervised-tokens: 1.227292E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.224657E-02 | lm-token-buffer-mib: 5.813536E+01 | global_load_balancing_loss: 9.992911E-01 | loss scale: 1.0 | grad norm: 0.047 | params norm: 11357.477 | params rms: 0.061497 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 9:32:37.654824 | finish at 2026-10-10 17:15:10 + [2026-10-08 07:44:00] iteration 4455/ 6855 | consumed samples: 570240 | elapsed time per iteration (ms): 87421.4 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.76% | FLOPs estimator: workload-aware | logical tokens/s: 191911.3 | valid tokens/s: 191911.3 | physical tokens/s: 192372.0 | loss tokens/s: 145628.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.763415E-01 | lm-global-supervised-tokens: 1.273100E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.186817E-02 | lm-token-buffer-mib: 5.821558E+01 | global_load_balancing_loss: 1.000206E+00 | loss scale: 1.0 | grad norm: 0.138 | params norm: 11357.423 | params rms: 0.061497 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 10:16:51.322975 | finish at 2026-10-10 18:00:51 + [2026-10-08 07:45:30] iteration 4456/ 6855 | consumed samples: 570368 | elapsed time per iteration (ms): 90342.4 | throughput per GPU (TFLOP/s/GPU): 40.9 | MFU 13.11% | FLOPs estimator: workload-aware | logical tokens/s: 185707.0 | valid tokens/s: 185707.0 | physical tokens/s: 186121.7 | loss tokens/s: 136342.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.027321E-01 | lm-global-supervised-tokens: 1.231751E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.244694E-02 | lm-token-buffer-mib: 5.803944E+01 | global_load_balancing_loss: 1.000957E+00 | loss scale: 1.0 | grad norm: 0.105 | params norm: 11357.370 | params rms: 0.061496 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 12:12:11.299381 | finish at 2026-10-10 19:57:41 + [2026-10-08 07:46:57] iteration 4457/ 6855 | consumed samples: 570496 | elapsed time per iteration (ms): 87220.5 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.69% | FLOPs estimator: workload-aware | logical tokens/s: 192353.2 | valid tokens/s: 192353.1 | physical tokens/s: 192757.0 | loss tokens/s: 145400.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.012687E-01 | lm-global-supervised-tokens: 1.268189E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.169598E-02 | lm-token-buffer-mib: 5.815305E+01 | global_load_balancing_loss: 1.000526E+00 | loss scale: 1.0 | grad norm: 0.090 | params norm: 11357.317 | params rms: 0.061496 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 10:05:54.685229 | finish at 2026-10-10 17:52:52 + [2026-10-08 07:48:27] iteration 4458/ 6855 | consumed samples: 570624 | elapsed time per iteration (ms): 89400.3 | throughput per GPU (TFLOP/s/GPU): 42.8 | MFU 13.73% | FLOPs estimator: workload-aware | logical tokens/s: 187663.9 | valid tokens/s: 187663.9 | physical tokens/s: 188074.8 | loss tokens/s: 135016.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.913542E-01 | lm-global-supervised-tokens: 1.207050E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.444395E-02 | lm-token-buffer-mib: 5.809016E+01 | global_load_balancing_loss: 1.000493E+00 | loss scale: 1.0 | grad norm: 0.090 | params norm: 11357.263 | params rms: 0.061496 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 11:31:32.594027 | finish at 2026-10-10 19:19:59 + [2026-10-08 07:49:53] iteration 4459/ 6855 | consumed samples: 570752 | elapsed time per iteration (ms): 86515.2 | throughput per GPU (TFLOP/s/GPU): 42.8 | MFU 13.73% | FLOPs estimator: workload-aware | logical tokens/s: 193921.3 | valid tokens/s: 193921.3 | physical tokens/s: 194421.0 | loss tokens/s: 140034.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.199242E-01 | lm-global-supervised-tokens: 1.211506E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.093835E-02 | lm-token-buffer-mib: 5.822889E+01 | global_load_balancing_loss: 1.000588E+00 | loss scale: 1.0 | grad norm: 0.114 | params norm: 11357.207 | params rms: 0.061496 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 9:34:50.366405 | finish at 2026-10-10 17:24:44 + [2026-10-08 07:51:44] iteration 4460/ 6855 | consumed samples: 570880 | elapsed time per iteration (ms): 110306.1 | throughput per GPU (TFLOP/s/GPU): 33.2 | MFU 10.65% | FLOPs estimator: workload-aware | logical tokens/s: 152096.8 | valid tokens/s: 152096.8 | physical tokens/s: 152392.8 | loss tokens/s: 112306.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.099707E-01 | lm-global-supervised-tokens: 1.238808E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.165929E-02 | lm-token-buffer-mib: 5.821388E+01 | global_load_balancing_loss: 9.998906E-01 | loss scale: 1.0 | grad norm: 0.078 | params norm: 11357.152 | params rms: 0.061495 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 1:23:03.133223 | finish at 2026-10-11 09:14:47 + [2026-10-08 07:53:12] iteration 4461/ 6855 | consumed samples: 571008 | elapsed time per iteration (ms): 88371.3 | throughput per GPU (TFLOP/s/GPU): 41.7 | MFU 13.36% | FLOPs estimator: workload-aware | logical tokens/s: 189848.8 | valid tokens/s: 189848.8 | physical tokens/s: 190335.1 | loss tokens/s: 135434.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.655896E-01 | lm-global-supervised-tokens: 1.196850E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.328507E-02 | lm-token-buffer-mib: 5.834773E+01 | global_load_balancing_loss: 9.994761E-01 | loss scale: 1.0 | grad norm: 0.113 | params norm: 11357.098 | params rms: 0.061495 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 10:46:00.860765 | finish at 2026-10-10 18:39:13 + [2026-10-08 07:54:38] iteration 4462/ 6855 | consumed samples: 571136 | elapsed time per iteration (ms): 85686.4 | throughput per GPU (TFLOP/s/GPU): 43.1 | MFU 13.81% | FLOPs estimator: workload-aware | logical tokens/s: 195797.9 | valid tokens/s: 195797.9 | physical tokens/s: 196292.6 | loss tokens/s: 145632.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.557272E-01 | lm-global-supervised-tokens: 1.247875E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.132431E-02 | lm-token-buffer-mib: 5.820832E+01 | global_load_balancing_loss: 9.998596E-01 | loss scale: 1.0 | grad norm: 0.056 | params norm: 11357.042 | params rms: 0.061495 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 8:57:27.503700 | finish at 2026-10-10 16:52:05 + [2026-10-08 07:56:07] iteration 4463/ 6855 | consumed samples: 571264 | elapsed time per iteration (ms): 88948.1 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.70% | FLOPs estimator: workload-aware | logical tokens/s: 188617.2 | valid tokens/s: 188617.2 | physical tokens/s: 189008.6 | loss tokens/s: 132491.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.620618E-01 | lm-global-supervised-tokens: 1.178486E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.465293E-02 | lm-token-buffer-mib: 5.818537E+01 | global_load_balancing_loss: 9.999819E-01 | loss scale: 1.0 | grad norm: 0.125 | params norm: 11356.985 | params rms: 0.061494 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 11:06:03.805229 | finish at 2026-10-10 19:02:10 + [2026-10-08 07:57:34] iteration 4464/ 6855 | consumed samples: 571392 | elapsed time per iteration (ms): 87171.0 | throughput per GPU (TFLOP/s/GPU): 41.7 | MFU 13.37% | FLOPs estimator: workload-aware | logical tokens/s: 192463.0 | valid tokens/s: 192463.0 | physical tokens/s: 192859.8 | loss tokens/s: 147295.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.713735E-01 | lm-global-supervised-tokens: 1.283993E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.174147E-02 | lm-token-buffer-mib: 5.810898E+01 | global_load_balancing_loss: 1.000477E+00 | loss scale: 1.0 | grad norm: 0.093 | params norm: 11356.928 | params rms: 0.061494 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 9:53:45.979011 | finish at 2026-10-10 17:51:20 + [2026-10-08 07:59:03] iteration 4465/ 6855 | consumed samples: 571520 | elapsed time per iteration (ms): 89651.8 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.61% | FLOPs estimator: workload-aware | logical tokens/s: 187137.4 | valid tokens/s: 187137.4 | physical tokens/s: 187489.9 | loss tokens/s: 143487.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.271661E-01 | lm-global-supervised-tokens: 1.286391E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.140349E-02 | lm-token-buffer-mib: 5.812425E+01 | global_load_balancing_loss: 9.999849E-01 | loss scale: 1.0 | grad norm: 0.070 | params norm: 11356.873 | params rms: 0.061494 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 11:31:07.912347 | finish at 2026-10-10 19:30:11 + [2026-10-08 08:00:32] iteration 4466/ 6855 | consumed samples: 571648 | elapsed time per iteration (ms): 89139.9 | throughput per GPU (TFLOP/s/GPU): 42.8 | MFU 13.70% | FLOPs estimator: workload-aware | logical tokens/s: 188211.7 | valid tokens/s: 188211.7 | physical tokens/s: 188672.8 | loss tokens/s: 137846.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.072906E-01 | lm-global-supervised-tokens: 1.228763E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.206046E-02 | lm-token-buffer-mib: 5.811855E+01 | global_load_balancing_loss: 1.001092E+00 | loss scale: 1.0 | grad norm: 0.102 | params norm: 11356.816 | params rms: 0.061493 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 11:09:15.239253 | finish at 2026-10-10 19:09:48 + [2026-10-08 08:01:58] iteration 4467/ 6855 | consumed samples: 571776 | elapsed time per iteration (ms): 85627.7 | throughput per GPU (TFLOP/s/GPU): 43.6 | MFU 13.96% | FLOPs estimator: workload-aware | logical tokens/s: 195931.2 | valid tokens/s: 195931.2 | physical tokens/s: 196349.1 | loss tokens/s: 148880.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.912144E-01 | lm-global-supervised-tokens: 1.274827E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.157195E-02 | lm-token-buffer-mib: 5.806679E+01 | global_load_balancing_loss: 1.000986E+00 | loss scale: 1.0 | grad norm: 0.088 | params norm: 11356.758 | params rms: 0.061493 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 8:47:59.021831 | finish at 2026-10-10 16:49:57 + [2026-10-08 08:03:28] iteration 4468/ 6855 | consumed samples: 571904 | elapsed time per iteration (ms): 89947.5 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.38% | FLOPs estimator: workload-aware | logical tokens/s: 186521.9 | valid tokens/s: 186521.9 | physical tokens/s: 186920.3 | loss tokens/s: 132791.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.869358E-01 | lm-global-supervised-tokens: 1.194427E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.287843E-02 | lm-token-buffer-mib: 5.812390E+01 | global_load_balancing_loss: 1.000981E+00 | loss scale: 1.0 | grad norm: 0.117 | params norm: 11356.701 | params rms: 0.061493 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 11:38:24.762152 | finish at 2026-10-10 19:41:53 + [2026-10-08 08:04:55] iteration 4469/ 6855 | consumed samples: 572032 | elapsed time per iteration (ms): 86887.1 | throughput per GPU (TFLOP/s/GPU): 43.2 | MFU 13.84% | FLOPs estimator: workload-aware | logical tokens/s: 193091.9 | valid tokens/s: 193091.9 | physical tokens/s: 193597.3 | loss tokens/s: 143762.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.652798E-01 | lm-global-supervised-tokens: 1.249113E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.006354E-02 | lm-token-buffer-mib: 5.812759E+01 | global_load_balancing_loss: 1.000350E+00 | loss scale: 1.0 | grad norm: 0.041 | params norm: 11356.644 | params rms: 0.061493 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 9:35:12.607471 | finish at 2026-10-10 17:40:08 + [2026-10-08 08:06:46] iteration 4470/ 6855 | consumed samples: 572160 | elapsed time per iteration (ms): 111168.0 | throughput per GPU (TFLOP/s/GPU): 33.7 | MFU 10.81% | FLOPs estimator: workload-aware | logical tokens/s: 150916.6 | valid tokens/s: 150916.6 | physical tokens/s: 151218.7 | loss tokens/s: 110026.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.067879E-01 | lm-global-supervised-tokens: 1.223140E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.475897E-02 | lm-token-buffer-mib: 5.808147E+01 | global_load_balancing_loss: 9.991797E-01 | loss scale: 1.0 | grad norm: 0.079 | params norm: 11356.585 | params rms: 0.061492 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 1:38:55.563959 | finish at 2026-10-11 09:45:42 + [2026-10-08 08:08:13] iteration 4471/ 6855 | consumed samples: 572288 | elapsed time per iteration (ms): 87188.7 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.48% | FLOPs estimator: workload-aware | logical tokens/s: 192423.4 | valid tokens/s: 192423.4 | physical tokens/s: 192844.9 | loss tokens/s: 141099.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.517618E-01 | lm-global-supervised-tokens: 1.230223E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.181292E-02 | lm-token-buffer-mib: 5.812427E+01 | global_load_balancing_loss: 9.995100E-01 | loss scale: 1.0 | grad norm: 0.057 | params norm: 11356.527 | params rms: 0.061492 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 9:44:17.808414 | finish at 2026-10-10 17:52:31 + [2026-10-08 08:09:41] iteration 4472/ 6855 | consumed samples: 572416 | elapsed time per iteration (ms): 87609.1 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.70% | FLOPs estimator: workload-aware | logical tokens/s: 191500.1 | valid tokens/s: 191500.1 | physical tokens/s: 191882.7 | loss tokens/s: 142339.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.400529E-01 | lm-global-supervised-tokens: 1.247022E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.156448E-02 | lm-token-buffer-mib: 5.805809E+01 | global_load_balancing_loss: 9.999955E-01 | loss scale: 1.0 | grad norm: 0.037 | params norm: 11356.470 | params rms: 0.061492 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 9:59:32.536680 | finish at 2026-10-10 18:09:13 + [2026-10-08 08:11:12] iteration 4473/ 6855 | consumed samples: 572544 | elapsed time per iteration (ms): 91103.7 | throughput per GPU (TFLOP/s/GPU): 41.7 | MFU 13.38% | FLOPs estimator: workload-aware | logical tokens/s: 184154.5 | valid tokens/s: 184154.5 | physical tokens/s: 184409.6 | loss tokens/s: 130959.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.190828E-01 | lm-global-supervised-tokens: 1.193090E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.146096E-02 | lm-token-buffer-mib: 5.811068E+01 | global_load_balancing_loss: 1.000122E+00 | loss scale: 1.0 | grad norm: 0.042 | params norm: 11356.413 | params rms: 0.061491 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 12:16:49.108625 | finish at 2026-10-10 20:28:01 + [2026-10-08 08:12:39] iteration 4474/ 6855 | consumed samples: 572672 | elapsed time per iteration (ms): 86794.0 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.65% | FLOPs estimator: workload-aware | logical tokens/s: 193298.7 | valid tokens/s: 193298.7 | physical tokens/s: 193862.8 | loss tokens/s: 141886.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.549736E-01 | lm-global-supervised-tokens: 1.231487E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.161407E-02 | lm-token-buffer-mib: 5.825421E+01 | global_load_balancing_loss: 1.000332E+00 | loss scale: 1.0 | grad norm: 0.043 | params norm: 11356.354 | params rms: 0.061491 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 9:24:16.496756 | finish at 2026-10-10 17:36:55 + [2026-10-08 08:14:06] iteration 4475/ 6855 | consumed samples: 572800 | elapsed time per iteration (ms): 87647.7 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.76% | FLOPs estimator: workload-aware | logical tokens/s: 191415.8 | valid tokens/s: 191415.8 | physical tokens/s: 191808.5 | loss tokens/s: 137663.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.208955E-01 | lm-global-supervised-tokens: 1.206592E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.155446E-02 | lm-token-buffer-mib: 5.813800E+01 | global_load_balancing_loss: 1.000317E+00 | loss scale: 1.0 | grad norm: 0.042 | params norm: 11356.295 | params rms: 0.061491 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 9:56:41.636820 | finish at 2026-10-10 18:10:48 + [2026-10-08 08:15:33] iteration 4476/ 6855 | consumed samples: 572928 | elapsed time per iteration (ms): 86122.5 | throughput per GPU (TFLOP/s/GPU): 43.8 | MFU 14.05% | FLOPs estimator: workload-aware | logical tokens/s: 194806.3 | valid tokens/s: 194806.3 | physical tokens/s: 195376.6 | loss tokens/s: 144241.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.942123E-01 | lm-global-supervised-tokens: 1.242247E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.525882E-02 | lm-token-buffer-mib: 5.816066E+01 | global_load_balancing_loss: 1.000708E+00 | loss scale: 1.0 | grad norm: 0.051 | params norm: 11356.236 | params rms: 0.061490 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 8:54:45.389929 | finish at 2026-10-10 17:10:18 + [2026-10-08 08:16:58] iteration 4477/ 6855 | consumed samples: 573056 | elapsed time per iteration (ms): 85562.2 | throughput per GPU (TFLOP/s/GPU): 43.5 | MFU 13.95% | FLOPs estimator: workload-aware | logical tokens/s: 196081.1 | valid tokens/s: 196081.1 | physical tokens/s: 196546.7 | loss tokens/s: 145422.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.718593E-01 | lm-global-supervised-tokens: 1.244268E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.226116E-02 | lm-token-buffer-mib: 5.822309E+01 | global_load_balancing_loss: 1.000148E+00 | loss scale: 1.0 | grad norm: 0.067 | params norm: 11356.178 | params rms: 0.061490 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 8:31:06.830690 | finish at 2026-10-10 16:48:05 + [2026-10-08 08:18:25] iteration 4478/ 6855 | consumed samples: 573184 | elapsed time per iteration (ms): 87205.4 | throughput per GPU (TFLOP/s/GPU): 42.8 | MFU 13.72% | FLOPs estimator: workload-aware | logical tokens/s: 192386.7 | valid tokens/s: 192386.6 | physical tokens/s: 192749.9 | loss tokens/s: 133788.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.508260E-01 | lm-global-supervised-tokens: 1.166707E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.215042E-02 | lm-token-buffer-mib: 5.815581E+01 | global_load_balancing_loss: 1.001253E+00 | loss scale: 1.0 | grad norm: 0.037 | params norm: 11356.120 | params rms: 0.061490 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 9:34:47.267513 | finish at 2026-10-10 17:53:13 + [2026-10-08 08:19:53] iteration 4479/ 6855 | consumed samples: 573312 | elapsed time per iteration (ms): 87939.3 | throughput per GPU (TFLOP/s/GPU): 42.8 | MFU 13.71% | FLOPs estimator: workload-aware | logical tokens/s: 190779.8 | valid tokens/s: 190779.7 | physical tokens/s: 191225.5 | loss tokens/s: 142082.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.801608E-01 | lm-global-supervised-tokens: 1.249466E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.171169E-02 | lm-token-buffer-mib: 5.804260E+01 | global_load_balancing_loss: 1.000067E+00 | loss scale: 1.0 | grad norm: 0.069 | params norm: 11356.064 | params rms: 0.061489 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 10:02:23.873245 | finish at 2026-10-10 18:22:17 + [2026-10-08 08:21:44] iteration 4480/ 6855 | consumed samples: 573440 | elapsed time per iteration (ms): 110409.3 | throughput per GPU (TFLOP/s/GPU): 34.2 | MFU 10.95% | FLOPs estimator: workload-aware | logical tokens/s: 151954.3 | valid tokens/s: 151954.3 | physical tokens/s: 152354.7 | loss tokens/s: 115362.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.709409E-01 | lm-global-supervised-tokens: 1.273703E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.147685E-02 | lm-token-buffer-mib: 5.817494E+01 | global_load_balancing_loss: 9.997773E-01 | loss scale: 1.0 | grad norm: 0.061 | params norm: 11356.008 | params rms: 0.061489 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 0:50:22.100735 | finish at 2026-10-11 09:12:06 +times across ranks (ms): + save-checkpoint: + rank 0: 25066.82 + rank 1: 25066.86 + rank 2: 25066.84 + rank 3: 25066.82 + rank 4: 25066.81 + rank 5: 25066.79 + rank 6: 25066.79 + rank 7: 25066.82 + rank 8: 25067.34 + rank 9: 25067.29 + rank 10: 25067.30 + rank 11: 25067.30 + rank 12: 25067.33 + rank 13: 25067.29 + rank 14: 25067.33 + rank 15: 25067.29 + rank 16: 25066.54 + rank 17: 25066.54 + rank 18: 25066.52 + rank 19: 25066.54 + rank 20: 25066.53 + rank 21: 25066.54 + rank 22: 25066.46 + rank 23: 25066.51 + rank 24: 25067.86 + rank 25: 25068.31 + rank 26: 25067.97 + rank 27: 25067.88 + rank 28: 25067.82 + rank 29: 25067.86 + rank 30: 25067.78 + rank 31: 25067.85 + rank 32: 25067.96 + rank 33: 25067.97 + rank 34: 25067.93 + rank 35: 25067.92 + rank 36: 25067.84 + rank 37: 25067.83 + rank 38: 25067.87 + rank 39: 25067.85 + rank 40: 25067.83 + rank 41: 25067.75 + rank 42: 25067.77 + rank 43: 25067.82 + rank 44: 25067.76 + rank 45: 25067.80 + rank 46: 25067.71 + rank 47: 25067.81 + rank 48: 25068.16 + rank 49: 25068.32 + rank 50: 25068.28 + rank 51: 25068.12 + rank 52: 25068.28 + rank 53: 25068.28 + rank 54: 25068.27 + rank 55: 25068.30 + rank 56: 25067.89 + rank 57: 25067.90 + rank 58: 25067.85 + rank 59: 25067.87 + rank 60: 25067.88 + rank 61: 25067.87 + rank 62: 25068.01 + rank 63: 25067.92 + [2026-10-08 08:23:54] iteration 4481/ 6855 | consumed samples: 573568 | elapsed time per iteration (ms): 101005.6 | throughput per GPU (TFLOP/s/GPU): 35.9 | MFU 11.51% | FLOPs estimator: workload-aware | logical tokens/s: 166101.8 | valid tokens/s: 166101.8 | physical tokens/s: 166514.0 | loss tokens/s: 120986.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.532456E-01 | lm-global-supervised-tokens: 1.222034E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.353198E+00 | lm-token-buffer-mib: 5.809897E+01 | global_load_balancing_loss: 9.998949E-01 | loss scale: 1.0 | grad norm: 0.058 | params norm: 11355.952 | params rms: 0.061489 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 18:36:27.301134 | finish at 2026-10-11 03:00:21 + [2026-10-08 08:25:31] iteration 4482/ 6855 | consumed samples: 573696 | elapsed time per iteration (ms): 96794.4 | throughput per GPU (TFLOP/s/GPU): 39.2 | MFU 12.56% | FLOPs estimator: workload-aware | logical tokens/s: 173328.0 | valid tokens/s: 173328.0 | physical tokens/s: 173731.8 | loss tokens/s: 123174.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.318139E-01 | lm-global-supervised-tokens: 1.192262E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.336600E-02 | lm-token-buffer-mib: 5.815378E+01 | global_load_balancing_loss: 1.000156E+00 | loss scale: 1.0 | grad norm: 0.092 | params norm: 11355.896 | params rms: 0.061488 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 15:48:13.172248 | finish at 2026-10-11 00:13:44 + [2026-10-08 08:27:00] iteration 4483/ 6855 | consumed samples: 573824 | elapsed time per iteration (ms): 88750.7 | throughput per GPU (TFLOP/s/GPU): 41.2 | MFU 13.21% | FLOPs estimator: workload-aware | logical tokens/s: 189036.8 | valid tokens/s: 189036.8 | physical tokens/s: 189591.8 | loss tokens/s: 138667.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.929095E-01 | lm-global-supervised-tokens: 1.230686E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.808984E-02 | lm-token-buffer-mib: 5.815712E+01 | global_load_balancing_loss: 9.993317E-01 | loss scale: 1.0 | grad norm: 0.035 | params norm: 11355.839 | params rms: 0.061488 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 10:28:36.545025 | finish at 2026-10-10 18:55:36 + [2026-10-08 08:28:26] iteration 4484/ 6855 | consumed samples: 573952 | elapsed time per iteration (ms): 86228.2 | throughput per GPU (TFLOP/s/GPU): 43.7 | MFU 13.99% | FLOPs estimator: workload-aware | logical tokens/s: 194567.2 | valid tokens/s: 194567.2 | physical tokens/s: 195027.2 | loss tokens/s: 146337.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.904558E-01 | lm-global-supervised-tokens: 1.261843E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.283854E-02 | lm-token-buffer-mib: 5.823667E+01 | global_load_balancing_loss: 9.997737E-01 | loss scale: 1.0 | grad norm: 0.078 | params norm: 11355.783 | params rms: 0.061488 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 8:47:27.139547 | finish at 2026-10-10 17:15:53 + [2026-10-08 08:29:50] iteration 4485/ 6855 | consumed samples: 574080 | elapsed time per iteration (ms): 84417.3 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.66% | FLOPs estimator: workload-aware | logical tokens/s: 198740.8 | valid tokens/s: 198740.8 | physical tokens/s: 199340.6 | loss tokens/s: 146770.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.150713E-01 | lm-global-supervised-tokens: 1.239000E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.302359E-02 | lm-token-buffer-mib: 5.811665E+01 | global_load_balancing_loss: 9.997609E-01 | loss scale: 1.0 | grad norm: 0.118 | params norm: 11355.727 | params rms: 0.061488 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 7:34:29.107196 | finish at 2026-10-10 16:04:19 + [2026-10-08 08:31:14] iteration 4486/ 6855 | consumed samples: 574208 | elapsed time per iteration (ms): 84128.1 | throughput per GPU (TFLOP/s/GPU): 43.1 | MFU 13.81% | FLOPs estimator: workload-aware | logical tokens/s: 199424.4 | valid tokens/s: 199424.4 | physical tokens/s: 199932.0 | loss tokens/s: 147001.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.295049E-01 | lm-global-supervised-tokens: 1.236695E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.161407E-02 | lm-token-buffer-mib: 5.810419E+01 | global_load_balancing_loss: 1.001054E+00 | loss scale: 1.0 | grad norm: 0.155 | params norm: 11355.671 | params rms: 0.061487 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 7:21:39.516151 | finish at 2026-10-10 15:52:54 + [2026-10-08 08:32:41] iteration 4487/ 6855 | consumed samples: 574336 | elapsed time per iteration (ms): 86092.3 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.67% | FLOPs estimator: workload-aware | logical tokens/s: 194874.2 | valid tokens/s: 194874.2 | physical tokens/s: 195468.3 | loss tokens/s: 146091.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.576731E-01 | lm-global-supervised-tokens: 1.257739E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.339222E-02 | lm-token-buffer-mib: 5.822803E+01 | global_load_balancing_loss: 1.000669E+00 | loss scale: 1.0 | grad norm: 0.165 | params norm: 11355.616 | params rms: 0.061487 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 8:37:46.617065 | finish at 2026-10-10 17:10:27 + [2026-10-08 08:34:10] iteration 4488/ 6855 | consumed samples: 574464 | elapsed time per iteration (ms): 89333.7 | throughput per GPU (TFLOP/s/GPU): 41.4 | MFU 13.26% | FLOPs estimator: workload-aware | logical tokens/s: 187803.7 | valid tokens/s: 187803.7 | physical tokens/s: 188314.6 | loss tokens/s: 141451.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.038796E-01 | lm-global-supervised-tokens: 1.263633E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.280681E-02 | lm-token-buffer-mib: 5.827562E+01 | global_load_balancing_loss: 9.995399E-01 | loss scale: 1.0 | grad norm: 0.070 | params norm: 11355.559 | params rms: 0.061487 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 10:44:12.818100 | finish at 2026-10-10 19:18:23 + [2026-10-08 08:35:37] iteration 4489/ 6855 | consumed samples: 574592 | elapsed time per iteration (ms): 87412.6 | throughput per GPU (TFLOP/s/GPU): 43.2 | MFU 13.86% | FLOPs estimator: workload-aware | logical tokens/s: 191931.2 | valid tokens/s: 191931.2 | physical tokens/s: 192304.5 | loss tokens/s: 142233.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.558157E-01 | lm-global-supervised-tokens: 1.243303E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.410485E-02 | lm-token-buffer-mib: 5.810496E+01 | global_load_balancing_loss: 1.001228E+00 | loss scale: 1.0 | grad norm: 0.066 | params norm: 11355.501 | params rms: 0.061486 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 9:26:58.245542 | finish at 2026-10-10 18:02:36 + [2026-10-08 08:37:26] iteration 4490/ 6855 | consumed samples: 574720 | elapsed time per iteration (ms): 108605.5 | throughput per GPU (TFLOP/s/GPU): 34.0 | MFU 10.90% | FLOPs estimator: workload-aware | logical tokens/s: 154478.4 | valid tokens/s: 154478.4 | physical tokens/s: 154904.9 | loss tokens/s: 111808.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.701430E-01 | lm-global-supervised-tokens: 1.214302E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.856923E-02 | lm-token-buffer-mib: 5.813265E+01 | global_load_balancing_loss: 1.000739E+00 | loss scale: 1.0 | grad norm: 0.070 | params norm: 11355.444 | params rms: 0.061486 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 23:20:51.933030 | finish at 2026-10-11 07:58:18 + [2026-10-08 08:38:51] iteration 4491/ 6855 | consumed samples: 574848 | elapsed time per iteration (ms): 85174.7 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.77% | FLOPs estimator: workload-aware | logical tokens/s: 196973.7 | valid tokens/s: 196973.7 | physical tokens/s: 197613.1 | loss tokens/s: 150507.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.692154E-01 | lm-global-supervised-tokens: 1.281940E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.216939E-02 | lm-token-buffer-mib: 5.810935E+01 | global_load_balancing_loss: 1.000195E+00 | loss scale: 1.0 | grad norm: 0.034 | params norm: 11355.386 | params rms: 0.061486 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 7:55:52.957598 | finish at 2026-10-10 16:34:44 + [2026-10-08 08:40:21] iteration 4492/ 6855 | consumed samples: 574976 | elapsed time per iteration (ms): 89992.0 | throughput per GPU (TFLOP/s/GPU): 40.8 | MFU 13.07% | FLOPs estimator: workload-aware | logical tokens/s: 186429.7 | valid tokens/s: 186429.7 | physical tokens/s: 186837.1 | loss tokens/s: 133061.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.241170E-01 | lm-global-supervised-tokens: 1.197443E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.154077E-02 | lm-token-buffer-mib: 5.810175E+01 | global_load_balancing_loss: 1.000674E+00 | loss scale: 1.0 | grad norm: 0.056 | params norm: 11355.327 | params rms: 0.061485 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 11:04:11.035398 | finish at 2026-10-10 19:44:32 + [2026-10-08 08:41:48] iteration 4493/ 6855 | consumed samples: 575104 | elapsed time per iteration (ms): 87214.7 | throughput per GPU (TFLOP/s/GPU): 42.8 | MFU 13.73% | FLOPs estimator: workload-aware | logical tokens/s: 192366.7 | valid tokens/s: 192366.7 | physical tokens/s: 192763.4 | loss tokens/s: 142076.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.260157E-01 | lm-global-supervised-tokens: 1.239114E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.199314E-02 | lm-token-buffer-mib: 5.805278E+01 | global_load_balancing_loss: 9.988449E-01 | loss scale: 1.0 | grad norm: 0.038 | params norm: 11355.268 | params rms: 0.061485 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 9:13:21.179928 | finish at 2026-10-10 17:55:10 + [2026-10-08 08:43:14] iteration 4494/ 6855 | consumed samples: 575232 | elapsed time per iteration (ms): 85380.7 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.44% | FLOPs estimator: workload-aware | logical tokens/s: 196498.5 | valid tokens/s: 196498.4 | physical tokens/s: 196935.1 | loss tokens/s: 144617.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.404499E-01 | lm-global-supervised-tokens: 1.234759E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.329104E-02 | lm-token-buffer-mib: 5.808212E+01 | global_load_balancing_loss: 1.000091E+00 | loss scale: 1.0 | grad norm: 0.062 | params norm: 11355.210 | params rms: 0.061485 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 7:59:43.914021 | finish at 2026-10-10 16:42:58 + [2026-10-08 08:44:44] iteration 4495/ 6855 | consumed samples: 575360 | elapsed time per iteration (ms): 90318.5 | throughput per GPU (TFLOP/s/GPU): 41.3 | MFU 13.23% | FLOPs estimator: workload-aware | logical tokens/s: 185755.8 | valid tokens/s: 185755.8 | physical tokens/s: 186153.2 | loss tokens/s: 135193.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.808416E-01 | lm-global-supervised-tokens: 1.221044E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.197487E-02 | lm-token-buffer-mib: 5.810279E+01 | global_load_balancing_loss: 9.998752E-01 | loss scale: 1.0 | grad norm: 0.032 | params norm: 11355.152 | params rms: 0.061484 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 11:12:31.737185 | finish at 2026-10-10 19:57:16 + [2026-10-08 08:46:14] iteration 4496/ 6855 | consumed samples: 575488 | elapsed time per iteration (ms): 90313.1 | throughput per GPU (TFLOP/s/GPU): 41.4 | MFU 13.27% | FLOPs estimator: workload-aware | logical tokens/s: 185767.2 | valid tokens/s: 185767.2 | physical tokens/s: 186116.7 | loss tokens/s: 134876.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.388656E-01 | lm-global-supervised-tokens: 1.218107E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.242430E-02 | lm-token-buffer-mib: 5.809878E+01 | global_load_balancing_loss: 1.000861E+00 | loss scale: 1.0 | grad norm: 0.042 | params norm: 11355.094 | params rms: 0.061484 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 11:10:48.679625 | finish at 2026-10-10 19:57:03 + [2026-10-08 08:47:46] iteration 4497/ 6855 | consumed samples: 575616 | elapsed time per iteration (ms): 91799.3 | throughput per GPU (TFLOP/s/GPU): 41.4 | MFU 13.25% | FLOPs estimator: workload-aware | logical tokens/s: 182759.7 | valid tokens/s: 182759.7 | physical tokens/s: 183329.3 | loss tokens/s: 135604.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.951711E-01 | lm-global-supervised-tokens: 1.244836E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.815098E-02 | lm-token-buffer-mib: 5.825381E+01 | global_load_balancing_loss: 9.994826E-01 | loss scale: 1.0 | grad norm: 0.045 | params norm: 11355.037 | params rms: 0.061484 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 12:07:42.813385 | finish at 2026-10-10 20:55:29 + [2026-10-08 08:49:11] iteration 4498/ 6855 | consumed samples: 575744 | elapsed time per iteration (ms): 84404.8 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.77% | FLOPs estimator: workload-aware | logical tokens/s: 198770.8 | valid tokens/s: 198770.8 | physical tokens/s: 199144.3 | loss tokens/s: 148726.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.306645E-01 | lm-global-supervised-tokens: 1.255326E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.169820E-02 | lm-token-buffer-mib: 5.806330E+01 | global_load_balancing_loss: 9.993609E-01 | loss scale: 1.0 | grad norm: 0.041 | params norm: 11354.979 | params rms: 0.061484 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 7:15:42.125255 | finish at 2026-10-10 16:04:53 + [2026-10-08 08:50:35] iteration 4499/ 6855 | consumed samples: 575872 | elapsed time per iteration (ms): 84110.4 | throughput per GPU (TFLOP/s/GPU): 44.0 | MFU 14.09% | FLOPs estimator: workload-aware | logical tokens/s: 199466.5 | valid tokens/s: 199466.5 | physical tokens/s: 199801.8 | loss tokens/s: 143853.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.755376E-01 | lm-global-supervised-tokens: 1.209953E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.150832E-02 | lm-token-buffer-mib: 5.810616E+01 | global_load_balancing_loss: 1.000525E+00 | loss scale: 1.0 | grad norm: 0.049 | params norm: 11354.921 | params rms: 0.061483 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 7:02:44.115229 | finish at 2026-10-10 15:53:19 + [2026-10-08 08:52:22] iteration 4500/ 6855 | consumed samples: 576000 | elapsed time per iteration (ms): 107548.9 | throughput per GPU (TFLOP/s/GPU): 34.7 | MFU 11.14% | FLOPs estimator: workload-aware | logical tokens/s: 155996.1 | valid tokens/s: 155996.1 | physical tokens/s: 156290.7 | loss tokens/s: 109789.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.683329E-01 | lm-global-supervised-tokens: 1.180769E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.397942E-02 | lm-token-buffer-mib: 5.806800E+01 | global_load_balancing_loss: 1.000623E+00 | loss scale: 1.0 | grad norm: 0.032 | params norm: 11354.865 | params rms: 0.061483 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 22:21:17.735038 | finish at 2026-10-11 07:13:40 + [2026-10-08 08:53:48] iteration 4501/ 6855 | consumed samples: 576128 | elapsed time per iteration (ms): 85534.5 | throughput per GPU (TFLOP/s/GPU): 43.1 | MFU 13.81% | FLOPs estimator: workload-aware | logical tokens/s: 196144.8 | valid tokens/s: 196144.8 | physical tokens/s: 196656.9 | loss tokens/s: 147803.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.741834E-01 | lm-global-supervised-tokens: 1.264229E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.474711E-02 | lm-token-buffer-mib: 5.821883E+01 | global_load_balancing_loss: 9.991623E-01 | loss scale: 1.0 | grad norm: 0.052 | params norm: 11354.807 | params rms: 0.061483 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 7:55:48.154919 | finish at 2026-10-10 16:49:36 + [2026-10-08 08:55:16] iteration 4502/ 6855 | consumed samples: 576256 | elapsed time per iteration (ms): 88113.9 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.74% | FLOPs estimator: workload-aware | logical tokens/s: 190403.3 | valid tokens/s: 190403.3 | physical tokens/s: 190766.1 | loss tokens/s: 138996.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.550236E-01 | lm-global-supervised-tokens: 1.224748E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.170473E-02 | lm-token-buffer-mib: 5.821951E+01 | global_load_balancing_loss: 1.001078E+00 | loss scale: 1.0 | grad norm: 0.039 | params norm: 11354.749 | params rms: 0.061482 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 9:35:32.073893 | finish at 2026-10-10 18:30:48 + [2026-10-08 08:56:43] iteration 4503/ 6855 | consumed samples: 576384 | elapsed time per iteration (ms): 87068.4 | throughput per GPU (TFLOP/s/GPU): 42.8 | MFU 13.71% | FLOPs estimator: workload-aware | logical tokens/s: 192689.7 | valid tokens/s: 192689.7 | physical tokens/s: 193082.4 | loss tokens/s: 146430.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.002306E-01 | lm-global-supervised-tokens: 1.274949E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.233523E-02 | lm-token-buffer-mib: 5.814896E+01 | global_load_balancing_loss: 1.000739E+00 | loss scale: 1.0 | grad norm: 0.037 | params norm: 11354.690 | params rms: 0.061482 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 8:53:04.907982 | finish at 2026-10-10 17:49:48 + [2026-10-08 08:58:09] iteration 4504/ 6855 | consumed samples: 576512 | elapsed time per iteration (ms): 86556.3 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.59% | FLOPs estimator: workload-aware | logical tokens/s: 193830.1 | valid tokens/s: 193830.1 | physical tokens/s: 194247.2 | loss tokens/s: 152069.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.462531E-01 | lm-global-supervised-tokens: 1.316260E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.374176E-02 | lm-token-buffer-mib: 5.815514E+01 | global_load_balancing_loss: 9.996197E-01 | loss scale: 1.0 | grad norm: 0.057 | params norm: 11354.635 | params rms: 0.061482 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 8:31:33.820205 | finish at 2026-10-10 17:29:43 + [2026-10-08 08:59:37] iteration 4505/ 6855 | consumed samples: 576640 | elapsed time per iteration (ms): 87768.8 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.51% | FLOPs estimator: workload-aware | logical tokens/s: 191152.4 | valid tokens/s: 191152.4 | physical tokens/s: 191665.6 | loss tokens/s: 143549.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.447220E-01 | lm-global-supervised-tokens: 1.259917E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.471596E-02 | lm-token-buffer-mib: 5.827874E+01 | global_load_balancing_loss: 1.001085E+00 | loss scale: 1.0 | grad norm: 0.043 | params norm: 11354.579 | params rms: 0.061481 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 9:17:36.587040 | finish at 2026-10-10 18:17:14 + [2026-10-08 09:01:03] iteration 4506/ 6855 | consumed samples: 576768 | elapsed time per iteration (ms): 85595.0 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.62% | FLOPs estimator: workload-aware | logical tokens/s: 196006.8 | valid tokens/s: 196006.8 | physical tokens/s: 196524.9 | loss tokens/s: 145496.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 5.843204E-01 | lm-global-supervised-tokens: 1.245375E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.118760E-02 | lm-token-buffer-mib: 5.828870E+01 | global_load_balancing_loss: 1.000175E+00 | loss scale: 1.0 | grad norm: 0.061 | params norm: 11354.522 | params rms: 0.061481 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 7:51:02.682509 | finish at 2026-10-10 16:52:06 + [2026-10-08 09:02:33] iteration 4507/ 6855 | consumed samples: 576896 | elapsed time per iteration (ms): 90398.3 | throughput per GPU (TFLOP/s/GPU): 40.6 | MFU 13.01% | FLOPs estimator: workload-aware | logical tokens/s: 185592.0 | valid tokens/s: 185592.0 | physical tokens/s: 186027.5 | loss tokens/s: 135682.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.460942E-01 | lm-global-supervised-tokens: 1.226546E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.112747E-02 | lm-token-buffer-mib: 5.819254E+01 | global_load_balancing_loss: 1.000022E+00 | loss scale: 1.0 | grad norm: 0.183 | params norm: 11354.467 | params rms: 0.061481 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 10:57:35.253026 | finish at 2026-10-10 20:00:08 + [2026-10-08 09:04:02] iteration 4508/ 6855 | consumed samples: 577024 | elapsed time per iteration (ms): 88588.7 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.42% | FLOPs estimator: workload-aware | logical tokens/s: 189383.0 | valid tokens/s: 189382.9 | physical tokens/s: 189854.5 | loss tokens/s: 138012.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.216518E-01 | lm-global-supervised-tokens: 1.222634E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.168747E-02 | lm-token-buffer-mib: 5.807534E+01 | global_load_balancing_loss: 1.000231E+00 | loss scale: 1.0 | grad norm: 0.290 | params norm: 11354.412 | params rms: 0.061480 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 9:45:17.689104 | finish at 2026-10-10 18:49:20 + [2026-10-08 09:05:28] iteration 4509/ 6855 | consumed samples: 577152 | elapsed time per iteration (ms): 86194.9 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.40% | FLOPs estimator: workload-aware | logical tokens/s: 194642.1 | valid tokens/s: 194642.0 | physical tokens/s: 195117.2 | loss tokens/s: 149329.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.613322E-01 | lm-global-supervised-tokens: 1.287140E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.133238E-02 | lm-token-buffer-mib: 5.816730E+01 | global_load_balancing_loss: 9.994010E-01 | loss scale: 1.0 | grad norm: 0.277 | params norm: 11354.357 | params rms: 0.061480 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 8:10:13.323858 | finish at 2026-10-10 17:15:41 + [2026-10-08 09:07:20] iteration 4510/ 6855 | consumed samples: 577280 | elapsed time per iteration (ms): 112390.0 | throughput per GPU (TFLOP/s/GPU): 33.3 | MFU 10.67% | FLOPs estimator: workload-aware | logical tokens/s: 149276.0 | valid tokens/s: 149276.0 | physical tokens/s: 149714.0 | loss tokens/s: 114324.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.541567E-01 | lm-global-supervised-tokens: 1.284889E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.474843E-02 | lm-token-buffer-mib: 5.824644E+01 | global_load_balancing_loss: 1.000094E+00 | loss scale: 1.0 | grad norm: 0.068 | params norm: 11354.300 | params rms: 0.061480 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 1:12:34.663742 | finish at 2026-10-11 10:19:55 + [2026-10-08 09:08:46] iteration 4511/ 6855 | consumed samples: 577408 | elapsed time per iteration (ms): 85451.2 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.66% | FLOPs estimator: workload-aware | logical tokens/s: 196336.4 | valid tokens/s: 196336.4 | physical tokens/s: 196767.0 | loss tokens/s: 146210.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 5.881278E-01 | lm-global-supervised-tokens: 1.249385E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.200080E-02 | lm-token-buffer-mib: 5.804275E+01 | global_load_balancing_loss: 1.000192E+00 | loss scale: 1.0 | grad norm: 0.243 | params norm: 11354.244 | params rms: 0.061480 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 7:38:17.508314 | finish at 2026-10-10 16:47:03 + [2026-10-08 09:10:13] iteration 4512/ 6855 | consumed samples: 577536 | elapsed time per iteration (ms): 87068.9 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.66% | FLOPs estimator: workload-aware | logical tokens/s: 192688.7 | valid tokens/s: 192688.7 | physical tokens/s: 193138.6 | loss tokens/s: 143130.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.127287E-01 | lm-global-supervised-tokens: 1.246225E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.893820E-02 | lm-token-buffer-mib: 5.810759E+01 | global_load_balancing_loss: 9.990563E-01 | loss scale: 1.0 | grad norm: 0.098 | params norm: 11354.187 | params rms: 0.061479 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 8:40:02.459767 | finish at 2026-10-10 17:50:15 + [2026-10-08 09:11:39] iteration 4513/ 6855 | consumed samples: 577664 | elapsed time per iteration (ms): 86491.2 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.38% | FLOPs estimator: workload-aware | logical tokens/s: 193975.7 | valid tokens/s: 193975.7 | physical tokens/s: 194382.5 | loss tokens/s: 141034.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.468326E-01 | lm-global-supervised-tokens: 1.219821E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.149568E-02 | lm-token-buffer-mib: 5.819843E+01 | global_load_balancing_loss: 1.001685E+00 | loss scale: 1.0 | grad norm: 0.308 | params norm: 11354.129 | params rms: 0.061479 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 8:16:02.334493 | finish at 2026-10-10 17:27:42 + [2026-10-08 09:13:05] iteration 4514/ 6855 | consumed samples: 577792 | elapsed time per iteration (ms): 85377.8 | throughput per GPU (TFLOP/s/GPU): 43.2 | MFU 13.84% | FLOPs estimator: workload-aware | logical tokens/s: 196505.5 | valid tokens/s: 196505.5 | physical tokens/s: 196931.8 | loss tokens/s: 145263.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.684716E-01 | lm-global-supervised-tokens: 1.240225E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.144796E-02 | lm-token-buffer-mib: 5.813390E+01 | global_load_balancing_loss: 1.001402E+00 | loss scale: 1.0 | grad norm: 0.118 | params norm: 11354.070 | params rms: 0.061479 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 7:31:09.371725 | finish at 2026-10-10 16:44:14 + [2026-10-08 09:14:32] iteration 4515/ 6855 | consumed samples: 577920 | elapsed time per iteration (ms): 86729.9 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.61% | FLOPs estimator: workload-aware | logical tokens/s: 193441.5 | valid tokens/s: 193441.5 | physical tokens/s: 193915.6 | loss tokens/s: 142425.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.651101E-01 | lm-global-supervised-tokens: 1.235251E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.748034E-02 | lm-token-buffer-mib: 5.827580E+01 | global_load_balancing_loss: 9.998706E-01 | loss scale: 1.0 | grad norm: 0.205 | params norm: 11354.012 | params rms: 0.061478 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 8:22:27.907147 | finish at 2026-10-10 17:36:59 + [2026-10-08 09:15:57] iteration 4516/ 6855 | consumed samples: 578048 | elapsed time per iteration (ms): 85913.2 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.53% | FLOPs estimator: workload-aware | logical tokens/s: 195281.0 | valid tokens/s: 195281.0 | physical tokens/s: 195728.6 | loss tokens/s: 142634.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.489160E-01 | lm-global-supervised-tokens: 1.225420E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.930132E-02 | lm-token-buffer-mib: 5.803123E+01 | global_load_balancing_loss: 9.999691E-01 | loss scale: 1.0 | grad norm: 0.243 | params norm: 11353.954 | params rms: 0.061478 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 7:49:10.881998 | finish at 2026-10-10 17:05:08 + [2026-10-08 09:17:24] iteration 4517/ 6855 | consumed samples: 578176 | elapsed time per iteration (ms): 86782.6 | throughput per GPU (TFLOP/s/GPU): 43.1 | MFU 13.80% | FLOPs estimator: workload-aware | logical tokens/s: 193323.4 | valid tokens/s: 193323.4 | physical tokens/s: 193716.9 | loss tokens/s: 141465.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.469437E-01 | lm-global-supervised-tokens: 1.227676E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.164362E-02 | lm-token-buffer-mib: 5.813302E+01 | global_load_balancing_loss: 9.999590E-01 | loss scale: 1.0 | grad norm: 0.241 | params norm: 11353.894 | params rms: 0.061478 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 8:21:37.613832 | finish at 2026-10-10 17:39:02 + [2026-10-08 09:18:50] iteration 4518/ 6855 | consumed samples: 578304 | elapsed time per iteration (ms): 85833.9 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.46% | FLOPs estimator: workload-aware | logical tokens/s: 195460.7 | valid tokens/s: 195460.7 | physical tokens/s: 195986.5 | loss tokens/s: 140906.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.115547E-01 | lm-global-supervised-tokens: 1.209453E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.567011E-02 | lm-token-buffer-mib: 5.818456E+01 | global_load_balancing_loss: 1.000324E+00 | loss scale: 1.0 | grad norm: 0.190 | params norm: 11353.835 | params rms: 0.061477 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 7:43:13.824798 | finish at 2026-10-10 17:02:04 + [2026-10-08 09:20:20] iteration 4519/ 6855 | consumed samples: 578432 | elapsed time per iteration (ms): 90348.5 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.56% | FLOPs estimator: workload-aware | logical tokens/s: 185694.3 | valid tokens/s: 185694.3 | physical tokens/s: 186150.5 | loss tokens/s: 139514.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.335498E-01 | lm-global-supervised-tokens: 1.260491E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.288569E-02 | lm-token-buffer-mib: 5.814826E+01 | global_load_balancing_loss: 9.998780E-01 | loss scale: 1.0 | grad norm: 0.163 | params norm: 11353.777 | params rms: 0.061477 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 10:37:34.059273 | finish at 2026-10-10 19:57:54 + [2026-10-08 09:22:09] iteration 4520/ 6855 | consumed samples: 578560 | elapsed time per iteration (ms): 108774.1 | throughput per GPU (TFLOP/s/GPU): 34.4 | MFU 11.02% | FLOPs estimator: workload-aware | logical tokens/s: 154239.0 | valid tokens/s: 154239.0 | physical tokens/s: 154478.5 | loss tokens/s: 105382.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.786867E-01 | lm-global-supervised-tokens: 1.146291E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.304519E-02 | lm-token-buffer-mib: 5.817377E+01 | global_load_balancing_loss: 1.000210E+00 | loss scale: 1.0 | grad norm: 0.047 | params norm: 11353.717 | params rms: 0.061477 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 22:33:07.567632 | finish at 2026-10-11 07:55:17 +times across ranks (ms): + save-checkpoint: + rank 0: 25859.29 + rank 1: 25859.32 + rank 2: 25859.32 + rank 3: 25859.29 + rank 4: 25859.29 + rank 5: 25859.25 + rank 6: 25859.24 + rank 7: 25859.28 + rank 8: 25858.25 + rank 9: 25858.23 + rank 10: 25858.23 + rank 11: 25858.32 + rank 12: 25858.23 + rank 13: 25858.24 + rank 14: 25858.23 + rank 15: 25858.22 + rank 16: 25859.38 + rank 17: 25859.39 + rank 18: 25859.34 + rank 19: 25859.38 + rank 20: 25859.35 + rank 21: 25859.39 + rank 22: 25859.35 + rank 23: 25859.34 + rank 24: 25859.96 + rank 25: 25860.40 + rank 26: 25860.00 + rank 27: 25860.01 + rank 28: 25859.91 + rank 29: 25859.95 + rank 30: 25859.94 + rank 31: 25860.01 + rank 32: 25860.51 + rank 33: 25860.48 + rank 34: 25860.51 + rank 35: 25860.54 + rank 36: 25860.44 + rank 37: 25860.40 + rank 38: 25860.38 + rank 39: 25860.41 + rank 40: 25858.73 + rank 41: 25858.79 + rank 42: 25858.73 + rank 43: 25858.75 + rank 44: 25858.72 + rank 45: 25858.74 + rank 46: 25858.72 + rank 47: 25858.70 + rank 48: 25859.67 + rank 49: 25859.83 + rank 50: 25859.76 + rank 51: 25859.60 + rank 52: 25859.79 + rank 53: 25859.75 + rank 54: 25859.79 + rank 55: 25859.80 + rank 56: 25859.86 + rank 57: 25859.83 + rank 58: 25859.85 + rank 59: 25859.80 + rank 60: 25859.85 + rank 61: 25859.80 + rank 62: 25859.82 + rank 63: 25859.84 + [2026-10-08 09:24:07] iteration 4521/ 6855 | consumed samples: 578688 | elapsed time per iteration (ms): 92167.5 | throughput per GPU (TFLOP/s/GPU): 40.9 | MFU 13.11% | FLOPs estimator: workload-aware | logical tokens/s: 182029.1 | valid tokens/s: 182029.1 | physical tokens/s: 182328.6 | loss tokens/s: 134063.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.928120E-01 | lm-global-supervised-tokens: 1.235632E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.817798E+00 | lm-token-buffer-mib: 5.804620E+01 | global_load_balancing_loss: 9.995220E-01 | loss scale: 1.0 | grad norm: 0.117 | params norm: 11353.657 | params rms: 0.061476 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 11:45:18.882164 | finish at 2026-10-10 21:09:26 + [2026-10-08 09:25:38] iteration 4522/ 6855 | consumed samples: 578816 | elapsed time per iteration (ms): 91020.4 | throughput per GPU (TFLOP/s/GPU): 40.5 | MFU 12.97% | FLOPs estimator: workload-aware | logical tokens/s: 184323.6 | valid tokens/s: 184323.6 | physical tokens/s: 184728.4 | loss tokens/s: 138182.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.445125E-01 | lm-global-supervised-tokens: 1.257740E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.961415E-02 | lm-token-buffer-mib: 5.813792E+01 | global_load_balancing_loss: 1.000332E+00 | loss scale: 1.0 | grad norm: 0.134 | params norm: 11353.599 | params rms: 0.061476 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 10:59:10.693432 | finish at 2026-10-10 20:24:49 + [2026-10-08 09:27:02] iteration 4523/ 6855 | consumed samples: 578944 | elapsed time per iteration (ms): 83449.1 | throughput per GPU (TFLOP/s/GPU): 43.8 | MFU 14.04% | FLOPs estimator: workload-aware | logical tokens/s: 201047.0 | valid tokens/s: 201047.0 | physical tokens/s: 201500.9 | loss tokens/s: 150383.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.214562E-01 | lm-global-supervised-tokens: 1.254934E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.170094E-02 | lm-token-buffer-mib: 5.803889E+01 | global_load_balancing_loss: 9.989671E-01 | loss scale: 1.0 | grad norm: 0.046 | params norm: 11353.544 | params rms: 0.061476 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 6:03:23.294013 | finish at 2026-10-10 15:30:25 + [2026-10-08 09:28:28] iteration 4524/ 6855 | consumed samples: 579072 | elapsed time per iteration (ms): 86117.6 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.70% | FLOPs estimator: workload-aware | logical tokens/s: 194817.1 | valid tokens/s: 194817.1 | physical tokens/s: 195442.8 | loss tokens/s: 146102.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.715510E-01 | lm-global-supervised-tokens: 1.258195E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.208654E-02 | lm-token-buffer-mib: 5.849472E+01 | global_load_balancing_loss: 1.000731E+00 | loss scale: 1.0 | grad norm: 0.113 | params norm: 11353.491 | params rms: 0.061475 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 7:45:40.022702 | finish at 2026-10-10 17:14:08 + [2026-10-08 09:29:55] iteration 4525/ 6855 | consumed samples: 579200 | elapsed time per iteration (ms): 87015.9 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.58% | FLOPs estimator: workload-aware | logical tokens/s: 192805.5 | valid tokens/s: 192805.5 | physical tokens/s: 193355.3 | loss tokens/s: 136465.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.295673E-01 | lm-global-supervised-tokens: 1.187468E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.583597E-02 | lm-token-buffer-mib: 5.814270E+01 | global_load_balancing_loss: 1.002031E+00 | loss scale: 1.0 | grad norm: 0.215 | params norm: 11353.436 | params rms: 0.061475 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 8:19:07.132864 | finish at 2026-10-10 17:49:02 + [2026-10-08 09:31:23] iteration 4526/ 6855 | consumed samples: 579328 | elapsed time per iteration (ms): 87635.7 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.65% | FLOPs estimator: workload-aware | logical tokens/s: 191442.6 | valid tokens/s: 191442.6 | physical tokens/s: 191886.5 | loss tokens/s: 142215.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.649822E-01 | lm-global-supervised-tokens: 1.246312E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.315717E-02 | lm-token-buffer-mib: 5.814066E+01 | global_load_balancing_loss: 1.000364E+00 | loss scale: 1.0 | grad norm: 0.138 | params norm: 11353.381 | params rms: 0.061475 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 8:41:43.599133 | finish at 2026-10-10 18:13:06 + [2026-10-08 09:32:51] iteration 4527/ 6855 | consumed samples: 579456 | elapsed time per iteration (ms): 88771.8 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.42% | FLOPs estimator: workload-aware | logical tokens/s: 188991.8 | valid tokens/s: 188991.8 | physical tokens/s: 189681.3 | loss tokens/s: 138622.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.074451E-01 | lm-global-supervised-tokens: 1.230577E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.761766E-02 | lm-token-buffer-mib: 5.816022E+01 | global_load_balancing_loss: 1.000167E+00 | loss scale: 1.0 | grad norm: 0.178 | params norm: 11353.328 | params rms: 0.061475 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 9:24:20.820431 | finish at 2026-10-10 18:57:12 + [2026-10-08 09:34:20] iteration 4528/ 6855 | consumed samples: 579584 | elapsed time per iteration (ms): 88690.5 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.57% | FLOPs estimator: workload-aware | logical tokens/s: 189165.1 | valid tokens/s: 189165.1 | physical tokens/s: 189489.7 | loss tokens/s: 136702.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.072649E-01 | lm-global-supervised-tokens: 1.212424E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.322768E-02 | lm-token-buffer-mib: 5.811227E+01 | global_load_balancing_loss: 1.000208E+00 | loss scale: 1.0 | grad norm: 0.044 | params norm: 11353.277 | params rms: 0.061474 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 9:19:42.840707 | finish at 2026-10-10 18:54:03 + [2026-10-08 09:35:48] iteration 4529/ 6855 | consumed samples: 579712 | elapsed time per iteration (ms): 88151.0 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.53% | FLOPs estimator: workload-aware | logical tokens/s: 190323.1 | valid tokens/s: 190323.1 | physical tokens/s: 190721.2 | loss tokens/s: 140824.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 5.961461E-01 | lm-global-supervised-tokens: 1.241386E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.245624E-02 | lm-token-buffer-mib: 5.816048E+01 | global_load_balancing_loss: 1.000514E+00 | loss scale: 1.0 | grad norm: 0.263 | params norm: 11353.223 | params rms: 0.061474 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 8:57:19.278737 | finish at 2026-10-10 18:33:08 + [2026-10-08 09:37:40] iteration 4530/ 6855 | consumed samples: 579840 | elapsed time per iteration (ms): 111690.8 | throughput per GPU (TFLOP/s/GPU): 33.8 | MFU 10.82% | FLOPs estimator: workload-aware | logical tokens/s: 150211.0 | valid tokens/s: 150211.0 | physical tokens/s: 150498.4 | loss tokens/s: 109998.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.870010E-01 | lm-global-supervised-tokens: 1.228586E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.143742E-02 | lm-token-buffer-mib: 5.816618E+01 | global_load_balancing_loss: 9.994353E-01 | loss scale: 1.0 | grad norm: 0.043 | params norm: 11353.170 | params rms: 0.061474 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 0:08:01.066096 | finish at 2026-10-11 09:45:41 + [2026-10-08 09:39:09] iteration 4531/ 6855 | consumed samples: 579968 | elapsed time per iteration (ms): 88760.4 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.65% | FLOPs estimator: workload-aware | logical tokens/s: 189016.3 | valid tokens/s: 189016.3 | physical tokens/s: 189626.0 | loss tokens/s: 139356.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.487402E-01 | lm-global-supervised-tokens: 1.236933E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.624661E-02 | lm-token-buffer-mib: 5.808440E+01 | global_load_balancing_loss: 9.995349E-01 | loss scale: 1.0 | grad norm: 0.218 | params norm: 11353.115 | params rms: 0.061473 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 9:17:59.142028 | finish at 2026-10-10 18:57:08 + [2026-10-08 09:40:37] iteration 4532/ 6855 | consumed samples: 580096 | elapsed time per iteration (ms): 88674.9 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.55% | FLOPs estimator: workload-aware | logical tokens/s: 189198.4 | valid tokens/s: 189198.4 | physical tokens/s: 189519.5 | loss tokens/s: 143532.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.901726E-01 | lm-global-supervised-tokens: 1.272770E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.286833E-02 | lm-token-buffer-mib: 5.806425E+01 | global_load_balancing_loss: 9.987107E-01 | loss scale: 1.0 | grad norm: 0.070 | params norm: 11353.062 | params rms: 0.061473 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 9:13:11.748520 | finish at 2026-10-10 18:53:49 + [2026-10-08 09:42:07] iteration 4533/ 6855 | consumed samples: 580224 | elapsed time per iteration (ms): 89710.8 | throughput per GPU (TFLOP/s/GPU): 41.4 | MFU 13.28% | FLOPs estimator: workload-aware | logical tokens/s: 187014.3 | valid tokens/s: 187014.3 | physical tokens/s: 187426.6 | loss tokens/s: 139292.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.591944E-01 | lm-global-supervised-tokens: 1.249601E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.266349E-02 | lm-token-buffer-mib: 5.812536E+01 | global_load_balancing_loss: 1.000374E+00 | loss scale: 1.0 | grad norm: 0.118 | params norm: 11353.007 | params rms: 0.061473 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 9:51:48.499588 | finish at 2026-10-10 19:33:56 + [2026-10-08 09:43:35] iteration 4534/ 6855 | consumed samples: 580352 | elapsed time per iteration (ms): 87748.9 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.66% | FLOPs estimator: workload-aware | logical tokens/s: 191195.6 | valid tokens/s: 191195.6 | physical tokens/s: 191650.9 | loss tokens/s: 139116.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.555818E-01 | lm-global-supervised-tokens: 1.220731E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.455667E-02 | lm-token-buffer-mib: 5.805407E+01 | global_load_balancing_loss: 1.001513E+00 | loss scale: 1.0 | grad norm: 0.086 | params norm: 11352.952 | params rms: 0.061473 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 8:34:25.102127 | finish at 2026-10-10 18:18:00 + [2026-10-08 09:44:59] iteration 4535/ 6855 | consumed samples: 580480 | elapsed time per iteration (ms): 83878.7 | throughput per GPU (TFLOP/s/GPU): 43.2 | MFU 13.84% | FLOPs estimator: workload-aware | logical tokens/s: 200017.4 | valid tokens/s: 200017.4 | physical tokens/s: 200509.6 | loss tokens/s: 144052.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.112034E-01 | lm-global-supervised-tokens: 1.208298E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.751108E-02 | lm-token-buffer-mib: 5.810840E+01 | global_load_balancing_loss: 1.001120E+00 | loss scale: 1.0 | grad norm: 0.063 | params norm: 11352.897 | params rms: 0.061472 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 6:03:18.681946 | finish at 2026-10-10 15:48:17 + [2026-10-08 09:46:24] iteration 4536/ 6855 | consumed samples: 580608 | elapsed time per iteration (ms): 85041.4 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.77% | FLOPs estimator: workload-aware | logical tokens/s: 197282.6 | valid tokens/s: 197282.6 | physical tokens/s: 197733.8 | loss tokens/s: 147202.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.668435E-01 | lm-global-supervised-tokens: 1.251834E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.103961E-02 | lm-token-buffer-mib: 5.815112E+01 | global_load_balancing_loss: 1.000098E+00 | loss scale: 1.0 | grad norm: 0.066 | params norm: 11352.841 | params rms: 0.061472 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 6:46:51.104912 | finish at 2026-10-10 16:33:15 + [2026-10-08 09:47:53] iteration 4537/ 6855 | consumed samples: 580736 | elapsed time per iteration (ms): 88790.0 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.50% | FLOPs estimator: workload-aware | logical tokens/s: 188953.3 | valid tokens/s: 188953.3 | physical tokens/s: 189382.2 | loss tokens/s: 139746.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.076458E-01 | lm-global-supervised-tokens: 1.240807E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.912676E-02 | lm-token-buffer-mib: 5.815152E+01 | global_load_balancing_loss: 1.000049E+00 | loss scale: 1.0 | grad norm: 0.058 | params norm: 11352.786 | params rms: 0.061472 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 9:10:15.235939 | finish at 2026-10-10 18:58:08 + [2026-10-08 09:49:20] iteration 4538/ 6855 | consumed samples: 580864 | elapsed time per iteration (ms): 87780.7 | throughput per GPU (TFLOP/s/GPU): 43.1 | MFU 13.82% | FLOPs estimator: workload-aware | logical tokens/s: 191126.2 | valid tokens/s: 191126.2 | physical tokens/s: 191438.5 | loss tokens/s: 130859.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.087390E-01 | lm-global-supervised-tokens: 1.148697E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.475011E-02 | lm-token-buffer-mib: 5.828272E+01 | global_load_balancing_loss: 1.000260E+00 | loss scale: 1.0 | grad norm: 0.048 | params norm: 11352.730 | params rms: 0.061471 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 8:29:47.947564 | finish at 2026-10-10 18:19:08 + [2026-10-08 09:50:50] iteration 4539/ 6855 | consumed samples: 580992 | elapsed time per iteration (ms): 89578.4 | throughput per GPU (TFLOP/s/GPU): 41.1 | MFU 13.17% | FLOPs estimator: workload-aware | logical tokens/s: 187290.1 | valid tokens/s: 187290.1 | physical tokens/s: 187852.6 | loss tokens/s: 142226.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.016085E-01 | lm-global-supervised-tokens: 1.274043E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.336673E-02 | lm-token-buffer-mib: 5.817011E+01 | global_load_balancing_loss: 9.987011E-01 | loss scale: 1.0 | grad norm: 0.096 | params norm: 11352.675 | params rms: 0.061471 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 9:37:43.578578 | finish at 2026-10-10 19:28:34 + [2026-10-08 09:52:36] iteration 4540/ 6855 | consumed samples: 581120 | elapsed time per iteration (ms): 105603.7 | throughput per GPU (TFLOP/s/GPU): 34.6 | MFU 11.10% | FLOPs estimator: workload-aware | logical tokens/s: 158869.4 | valid tokens/s: 158869.4 | physical tokens/s: 159314.9 | loss tokens/s: 119827.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.986720E-01 | lm-global-supervised-tokens: 1.265420E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.388620E-02 | lm-token-buffer-mib: 5.810541E+01 | global_load_balancing_loss: 1.000626E+00 | loss scale: 1.0 | grad norm: 0.067 | params norm: 11352.620 | params rms: 0.061471 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 19:54:32.598437 | finish at 2026-10-11 05:47:08 + [2026-10-08 09:54:00] iteration 4541/ 6855 | consumed samples: 581248 | elapsed time per iteration (ms): 84144.3 | throughput per GPU (TFLOP/s/GPU): 43.3 | MFU 13.87% | FLOPs estimator: workload-aware | logical tokens/s: 199385.0 | valid tokens/s: 199385.0 | physical tokens/s: 199781.0 | loss tokens/s: 156486.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.429502E-01 | lm-global-supervised-tokens: 1.316748E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.392929E-02 | lm-token-buffer-mib: 5.803246E+01 | global_load_balancing_loss: 1.000667E+00 | loss scale: 1.0 | grad norm: 0.063 | params norm: 11352.563 | params rms: 0.061470 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 6:05:09.942162 | finish at 2026-10-10 15:59:10 + [2026-10-08 09:55:27] iteration 4542/ 6855 | consumed samples: 581376 | elapsed time per iteration (ms): 87333.5 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.55% | FLOPs estimator: workload-aware | logical tokens/s: 192104.7 | valid tokens/s: 192104.7 | physical tokens/s: 192572.6 | loss tokens/s: 140487.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 5.987782E-01 | lm-global-supervised-tokens: 1.226930E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.134712E-02 | lm-token-buffer-mib: 5.817072E+01 | global_load_balancing_loss: 1.000826E+00 | loss scale: 1.0 | grad norm: 0.068 | params norm: 11352.507 | params rms: 0.061470 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 8:06:42.330693 | finish at 2026-10-10 18:02:09 + [2026-10-08 09:56:55] iteration 4543/ 6855 | consumed samples: 581504 | elapsed time per iteration (ms): 88068.2 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.54% | FLOPs estimator: workload-aware | logical tokens/s: 190502.1 | valid tokens/s: 190502.1 | physical tokens/s: 191034.1 | loss tokens/s: 143032.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.725259E-01 | lm-global-supervised-tokens: 1.259660E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.337175E-02 | lm-token-buffer-mib: 5.831204E+01 | global_load_balancing_loss: 1.000657E+00 | loss scale: 1.0 | grad norm: 0.050 | params norm: 11352.450 | params rms: 0.061470 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 8:33:33.672594 | finish at 2026-10-10 18:30:29 + [2026-10-08 09:58:22] iteration 4544/ 6855 | consumed samples: 581632 | elapsed time per iteration (ms): 87194.5 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.50% | FLOPs estimator: workload-aware | logical tokens/s: 192411.5 | valid tokens/s: 192411.5 | physical tokens/s: 193015.2 | loss tokens/s: 141232.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.625755E-01 | lm-global-supervised-tokens: 1.231472E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.588240E-02 | lm-token-buffer-mib: 5.816798E+01 | global_load_balancing_loss: 1.000299E+00 | loss scale: 1.0 | grad norm: 0.067 | params norm: 11352.392 | params rms: 0.061470 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 7:58:26.423862 | finish at 2026-10-10 17:56:49 + [2026-10-08 09:59:53] iteration 4545/ 6855 | consumed samples: 581760 | elapsed time per iteration (ms): 91040.4 | throughput per GPU (TFLOP/s/GPU): 40.6 | MFU 13.00% | FLOPs estimator: workload-aware | logical tokens/s: 184282.7 | valid tokens/s: 184282.7 | physical tokens/s: 184834.0 | loss tokens/s: 131763.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.157759E-01 | lm-global-supervised-tokens: 1.199579E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.450260E-02 | lm-token-buffer-mib: 5.818844E+01 | global_load_balancing_loss: 1.001159E+00 | loss scale: 1.0 | grad norm: 0.063 | params norm: 11352.332 | params rms: 0.061469 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 10:25:03.253570 | finish at 2026-10-10 20:24:57 + [2026-10-08 10:01:21] iteration 4546/ 6855 | consumed samples: 581888 | elapsed time per iteration (ms): 88099.7 | throughput per GPU (TFLOP/s/GPU): 41.5 | MFU 13.31% | FLOPs estimator: workload-aware | logical tokens/s: 190434.4 | valid tokens/s: 190434.3 | physical tokens/s: 190936.9 | loss tokens/s: 143725.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.560906E-01 | lm-global-supervised-tokens: 1.266219E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.158148E-02 | lm-token-buffer-mib: 5.818356E+01 | global_load_balancing_loss: 1.000449E+00 | loss scale: 1.0 | grad norm: 0.197 | params norm: 11352.273 | params rms: 0.061469 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 8:30:22.248528 | finish at 2026-10-10 18:31:44 + [2026-10-08 10:02:46] iteration 4547/ 6855 | consumed samples: 582016 | elapsed time per iteration (ms): 84389.6 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.62% | FLOPs estimator: workload-aware | logical tokens/s: 198805.7 | valid tokens/s: 198805.7 | physical tokens/s: 199254.1 | loss tokens/s: 152436.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.012075E-01 | lm-global-supervised-tokens: 1.286407E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.550408E-02 | lm-token-buffer-mib: 5.808736E+01 | global_load_balancing_loss: 9.990993E-01 | loss scale: 1.0 | grad norm: 0.165 | params norm: 11352.215 | params rms: 0.061469 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 6:06:11.253016 | finish at 2026-10-10 16:08:57 + [2026-10-08 10:04:14] iteration 4548/ 6855 | consumed samples: 582144 | elapsed time per iteration (ms): 88470.0 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.43% | FLOPs estimator: workload-aware | logical tokens/s: 189637.0 | valid tokens/s: 189637.0 | physical tokens/s: 190043.0 | loss tokens/s: 149420.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.576653E-01 | lm-global-supervised-tokens: 1.321924E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.365030E-02 | lm-token-buffer-mib: 5.807369E+01 | global_load_balancing_loss: 9.978569E-01 | loss scale: 1.0 | grad norm: 0.085 | params norm: 11352.154 | params rms: 0.061468 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 8:41:40.321968 | finish at 2026-10-10 18:45:55 + [2026-10-08 10:05:43] iteration 4549/ 6855 | consumed samples: 582272 | elapsed time per iteration (ms): 88364.5 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.41% | FLOPs estimator: workload-aware | logical tokens/s: 189863.3 | valid tokens/s: 189863.3 | physical tokens/s: 190251.9 | loss tokens/s: 139941.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.567335E-01 | lm-global-supervised-tokens: 1.236588E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.334683E-02 | lm-token-buffer-mib: 5.816279E+01 | global_load_balancing_loss: 1.000257E+00 | loss scale: 1.0 | grad norm: 0.186 | params norm: 11352.093 | params rms: 0.061468 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 8:36:08.646514 | finish at 2026-10-10 18:41:51 + [2026-10-08 10:07:30] iteration 4550/ 6855 | consumed samples: 582400 | elapsed time per iteration (ms): 107526.0 | throughput per GPU (TFLOP/s/GPU): 34.2 | MFU 10.95% | FLOPs estimator: workload-aware | logical tokens/s: 156029.0 | valid tokens/s: 156029.0 | physical tokens/s: 156399.9 | loss tokens/s: 115887.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.060383E-01 | lm-global-supervised-tokens: 1.246087E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.143013E-02 | lm-token-buffer-mib: 5.818418E+01 | global_load_balancing_loss: 1.001043E+00 | loss scale: 1.0 | grad norm: 0.038 | params norm: 11352.031 | params rms: 0.061468 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 20:50:47.464125 | finish at 2026-10-11 06:58:18 + [2026-10-08 10:08:55] iteration 4551/ 6855 | consumed samples: 582528 | elapsed time per iteration (ms): 84448.6 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.77% | FLOPs estimator: workload-aware | logical tokens/s: 198667.4 | valid tokens/s: 198667.3 | physical tokens/s: 199122.2 | loss tokens/s: 147552.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.672500E-01 | lm-global-supervised-tokens: 1.246062E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.615214E-02 | lm-token-buffer-mib: 5.804617E+01 | global_load_balancing_loss: 1.001047E+00 | loss scale: 1.0 | grad norm: 0.225 | params norm: 11351.969 | params rms: 0.061467 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 6:02:49.486633 | finish at 2026-10-10 16:11:44 + [2026-10-08 10:10:23] iteration 4552/ 6855 | consumed samples: 582656 | elapsed time per iteration (ms): 87897.0 | throughput per GPU (TFLOP/s/GPU): 41.4 | MFU 13.27% | FLOPs estimator: workload-aware | logical tokens/s: 190873.5 | valid tokens/s: 190873.5 | physical tokens/s: 191507.2 | loss tokens/s: 143594.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.730290E-01 | lm-global-supervised-tokens: 1.262157E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.128248E-02 | lm-token-buffer-mib: 5.809422E+01 | global_load_balancing_loss: 1.000583E+00 | loss scale: 1.0 | grad norm: 0.128 | params norm: 11351.906 | params rms: 0.061467 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 8:13:46.849374 | finish at 2026-10-10 18:24:09 + [2026-10-08 10:11:51] iteration 4553/ 6855 | consumed samples: 582784 | elapsed time per iteration (ms): 88811.7 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.55% | FLOPs estimator: workload-aware | logical tokens/s: 188907.4 | valid tokens/s: 188907.4 | physical tokens/s: 189398.9 | loss tokens/s: 135097.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.456569E-01 | lm-global-supervised-tokens: 1.199827E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.164121E-02 | lm-token-buffer-mib: 5.811773E+01 | global_load_balancing_loss: 1.000973E+00 | loss scale: 1.0 | grad norm: 0.136 | params norm: 11351.844 | params rms: 0.061467 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 8:47:24.462294 | finish at 2026-10-10 18:59:16 + [2026-10-08 10:13:18] iteration 4554/ 6855 | consumed samples: 582912 | elapsed time per iteration (ms): 86312.7 | throughput per GPU (TFLOP/s/GPU): 42.8 | MFU 13.70% | FLOPs estimator: workload-aware | logical tokens/s: 194376.6 | valid tokens/s: 194376.6 | physical tokens/s: 194941.5 | loss tokens/s: 141774.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.434840E-01 | lm-global-supervised-tokens: 1.223697E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.336302E-02 | lm-token-buffer-mib: 5.823520E+01 | global_load_balancing_loss: 1.000069E+00 | loss scale: 1.0 | grad norm: 0.078 | params norm: 11351.783 | params rms: 0.061466 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 7:10:05.432806 | finish at 2026-10-10 17:23:23 + [2026-10-08 10:14:47] iteration 4555/ 6855 | consumed samples: 583040 | elapsed time per iteration (ms): 89504.8 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.40% | FLOPs estimator: workload-aware | logical tokens/s: 187444.0 | valid tokens/s: 187444.0 | physical tokens/s: 187815.3 | loss tokens/s: 137843.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.942095E-01 | lm-global-supervised-tokens: 1.233762E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.173607E-02 | lm-token-buffer-mib: 5.819656E+01 | global_load_balancing_loss: 1.000266E+00 | loss scale: 1.0 | grad norm: 0.120 | params norm: 11351.722 | params rms: 0.061466 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 9:11:01.006737 | finish at 2026-10-10 19:25:48 + [2026-10-08 10:16:15] iteration 4556/ 6855 | consumed samples: 583168 | elapsed time per iteration (ms): 87907.5 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.61% | FLOPs estimator: workload-aware | logical tokens/s: 190850.1 | valid tokens/s: 190850.1 | physical tokens/s: 191308.1 | loss tokens/s: 141793.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.872162E-01 | lm-global-supervised-tokens: 1.246472E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.143087E-02 | lm-token-buffer-mib: 5.812384E+01 | global_load_balancing_loss: 9.991390E-01 | loss scale: 1.0 | grad norm: 0.060 | params norm: 11351.660 | params rms: 0.061466 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 8:08:19.247740 | finish at 2026-10-10 18:24:34 + [2026-10-08 10:17:39] iteration 4557/ 6855 | consumed samples: 583296 | elapsed time per iteration (ms): 84179.4 | throughput per GPU (TFLOP/s/GPU): 43.3 | MFU 13.87% | FLOPs estimator: workload-aware | logical tokens/s: 199303.1 | valid tokens/s: 199303.1 | physical tokens/s: 199797.1 | loss tokens/s: 150108.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.744592E-01 | lm-global-supervised-tokens: 1.263604E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.352285E-02 | lm-token-buffer-mib: 5.810112E+01 | global_load_balancing_loss: 9.996881E-01 | loss scale: 1.0 | grad norm: 0.054 | params norm: 11351.599 | params rms: 0.061465 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 5:44:04.269343 | finish at 2026-10-10 16:01:44 + [2026-10-08 10:19:05] iteration 4558/ 6855 | consumed samples: 583424 | elapsed time per iteration (ms): 85531.5 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.65% | FLOPs estimator: workload-aware | logical tokens/s: 196152.5 | valid tokens/s: 196152.5 | physical tokens/s: 196633.9 | loss tokens/s: 146584.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.252061E-01 | lm-global-supervised-tokens: 1.253762E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.412915E-02 | lm-token-buffer-mib: 5.811326E+01 | global_load_balancing_loss: 9.993948E-01 | loss scale: 1.0 | grad norm: 0.104 | params norm: 11351.538 | params rms: 0.061465 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 6:34:25.796039 | finish at 2026-10-10 16:53:31 + [2026-10-08 10:20:34] iteration 4559/ 6855 | consumed samples: 583552 | elapsed time per iteration (ms): 89014.7 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.65% | FLOPs estimator: workload-aware | logical tokens/s: 188477.0 | valid tokens/s: 188477.0 | physical tokens/s: 188780.8 | loss tokens/s: 138728.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.155667E-01 | lm-global-supervised-tokens: 1.234884E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.271212E-02 | lm-token-buffer-mib: 5.807716E+01 | global_load_balancing_loss: 1.000276E+00 | loss scale: 1.0 | grad norm: 0.040 | params norm: 11351.476 | params rms: 0.061465 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 8:46:17.646498 | finish at 2026-10-10 19:06:51 + [2026-10-08 10:22:25] iteration 4560/ 6855 | consumed samples: 583680 | elapsed time per iteration (ms): 111344.4 | throughput per GPU (TFLOP/s/GPU): 33.3 | MFU 10.66% | FLOPs estimator: workload-aware | logical tokens/s: 150678.1 | valid tokens/s: 150678.1 | physical tokens/s: 150972.4 | loss tokens/s: 113656.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.814115E-01 | lm-global-supervised-tokens: 1.265503E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.053064E-02 | lm-token-buffer-mib: 5.810802E+01 | global_load_balancing_loss: 1.001107E+00 | loss scale: 1.0 | grad norm: 0.057 | params norm: 11351.415 | params rms: 0.061464 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 22:58:55.425196 | finish at 2026-10-11 09:21:21 +times across ranks (ms): + save-checkpoint: + rank 0: 25840.92 + rank 1: 25840.93 + rank 2: 25840.99 + rank 3: 25840.76 + rank 4: 25840.85 + rank 5: 25840.75 + rank 6: 25840.74 + rank 7: 25840.76 + rank 8: 25841.12 + rank 9: 25841.04 + rank 10: 25841.08 + rank 11: 25841.07 + rank 12: 25893.62 + rank 13: 25841.06 + rank 14: 25841.08 + rank 15: 25841.06 + rank 16: 25841.11 + rank 17: 25841.10 + rank 18: 25841.09 + rank 19: 25841.10 + rank 20: 25841.07 + rank 21: 25841.12 + rank 22: 25841.07 + rank 23: 25841.06 + rank 24: 25841.41 + rank 25: 25841.23 + rank 26: 25841.35 + rank 27: 25841.34 + rank 28: 25841.36 + rank 29: 25841.38 + rank 30: 25841.91 + rank 31: 25841.24 + rank 32: 25841.11 + rank 33: 25841.07 + rank 34: 25841.12 + rank 35: 25841.01 + rank 36: 25841.05 + rank 37: 25841.01 + rank 38: 25841.06 + rank 39: 25841.06 + rank 40: 25839.80 + rank 41: 25839.77 + rank 42: 25839.78 + rank 43: 25839.81 + rank 44: 25839.77 + rank 45: 25839.79 + rank 46: 25839.79 + rank 47: 25839.62 + rank 48: 25840.87 + rank 49: 25840.88 + rank 50: 25840.97 + rank 51: 25840.82 + rank 52: 25840.81 + rank 53: 25840.81 + rank 54: 25840.85 + rank 55: 25841.01 + rank 56: 25841.19 + rank 57: 25841.18 + rank 58: 25841.16 + rank 59: 25841.15 + rank 60: 25841.15 + rank 61: 25841.19 + rank 62: 25841.17 + rank 63: 25841.18 + [2026-10-08 10:24:44] iteration 4561/ 6855 | consumed samples: 583808 | elapsed time per iteration (ms): 112930.7 | throughput per GPU (TFLOP/s/GPU): 33.2 | MFU 10.63% | FLOPs estimator: workload-aware | logical tokens/s: 148562.0 | valid tokens/s: 148562.0 | physical tokens/s: 148939.6 | loss tokens/s: 110770.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.068712E-01 | lm-global-supervised-tokens: 1.250938E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 4.244767E-01 | lm-token-buffer-mib: 5.811388E+01 | global_load_balancing_loss: 1.000474E+00 | loss scale: 1.0 | grad norm: 0.077 | params norm: 11351.356 | params rms: 0.061464 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 23:57:42.981645 | finish at 2026-10-11 10:22:27 + [2026-10-08 10:26:12] iteration 4562/ 6855 | consumed samples: 583936 | elapsed time per iteration (ms): 88373.1 | throughput per GPU (TFLOP/s/GPU): 40.6 | MFU 13.01% | FLOPs estimator: workload-aware | logical tokens/s: 189844.7 | valid tokens/s: 189844.7 | physical tokens/s: 190419.6 | loss tokens/s: 145226.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.633542E-01 | lm-global-supervised-tokens: 1.283412E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.322951E-02 | lm-token-buffer-mib: 5.820672E+01 | global_load_balancing_loss: 1.001033E+00 | loss scale: 1.0 | grad norm: 0.050 | params norm: 11351.296 | params rms: 0.061464 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 8:17:19.445140 | finish at 2026-10-10 18:43:32 + [2026-10-08 10:27:38] iteration 4563/ 6855 | consumed samples: 584064 | elapsed time per iteration (ms): 85424.8 | throughput per GPU (TFLOP/s/GPU): 43.3 | MFU 13.87% | FLOPs estimator: workload-aware | logical tokens/s: 196396.4 | valid tokens/s: 196396.3 | physical tokens/s: 196812.6 | loss tokens/s: 144290.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.732826E-01 | lm-global-supervised-tokens: 1.232598E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.394839E-02 | lm-token-buffer-mib: 5.829346E+01 | global_load_balancing_loss: 1.000155E+00 | loss scale: 1.0 | grad norm: 0.045 | params norm: 11351.237 | params rms: 0.061463 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 6:23:13.735951 | finish at 2026-10-10 16:50:52 + [2026-10-08 10:29:05] iteration 4564/ 6855 | consumed samples: 584192 | elapsed time per iteration (ms): 87571.3 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.57% | FLOPs estimator: workload-aware | logical tokens/s: 191583.4 | valid tokens/s: 191583.4 | physical tokens/s: 192130.4 | loss tokens/s: 144929.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.047877E-01 | lm-global-supervised-tokens: 1.269166E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.393971E-02 | lm-token-buffer-mib: 5.813133E+01 | global_load_balancing_loss: 9.993884E-01 | loss scale: 1.0 | grad norm: 0.108 | params norm: 11351.178 | params rms: 0.061463 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 7:43:45.801394 | finish at 2026-10-10 18:12:51 + [2026-10-08 10:30:33] iteration 4565/ 6855 | consumed samples: 584320 | elapsed time per iteration (ms): 87624.7 | throughput per GPU (TFLOP/s/GPU): 41.7 | MFU 13.38% | FLOPs estimator: workload-aware | logical tokens/s: 191466.0 | valid tokens/s: 191466.0 | physical tokens/s: 192133.1 | loss tokens/s: 140440.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.613423E-01 | lm-global-supervised-tokens: 1.230604E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.536817E-02 | lm-token-buffer-mib: 5.835325E+01 | global_load_balancing_loss: 1.000351E+00 | loss scale: 1.0 | grad norm: 0.060 | params norm: 11351.119 | params rms: 0.061463 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 7:44:20.497642 | finish at 2026-10-10 18:14:54 + [2026-10-08 10:31:59] iteration 4566/ 6855 | consumed samples: 584448 | elapsed time per iteration (ms): 85657.4 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.65% | FLOPs estimator: workload-aware | logical tokens/s: 195864.1 | valid tokens/s: 195864.1 | physical tokens/s: 196284.0 | loss tokens/s: 153906.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.800008E-01 | lm-global-supervised-tokens: 1.318323E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.479415E-02 | lm-token-buffer-mib: 5.811056E+01 | global_load_balancing_loss: 9.988533E-01 | loss scale: 1.0 | grad norm: 0.043 | params norm: 11351.060 | params rms: 0.061462 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 6:27:49.863121 | finish at 2026-10-10 16:59:49 + [2026-10-08 10:34:14] iteration 4567/ 6855 | consumed samples: 584576 | elapsed time per iteration (ms): 135158.8 | throughput per GPU (TFLOP/s/GPU): 27.3 | MFU 8.74% | FLOPs estimator: workload-aware | logical tokens/s: 123396.4 | valid tokens/s: 123396.4 | physical tokens/s: 123769.2 | loss tokens/s: 91812.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.501969E-01 | lm-global-supervised-tokens: 1.240921E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.153301E-02 | lm-token-buffer-mib: 5.584840E+01 | global_load_balancing_loss: 1.000549E+00 | loss scale: 1.0 | grad norm: 0.050 | params norm: 11351.002 | params rms: 0.061462 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 13:54:03.438332 | finish at 2026-10-12 00:28:17 + [2026-10-08 10:35:43] iteration 4568/ 6855 | consumed samples: 584704 | elapsed time per iteration (ms): 89575.9 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.49% | FLOPs estimator: workload-aware | logical tokens/s: 187295.9 | valid tokens/s: 187295.9 | physical tokens/s: 187588.5 | loss tokens/s: 140021.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.787902E-01 | lm-global-supervised-tokens: 1.254258E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 3.131802E-02 | lm-token-buffer-mib: 5.806802E+01 | global_load_balancing_loss: 1.001492E+00 | loss scale: 1.0 | grad norm: 0.041 | params norm: 11350.946 | params rms: 0.061462 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 8:54:20.092747 | finish at 2026-10-10 19:30:04 + [2026-10-08 10:37:14] iteration 4569/ 6855 | consumed samples: 584832 | elapsed time per iteration (ms): 90328.1 | throughput per GPU (TFLOP/s/GPU): 41.2 | MFU 13.22% | FLOPs estimator: workload-aware | logical tokens/s: 185735.9 | valid tokens/s: 185735.9 | physical tokens/s: 186138.6 | loss tokens/s: 138445.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.221814E-01 | lm-global-supervised-tokens: 1.250551E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.386248E-02 | lm-token-buffer-mib: 5.805727E+01 | global_load_balancing_loss: 9.999279E-01 | loss scale: 1.0 | grad norm: 0.044 | params norm: 11350.891 | params rms: 0.061461 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 9:21:29.947683 | finish at 2026-10-10 19:58:44 + [2026-10-08 10:39:03] iteration 4570/ 6855 | consumed samples: 584960 | elapsed time per iteration (ms): 108802.3 | throughput per GPU (TFLOP/s/GPU): 33.8 | MFU 10.84% | FLOPs estimator: workload-aware | logical tokens/s: 154199.0 | valid tokens/s: 154199.0 | physical tokens/s: 154585.6 | loss tokens/s: 106547.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.644941E-01 | lm-global-supervised-tokens: 1.159265E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.632872E-02 | lm-token-buffer-mib: 5.815249E+01 | global_load_balancing_loss: 1.000300E+00 | loss scale: 1.0 | grad norm: 0.049 | params norm: 11350.835 | params rms: 0.061461 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 21:03:33.163377 | finish at 2026-10-11 07:42:36 + [2026-10-08 10:40:29] iteration 4571/ 6855 | consumed samples: 585088 | elapsed time per iteration (ms): 86208.0 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.45% | FLOPs estimator: workload-aware | logical tokens/s: 194612.1 | valid tokens/s: 194612.1 | physical tokens/s: 195110.0 | loss tokens/s: 147891.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.672689E-01 | lm-global-supervised-tokens: 1.274939E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.793128E-02 | lm-token-buffer-mib: 5.814568E+01 | global_load_balancing_loss: 9.987755E-01 | loss scale: 1.0 | grad norm: 0.075 | params norm: 11350.780 | params rms: 0.061461 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 6:41:39.105091 | finish at 2026-10-10 17:22:08 + [2026-10-08 10:41:56] iteration 4572/ 6855 | consumed samples: 585216 | elapsed time per iteration (ms): 86831.4 | throughput per GPU (TFLOP/s/GPU): 43.1 | MFU 13.80% | FLOPs estimator: workload-aware | logical tokens/s: 193215.6 | valid tokens/s: 193215.6 | physical tokens/s: 193572.0 | loss tokens/s: 142328.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.423990E-01 | lm-global-supervised-tokens: 1.235854E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.168774E-02 | lm-token-buffer-mib: 5.805470E+01 | global_load_balancing_loss: 9.992131E-01 | loss scale: 1.0 | grad norm: 0.035 | params norm: 11350.724 | params rms: 0.061460 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 7:03:56.103974 | finish at 2026-10-10 17:45:52 + [2026-10-08 10:43:25] iteration 4573/ 6855 | consumed samples: 585344 | elapsed time per iteration (ms): 88952.6 | throughput per GPU (TFLOP/s/GPU): 41.0 | MFU 13.14% | FLOPs estimator: workload-aware | logical tokens/s: 188608.2 | valid tokens/s: 188608.2 | physical tokens/s: 189382.3 | loss tokens/s: 141165.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.339026E-01 | lm-global-supervised-tokens: 1.255701E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.144054E-02 | lm-token-buffer-mib: 5.815392E+01 | global_load_balancing_loss: 1.001095E+00 | loss scale: 1.0 | grad norm: 0.092 | params norm: 11350.668 | params rms: 0.061460 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 8:23:09.788047 | finish at 2026-10-10 19:06:34 + [2026-10-08 10:44:51] iteration 4574/ 6855 | consumed samples: 585472 | elapsed time per iteration (ms): 86615.9 | throughput per GPU (TFLOP/s/GPU): 43.1 | MFU 13.82% | FLOPs estimator: workload-aware | logical tokens/s: 193696.3 | valid tokens/s: 193696.3 | physical tokens/s: 194211.1 | loss tokens/s: 147676.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.897719E-01 | lm-global-supervised-tokens: 1.279118E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.361354E-02 | lm-token-buffer-mib: 5.813664E+01 | global_load_balancing_loss: 1.000268E+00 | loss scale: 1.0 | grad norm: 0.060 | params norm: 11350.612 | params rms: 0.061460 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 6:52:50.921286 | finish at 2026-10-10 17:37:42 + [2026-10-08 10:46:20] iteration 4575/ 6855 | consumed samples: 585600 | elapsed time per iteration (ms): 88551.7 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.64% | FLOPs estimator: workload-aware | logical tokens/s: 189462.5 | valid tokens/s: 189462.5 | physical tokens/s: 189793.9 | loss tokens/s: 142632.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.882604E-01 | lm-global-supervised-tokens: 1.263037E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.519388E-02 | lm-token-buffer-mib: 5.812315E+01 | global_load_balancing_loss: 1.001556E+00 | loss scale: 1.0 | grad norm: 0.045 | params norm: 11350.556 | params rms: 0.061460 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 8:04:57.765369 | finish at 2026-10-10 18:51:18 + [2026-10-08 10:47:49] iteration 4576/ 6855 | consumed samples: 585728 | elapsed time per iteration (ms): 89664.5 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.60% | FLOPs estimator: workload-aware | logical tokens/s: 187110.3 | valid tokens/s: 187110.3 | physical tokens/s: 187520.7 | loss tokens/s: 139407.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.214536E-01 | lm-global-supervised-tokens: 1.249994E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.125254E-02 | lm-token-buffer-mib: 5.812018E+01 | global_load_balancing_loss: 9.997040E-01 | loss scale: 1.0 | grad norm: 0.109 | params norm: 11350.500 | params rms: 0.061459 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 8:45:45.440051 | finish at 2026-10-10 19:33:35 + [2026-10-08 10:49:12] iteration 4577/ 6855 | consumed samples: 585856 | elapsed time per iteration (ms): 82932.9 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.77% | FLOPs estimator: workload-aware | logical tokens/s: 202298.7 | valid tokens/s: 202298.7 | physical tokens/s: 202821.6 | loss tokens/s: 152786.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.683501E-01 | lm-global-supervised-tokens: 1.267105E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.425541E-02 | lm-token-buffer-mib: 5.809379E+01 | global_load_balancing_loss: 1.000439E+00 | loss scale: 1.0 | grad norm: 0.071 | params norm: 11350.444 | params rms: 0.061459 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 4:28:41.082003 | finish at 2026-10-10 15:17:53 + [2026-10-08 10:50:40] iteration 4578/ 6855 | consumed samples: 585984 | elapsed time per iteration (ms): 87406.0 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.44% | FLOPs estimator: workload-aware | logical tokens/s: 191945.1 | valid tokens/s: 191945.1 | physical tokens/s: 192390.5 | loss tokens/s: 145697.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.961467E-01 | lm-global-supervised-tokens: 1.273478E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.688148E-02 | lm-token-buffer-mib: 5.804691E+01 | global_load_balancing_loss: 1.000013E+00 | loss scale: 1.0 | grad norm: 0.106 | params norm: 11350.388 | params rms: 0.061459 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 7:17:03.380338 | finish at 2026-10-10 18:07:43 + [2026-10-08 10:52:13] iteration 4579/ 6855 | consumed samples: 586112 | elapsed time per iteration (ms): 93649.1 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.49% | FLOPs estimator: workload-aware | logical tokens/s: 179149.7 | valid tokens/s: 179149.7 | physical tokens/s: 179420.2 | loss tokens/s: 128263.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.862591E-01 | lm-global-supervised-tokens: 1.201176E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.683488E-02 | lm-token-buffer-mib: 5.813170E+01 | global_load_balancing_loss: 9.996586E-01 | loss scale: 1.0 | grad norm: 0.110 | params norm: 11350.332 | params rms: 0.061458 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 11:12:25.432602 | finish at 2026-10-10 22:04:39 + [2026-10-08 10:53:59] iteration 4580/ 6855 | consumed samples: 586240 | elapsed time per iteration (ms): 105852.9 | throughput per GPU (TFLOP/s/GPU): 34.2 | MFU 10.95% | FLOPs estimator: workload-aware | logical tokens/s: 158495.4 | valid tokens/s: 158495.4 | physical tokens/s: 158893.6 | loss tokens/s: 114410.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.038140E-01 | lm-global-supervised-tokens: 1.211072E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.372829E-02 | lm-token-buffer-mib: 5.820690E+01 | global_load_balancing_loss: 1.000340E+00 | loss scale: 1.0 | grad norm: 0.087 | params norm: 11350.275 | params rms: 0.061458 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 18:53:35.312308 | finish at 2026-10-11 05:47:35 + [2026-10-08 10:55:29] iteration 4581/ 6855 | consumed samples: 586368 | elapsed time per iteration (ms): 90155.8 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.53% | FLOPs estimator: workload-aware | logical tokens/s: 186090.7 | valid tokens/s: 186090.7 | physical tokens/s: 186383.0 | loss tokens/s: 131398.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.091145E-01 | lm-global-supervised-tokens: 1.184633E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.583189E-02 | lm-token-buffer-mib: 5.807396E+01 | global_load_balancing_loss: 1.001295E+00 | loss scale: 1.0 | grad norm: 0.114 | params norm: 11350.218 | params rms: 0.061458 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 8:56:54.277509 | finish at 2026-10-10 19:52:24 + [2026-10-08 10:56:53] iteration 4582/ 6855 | consumed samples: 586496 | elapsed time per iteration (ms): 83740.4 | throughput per GPU (TFLOP/s/GPU): 42.8 | MFU 13.72% | FLOPs estimator: workload-aware | logical tokens/s: 200347.2 | valid tokens/s: 200347.2 | physical tokens/s: 200868.6 | loss tokens/s: 150936.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.435567E-01 | lm-global-supervised-tokens: 1.263945E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.151075E-02 | lm-token-buffer-mib: 5.810538E+01 | global_load_balancing_loss: 1.000829E+00 | loss scale: 1.0 | grad norm: 0.134 | params norm: 11350.161 | params rms: 0.061457 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 4:52:21.961888 | finish at 2026-10-10 15:49:15 + [2026-10-08 10:58:22] iteration 4583/ 6855 | consumed samples: 586624 | elapsed time per iteration (ms): 89307.4 | throughput per GPU (TFLOP/s/GPU): 41.2 | MFU 13.19% | FLOPs estimator: workload-aware | logical tokens/s: 187858.8 | valid tokens/s: 187858.8 | physical tokens/s: 188334.9 | loss tokens/s: 138647.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.221154E-01 | lm-global-supervised-tokens: 1.238224E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.152151E-02 | lm-token-buffer-mib: 5.824540E+01 | global_load_balancing_loss: 1.000165E+00 | loss scale: 1.0 | grad norm: 0.274 | params norm: 11350.105 | params rms: 0.061457 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 8:21:46.398148 | finish at 2026-10-10 19:20:09 + [2026-10-08 10:59:53] iteration 4584/ 6855 | consumed samples: 586752 | elapsed time per iteration (ms): 90628.4 | throughput per GPU (TFLOP/s/GPU): 41.4 | MFU 13.28% | FLOPs estimator: workload-aware | logical tokens/s: 185120.3 | valid tokens/s: 185120.3 | physical tokens/s: 185584.8 | loss tokens/s: 136437.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.804605E-01 | lm-global-supervised-tokens: 1.236514E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.175132E-02 | lm-token-buffer-mib: 5.828207E+01 | global_load_balancing_loss: 9.995620E-01 | loss scale: 1.0 | grad norm: 0.068 | params norm: 11350.050 | params rms: 0.061457 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 9:10:17.189186 | finish at 2026-10-10 20:10:10 + [2026-10-08 11:01:24] iteration 4585/ 6855 | consumed samples: 586880 | elapsed time per iteration (ms): 91112.1 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.43% | FLOPs estimator: workload-aware | logical tokens/s: 184137.4 | valid tokens/s: 184137.4 | physical tokens/s: 184547.6 | loss tokens/s: 128894.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.988999E-01 | lm-global-supervised-tokens: 1.174383E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.132607E-02 | lm-token-buffer-mib: 5.813022E+01 | global_load_balancing_loss: 9.996741E-01 | loss scale: 1.0 | grad norm: 0.246 | params norm: 11349.994 | params rms: 0.061457 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 9:27:04.377983 | finish at 2026-10-10 20:28:29 + [2026-10-08 11:02:53] iteration 4586/ 6855 | consumed samples: 587008 | elapsed time per iteration (ms): 88387.0 | throughput per GPU (TFLOP/s/GPU): 43.1 | MFU 13.80% | FLOPs estimator: workload-aware | logical tokens/s: 189815.0 | valid tokens/s: 189815.0 | physical tokens/s: 190151.9 | loss tokens/s: 128934.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.114600E-01 | lm-global-supervised-tokens: 1.139610E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.572766E-02 | lm-token-buffer-mib: 5.810301E+01 | global_load_balancing_loss: 1.000797E+00 | loss scale: 1.0 | grad norm: 0.135 | params norm: 11349.938 | params rms: 0.061456 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 7:42:30.202188 | finish at 2026-10-10 18:45:23 + [2026-10-08 11:04:19] iteration 4587/ 6855 | consumed samples: 587136 | elapsed time per iteration (ms): 86609.6 | throughput per GPU (TFLOP/s/GPU): 41.7 | MFU 13.36% | FLOPs estimator: workload-aware | logical tokens/s: 193710.1 | valid tokens/s: 193710.1 | physical tokens/s: 194310.5 | loss tokens/s: 143029.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.301502E-01 | lm-global-supervised-tokens: 1.238773E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.249834E-02 | lm-token-buffer-mib: 5.805807E+01 | global_load_balancing_loss: 1.000721E+00 | loss scale: 1.0 | grad norm: 0.289 | params norm: 11349.882 | params rms: 0.061456 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 6:33:50.506983 | finish at 2026-10-10 17:38:10 + [2026-10-08 11:05:48] iteration 4588/ 6855 | consumed samples: 587264 | elapsed time per iteration (ms): 88767.2 | throughput per GPU (TFLOP/s/GPU): 41.1 | MFU 13.18% | FLOPs estimator: workload-aware | logical tokens/s: 189002.1 | valid tokens/s: 189002.1 | physical tokens/s: 189493.8 | loss tokens/s: 140569.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.184092E-01 | lm-global-supervised-tokens: 1.247794E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.449697E-02 | lm-token-buffer-mib: 5.821645E+01 | global_load_balancing_loss: 9.997914E-01 | loss scale: 1.0 | grad norm: 0.309 | params norm: 11349.827 | params rms: 0.061456 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 7:53:55.180768 | finish at 2026-10-10 18:59:43 + [2026-10-08 11:07:17] iteration 4589/ 6855 | consumed samples: 587392 | elapsed time per iteration (ms): 88822.8 | throughput per GPU (TFLOP/s/GPU): 41.7 | MFU 13.35% | FLOPs estimator: workload-aware | logical tokens/s: 188883.7 | valid tokens/s: 188883.7 | physical tokens/s: 189178.0 | loss tokens/s: 140014.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.033604E-01 | lm-global-supervised-tokens: 1.243643E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.675431E-02 | lm-token-buffer-mib: 5.807909E+01 | global_load_balancing_loss: 9.991192E-01 | loss scale: 1.0 | grad norm: 0.129 | params norm: 11349.774 | params rms: 0.061455 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 7:54:32.352788 | finish at 2026-10-10 19:01:49 + [2026-10-08 11:09:04] iteration 4590/ 6855 | consumed samples: 587520 | elapsed time per iteration (ms): 107158.3 | throughput per GPU (TFLOP/s/GPU): 33.5 | MFU 10.72% | FLOPs estimator: workload-aware | logical tokens/s: 156564.5 | valid tokens/s: 156564.5 | physical tokens/s: 156960.4 | loss tokens/s: 113131.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.512446E-01 | lm-global-supervised-tokens: 1.212293E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.302673E-02 | lm-token-buffer-mib: 5.817297E+01 | global_load_balancing_loss: 1.000210E+00 | loss scale: 1.0 | grad norm: 0.330 | params norm: 11349.721 | params rms: 0.061455 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 19:25:13.572006 | finish at 2026-10-11 06:34:18 + [2026-10-08 11:10:31] iteration 4591/ 6855 | consumed samples: 587648 | elapsed time per iteration (ms): 87184.1 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.57% | FLOPs estimator: workload-aware | logical tokens/s: 192434.3 | valid tokens/s: 192434.3 | physical tokens/s: 192919.4 | loss tokens/s: 142924.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.401153E-01 | lm-global-supervised-tokens: 1.246075E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.145783E-02 | lm-token-buffer-mib: 5.820380E+01 | global_load_balancing_loss: 1.000943E+00 | loss scale: 1.0 | grad norm: 0.089 | params norm: 11349.666 | params rms: 0.061455 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 6:49:44.806520 | finish at 2026-10-10 18:00:16 + [2026-10-08 11:12:02] iteration 4592/ 6855 | consumed samples: 587776 | elapsed time per iteration (ms): 91174.0 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.46% | FLOPs estimator: workload-aware | logical tokens/s: 184012.6 | valid tokens/s: 184012.6 | physical tokens/s: 184379.6 | loss tokens/s: 127084.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.639269E-01 | lm-global-supervised-tokens: 1.158676E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.339300E-02 | lm-token-buffer-mib: 5.816872E+01 | global_load_balancing_loss: 1.001179E+00 | loss scale: 1.0 | grad norm: 0.247 | params norm: 11349.612 | params rms: 0.061454 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 9:18:46.710260 | finish at 2026-10-10 20:30:49 + [2026-10-08 11:13:28] iteration 4593/ 6855 | consumed samples: 587904 | elapsed time per iteration (ms): 85246.7 | throughput per GPU (TFLOP/s/GPU): 43.2 | MFU 13.85% | FLOPs estimator: workload-aware | logical tokens/s: 196807.6 | valid tokens/s: 196807.6 | physical tokens/s: 197251.7 | loss tokens/s: 143634.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.328865E-01 | lm-global-supervised-tokens: 1.224439E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.155680E-02 | lm-token-buffer-mib: 5.814856E+01 | global_load_balancing_loss: 1.000761E+00 | loss scale: 1.0 | grad norm: 0.107 | params norm: 11349.558 | params rms: 0.061454 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 5:33:48.063553 | finish at 2026-10-10 16:47:16 + [2026-10-08 11:14:55] iteration 4594/ 6855 | consumed samples: 588032 | elapsed time per iteration (ms): 87653.4 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.53% | FLOPs estimator: workload-aware | logical tokens/s: 191403.1 | valid tokens/s: 191403.0 | physical tokens/s: 191924.2 | loss tokens/s: 138382.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.560041E-01 | lm-global-supervised-tokens: 1.212968E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.612503E-02 | lm-token-buffer-mib: 5.816473E+01 | global_load_balancing_loss: 1.000997E+00 | loss scale: 1.0 | grad norm: 0.181 | params norm: 11349.503 | params rms: 0.061454 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 7:03:04.372313 | finish at 2026-10-10 18:18:00 + [2026-10-08 11:16:22] iteration 4595/ 6855 | consumed samples: 588160 | elapsed time per iteration (ms): 86664.7 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.61% | FLOPs estimator: workload-aware | logical tokens/s: 193587.3 | valid tokens/s: 193587.3 | physical tokens/s: 193973.1 | loss tokens/s: 136688.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 5.941183E-01 | lm-global-supervised-tokens: 1.184606E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.520346E-02 | lm-token-buffer-mib: 5.820506E+01 | global_load_balancing_loss: 9.998428E-01 | loss scale: 1.0 | grad norm: 0.037 | params norm: 11349.448 | params rms: 0.061454 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 6:24:22.220993 | finish at 2026-10-10 17:40:44 + [2026-10-08 11:17:50] iteration 4596/ 6855 | consumed samples: 588288 | elapsed time per iteration (ms): 87709.2 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.78% | FLOPs estimator: workload-aware | logical tokens/s: 191282.1 | valid tokens/s: 191282.1 | physical tokens/s: 191721.1 | loss tokens/s: 143883.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.727543E-01 | lm-global-supervised-tokens: 1.261992E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.145251E-02 | lm-token-buffer-mib: 5.814168E+01 | global_load_balancing_loss: 9.985957E-01 | loss scale: 1.0 | grad norm: 0.135 | params norm: 11349.394 | params rms: 0.061453 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 7:02:15.122442 | finish at 2026-10-10 18:20:05 + [2026-10-08 11:19:13] iteration 4597/ 6855 | consumed samples: 588416 | elapsed time per iteration (ms): 83425.5 | throughput per GPU (TFLOP/s/GPU): 43.8 | MFU 14.05% | FLOPs estimator: workload-aware | logical tokens/s: 201104.0 | valid tokens/s: 201104.0 | physical tokens/s: 201684.8 | loss tokens/s: 153730.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.881636E-01 | lm-global-supervised-tokens: 1.282502E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.134880E-02 | lm-token-buffer-mib: 5.818690E+01 | global_load_balancing_loss: 9.986464E-01 | loss scale: 1.0 | grad norm: 0.074 | params norm: 11349.343 | params rms: 0.061453 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 4:19:34.717977 | finish at 2026-10-10 15:38:48 + [2026-10-08 11:20:40] iteration 4598/ 6855 | consumed samples: 588544 | elapsed time per iteration (ms): 86671.5 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.46% | FLOPs estimator: workload-aware | logical tokens/s: 193572.2 | valid tokens/s: 193572.2 | physical tokens/s: 193973.2 | loss tokens/s: 145878.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.674784E-01 | lm-global-supervised-tokens: 1.264352E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.154232E-02 | lm-token-buffer-mib: 5.808002E+01 | global_load_balancing_loss: 9.992830E-01 | loss scale: 1.0 | grad norm: 0.125 | params norm: 11349.290 | params rms: 0.061453 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 6:20:17.659910 | finish at 2026-10-10 17:40:57 + [2026-10-08 11:22:07] iteration 4599/ 6855 | consumed samples: 588672 | elapsed time per iteration (ms): 87122.6 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.45% | FLOPs estimator: workload-aware | logical tokens/s: 192569.5 | valid tokens/s: 192569.5 | physical tokens/s: 192980.0 | loss tokens/s: 142747.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.601688E-01 | lm-global-supervised-tokens: 1.243649E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.812621E-02 | lm-token-buffer-mib: 5.818382E+01 | global_load_balancing_loss: 1.001161E+00 | loss scale: 1.0 | grad norm: 0.053 | params norm: 11349.235 | params rms: 0.061452 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 6:35:48.542747 | finish at 2026-10-10 17:57:55 + [2026-10-08 11:23:55] iteration 4600/ 6855 | consumed samples: 588800 | elapsed time per iteration (ms): 107970.6 | throughput per GPU (TFLOP/s/GPU): 33.7 | MFU 10.81% | FLOPs estimator: workload-aware | logical tokens/s: 155386.7 | valid tokens/s: 155386.7 | physical tokens/s: 155702.4 | loss tokens/s: 114747.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.427233E-01 | lm-global-supervised-tokens: 1.238933E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.178891E-02 | lm-token-buffer-mib: 5.815419E+01 | global_load_balancing_loss: 1.001488E+00 | loss scale: 1.0 | grad norm: 0.069 | params norm: 11349.181 | params rms: 0.061452 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 19:37:53.660816 | finish at 2026-10-11 07:01:48 +times across ranks (ms): + save-checkpoint: + rank 0: 24076.65 + rank 1: 24076.60 + rank 2: 24076.63 + rank 3: 24076.59 + rank 4: 24076.62 + rank 5: 24076.59 + rank 6: 24076.57 + rank 7: 24076.57 + rank 8: 24076.66 + rank 9: 24076.63 + rank 10: 24076.67 + rank 11: 24076.63 + rank 12: 24076.64 + rank 13: 24076.63 + rank 14: 24076.64 + rank 15: 24076.62 + rank 16: 24076.67 + rank 17: 24076.62 + rank 18: 24076.63 + rank 19: 24076.63 + rank 20: 24076.62 + rank 21: 24076.64 + rank 22: 24076.62 + rank 23: 24076.62 + rank 24: 24076.72 + rank 25: 24076.72 + rank 26: 24076.72 + rank 27: 24076.80 + rank 28: 24076.70 + rank 29: 24076.73 + rank 30: 24076.70 + rank 31: 24076.72 + rank 32: 24076.60 + rank 33: 24076.61 + rank 34: 24076.62 + rank 35: 24076.59 + rank 36: 24076.57 + rank 37: 24076.57 + rank 38: 24076.61 + rank 39: 24076.56 + rank 40: 24076.50 + rank 41: 24076.47 + rank 42: 24076.48 + rank 43: 24076.49 + rank 44: 24076.45 + rank 45: 24076.45 + rank 46: 24076.48 + rank 47: 24076.43 + rank 48: 24076.12 + rank 49: 24076.10 + rank 50: 24076.09 + rank 51: 24076.09 + rank 52: 24076.12 + rank 53: 24076.08 + rank 54: 24076.13 + rank 55: 24076.08 + rank 56: 24075.70 + rank 57: 24075.69 + rank 58: 24075.68 + rank 59: 24075.67 + rank 60: 24075.67 + rank 61: 24075.69 + rank 62: 24075.67 + rank 63: 24075.70 + [2026-10-08 11:25:55] iteration 4601/ 6855 | consumed samples: 588928 | elapsed time per iteration (ms): 96171.3 | throughput per GPU (TFLOP/s/GPU): 38.1 | MFU 12.22% | FLOPs estimator: workload-aware | logical tokens/s: 174451.1 | valid tokens/s: 174451.1 | physical tokens/s: 174927.9 | loss tokens/s: 135027.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.944730E-01 | lm-global-supervised-tokens: 1.298580E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 5.879446E-01 | lm-token-buffer-mib: 5.823895E+01 | global_load_balancing_loss: 1.000940E+00 | loss scale: 1.0 | grad norm: 0.080 | params norm: 11349.128 | params rms: 0.061452 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 12:12:50.106828 | finish at 2026-10-10 23:38:45 + [2026-10-08 11:27:30] iteration 4602/ 6855 | consumed samples: 589056 | elapsed time per iteration (ms): 94173.0 | throughput per GPU (TFLOP/s/GPU): 40.1 | MFU 12.84% | FLOPs estimator: workload-aware | logical tokens/s: 178152.6 | valid tokens/s: 178152.6 | physical tokens/s: 178660.9 | loss tokens/s: 132085.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.919407E-01 | lm-global-supervised-tokens: 1.243884E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.470176E-02 | lm-token-buffer-mib: 5.817662E+01 | global_load_balancing_loss: 1.000393E+00 | loss scale: 1.0 | grad norm: 0.075 | params norm: 11349.076 | params rms: 0.061452 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 10:56:11.713892 | finish at 2026-10-10 22:23:41 + [2026-10-08 11:28:55] iteration 4603/ 6855 | consumed samples: 589184 | elapsed time per iteration (ms): 85399.2 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.65% | FLOPs estimator: workload-aware | logical tokens/s: 196456.4 | valid tokens/s: 196456.4 | physical tokens/s: 196847.4 | loss tokens/s: 141150.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.393084E-01 | lm-global-supervised-tokens: 1.205411E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.179623E-02 | lm-token-buffer-mib: 5.807494E+01 | global_load_balancing_loss: 1.001282E+00 | loss scale: 1.0 | grad norm: 0.032 | params norm: 11349.023 | params rms: 0.061451 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 5:25:19.007443 | finish at 2026-10-10 16:54:14 + [2026-10-08 11:30:25] iteration 4604/ 6855 | consumed samples: 589312 | elapsed time per iteration (ms): 90279.9 | throughput per GPU (TFLOP/s/GPU): 41.0 | MFU 13.14% | FLOPs estimator: workload-aware | logical tokens/s: 185834.9 | valid tokens/s: 185834.9 | physical tokens/s: 186342.6 | loss tokens/s: 135222.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.134103E-01 | lm-global-supervised-tokens: 1.220786E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.171188E-02 | lm-token-buffer-mib: 5.806642E+01 | global_load_balancing_loss: 1.000491E+00 | loss scale: 1.0 | grad norm: 0.092 | params norm: 11348.971 | params rms: 0.061451 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 8:27:00.032526 | finish at 2026-10-10 19:57:25 + [2026-10-08 11:31:49] iteration 4605/ 6855 | consumed samples: 589440 | elapsed time per iteration (ms): 84184.9 | throughput per GPU (TFLOP/s/GPU): 43.7 | MFU 14.01% | FLOPs estimator: workload-aware | logical tokens/s: 199289.8 | valid tokens/s: 199289.8 | physical tokens/s: 199677.7 | loss tokens/s: 143186.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.590409E-01 | lm-global-supervised-tokens: 1.205411E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.136185E-02 | lm-token-buffer-mib: 5.813030E+01 | global_load_balancing_loss: 9.993894E-01 | loss scale: 1.0 | grad norm: 0.051 | params norm: 11348.919 | params rms: 0.061451 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 4:36:55.981650 | finish at 2026-10-10 16:08:45 + [2026-10-08 11:33:18] iteration 4606/ 6855 | consumed samples: 589568 | elapsed time per iteration (ms): 88845.6 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.42% | FLOPs estimator: workload-aware | logical tokens/s: 188835.3 | valid tokens/s: 188835.3 | physical tokens/s: 189322.6 | loss tokens/s: 140548.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.805418E-01 | lm-global-supervised-tokens: 1.248715E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.310048E-02 | lm-token-buffer-mib: 5.806359E+01 | global_load_balancing_loss: 9.989516E-01 | loss scale: 1.0 | grad norm: 0.119 | params norm: 11348.867 | params rms: 0.061450 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 7:30:13.752543 | finish at 2026-10-10 19:03:32 + [2026-10-08 11:34:42] iteration 4607/ 6855 | consumed samples: 589696 | elapsed time per iteration (ms): 83919.4 | throughput per GPU (TFLOP/s/GPU): 43.8 | MFU 14.03% | FLOPs estimator: workload-aware | logical tokens/s: 199920.5 | valid tokens/s: 199920.5 | physical tokens/s: 200439.6 | loss tokens/s: 147469.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.593599E-01 | lm-global-supervised-tokens: 1.237557E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.597453E-02 | lm-token-buffer-mib: 5.825288E+01 | global_load_balancing_loss: 9.984604E-01 | loss scale: 1.0 | grad norm: 0.112 | params norm: 11348.813 | params rms: 0.061450 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 4:24:10.773630 | finish at 2026-10-10 15:58:53 + [2026-10-08 11:36:06] iteration 4608/ 6855 | consumed samples: 589824 | elapsed time per iteration (ms): 84351.9 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.62% | FLOPs estimator: workload-aware | logical tokens/s: 198895.5 | valid tokens/s: 198895.5 | physical tokens/s: 199601.9 | loss tokens/s: 150260.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.222698E-01 | lm-global-supervised-tokens: 1.267474E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.409667E-02 | lm-token-buffer-mib: 5.809339E+01 | global_load_balancing_loss: 1.001021E+00 | loss scale: 1.0 | grad norm: 0.108 | params norm: 11348.759 | params rms: 0.061450 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 4:38:58.732384 | finish at 2026-10-10 16:15:05 + [2026-10-08 11:37:34] iteration 4609/ 6855 | consumed samples: 589952 | elapsed time per iteration (ms): 87932.4 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.79% | FLOPs estimator: workload-aware | logical tokens/s: 190796.5 | valid tokens/s: 190796.5 | physical tokens/s: 191283.7 | loss tokens/s: 137970.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.501486E-01 | lm-global-supervised-tokens: 1.213210E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.532101E-02 | lm-token-buffer-mib: 5.806394E+01 | global_load_balancing_loss: 1.000551E+00 | loss scale: 1.0 | grad norm: 0.146 | params norm: 11348.708 | params rms: 0.061450 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 6:51:36.122180 | finish at 2026-10-10 18:29:11 + [2026-10-08 11:39:22] iteration 4610/ 6855 | consumed samples: 590080 | elapsed time per iteration (ms): 107516.6 | throughput per GPU (TFLOP/s/GPU): 33.9 | MFU 10.85% | FLOPs estimator: workload-aware | logical tokens/s: 156043.1 | valid tokens/s: 156043.1 | physical tokens/s: 156313.8 | loss tokens/s: 115175.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.366701E-01 | lm-global-supervised-tokens: 1.238326E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.433877E-02 | lm-token-buffer-mib: 5.810765E+01 | global_load_balancing_loss: 1.000848E+00 | loss scale: 1.0 | grad norm: 0.041 | params norm: 11348.655 | params rms: 0.061449 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 19:02:54.655964 | finish at 2026-10-11 06:42:17 + [2026-10-08 11:40:47] iteration 4611/ 6855 | consumed samples: 590208 | elapsed time per iteration (ms): 84964.2 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.62% | FLOPs estimator: workload-aware | logical tokens/s: 197462.1 | valid tokens/s: 197462.1 | physical tokens/s: 197866.5 | loss tokens/s: 152129.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.657706E-01 | lm-global-supervised-tokens: 1.292557E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.893062E-02 | lm-token-buffer-mib: 5.811379E+01 | global_load_balancing_loss: 1.000057E+00 | loss scale: 1.0 | grad norm: 0.104 | params norm: 11348.604 | params rms: 0.061449 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 4:57:39.649329 | finish at 2026-10-10 16:38:27 + [2026-10-08 11:42:15] iteration 4612/ 6855 | consumed samples: 590336 | elapsed time per iteration (ms): 88012.4 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.43% | FLOPs estimator: workload-aware | logical tokens/s: 190622.9 | valid tokens/s: 190622.9 | physical tokens/s: 191214.6 | loss tokens/s: 145771.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.850032E-01 | lm-global-supervised-tokens: 1.282970E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.326760E-02 | lm-token-buffer-mib: 5.814395E+01 | global_load_balancing_loss: 1.000154E+00 | loss scale: 1.0 | grad norm: 0.072 | params norm: 11348.552 | params rms: 0.061449 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 6:50:11.899636 | finish at 2026-10-10 18:32:27 + [2026-10-08 11:43:40] iteration 4613/ 6855 | consumed samples: 590464 | elapsed time per iteration (ms): 85187.8 | throughput per GPU (TFLOP/s/GPU): 43.4 | MFU 13.90% | FLOPs estimator: workload-aware | logical tokens/s: 196943.2 | valid tokens/s: 196943.2 | physical tokens/s: 197539.6 | loss tokens/s: 150703.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.478131E-01 | lm-global-supervised-tokens: 1.283808E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.991202E-02 | lm-token-buffer-mib: 5.804798E+01 | global_load_balancing_loss: 1.000038E+00 | loss scale: 1.0 | grad norm: 0.081 | params norm: 11348.500 | params rms: 0.061448 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 5:03:11.067757 | finish at 2026-10-10 16:46:51 + [2026-10-08 11:45:08] iteration 4614/ 6855 | consumed samples: 590592 | elapsed time per iteration (ms): 88031.1 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.48% | FLOPs estimator: workload-aware | logical tokens/s: 190582.2 | valid tokens/s: 190582.2 | physical tokens/s: 191096.4 | loss tokens/s: 138979.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.390344E-01 | lm-global-supervised-tokens: 1.223453E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.196812E-02 | lm-token-buffer-mib: 5.812408E+01 | global_load_balancing_loss: 1.001736E+00 | loss scale: 1.0 | grad norm: 0.136 | params norm: 11348.446 | params rms: 0.061448 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 6:47:57.710672 | finish at 2026-10-10 18:33:06 + [2026-10-08 11:46:35] iteration 4615/ 6855 | consumed samples: 590720 | elapsed time per iteration (ms): 87128.9 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.75% | FLOPs estimator: workload-aware | logical tokens/s: 192555.3 | valid tokens/s: 192555.3 | physical tokens/s: 192954.9 | loss tokens/s: 137869.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.477463E-01 | lm-global-supervised-tokens: 1.201243E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.402767E-02 | lm-token-buffer-mib: 5.807180E+01 | global_load_balancing_loss: 1.000200E+00 | loss scale: 1.0 | grad norm: 0.089 | params norm: 11348.392 | params rms: 0.061448 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 6:12:48.642120 | finish at 2026-10-10 17:59:24 + [2026-10-08 11:48:04] iteration 4616/ 6855 | consumed samples: 590848 | elapsed time per iteration (ms): 88359.3 | throughput per GPU (TFLOP/s/GPU): 42.8 | MFU 13.71% | FLOPs estimator: workload-aware | logical tokens/s: 189874.7 | valid tokens/s: 189874.7 | physical tokens/s: 190232.5 | loss tokens/s: 137529.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.396194E-01 | lm-global-supervised-tokens: 1.215199E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.323725E-02 | lm-token-buffer-mib: 5.803020E+01 | global_load_balancing_loss: 9.988314E-01 | loss scale: 1.0 | grad norm: 0.192 | params norm: 11348.336 | params rms: 0.061448 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 6:57:16.537064 | finish at 2026-10-10 18:45:20 + [2026-10-08 11:49:33] iteration 4617/ 6855 | consumed samples: 590976 | elapsed time per iteration (ms): 89389.3 | throughput per GPU (TFLOP/s/GPU): 41.1 | MFU 13.17% | FLOPs estimator: workload-aware | logical tokens/s: 187686.2 | valid tokens/s: 187686.2 | physical tokens/s: 188260.2 | loss tokens/s: 131144.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.369740E-01 | lm-global-supervised-tokens: 1.172287E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.271831E-02 | lm-token-buffer-mib: 5.816991E+01 | global_load_balancing_loss: 1.000937E+00 | loss scale: 1.0 | grad norm: 0.242 | params norm: 11348.280 | params rms: 0.061447 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 7:34:13.206686 | finish at 2026-10-10 19:23:46 + [2026-10-08 11:51:02] iteration 4618/ 6855 | consumed samples: 591104 | elapsed time per iteration (ms): 89042.2 | throughput per GPU (TFLOP/s/GPU): 41.0 | MFU 13.15% | FLOPs estimator: workload-aware | logical tokens/s: 188418.4 | valid tokens/s: 188418.4 | physical tokens/s: 188834.0 | loss tokens/s: 136864.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.037563E-01 | lm-global-supervised-tokens: 1.218669E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.344673E-02 | lm-token-buffer-mib: 5.807117E+01 | global_load_balancing_loss: 1.000522E+00 | loss scale: 1.0 | grad norm: 0.059 | params norm: 11348.224 | params rms: 0.061447 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 7:19:47.374931 | finish at 2026-10-10 19:10:49 + [2026-10-08 11:52:30] iteration 4619/ 6855 | consumed samples: 591232 | elapsed time per iteration (ms): 88078.7 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.57% | FLOPs estimator: workload-aware | logical tokens/s: 190479.4 | valid tokens/s: 190479.4 | physical tokens/s: 190822.9 | loss tokens/s: 144085.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.703052E-01 | lm-global-supervised-tokens: 1.269083E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.359530E-02 | lm-token-buffer-mib: 5.813017E+01 | global_load_balancing_loss: 9.989783E-01 | loss scale: 1.0 | grad norm: 0.177 | params norm: 11348.168 | params rms: 0.061447 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 6:42:23.888583 | finish at 2026-10-10 18:34:54 + [2026-10-08 11:54:20] iteration 4620/ 6855 | consumed samples: 591360 | elapsed time per iteration (ms): 110099.7 | throughput per GPU (TFLOP/s/GPU): 34.4 | MFU 11.04% | FLOPs estimator: workload-aware | logical tokens/s: 152381.9 | valid tokens/s: 152381.9 | physical tokens/s: 152648.1 | loss tokens/s: 113255.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.634155E-01 | lm-global-supervised-tokens: 1.246939E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.260978E-02 | lm-token-buffer-mib: 5.805788E+01 | global_load_balancing_loss: 9.997322E-01 | loss scale: 1.0 | grad norm: 0.070 | params norm: 11348.111 | params rms: 0.061446 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 20:21:12.922693 | finish at 2026-10-11 08:15:33 + [2026-10-08 11:55:48] iteration 4621/ 6855 | consumed samples: 591488 | elapsed time per iteration (ms): 87582.4 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.79% | FLOPs estimator: workload-aware | logical tokens/s: 191558.8 | valid tokens/s: 191558.8 | physical tokens/s: 191786.0 | loss tokens/s: 134706.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.806661E-01 | lm-global-supervised-tokens: 1.179790E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.573662E-02 | lm-token-buffer-mib: 5.806154E+01 | global_load_balancing_loss: 9.998960E-01 | loss scale: 1.0 | grad norm: 0.093 | params norm: 11348.054 | params rms: 0.061446 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 6:20:59.155289 | finish at 2026-10-10 18:16:47 + [2026-10-08 11:57:16] iteration 4622/ 6855 | consumed samples: 591616 | elapsed time per iteration (ms): 88539.2 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.57% | FLOPs estimator: workload-aware | logical tokens/s: 189488.5 | valid tokens/s: 189488.5 | physical tokens/s: 190023.4 | loss tokens/s: 139094.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.647213E-01 | lm-global-supervised-tokens: 1.231533E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.228677E-02 | lm-token-buffer-mib: 5.813860E+01 | global_load_balancing_loss: 1.000442E+00 | loss scale: 1.0 | grad norm: 0.313 | params norm: 11347.994 | params rms: 0.061446 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 6:55:07.923014 | finish at 2026-10-10 18:52:24 + [2026-10-08 11:58:45] iteration 4623/ 6855 | consumed samples: 591744 | elapsed time per iteration (ms): 88487.2 | throughput per GPU (TFLOP/s/GPU): 41.6 | MFU 13.34% | FLOPs estimator: workload-aware | logical tokens/s: 189599.8 | valid tokens/s: 189599.8 | physical tokens/s: 190068.7 | loss tokens/s: 143554.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.937087E-01 | lm-global-supervised-tokens: 1.270275E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.225969E-02 | lm-token-buffer-mib: 5.811044E+01 | global_load_balancing_loss: 1.000674E+00 | loss scale: 1.0 | grad norm: 0.222 | params norm: 11347.935 | params rms: 0.061445 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 6:51:43.345837 | finish at 2026-10-10 18:50:28 + [2026-10-08 12:00:12] iteration 4624/ 6855 | consumed samples: 591872 | elapsed time per iteration (ms): 87013.8 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.53% | FLOPs estimator: workload-aware | logical tokens/s: 192810.5 | valid tokens/s: 192810.5 | physical tokens/s: 193238.6 | loss tokens/s: 138141.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 5.859897E-01 | lm-global-supervised-tokens: 1.202021E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.139797E-02 | lm-token-buffer-mib: 5.821303E+01 | global_load_balancing_loss: 1.001784E+00 | loss scale: 1.0 | grad norm: 0.267 | params norm: 11347.876 | params rms: 0.061445 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 5:55:27.884398 | finish at 2026-10-10 17:55:40 + [2026-10-08 12:01:39] iteration 4625/ 6855 | consumed samples: 592000 | elapsed time per iteration (ms): 87164.7 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.77% | FLOPs estimator: workload-aware | logical tokens/s: 192477.0 | valid tokens/s: 192476.9 | physical tokens/s: 192887.1 | loss tokens/s: 144137.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.573876E-01 | lm-global-supervised-tokens: 1.256366E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.465062E-02 | lm-token-buffer-mib: 5.811961E+01 | global_load_balancing_loss: 1.000452E+00 | loss scale: 1.0 | grad norm: 0.212 | params norm: 11347.816 | params rms: 0.061445 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 5:59:37.208762 | finish at 2026-10-10 18:01:16 + [2026-10-08 12:03:06] iteration 4626/ 6855 | consumed samples: 592128 | elapsed time per iteration (ms): 87190.3 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.74% | FLOPs estimator: workload-aware | logical tokens/s: 192420.1 | valid tokens/s: 192420.1 | physical tokens/s: 192730.0 | loss tokens/s: 139864.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.329370E-01 | lm-global-supervised-tokens: 1.219482E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.645167E-02 | lm-token-buffer-mib: 5.819981E+01 | global_load_balancing_loss: 1.000283E+00 | loss scale: 1.0 | grad norm: 0.097 | params norm: 11347.757 | params rms: 0.061444 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 5:59:07.284428 | finish at 2026-10-10 18:02:14 + [2026-10-08 12:04:33] iteration 4627/ 6855 | consumed samples: 592256 | elapsed time per iteration (ms): 87232.8 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.70% | FLOPs estimator: workload-aware | logical tokens/s: 192326.7 | valid tokens/s: 192326.7 | physical tokens/s: 192855.7 | loss tokens/s: 132458.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.188425E-01 | lm-global-supervised-tokens: 1.155469E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.471063E-02 | lm-token-buffer-mib: 5.825217E+01 | global_load_balancing_loss: 1.000078E+00 | loss scale: 1.0 | grad norm: 0.182 | params norm: 11347.700 | params rms: 0.061444 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 5:59:14.664604 | finish at 2026-10-10 18:03:48 + [2026-10-08 12:06:01] iteration 4628/ 6855 | consumed samples: 592384 | elapsed time per iteration (ms): 87609.6 | throughput per GPU (TFLOP/s/GPU): 42.8 | MFU 13.71% | FLOPs estimator: workload-aware | logical tokens/s: 191499.1 | valid tokens/s: 191499.1 | physical tokens/s: 191994.8 | loss tokens/s: 143160.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.746212E-01 | lm-global-supervised-tokens: 1.254218E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.161834E-02 | lm-token-buffer-mib: 5.808630E+01 | global_load_balancing_loss: 9.991455E-01 | loss scale: 1.0 | grad norm: 0.058 | params norm: 11347.644 | params rms: 0.061444 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 6:11:46.478998 | finish at 2026-10-10 18:17:48 + [2026-10-08 12:07:25] iteration 4629/ 6855 | consumed samples: 592512 | elapsed time per iteration (ms): 83999.0 | throughput per GPU (TFLOP/s/GPU): 43.2 | MFU 13.83% | FLOPs estimator: workload-aware | logical tokens/s: 199730.7 | valid tokens/s: 199730.7 | physical tokens/s: 200188.0 | loss tokens/s: 147029.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.460308E-01 | lm-global-supervised-tokens: 1.235033E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.629429E-02 | lm-token-buffer-mib: 5.804747E+01 | global_load_balancing_loss: 9.992263E-01 | loss scale: 1.0 | grad norm: 0.135 | params norm: 11347.588 | params rms: 0.061443 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 3:56:21.879244 | finish at 2026-10-10 16:03:47 + [2026-10-08 12:09:12] iteration 4630/ 6855 | consumed samples: 592640 | elapsed time per iteration (ms): 106964.3 | throughput per GPU (TFLOP/s/GPU): 35.2 | MFU 11.30% | FLOPs estimator: workload-aware | logical tokens/s: 156848.5 | valid tokens/s: 156848.5 | physical tokens/s: 157087.1 | loss tokens/s: 114007.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.593575E-01 | lm-global-supervised-tokens: 1.219478E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.147301E-02 | lm-token-buffer-mib: 5.806483E+01 | global_load_balancing_loss: 9.994305E-01 | loss scale: 1.0 | grad norm: 0.082 | params norm: 11347.530 | params rms: 0.061443 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 18:06:35.642644 | finish at 2026-10-11 06:15:48 + [2026-10-08 12:10:38] iteration 4631/ 6855 | consumed samples: 592768 | elapsed time per iteration (ms): 86448.7 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.58% | FLOPs estimator: workload-aware | logical tokens/s: 194071.0 | valid tokens/s: 194071.0 | physical tokens/s: 194651.1 | loss tokens/s: 151521.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.969706E-01 | lm-global-supervised-tokens: 1.309884E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.756408E-01 | lm-token-buffer-mib: 5.828764E+01 | global_load_balancing_loss: 1.000635E+00 | loss scale: 1.0 | grad norm: 0.072 | params norm: 11347.473 | params rms: 0.061443 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 5:24:21.827564 | finish at 2026-10-10 17:35:00 + [2026-10-08 12:12:08] iteration 4632/ 6855 | consumed samples: 592896 | elapsed time per iteration (ms): 89460.2 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.41% | FLOPs estimator: workload-aware | logical tokens/s: 187537.8 | valid tokens/s: 187537.8 | physical tokens/s: 188081.6 | loss tokens/s: 139112.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.123489E-01 | lm-global-supervised-tokens: 1.244506E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.160239E-02 | lm-token-buffer-mib: 5.809122E+01 | global_load_balancing_loss: 1.000397E+00 | loss scale: 1.0 | grad norm: 0.182 | params norm: 11347.417 | params rms: 0.061443 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 7:14:30.114859 | finish at 2026-10-10 19:26:38 + [2026-10-08 12:13:37] iteration 4633/ 6855 | consumed samples: 593024 | elapsed time per iteration (ms): 88796.8 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.62% | FLOPs estimator: workload-aware | logical tokens/s: 188939.3 | valid tokens/s: 188939.3 | physical tokens/s: 189280.9 | loss tokens/s: 141200.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.012868E-01 | lm-global-supervised-tokens: 1.253813E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.481419E-02 | lm-token-buffer-mib: 5.823343E+01 | global_load_balancing_loss: 1.000067E+00 | loss scale: 1.0 | grad norm: 0.037 | params norm: 11347.361 | params rms: 0.061442 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 6:48:26.483016 | finish at 2026-10-10 19:02:03 + [2026-10-08 12:15:03] iteration 4634/ 6855 | consumed samples: 593152 | elapsed time per iteration (ms): 86223.9 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.60% | FLOPs estimator: workload-aware | logical tokens/s: 194577.4 | valid tokens/s: 194577.4 | physical tokens/s: 194937.0 | loss tokens/s: 144939.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.136400E-01 | lm-global-supervised-tokens: 1.249721E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.191425E-02 | lm-token-buffer-mib: 5.819402E+01 | global_load_balancing_loss: 1.001191E+00 | loss scale: 1.0 | grad norm: 0.087 | params norm: 11347.306 | params rms: 0.061442 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 5:11:43.240774 | finish at 2026-10-10 17:26:46 + [2026-10-08 12:16:29] iteration 4635/ 6855 | consumed samples: 593280 | elapsed time per iteration (ms): 86000.7 | throughput per GPU (TFLOP/s/GPU): 43.7 | MFU 14.01% | FLOPs estimator: workload-aware | logical tokens/s: 195081.9 | valid tokens/s: 195081.8 | physical tokens/s: 195455.9 | loss tokens/s: 141861.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.429868E-01 | lm-global-supervised-tokens: 1.220017E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.870754E-02 | lm-token-buffer-mib: 5.803132E+01 | global_load_balancing_loss: 1.001167E+00 | loss scale: 1.0 | grad norm: 0.090 | params norm: 11347.250 | params rms: 0.061442 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 5:02:01.537056 | finish at 2026-10-10 17:18:30 + [2026-10-08 12:17:54] iteration 4636/ 6855 | consumed samples: 593408 | elapsed time per iteration (ms): 84611.5 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.57% | FLOPs estimator: workload-aware | logical tokens/s: 198285.0 | valid tokens/s: 198285.0 | physical tokens/s: 198981.1 | loss tokens/s: 151865.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.129782E-01 | lm-global-supervised-tokens: 1.284953E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.149734E-02 | lm-token-buffer-mib: 5.812992E+01 | global_load_balancing_loss: 1.000048E+00 | loss scale: 1.0 | grad norm: 0.188 | params norm: 11347.194 | params rms: 0.061441 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 4:09:13.001616 | finish at 2026-10-10 16:27:07 + [2026-10-08 12:19:23] iteration 4637/ 6855 | consumed samples: 593536 | elapsed time per iteration (ms): 89721.9 | throughput per GPU (TFLOP/s/GPU): 41.2 | MFU 13.22% | FLOPs estimator: workload-aware | logical tokens/s: 186990.1 | valid tokens/s: 186990.1 | physical tokens/s: 187540.2 | loss tokens/s: 141111.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.783456E-01 | lm-global-supervised-tokens: 1.266082E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.150316E-02 | lm-token-buffer-mib: 5.820658E+01 | global_load_balancing_loss: 9.991705E-01 | loss scale: 1.0 | grad norm: 0.261 | params norm: 11347.142 | params rms: 0.061441 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 7:16:43.138210 | finish at 2026-10-10 19:36:06 + [2026-10-08 12:20:51] iteration 4638/ 6855 | consumed samples: 593664 | elapsed time per iteration (ms): 87541.1 | throughput per GPU (TFLOP/s/GPU): 41.7 | MFU 13.36% | FLOPs estimator: workload-aware | logical tokens/s: 191649.6 | valid tokens/s: 191649.6 | physical tokens/s: 192508.5 | loss tokens/s: 141300.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.602848E-01 | lm-global-supervised-tokens: 1.236959E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.223698E-02 | lm-token-buffer-mib: 5.809914E+01 | global_load_balancing_loss: 9.996941E-01 | loss scale: 1.0 | grad norm: 0.160 | params norm: 11347.088 | params rms: 0.061441 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 5:54:38.572770 | finish at 2026-10-10 18:15:29 + [2026-10-08 12:22:17] iteration 4639/ 6855 | consumed samples: 593792 | elapsed time per iteration (ms): 86175.9 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.59% | FLOPs estimator: workload-aware | logical tokens/s: 194685.3 | valid tokens/s: 194685.2 | physical tokens/s: 195241.1 | loss tokens/s: 142999.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.692691E-01 | lm-global-supervised-tokens: 1.232308E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.203722E-02 | lm-token-buffer-mib: 5.818138E+01 | global_load_balancing_loss: 1.000702E+00 | loss scale: 1.0 | grad norm: 0.072 | params norm: 11347.034 | params rms: 0.061440 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 5:02:45.883121 | finish at 2026-10-10 17:25:03 + [2026-10-08 12:24:06] iteration 4640/ 6855 | consumed samples: 593920 | elapsed time per iteration (ms): 109360.9 | throughput per GPU (TFLOP/s/GPU): 34.7 | MFU 11.13% | FLOPs estimator: workload-aware | logical tokens/s: 153410.8 | valid tokens/s: 153410.8 | physical tokens/s: 153703.2 | loss tokens/s: 112795.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.927478E-01 | lm-global-supervised-tokens: 1.233538E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.338764E-02 | lm-token-buffer-mib: 5.806351E+01 | global_load_balancing_loss: 1.000240E+00 | loss scale: 1.0 | grad norm: 0.090 | params norm: 11346.980 | params rms: 0.061440 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 19:17:14.422380 | finish at 2026-10-11 07:41:21 +times across ranks (ms): + save-checkpoint: + rank 0: 24232.36 + rank 1: 24232.42 + rank 2: 24232.38 + rank 3: 24232.39 + rank 4: 24232.37 + rank 5: 24232.36 + rank 6: 24232.36 + rank 7: 24232.37 + rank 8: 24232.50 + rank 9: 24232.49 + rank 10: 24232.49 + rank 11: 24232.48 + rank 12: 24232.48 + rank 13: 24232.49 + rank 14: 24232.50 + rank 15: 24232.50 + rank 16: 24232.57 + rank 17: 24232.55 + rank 18: 24232.56 + rank 19: 24232.57 + rank 20: 24232.55 + rank 21: 24232.59 + rank 22: 24232.56 + rank 23: 24232.55 + rank 24: 24230.09 + rank 25: 24230.06 + rank 26: 24230.17 + rank 27: 24230.10 + rank 28: 24230.06 + rank 29: 24230.07 + rank 30: 24230.07 + rank 31: 24230.08 + rank 32: 24232.74 + rank 33: 24232.76 + rank 34: 24232.76 + rank 35: 24232.71 + rank 36: 24232.70 + rank 37: 24232.73 + rank 38: 24232.70 + rank 39: 24232.71 + rank 40: 24232.72 + rank 41: 24232.72 + rank 42: 24232.71 + rank 43: 24232.74 + rank 44: 24232.71 + rank 45: 24232.74 + rank 46: 24232.72 + rank 47: 24232.70 + rank 48: 24232.19 + rank 49: 24232.23 + rank 50: 24232.14 + rank 51: 24232.16 + rank 52: 24232.16 + rank 53: 24232.16 + rank 54: 24232.18 + rank 55: 24232.17 + rank 56: 24232.51 + rank 57: 24232.49 + rank 58: 24232.49 + rank 59: 24232.49 + rank 60: 24232.49 + rank 61: 24232.48 + rank 62: 24232.49 + rank 63: 24232.48 + [2026-10-08 12:26:01] iteration 4641/ 6855 | consumed samples: 594048 | elapsed time per iteration (ms): 90427.1 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.51% | FLOPs estimator: workload-aware | logical tokens/s: 185532.5 | valid tokens/s: 185532.5 | physical tokens/s: 185800.5 | loss tokens/s: 132216.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.403221E-01 | lm-global-supervised-tokens: 1.195598E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 8.458242E-01 | lm-token-buffer-mib: 5.804830E+01 | global_load_balancing_loss: 1.000430E+00 | loss scale: 1.0 | grad norm: 0.030 | params norm: 11346.927 | params rms: 0.061440 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 7:36:45.678981 | finish at 2026-10-10 20:02:47 + [2026-10-08 12:27:34] iteration 4642/ 6855 | consumed samples: 594176 | elapsed time per iteration (ms): 92509.4 | throughput per GPU (TFLOP/s/GPU): 39.6 | MFU 12.68% | FLOPs estimator: workload-aware | logical tokens/s: 181356.3 | valid tokens/s: 181356.3 | physical tokens/s: 181734.4 | loss tokens/s: 142823.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.881096E-01 | lm-global-supervised-tokens: 1.321248E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.573382E-02 | lm-token-buffer-mib: 5.810851E+01 | global_load_balancing_loss: 9.992303E-01 | loss scale: 1.0 | grad norm: 0.049 | params norm: 11346.873 | params rms: 0.061440 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 8:52:03.203821 | finish at 2026-10-10 21:19:37 + [2026-10-08 12:28:59] iteration 4643/ 6855 | consumed samples: 594304 | elapsed time per iteration (ms): 85370.9 | throughput per GPU (TFLOP/s/GPU): 43.3 | MFU 13.88% | FLOPs estimator: workload-aware | logical tokens/s: 196520.9 | valid tokens/s: 196520.9 | physical tokens/s: 196981.7 | loss tokens/s: 148571.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.379415E-01 | lm-global-supervised-tokens: 1.268366E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.127188E-02 | lm-token-buffer-mib: 5.808248E+01 | global_load_balancing_loss: 1.000643E+00 | loss scale: 1.0 | grad norm: 0.041 | params norm: 11346.818 | params rms: 0.061439 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 4:27:20.493372 | finish at 2026-10-10 16:56:20 + [2026-10-08 12:30:24] iteration 4644/ 6855 | consumed samples: 594432 | elapsed time per iteration (ms): 84983.8 | throughput per GPU (TFLOP/s/GPU): 43.6 | MFU 13.97% | FLOPs estimator: workload-aware | logical tokens/s: 197416.7 | valid tokens/s: 197416.7 | physical tokens/s: 197881.7 | loss tokens/s: 144409.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.756668E-01 | lm-global-supervised-tokens: 1.227244E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.282463E-02 | lm-token-buffer-mib: 5.817803E+01 | global_load_balancing_loss: 1.000420E+00 | loss scale: 1.0 | grad norm: 0.064 | params norm: 11346.764 | params rms: 0.061439 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 4:11:39.151608 | finish at 2026-10-10 16:42:03 + [2026-10-08 12:31:49] iteration 4645/ 6855 | consumed samples: 594560 | elapsed time per iteration (ms): 84918.2 | throughput per GPU (TFLOP/s/GPU): 43.8 | MFU 14.05% | FLOPs estimator: workload-aware | logical tokens/s: 197568.2 | valid tokens/s: 197568.2 | physical tokens/s: 198076.9 | loss tokens/s: 146293.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.467738E-01 | lm-global-supervised-tokens: 1.242298E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.345837E-02 | lm-token-buffer-mib: 5.809753E+01 | global_load_balancing_loss: 9.993816E-01 | loss scale: 1.0 | grad norm: 0.045 | params norm: 11346.711 | params rms: 0.061439 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 4:07:49.249961 | finish at 2026-10-10 16:39:38 + [2026-10-08 12:33:15] iteration 4646/ 6855 | consumed samples: 594688 | elapsed time per iteration (ms): 86402.8 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.78% | FLOPs estimator: workload-aware | logical tokens/s: 194173.9 | valid tokens/s: 194173.9 | physical tokens/s: 194607.2 | loss tokens/s: 141815.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.777917E-01 | lm-global-supervised-tokens: 1.225328E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.359620E-02 | lm-token-buffer-mib: 5.817336E+01 | global_load_balancing_loss: 9.997172E-01 | loss scale: 1.0 | grad norm: 0.052 | params norm: 11346.657 | params rms: 0.061438 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 5:01:03.786964 | finish at 2026-10-10 17:34:19 + [2026-10-08 12:34:44] iteration 4647/ 6855 | consumed samples: 594816 | elapsed time per iteration (ms): 88843.8 | throughput per GPU (TFLOP/s/GPU): 41.7 | MFU 13.38% | FLOPs estimator: workload-aware | logical tokens/s: 188838.8 | valid tokens/s: 188838.8 | physical tokens/s: 189280.2 | loss tokens/s: 147966.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.009599E-01 | lm-global-supervised-tokens: 1.314592E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.131538E-02 | lm-token-buffer-mib: 5.815443E+01 | global_load_balancing_loss: 1.000549E+00 | loss scale: 1.0 | grad norm: 0.047 | params norm: 11346.604 | params rms: 0.061438 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 6:29:27.208992 | finish at 2026-10-10 19:04:11 + [2026-10-08 12:36:13] iteration 4648/ 6855 | consumed samples: 594944 | elapsed time per iteration (ms): 88311.2 | throughput per GPU (TFLOP/s/GPU): 41.5 | MFU 13.29% | FLOPs estimator: workload-aware | logical tokens/s: 189977.9 | valid tokens/s: 189977.9 | physical tokens/s: 190309.2 | loss tokens/s: 132007.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.266313E-01 | lm-global-supervised-tokens: 1.165770E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.587981E-02 | lm-token-buffer-mib: 5.818050E+01 | global_load_balancing_loss: 1.001252E+00 | loss scale: 1.0 | grad norm: 0.044 | params norm: 11346.552 | params rms: 0.061438 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 6:08:22.815556 | finish at 2026-10-10 18:44:35 + [2026-10-08 12:37:39] iteration 4649/ 6855 | consumed samples: 595072 | elapsed time per iteration (ms): 86166.2 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.63% | FLOPs estimator: workload-aware | logical tokens/s: 194707.7 | valid tokens/s: 194707.7 | physical tokens/s: 195226.3 | loss tokens/s: 141588.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.082828E-01 | lm-global-supervised-tokens: 1.220018E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.145281E-02 | lm-token-buffer-mib: 5.805967E+01 | global_load_balancing_loss: 1.000499E+00 | loss scale: 1.0 | grad norm: 0.035 | params norm: 11346.501 | params rms: 0.061438 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 4:48:02.593901 | finish at 2026-10-10 17:25:41 + [2026-10-08 12:39:29] iteration 4650/ 6855 | consumed samples: 595200 | elapsed time per iteration (ms): 109922.6 | throughput per GPU (TFLOP/s/GPU): 34.2 | MFU 10.97% | FLOPs estimator: workload-aware | logical tokens/s: 152627.4 | valid tokens/s: 152627.4 | physical tokens/s: 152959.2 | loss tokens/s: 113596.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.141137E-01 | lm-global-supervised-tokens: 1.248681E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.148858E-02 | lm-token-buffer-mib: 5.816456E+01 | global_load_balancing_loss: 9.990041E-01 | loss scale: 1.0 | grad norm: 0.052 | params norm: 11346.449 | params rms: 0.061437 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 19:19:39.381434 | finish at 2026-10-11 07:59:08 + [2026-10-08 12:40:54] iteration 4651/ 6855 | consumed samples: 595328 | elapsed time per iteration (ms): 85739.9 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.75% | FLOPs estimator: workload-aware | logical tokens/s: 195675.2 | valid tokens/s: 195675.2 | physical tokens/s: 196011.9 | loss tokens/s: 150075.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.098528E-01 | lm-global-supervised-tokens: 1.286742E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.829262E-02 | lm-token-buffer-mib: 5.807924E+01 | global_load_balancing_loss: 1.000636E+00 | loss scale: 1.0 | grad norm: 0.082 | params norm: 11346.395 | params rms: 0.061437 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 4:29:30.792920 | finish at 2026-10-10 17:10:25 + [2026-10-08 12:42:23] iteration 4652/ 6855 | consumed samples: 595456 | elapsed time per iteration (ms): 88414.6 | throughput per GPU (TFLOP/s/GPU): 42.8 | MFU 13.72% | FLOPs estimator: workload-aware | logical tokens/s: 189755.7 | valid tokens/s: 189755.7 | physical tokens/s: 190333.5 | loss tokens/s: 137715.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.755403E-01 | lm-global-supervised-tokens: 1.217605E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.431149E-02 | lm-token-buffer-mib: 5.816004E+01 | global_load_balancing_loss: 1.000961E+00 | loss scale: 1.0 | grad norm: 0.045 | params norm: 11346.342 | params rms: 0.061437 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 6:06:17.274280 | finish at 2026-10-10 18:48:40 + [2026-10-08 12:43:50] iteration 4653/ 6855 | consumed samples: 595584 | elapsed time per iteration (ms): 87119.1 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.76% | FLOPs estimator: workload-aware | logical tokens/s: 192577.9 | valid tokens/s: 192577.9 | physical tokens/s: 192879.1 | loss tokens/s: 132978.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.486133E-01 | lm-global-supervised-tokens: 1.158500E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.905680E-02 | lm-token-buffer-mib: 5.823649E+01 | global_load_balancing_loss: 1.000091E+00 | loss scale: 1.0 | grad norm: 0.090 | params norm: 11346.291 | params rms: 0.061436 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 5:17:16.275732 | finish at 2026-10-10 18:01:06 + [2026-10-08 12:45:16] iteration 4654/ 6855 | consumed samples: 595712 | elapsed time per iteration (ms): 86361.2 | throughput per GPU (TFLOP/s/GPU): 41.5 | MFU 13.29% | FLOPs estimator: workload-aware | logical tokens/s: 194266.1 | valid tokens/s: 194266.1 | physical tokens/s: 194936.3 | loss tokens/s: 145700.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.222699E-01 | lm-global-supervised-tokens: 1.258283E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.364491E-02 | lm-token-buffer-mib: 5.817062E+01 | global_load_balancing_loss: 1.001488E+00 | loss scale: 1.0 | grad norm: 0.070 | params norm: 11346.238 | params rms: 0.061436 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 4:48:00.975169 | finish at 2026-10-10 17:33:17 + [2026-10-08 12:46:48] iteration 4655/ 6855 | consumed samples: 595840 | elapsed time per iteration (ms): 91492.4 | throughput per GPU (TFLOP/s/GPU): 40.3 | MFU 12.93% | FLOPs estimator: workload-aware | logical tokens/s: 183372.4 | valid tokens/s: 183372.4 | physical tokens/s: 183775.1 | loss tokens/s: 133379.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.705809E-01 | lm-global-supervised-tokens: 1.220323E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.680472E-02 | lm-token-buffer-mib: 5.811759E+01 | global_load_balancing_loss: 9.989662E-01 | loss scale: 1.0 | grad norm: 0.136 | params norm: 11346.185 | params rms: 0.061436 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 7:54:43.362722 | finish at 2026-10-10 20:41:31 + [2026-10-08 12:48:13] iteration 4656/ 6855 | consumed samples: 595968 | elapsed time per iteration (ms): 85546.8 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.48% | FLOPs estimator: workload-aware | logical tokens/s: 196117.2 | valid tokens/s: 196117.2 | physical tokens/s: 196749.9 | loss tokens/s: 145652.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.512076E-01 | lm-global-supervised-tokens: 1.246010E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.729857E-02 | lm-token-buffer-mib: 5.807694E+01 | global_load_balancing_loss: 1.000001E+00 | loss scale: 1.0 | grad norm: 0.085 | params norm: 11346.131 | params rms: 0.061436 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 4:15:17.470647 | finish at 2026-10-10 17:03:31 + [2026-10-08 12:49:44] iteration 4657/ 6855 | consumed samples: 596096 | elapsed time per iteration (ms): 91037.9 | throughput per GPU (TFLOP/s/GPU): 40.0 | MFU 12.83% | FLOPs estimator: workload-aware | logical tokens/s: 184287.8 | valid tokens/s: 184287.8 | physical tokens/s: 184834.9 | loss tokens/s: 132588.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.651229E-01 | lm-global-supervised-tokens: 1.207055E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.604873E-02 | lm-token-buffer-mib: 5.818752E+01 | global_load_balancing_loss: 9.999703E-01 | loss scale: 1.0 | grad norm: 0.037 | params norm: 11346.076 | params rms: 0.061435 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 7:35:01.294703 | finish at 2026-10-10 20:24:46 + [2026-10-08 12:51:14] iteration 4658/ 6855 | consumed samples: 596224 | elapsed time per iteration (ms): 89306.3 | throughput per GPU (TFLOP/s/GPU): 41.0 | MFU 13.15% | FLOPs estimator: workload-aware | logical tokens/s: 187861.0 | valid tokens/s: 187861.0 | physical tokens/s: 188255.5 | loss tokens/s: 138829.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.169640E-01 | lm-global-supervised-tokens: 1.239832E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.141530E-02 | lm-token-buffer-mib: 5.818227E+01 | global_load_balancing_loss: 9.999007E-01 | loss scale: 1.0 | grad norm: 0.114 | params norm: 11346.021 | params rms: 0.061435 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 6:30:05.875459 | finish at 2026-10-10 19:21:20 + [2026-10-08 12:52:42] iteration 4659/ 6855 | consumed samples: 596352 | elapsed time per iteration (ms): 88549.0 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.50% | FLOPs estimator: workload-aware | logical tokens/s: 189468.1 | valid tokens/s: 189468.1 | physical tokens/s: 189780.2 | loss tokens/s: 140643.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.343453E-01 | lm-global-supervised-tokens: 1.245386E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.512579E-02 | lm-token-buffer-mib: 5.815868E+01 | global_load_balancing_loss: 1.000478E+00 | loss scale: 1.0 | grad norm: 0.107 | params norm: 11345.967 | params rms: 0.061435 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 6:00:53.635468 | finish at 2026-10-10 18:53:36 + [2026-10-08 12:54:28] iteration 4660/ 6855 | consumed samples: 596480 | elapsed time per iteration (ms): 105634.4 | throughput per GPU (TFLOP/s/GPU): 34.1 | MFU 10.93% | FLOPs estimator: workload-aware | logical tokens/s: 158823.1 | valid tokens/s: 158823.1 | physical tokens/s: 159259.3 | loss tokens/s: 119455.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.184200E-01 | lm-global-supervised-tokens: 1.261864E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.354741E-02 | lm-token-buffer-mib: 5.812669E+01 | global_load_balancing_loss: 1.000718E+00 | loss scale: 1.0 | grad norm: 0.095 | params norm: 11345.910 | params rms: 0.061434 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 16:24:27.502527 | finish at 2026-10-11 05:18:55 + [2026-10-08 12:55:54] iteration 4661/ 6855 | consumed samples: 596608 | elapsed time per iteration (ms): 86315.4 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.77% | FLOPs estimator: workload-aware | logical tokens/s: 194370.5 | valid tokens/s: 194370.5 | physical tokens/s: 194964.6 | loss tokens/s: 143692.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.441361E-01 | lm-global-supervised-tokens: 1.240291E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.148743E-02 | lm-token-buffer-mib: 5.809561E+01 | global_load_balancing_loss: 1.000150E+00 | loss scale: 1.0 | grad norm: 0.135 | params norm: 11345.854 | params rms: 0.061434 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 4:36:15.971132 | finish at 2026-10-10 17:32:10 + [2026-10-08 12:57:19] iteration 4662/ 6855 | consumed samples: 596736 | elapsed time per iteration (ms): 84514.0 | throughput per GPU (TFLOP/s/GPU): 42.8 | MFU 13.71% | FLOPs estimator: workload-aware | logical tokens/s: 198513.5 | valid tokens/s: 198513.5 | physical tokens/s: 198974.4 | loss tokens/s: 149310.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.083077E-01 | lm-global-supervised-tokens: 1.261883E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.401489E-02 | lm-token-buffer-mib: 5.816322E+01 | global_load_balancing_loss: 1.000288E+00 | loss scale: 1.0 | grad norm: 0.076 | params norm: 11345.798 | params rms: 0.061434 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 3:28:59.286045 | finish at 2026-10-10 16:26:18 + [2026-10-08 12:58:43] iteration 4663/ 6855 | consumed samples: 596864 | elapsed time per iteration (ms): 84763.4 | throughput per GPU (TFLOP/s/GPU): 43.2 | MFU 13.85% | FLOPs estimator: workload-aware | logical tokens/s: 197929.3 | valid tokens/s: 197929.3 | physical tokens/s: 198563.8 | loss tokens/s: 152483.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.748280E-01 | lm-global-supervised-tokens: 1.292497E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.274372E-02 | lm-token-buffer-mib: 5.825123E+01 | global_load_balancing_loss: 1.000016E+00 | loss scale: 1.0 | grad norm: 0.094 | params norm: 11345.742 | params rms: 0.061434 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 3:36:41.365131 | finish at 2026-10-10 16:35:25 + [2026-10-08 13:00:12] iteration 4664/ 6855 | consumed samples: 596992 | elapsed time per iteration (ms): 88150.2 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.69% | FLOPs estimator: workload-aware | logical tokens/s: 190325.0 | valid tokens/s: 190325.0 | physical tokens/s: 190683.3 | loss tokens/s: 132382.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.385888E-01 | lm-global-supervised-tokens: 1.166952E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.701154E-02 | lm-token-buffer-mib: 5.810299E+01 | global_load_balancing_loss: 1.000579E+00 | loss scale: 1.0 | grad norm: 0.052 | params norm: 11345.688 | params rms: 0.061433 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 5:38:57.080646 | finish at 2026-10-10 18:39:09 + [2026-10-08 13:01:36] iteration 4665/ 6855 | consumed samples: 597120 | elapsed time per iteration (ms): 84539.0 | throughput per GPU (TFLOP/s/GPU): 43.6 | MFU 13.99% | FLOPs estimator: workload-aware | logical tokens/s: 198455.1 | valid tokens/s: 198455.1 | physical tokens/s: 198906.4 | loss tokens/s: 141533.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.987514E-01 | lm-global-supervised-tokens: 1.196509E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.132368E-02 | lm-token-buffer-mib: 5.826462E+01 | global_load_balancing_loss: 9.994598E-01 | loss scale: 1.0 | grad norm: 0.062 | params norm: 11345.633 | params rms: 0.061433 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 3:25:40.405376 | finish at 2026-10-10 16:27:17 + [2026-10-08 13:03:05] iteration 4666/ 6855 | consumed samples: 597248 | elapsed time per iteration (ms): 88946.9 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.61% | FLOPs estimator: workload-aware | logical tokens/s: 188619.8 | valid tokens/s: 188619.8 | physical tokens/s: 189163.3 | loss tokens/s: 136081.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.475685E-01 | lm-global-supervised-tokens: 1.210399E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.261004E-02 | lm-token-buffer-mib: 5.806769E+01 | global_load_balancing_loss: 1.000447E+00 | loss scale: 1.0 | grad norm: 0.097 | params norm: 11345.577 | params rms: 0.061433 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 6:05:04.757598 | finish at 2026-10-10 19:08:10 + [2026-10-08 13:04:35] iteration 4667/ 6855 | consumed samples: 597376 | elapsed time per iteration (ms): 89565.5 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.47% | FLOPs estimator: workload-aware | logical tokens/s: 187317.3 | valid tokens/s: 187317.3 | physical tokens/s: 187863.5 | loss tokens/s: 131145.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.379669E-01 | lm-global-supervised-tokens: 1.174612E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.886208E-02 | lm-token-buffer-mib: 5.811215E+01 | global_load_balancing_loss: 1.000736E+00 | loss scale: 1.0 | grad norm: 0.099 | params norm: 11345.521 | params rms: 0.061432 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 6:26:09.415770 | finish at 2026-10-10 19:30:44 + [2026-10-08 13:06:01] iteration 4668/ 6855 | consumed samples: 597504 | elapsed time per iteration (ms): 86121.6 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.54% | FLOPs estimator: workload-aware | logical tokens/s: 194808.4 | valid tokens/s: 194808.4 | physical tokens/s: 195279.9 | loss tokens/s: 143241.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.224476E-01 | lm-global-supervised-tokens: 1.233623E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.153675E-02 | lm-token-buffer-mib: 5.805974E+01 | global_load_balancing_loss: 9.995677E-01 | loss scale: 1.0 | grad norm: 0.091 | params norm: 11345.466 | params rms: 0.061432 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 4:19:08.033908 | finish at 2026-10-10 17:25:09 + [2026-10-08 13:07:27] iteration 4669/ 6855 | consumed samples: 597632 | elapsed time per iteration (ms): 86379.6 | throughput per GPU (TFLOP/s/GPU): 43.3 | MFU 13.89% | FLOPs estimator: workload-aware | logical tokens/s: 194226.3 | valid tokens/s: 194226.3 | physical tokens/s: 194710.9 | loss tokens/s: 141956.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.758840E-01 | lm-global-supervised-tokens: 1.226215E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.472126E-02 | lm-token-buffer-mib: 5.809285E+01 | global_load_balancing_loss: 1.000773E+00 | loss scale: 1.0 | grad norm: 0.170 | params norm: 11345.410 | params rms: 0.061432 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 4:27:05.880755 | finish at 2026-10-10 17:34:33 + [2026-10-08 13:09:17] iteration 4670/ 6855 | consumed samples: 597760 | elapsed time per iteration (ms): 110008.3 | throughput per GPU (TFLOP/s/GPU): 33.4 | MFU 10.70% | FLOPs estimator: workload-aware | logical tokens/s: 152508.4 | valid tokens/s: 152508.4 | physical tokens/s: 152939.5 | loss tokens/s: 108079.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.364847E-01 | lm-global-supervised-tokens: 1.188968E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.182385E-02 | lm-token-buffer-mib: 5.818987E+01 | global_load_balancing_loss: 9.999793E-01 | loss scale: 1.0 | grad norm: 0.078 | params norm: 11345.353 | params rms: 0.061431 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 18:46:08.207014 | finish at 2026-10-11 07:55:25 + [2026-10-08 13:10:45] iteration 4671/ 6855 | consumed samples: 597888 | elapsed time per iteration (ms): 87506.8 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.58% | FLOPs estimator: workload-aware | logical tokens/s: 191724.6 | valid tokens/s: 191724.6 | physical tokens/s: 192127.1 | loss tokens/s: 142147.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.901645E-01 | lm-global-supervised-tokens: 1.243885E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.669454E-02 | lm-token-buffer-mib: 5.817177E+01 | global_load_balancing_loss: 9.998146E-01 | loss scale: 1.0 | grad norm: 0.116 | params norm: 11345.298 | params rms: 0.061431 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 5:05:14.943226 | finish at 2026-10-10 18:16:00 + [2026-10-08 13:12:13] iteration 4672/ 6855 | consumed samples: 598016 | elapsed time per iteration (ms): 88777.2 | throughput per GPU (TFLOP/s/GPU): 41.2 | MFU 13.22% | FLOPs estimator: workload-aware | logical tokens/s: 188980.5 | valid tokens/s: 188980.5 | physical tokens/s: 189484.6 | loss tokens/s: 142101.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.497564E-01 | lm-global-supervised-tokens: 1.261532E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.279623E-02 | lm-token-buffer-mib: 5.820953E+01 | global_load_balancing_loss: 9.994625E-01 | loss scale: 1.0 | grad norm: 0.051 | params norm: 11345.245 | params rms: 0.061431 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 5:50:00.575517 | finish at 2026-10-10 19:02:14 + [2026-10-08 13:13:41] iteration 4673/ 6855 | consumed samples: 598144 | elapsed time per iteration (ms): 87289.6 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.51% | FLOPs estimator: workload-aware | logical tokens/s: 192200.7 | valid tokens/s: 192200.6 | physical tokens/s: 192585.4 | loss tokens/s: 146269.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.894329E-01 | lm-global-supervised-tokens: 1.276781E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.186251E-02 | lm-token-buffer-mib: 5.806670E+01 | global_load_balancing_loss: 1.000080E+00 | loss scale: 1.0 | grad norm: 0.117 | params norm: 11345.190 | params rms: 0.061431 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 4:54:25.952752 | finish at 2026-10-10 18:08:07 + [2026-10-08 13:15:07] iteration 4674/ 6855 | consumed samples: 598272 | elapsed time per iteration (ms): 86705.2 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.66% | FLOPs estimator: workload-aware | logical tokens/s: 193497.1 | valid tokens/s: 193497.1 | physical tokens/s: 193942.6 | loss tokens/s: 141479.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.299760E-01 | lm-global-supervised-tokens: 1.226700E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.145296E-02 | lm-token-buffer-mib: 5.808395E+01 | global_load_balancing_loss: 1.000529E+00 | loss scale: 1.0 | grad norm: 0.066 | params norm: 11345.134 | params rms: 0.061430 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 4:31:43.950628 | finish at 2026-10-10 17:46:51 + [2026-10-08 13:16:39] iteration 4675/ 6855 | consumed samples: 598400 | elapsed time per iteration (ms): 91668.5 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.59% | FLOPs estimator: workload-aware | logical tokens/s: 183020.3 | valid tokens/s: 183020.3 | physical tokens/s: 183368.9 | loss tokens/s: 126267.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.518993E-01 | lm-global-supervised-tokens: 1.157473E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.280073E-02 | lm-token-buffer-mib: 5.824651E+01 | global_load_balancing_loss: 1.001631E+00 | loss scale: 1.0 | grad norm: 0.063 | params norm: 11345.079 | params rms: 0.061430 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 7:30:37.293501 | finish at 2026-10-10 20:47:16 + [2026-10-08 13:18:05] iteration 4676/ 6855 | consumed samples: 598528 | elapsed time per iteration (ms): 86242.6 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.70% | FLOPs estimator: workload-aware | logical tokens/s: 194534.8 | valid tokens/s: 194534.8 | physical tokens/s: 195210.3 | loss tokens/s: 137452.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.533664E-01 | lm-global-supervised-tokens: 1.185425E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.271289E-02 | lm-token-buffer-mib: 5.813316E+01 | global_load_balancing_loss: 1.001306E+00 | loss scale: 1.0 | grad norm: 0.078 | params norm: 11345.024 | params rms: 0.061430 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 4:12:02.549992 | finish at 2026-10-10 17:30:08 + [2026-10-08 13:19:32] iteration 4677/ 6855 | consumed samples: 598656 | elapsed time per iteration (ms): 87104.7 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.58% | FLOPs estimator: workload-aware | logical tokens/s: 192609.8 | valid tokens/s: 192609.8 | physical tokens/s: 193209.6 | loss tokens/s: 145259.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.552135E-01 | lm-global-supervised-tokens: 1.265280E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.330655E-02 | lm-token-buffer-mib: 5.806239E+01 | global_load_balancing_loss: 9.996600E-01 | loss scale: 1.0 | grad norm: 0.054 | params norm: 11344.969 | params rms: 0.061429 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 4:41:53.962017 | finish at 2026-10-10 18:01:26 + [2026-10-08 13:21:00] iteration 4678/ 6855 | consumed samples: 598784 | elapsed time per iteration (ms): 87801.7 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.66% | FLOPs estimator: workload-aware | logical tokens/s: 191080.7 | valid tokens/s: 191080.7 | physical tokens/s: 191527.9 | loss tokens/s: 142031.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.399049E-01 | lm-global-supervised-tokens: 1.247060E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.393731E-02 | lm-token-buffer-mib: 5.806425E+01 | global_load_balancing_loss: 9.986157E-01 | loss scale: 1.0 | grad norm: 0.039 | params norm: 11344.912 | params rms: 0.061429 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 5:05:44.373297 | finish at 2026-10-10 18:26:45 + [2026-10-08 13:22:26] iteration 4679/ 6855 | consumed samples: 598912 | elapsed time per iteration (ms): 85578.0 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.74% | FLOPs estimator: workload-aware | logical tokens/s: 196045.1 | valid tokens/s: 196045.1 | physical tokens/s: 196516.8 | loss tokens/s: 148257.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.408378E-01 | lm-global-supervised-tokens: 1.268758E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.342677E-02 | lm-token-buffer-mib: 5.804131E+01 | global_load_balancing_loss: 9.986246E-01 | loss scale: 1.0 | grad norm: 0.072 | params norm: 11344.855 | params rms: 0.061429 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 3:43:37.776917 | finish at 2026-10-10 17:06:04 + [2026-10-08 13:24:15] iteration 4680/ 6855 | consumed samples: 599040 | elapsed time per iteration (ms): 108909.1 | throughput per GPU (TFLOP/s/GPU): 34.3 | MFU 11.00% | FLOPs estimator: workload-aware | logical tokens/s: 154047.7 | valid tokens/s: 154047.7 | physical tokens/s: 154357.1 | loss tokens/s: 110928.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.842231E-01 | lm-global-supervised-tokens: 1.208108E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.910320E-02 | lm-token-buffer-mib: 5.811875E+01 | global_load_balancing_loss: 9.998611E-01 | loss scale: 1.0 | grad norm: 0.035 | params norm: 11344.799 | params rms: 0.061428 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 17:47:57.393222 | finish at 2026-10-11 07:12:12 +times across ranks (ms): + save-checkpoint: + rank 0: 25042.01 + rank 1: 25042.05 + rank 2: 25042.04 + rank 3: 25042.03 + rank 4: 25042.03 + rank 5: 25042.02 + rank 6: 25042.00 + rank 7: 25042.02 + rank 8: 25041.34 + rank 9: 25041.31 + rank 10: 25041.32 + rank 11: 25041.30 + rank 12: 25041.32 + rank 13: 25041.30 + rank 14: 25041.34 + rank 15: 25041.29 + rank 16: 25041.50 + rank 17: 25041.47 + rank 18: 25041.48 + rank 19: 25041.48 + rank 20: 25041.46 + rank 21: 25041.48 + rank 22: 25041.46 + rank 23: 25041.48 + rank 24: 25041.85 + rank 25: 25041.78 + rank 26: 25041.82 + rank 27: 25041.85 + rank 28: 25041.86 + rank 29: 25041.84 + rank 30: 25041.77 + rank 31: 25041.74 + rank 32: 25042.18 + rank 33: 25042.23 + rank 34: 25042.22 + rank 35: 25042.20 + rank 36: 25042.13 + rank 37: 25042.15 + rank 38: 25042.19 + rank 39: 25042.16 + rank 40: 25041.57 + rank 41: 25041.61 + rank 42: 25041.58 + rank 43: 25041.58 + rank 44: 25041.54 + rank 45: 25041.55 + rank 46: 25041.56 + rank 47: 25041.54 + rank 48: 25041.73 + rank 49: 25041.76 + rank 50: 25041.74 + rank 51: 25041.77 + rank 52: 25041.85 + rank 53: 25041.80 + rank 54: 25041.77 + rank 55: 25041.78 + rank 56: 25041.77 + rank 57: 25041.73 + rank 58: 25041.70 + rank 59: 25041.71 + rank 60: 25041.63 + rank 61: 25041.62 + rank 62: 25041.70 + rank 63: 25041.62 + [2026-10-08 13:26:08] iteration 4681/ 6855 | consumed samples: 599168 | elapsed time per iteration (ms): 88150.5 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.51% | FLOPs estimator: workload-aware | logical tokens/s: 190324.7 | valid tokens/s: 190324.7 | physical tokens/s: 190750.2 | loss tokens/s: 136010.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.850482E-01 | lm-global-supervised-tokens: 1.198942E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.108343E-02 | lm-token-buffer-mib: 5.813001E+01 | global_load_balancing_loss: 1.001103E+00 | loss scale: 1.0 | grad norm: 0.060 | params norm: 11344.745 | params rms: 0.061428 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 5:13:59.139443 | finish at 2026-10-10 18:40:07 + [2026-10-08 13:27:41] iteration 4682/ 6855 | consumed samples: 599296 | elapsed time per iteration (ms): 92658.5 | throughput per GPU (TFLOP/s/GPU): 39.7 | MFU 12.71% | FLOPs estimator: workload-aware | logical tokens/s: 181064.9 | valid tokens/s: 181064.9 | physical tokens/s: 181455.8 | loss tokens/s: 128970.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.627063E-01 | lm-global-supervised-tokens: 1.195022E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.219556E-02 | lm-token-buffer-mib: 5.820568E+01 | global_load_balancing_loss: 1.001230E+00 | loss scale: 1.0 | grad norm: 0.062 | params norm: 11344.691 | params rms: 0.061428 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 7:55:46.922477 | finish at 2026-10-10 21:23:28 + [2026-10-08 13:29:09] iteration 4683/ 6855 | consumed samples: 599424 | elapsed time per iteration (ms): 87740.5 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.79% | FLOPs estimator: workload-aware | logical tokens/s: 191214.1 | valid tokens/s: 191214.1 | physical tokens/s: 191552.2 | loss tokens/s: 139400.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.453172E-01 | lm-global-supervised-tokens: 1.223108E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.215490E-02 | lm-token-buffer-mib: 5.805799E+01 | global_load_balancing_loss: 1.001287E+00 | loss scale: 1.0 | grad norm: 0.065 | params norm: 11344.636 | params rms: 0.061428 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 4:56:12.262373 | finish at 2026-10-10 18:25:21 + [2026-10-08 13:30:34] iteration 4684/ 6855 | consumed samples: 599552 | elapsed time per iteration (ms): 85870.8 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.76% | FLOPs estimator: workload-aware | logical tokens/s: 195376.9 | valid tokens/s: 195376.9 | physical tokens/s: 195815.4 | loss tokens/s: 141003.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.907225E-01 | lm-global-supervised-tokens: 1.210811E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.307076E-02 | lm-token-buffer-mib: 5.812543E+01 | global_load_balancing_loss: 1.000621E+00 | loss scale: 1.0 | grad norm: 0.131 | params norm: 11344.582 | params rms: 0.061427 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 3:47:05.576199 | finish at 2026-10-10 17:17:40 + [2026-10-08 13:32:02] iteration 4685/ 6855 | consumed samples: 599680 | elapsed time per iteration (ms): 87699.6 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.44% | FLOPs estimator: workload-aware | logical tokens/s: 191302.5 | valid tokens/s: 191302.5 | physical tokens/s: 191759.8 | loss tokens/s: 141256.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.572591E-01 | lm-global-supervised-tokens: 1.238818E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.257862E-02 | lm-token-buffer-mib: 5.818134E+01 | global_load_balancing_loss: 9.999996E-01 | loss scale: 1.0 | grad norm: 0.194 | params norm: 11344.527 | params rms: 0.061427 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 4:51:48.215773 | finish at 2026-10-10 18:23:50 + [2026-10-08 13:33:29] iteration 4686/ 6855 | consumed samples: 599808 | elapsed time per iteration (ms): 86753.1 | throughput per GPU (TFLOP/s/GPU): 43.3 | MFU 13.88% | FLOPs estimator: workload-aware | logical tokens/s: 193389.7 | valid tokens/s: 193389.7 | physical tokens/s: 193828.7 | loss tokens/s: 136290.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.893327E-01 | lm-global-supervised-tokens: 1.182362E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.728573E-02 | lm-token-buffer-mib: 5.838186E+01 | global_load_balancing_loss: 1.000312E+00 | loss scale: 1.0 | grad norm: 0.295 | params norm: 11344.474 | params rms: 0.061427 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 4:16:07.395119 | finish at 2026-10-10 17:49:36 + [2026-10-08 13:34:53] iteration 4687/ 6855 | consumed samples: 599936 | elapsed time per iteration (ms): 84560.3 | throughput per GPU (TFLOP/s/GPU): 43.2 | MFU 13.83% | FLOPs estimator: workload-aware | logical tokens/s: 198405.0 | valid tokens/s: 198404.9 | physical tokens/s: 198737.3 | loss tokens/s: 151552.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.251985E-01 | lm-global-supervised-tokens: 1.281534E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.205785E-02 | lm-token-buffer-mib: 5.810139E+01 | global_load_balancing_loss: 1.000807E+00 | loss scale: 1.0 | grad norm: 0.183 | params norm: 11344.418 | params rms: 0.061426 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 2:55:26.711517 | finish at 2026-10-10 16:30:20 + [2026-10-08 13:36:22] iteration 4688/ 6855 | consumed samples: 600064 | elapsed time per iteration (ms): 88979.8 | throughput per GPU (TFLOP/s/GPU): 40.7 | MFU 13.06% | FLOPs estimator: workload-aware | logical tokens/s: 188550.6 | valid tokens/s: 188550.6 | physical tokens/s: 189036.5 | loss tokens/s: 140252.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.495139E-01 | lm-global-supervised-tokens: 1.247968E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.157869E-02 | lm-token-buffer-mib: 5.822939E+01 | global_load_balancing_loss: 1.000261E+00 | loss scale: 1.0 | grad norm: 0.230 | params norm: 11344.366 | params rms: 0.061426 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 5:33:39.297868 | finish at 2026-10-10 19:10:02 + [2026-10-08 13:37:47] iteration 4689/ 6855 | consumed samples: 600192 | elapsed time per iteration (ms): 84666.0 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.68% | FLOPs estimator: workload-aware | logical tokens/s: 198157.2 | valid tokens/s: 198157.2 | physical tokens/s: 198663.6 | loss tokens/s: 149507.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.577147E-01 | lm-global-supervised-tokens: 1.265822E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.151586E-02 | lm-token-buffer-mib: 5.808408E+01 | global_load_balancing_loss: 1.000408E+00 | loss scale: 1.0 | grad norm: 0.222 | params norm: 11344.315 | params rms: 0.061426 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 2:56:26.523226 | finish at 2026-10-10 16:34:14 + [2026-10-08 13:39:34] iteration 4690/ 6855 | consumed samples: 600320 | elapsed time per iteration (ms): 106823.7 | throughput per GPU (TFLOP/s/GPU): 34.3 | MFU 11.00% | FLOPs estimator: workload-aware | logical tokens/s: 157054.3 | valid tokens/s: 157054.3 | physical tokens/s: 157484.9 | loss tokens/s: 117315.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.830708E-01 | lm-global-supervised-tokens: 1.253211E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.161345E-02 | lm-token-buffer-mib: 5.830798E+01 | global_load_balancing_loss: 1.000127E+00 | loss scale: 1.0 | grad norm: 0.230 | params norm: 11344.263 | params rms: 0.061425 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 16:14:33.333622 | finish at 2026-10-11 05:54:07 + [2026-10-08 13:41:04] iteration 4691/ 6855 | consumed samples: 600448 | elapsed time per iteration (ms): 89737.5 | throughput per GPU (TFLOP/s/GPU): 41.6 | MFU 13.34% | FLOPs estimator: workload-aware | logical tokens/s: 186958.3 | valid tokens/s: 186958.3 | physical tokens/s: 187273.7 | loss tokens/s: 135917.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.827753E-01 | lm-global-supervised-tokens: 1.219690E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 5.814164E-02 | lm-token-buffer-mib: 5.821725E+01 | global_load_balancing_loss: 9.991150E-01 | loss scale: 1.0 | grad norm: 0.132 | params norm: 11344.212 | params rms: 0.061425 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 5:56:31.935966 | finish at 2026-10-10 19:37:36 + [2026-10-08 13:42:28] iteration 4692/ 6855 | consumed samples: 600576 | elapsed time per iteration (ms): 84758.5 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.65% | FLOPs estimator: workload-aware | logical tokens/s: 197941.0 | valid tokens/s: 197941.0 | physical tokens/s: 198713.8 | loss tokens/s: 146710.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.425140E-01 | lm-global-supervised-tokens: 1.243497E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.131166E-02 | lm-token-buffer-mib: 5.807350E+01 | global_load_balancing_loss: 9.995644E-01 | loss scale: 1.0 | grad norm: 0.212 | params norm: 11344.160 | params rms: 0.061425 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 2:55:32.654280 | finish at 2026-10-10 16:38:01 + [2026-10-08 13:43:56] iteration 4693/ 6855 | consumed samples: 600704 | elapsed time per iteration (ms): 87668.7 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.49% | FLOPs estimator: workload-aware | logical tokens/s: 191370.6 | valid tokens/s: 191370.6 | physical tokens/s: 191761.9 | loss tokens/s: 144950.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.734866E-01 | lm-global-supervised-tokens: 1.270765E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.396661E-02 | lm-token-buffer-mib: 5.817281E+01 | global_load_balancing_loss: 9.999958E-01 | loss scale: 1.0 | grad norm: 0.051 | params norm: 11344.108 | params rms: 0.061425 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 4:38:59.743274 | finish at 2026-10-10 18:22:56 + [2026-10-08 13:45:25] iteration 4694/ 6855 | consumed samples: 600832 | elapsed time per iteration (ms): 88566.7 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.57% | FLOPs estimator: workload-aware | logical tokens/s: 189430.2 | valid tokens/s: 189430.2 | physical tokens/s: 189745.5 | loss tokens/s: 141440.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.649430E-01 | lm-global-supervised-tokens: 1.252695E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.499135E-02 | lm-token-buffer-mib: 5.806207E+01 | global_load_balancing_loss: 9.997725E-01 | loss scale: 1.0 | grad norm: 0.121 | params norm: 11344.057 | params rms: 0.061424 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 5:09:52.722642 | finish at 2026-10-10 18:55:17 + [2026-10-08 13:46:55] iteration 4695/ 6855 | consumed samples: 600960 | elapsed time per iteration (ms): 89988.1 | throughput per GPU (TFLOP/s/GPU): 41.5 | MFU 13.29% | FLOPs estimator: workload-aware | logical tokens/s: 186437.8 | valid tokens/s: 186437.8 | physical tokens/s: 186732.2 | loss tokens/s: 141062.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.528211E-01 | lm-global-supervised-tokens: 1.269398E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.592533E-02 | lm-token-buffer-mib: 5.811388E+01 | global_load_balancing_loss: 9.987876E-01 | loss scale: 1.0 | grad norm: 0.168 | params norm: 11344.003 | params rms: 0.061424 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 5:59:34.322891 | finish at 2026-10-10 19:46:29 + [2026-10-08 13:48:23] iteration 4696/ 6855 | consumed samples: 601088 | elapsed time per iteration (ms): 88102.0 | throughput per GPU (TFLOP/s/GPU): 43.2 | MFU 13.85% | FLOPs estimator: workload-aware | logical tokens/s: 190428.6 | valid tokens/s: 190428.6 | physical tokens/s: 190848.8 | loss tokens/s: 138549.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.883419E-01 | lm-global-supervised-tokens: 1.220652E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.303906E-02 | lm-token-buffer-mib: 5.814149E+01 | global_load_balancing_loss: 1.001312E+00 | loss scale: 1.0 | grad norm: 0.131 | params norm: 11343.950 | params rms: 0.061424 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 4:50:12.243218 | finish at 2026-10-10 18:38:35 + [2026-10-08 13:49:49] iteration 4697/ 6855 | consumed samples: 601216 | elapsed time per iteration (ms): 86199.5 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.57% | FLOPs estimator: workload-aware | logical tokens/s: 194631.4 | valid tokens/s: 194631.4 | physical tokens/s: 195136.3 | loss tokens/s: 146604.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.580919E-01 | lm-global-supervised-tokens: 1.263722E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.482231E-02 | lm-token-buffer-mib: 5.819920E+01 | global_load_balancing_loss: 1.001878E+00 | loss scale: 1.0 | grad norm: 0.140 | params norm: 11343.896 | params rms: 0.061424 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 3:40:18.608647 | finish at 2026-10-10 17:30:08 + [2026-10-08 13:51:15] iteration 4698/ 6855 | consumed samples: 601344 | elapsed time per iteration (ms): 85888.1 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.77% | FLOPs estimator: workload-aware | logical tokens/s: 195337.9 | valid tokens/s: 195337.9 | physical tokens/s: 195736.7 | loss tokens/s: 143981.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.598543E-01 | lm-global-supervised-tokens: 1.236629E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.858530E-02 | lm-token-buffer-mib: 5.821861E+01 | global_load_balancing_loss: 1.001670E+00 | loss scale: 1.0 | grad norm: 0.093 | params norm: 11343.842 | params rms: 0.061423 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 3:27:40.703501 | finish at 2026-10-10 17:18:56 + [2026-10-08 13:52:40] iteration 4699/ 6855 | consumed samples: 601472 | elapsed time per iteration (ms): 85200.2 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.67% | FLOPs estimator: workload-aware | logical tokens/s: 196915.1 | valid tokens/s: 196915.1 | physical tokens/s: 197468.7 | loss tokens/s: 148608.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.173804E-01 | lm-global-supervised-tokens: 1.266150E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.329076E-02 | lm-token-buffer-mib: 5.820681E+01 | global_load_balancing_loss: 1.000422E+00 | loss scale: 1.0 | grad norm: 0.245 | params norm: 11343.788 | params rms: 0.061423 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 3:01:31.579457 | finish at 2026-10-10 16:54:12 + [2026-10-08 13:54:26] iteration 4700/ 6855 | consumed samples: 601600 | elapsed time per iteration (ms): 106283.2 | throughput per GPU (TFLOP/s/GPU): 34.8 | MFU 11.14% | FLOPs estimator: workload-aware | logical tokens/s: 157853.5 | valid tokens/s: 157853.5 | physical tokens/s: 158194.3 | loss tokens/s: 117330.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.994197E-01 | lm-global-supervised-tokens: 1.247023E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.412444E-02 | lm-token-buffer-mib: 5.802751E+01 | global_load_balancing_loss: 9.991100E-01 | loss scale: 1.0 | grad norm: 0.039 | params norm: 11343.736 | params rms: 0.061423 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 15:37:20.388024 | finish at 2026-10-11 05:31:47 + [2026-10-08 13:55:54] iteration 4701/ 6855 | consumed samples: 601728 | elapsed time per iteration (ms): 88103.0 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.66% | FLOPs estimator: workload-aware | logical tokens/s: 190426.8 | valid tokens/s: 190426.8 | physical tokens/s: 190967.5 | loss tokens/s: 140649.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.481691E-01 | lm-global-supervised-tokens: 1.239167E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.235165E-02 | lm-token-buffer-mib: 5.812083E+01 | global_load_balancing_loss: 9.989470E-01 | loss scale: 1.0 | grad norm: 0.249 | params norm: 11343.683 | params rms: 0.061422 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 4:42:53.950171 | finish at 2026-10-10 18:38:48 + [2026-10-08 13:57:22] iteration 4702/ 6855 | consumed samples: 601856 | elapsed time per iteration (ms): 87365.5 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.66% | FLOPs estimator: workload-aware | logical tokens/s: 192034.4 | valid tokens/s: 192034.4 | physical tokens/s: 192491.7 | loss tokens/s: 137730.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.005321E-01 | lm-global-supervised-tokens: 1.203285E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.502424E-02 | lm-token-buffer-mib: 5.807885E+01 | global_load_balancing_loss: 9.999186E-01 | loss scale: 1.0 | grad norm: 0.053 | params norm: 11343.630 | params rms: 0.061422 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 4:14:57.920416 | finish at 2026-10-10 18:12:20 + [2026-10-08 13:58:47] iteration 4703/ 6855 | consumed samples: 601984 | elapsed time per iteration (ms): 85277.3 | throughput per GPU (TFLOP/s/GPU): 43.6 | MFU 13.98% | FLOPs estimator: workload-aware | logical tokens/s: 196737.1 | valid tokens/s: 196737.1 | physical tokens/s: 197266.0 | loss tokens/s: 140867.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.504532E-01 | lm-global-supervised-tokens: 1.201282E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.537293E-02 | lm-token-buffer-mib: 5.837440E+01 | global_load_balancing_loss: 1.000136E+00 | loss scale: 1.0 | grad norm: 0.146 | params norm: 11343.578 | params rms: 0.061422 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 2:58:36.806295 | finish at 2026-10-10 16:57:24 + [2026-10-08 14:00:11] iteration 4704/ 6855 | consumed samples: 602112 | elapsed time per iteration (ms): 83731.4 | throughput per GPU (TFLOP/s/GPU): 43.3 | MFU 13.86% | FLOPs estimator: workload-aware | logical tokens/s: 200368.7 | valid tokens/s: 200368.7 | physical tokens/s: 200928.0 | loss tokens/s: 148656.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.507936E-01 | lm-global-supervised-tokens: 1.244725E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.169962E-02 | lm-token-buffer-mib: 5.804097E+01 | global_load_balancing_loss: 1.000358E+00 | loss scale: 1.0 | grad norm: 0.131 | params norm: 11343.528 | params rms: 0.061422 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 2:01:46.253736 | finish at 2026-10-10 16:01:57 + [2026-10-08 14:01:34] iteration 4705/ 6855 | consumed samples: 602240 | elapsed time per iteration (ms): 83217.8 | throughput per GPU (TFLOP/s/GPU): 43.3 | MFU 13.88% | FLOPs estimator: workload-aware | logical tokens/s: 201605.9 | valid tokens/s: 201605.9 | physical tokens/s: 202084.6 | loss tokens/s: 156265.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.864136E-01 | lm-global-supervised-tokens: 1.300409E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.536916E-02 | lm-token-buffer-mib: 5.810547E+01 | global_load_balancing_loss: 9.993382E-01 | loss scale: 1.0 | grad norm: 0.081 | params norm: 11343.477 | params rms: 0.061421 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 1:41:58.352318 | finish at 2026-10-10 15:43:32 + [2026-10-08 14:03:01] iteration 4706/ 6855 | consumed samples: 602368 | elapsed time per iteration (ms): 86974.4 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.78% | FLOPs estimator: workload-aware | logical tokens/s: 192897.5 | valid tokens/s: 192897.5 | physical tokens/s: 193348.4 | loss tokens/s: 141991.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.909323E-01 | lm-global-supervised-tokens: 1.234964E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.342999E-02 | lm-token-buffer-mib: 5.823149E+01 | global_load_balancing_loss: 1.001170E+00 | loss scale: 1.0 | grad norm: 0.114 | params norm: 11343.425 | params rms: 0.061421 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 3:55:08.085092 | finish at 2026-10-10 17:58:09 + [2026-10-08 14:04:28] iteration 4707/ 6855 | consumed samples: 602496 | elapsed time per iteration (ms): 86713.0 | throughput per GPU (TFLOP/s/GPU): 43.5 | MFU 13.95% | FLOPs estimator: workload-aware | logical tokens/s: 193479.4 | valid tokens/s: 193479.3 | physical tokens/s: 193970.6 | loss tokens/s: 139034.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.355026E-01 | lm-global-supervised-tokens: 1.205609E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.943363E-02 | lm-token-buffer-mib: 5.803518E+01 | global_load_balancing_loss: 1.001443E+00 | loss scale: 1.0 | grad norm: 0.072 | params norm: 11343.373 | params rms: 0.061421 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 3:44:19.567657 | finish at 2026-10-10 17:48:47 + [2026-10-08 14:06:01] iteration 4708/ 6855 | consumed samples: 602624 | elapsed time per iteration (ms): 93284.2 | throughput per GPU (TFLOP/s/GPU): 40.6 | MFU 13.00% | FLOPs estimator: workload-aware | logical tokens/s: 179850.6 | valid tokens/s: 179850.6 | physical tokens/s: 180200.2 | loss tokens/s: 137442.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.555204E-01 | lm-global-supervised-tokens: 1.282119E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.174741E-02 | lm-token-buffer-mib: 5.814803E+01 | global_load_balancing_loss: 1.001984E+00 | loss scale: 1.0 | grad norm: 0.087 | params norm: 11343.319 | params rms: 0.061420 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 7:38:01.077482 | finish at 2026-10-10 21:44:02 + [2026-10-08 14:07:28] iteration 4709/ 6855 | consumed samples: 602752 | elapsed time per iteration (ms): 86515.1 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.51% | FLOPs estimator: workload-aware | logical tokens/s: 193922.3 | valid tokens/s: 193922.3 | physical tokens/s: 194350.1 | loss tokens/s: 147695.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.504354E-01 | lm-global-supervised-tokens: 1.277789E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.154298E-02 | lm-token-buffer-mib: 5.808620E+01 | global_load_balancing_loss: 1.000614E+00 | loss scale: 1.0 | grad norm: 0.089 | params norm: 11343.268 | params rms: 0.061420 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 3:34:21.442978 | finish at 2026-10-10 17:41:49 + [2026-10-08 14:09:15] iteration 4710/ 6855 | consumed samples: 602880 | elapsed time per iteration (ms): 107950.9 | throughput per GPU (TFLOP/s/GPU): 34.2 | MFU 10.95% | FLOPs estimator: workload-aware | logical tokens/s: 155415.2 | valid tokens/s: 155415.2 | physical tokens/s: 155706.7 | loss tokens/s: 109721.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.468478E-01 | lm-global-supervised-tokens: 1.184455E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.166714E-02 | lm-token-buffer-mib: 5.810728E+01 | global_load_balancing_loss: 9.995701E-01 | loss scale: 1.0 | grad norm: 0.099 | params norm: 11343.218 | params rms: 0.061420 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 16:19:14.616230 | finish at 2026-10-11 06:28:30 + [2026-10-08 14:10:41] iteration 4711/ 6855 | consumed samples: 603008 | elapsed time per iteration (ms): 85256.3 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.79% | FLOPs estimator: workload-aware | logical tokens/s: 196785.1 | valid tokens/s: 196785.1 | physical tokens/s: 197246.7 | loss tokens/s: 144415.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.386738E-01 | lm-global-supervised-tokens: 1.231231E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.129846E-02 | lm-token-buffer-mib: 5.805533E+01 | global_load_balancing_loss: 9.983250E-01 | loss scale: 1.0 | grad norm: 0.257 | params norm: 11343.167 | params rms: 0.061420 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 2:46:29.523499 | finish at 2026-10-10 16:57:10 + [2026-10-08 14:12:06] iteration 4712/ 6855 | consumed samples: 603136 | elapsed time per iteration (ms): 85262.6 | throughput per GPU (TFLOP/s/GPU): 43.5 | MFU 13.93% | FLOPs estimator: workload-aware | logical tokens/s: 196771.2 | valid tokens/s: 196771.2 | physical tokens/s: 197357.0 | loss tokens/s: 146690.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.606001E-01 | lm-global-supervised-tokens: 1.250722E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.256675E-02 | lm-token-buffer-mib: 5.820566E+01 | global_load_balancing_loss: 9.986159E-01 | loss scale: 1.0 | grad norm: 0.062 | params norm: 11343.117 | params rms: 0.061419 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 2:45:17.676575 | finish at 2026-10-10 16:57:24 + [2026-10-08 14:13:35] iteration 4713/ 6855 | consumed samples: 603264 | elapsed time per iteration (ms): 89123.7 | throughput per GPU (TFLOP/s/GPU): 43.1 | MFU 13.80% | FLOPs estimator: workload-aware | logical tokens/s: 188245.6 | valid tokens/s: 188245.6 | physical tokens/s: 188427.1 | loss tokens/s: 137410.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.014152E-01 | lm-global-supervised-tokens: 1.224655E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.106874E-02 | lm-token-buffer-mib: 5.807301E+01 | global_load_balancing_loss: 9.997690E-01 | loss scale: 1.0 | grad norm: 0.122 | params norm: 11343.065 | params rms: 0.061419 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 5:01:43.013198 | finish at 2026-10-10 19:15:18 + [2026-10-08 14:15:05] iteration 4714/ 6855 | consumed samples: 603392 | elapsed time per iteration (ms): 89566.1 | throughput per GPU (TFLOP/s/GPU): 41.5 | MFU 13.31% | FLOPs estimator: workload-aware | logical tokens/s: 187315.6 | valid tokens/s: 187315.6 | physical tokens/s: 187671.9 | loss tokens/s: 135718.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.711623E-01 | lm-global-supervised-tokens: 1.215575E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.368771E-02 | lm-token-buffer-mib: 5.813636E+01 | global_load_balancing_loss: 1.000064E+00 | loss scale: 1.0 | grad norm: 0.104 | params norm: 11343.012 | params rms: 0.061419 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 5:16:01.010149 | finish at 2026-10-10 19:31:06 + [2026-10-08 14:16:32] iteration 4715/ 6855 | consumed samples: 603520 | elapsed time per iteration (ms): 87747.1 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.56% | FLOPs estimator: workload-aware | logical tokens/s: 191199.4 | valid tokens/s: 191199.4 | physical tokens/s: 191614.6 | loss tokens/s: 137868.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.762943E-01 | lm-global-supervised-tokens: 1.209758E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.388150E-02 | lm-token-buffer-mib: 5.813597E+01 | global_load_balancing_loss: 1.002206E+00 | loss scale: 1.0 | grad norm: 0.084 | params norm: 11342.958 | params rms: 0.061418 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 4:09:38.692203 | finish at 2026-10-10 18:26:11 + [2026-10-08 14:18:00] iteration 4716/ 6855 | consumed samples: 603648 | elapsed time per iteration (ms): 87542.8 | throughput per GPU (TFLOP/s/GPU): 41.6 | MFU 13.32% | FLOPs estimator: workload-aware | logical tokens/s: 191645.3 | valid tokens/s: 191645.3 | physical tokens/s: 192185.3 | loss tokens/s: 141243.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.064032E-01 | lm-global-supervised-tokens: 1.236486E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.156476E-02 | lm-token-buffer-mib: 5.813111E+01 | global_load_balancing_loss: 1.001385E+00 | loss scale: 1.0 | grad norm: 0.179 | params norm: 11342.905 | params rms: 0.061418 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 4:00:54.112839 | finish at 2026-10-10 18:18:54 + [2026-10-08 14:19:23] iteration 4717/ 6855 | consumed samples: 603776 | elapsed time per iteration (ms): 83294.0 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.61% | FLOPs estimator: workload-aware | logical tokens/s: 201421.3 | valid tokens/s: 201421.3 | physical tokens/s: 202111.7 | loss tokens/s: 153683.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.161380E-01 | lm-global-supervised-tokens: 1.280092E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.547159E-02 | lm-token-buffer-mib: 5.821433E+01 | global_load_balancing_loss: 1.001438E+00 | loss scale: 1.0 | grad norm: 0.154 | params norm: 11342.853 | params rms: 0.061418 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 1:28:02.566201 | finish at 2026-10-10 15:47:26 + [2026-10-08 14:20:55] iteration 4718/ 6855 | consumed samples: 603904 | elapsed time per iteration (ms): 91491.7 | throughput per GPU (TFLOP/s/GPU): 41.2 | MFU 13.20% | FLOPs estimator: workload-aware | logical tokens/s: 183373.6 | valid tokens/s: 183373.6 | physical tokens/s: 183701.3 | loss tokens/s: 133312.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.703952E-01 | lm-global-supervised-tokens: 1.219702E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.189442E-02 | lm-token-buffer-mib: 5.803986E+01 | global_load_balancing_loss: 9.997558E-01 | loss scale: 1.0 | grad norm: 0.111 | params norm: 11342.800 | params rms: 0.061418 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 6:18:37.857017 | finish at 2026-10-10 20:39:33 + [2026-10-08 14:22:21] iteration 4719/ 6855 | consumed samples: 604032 | elapsed time per iteration (ms): 86571.8 | throughput per GPU (TFLOP/s/GPU): 43.5 | MFU 13.94% | FLOPs estimator: workload-aware | logical tokens/s: 193795.1 | valid tokens/s: 193795.1 | physical tokens/s: 194198.8 | loss tokens/s: 139543.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.254570E-01 | lm-global-supervised-tokens: 1.208055E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.302849E-02 | lm-token-buffer-mib: 5.811172E+01 | global_load_balancing_loss: 9.999457E-01 | loss scale: 1.0 | grad norm: 0.060 | params norm: 11342.746 | params rms: 0.061417 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 3:21:57.413166 | finish at 2026-10-10 17:44:19 + [2026-10-08 14:24:10] iteration 4720/ 6855 | consumed samples: 604160 | elapsed time per iteration (ms): 108948.9 | throughput per GPU (TFLOP/s/GPU): 34.0 | MFU 10.88% | FLOPs estimator: workload-aware | logical tokens/s: 153991.4 | valid tokens/s: 153991.3 | physical tokens/s: 154531.2 | loss tokens/s: 110894.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.378930E-01 | lm-global-supervised-tokens: 1.208185E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.243849E-02 | lm-token-buffer-mib: 5.851997E+01 | global_load_balancing_loss: 1.000689E+00 | loss scale: 1.0 | grad norm: 0.127 | params norm: 11342.693 | params rms: 0.061417 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 16:36:45.828167 | finish at 2026-10-11 07:00:56 +times across ranks (ms): + save-checkpoint: + rank 0: 25178.40 + rank 1: 25178.47 + rank 2: 25178.47 + rank 3: 25178.44 + rank 4: 25178.41 + rank 5: 25178.40 + rank 6: 25178.38 + rank 7: 25178.40 + rank 8: 25176.64 + rank 9: 25176.61 + rank 10: 25176.62 + rank 11: 25176.60 + rank 12: 25176.60 + rank 13: 25176.63 + rank 14: 25176.59 + rank 15: 25176.62 + rank 16: 25178.50 + rank 17: 25178.51 + rank 18: 25178.50 + rank 19: 25178.49 + rank 20: 25178.48 + rank 21: 25178.52 + rank 22: 25178.50 + rank 23: 25178.48 + rank 24: 25178.47 + rank 25: 25178.45 + rank 26: 25178.44 + rank 27: 25178.45 + rank 28: 25178.41 + rank 29: 25178.47 + rank 30: 25178.41 + rank 31: 25178.43 + rank 32: 25180.39 + rank 33: 25180.55 + rank 34: 25180.42 + rank 35: 25180.40 + rank 36: 25180.36 + rank 37: 25180.38 + rank 38: 25180.39 + rank 39: 25180.38 + rank 40: 25180.30 + rank 41: 25180.29 + rank 42: 25180.29 + rank 43: 25180.29 + rank 44: 25180.31 + rank 45: 25180.29 + rank 46: 25180.28 + rank 47: 25180.28 + rank 48: 25178.63 + rank 49: 25178.62 + rank 50: 25178.57 + rank 51: 25178.60 + rank 52: 25178.61 + rank 53: 25178.61 + rank 54: 25178.59 + rank 55: 25178.61 + rank 56: 25179.08 + rank 57: 25179.07 + rank 58: 25179.08 + rank 59: 25179.05 + rank 60: 25179.07 + rank 61: 25179.07 + rank 62: 25179.06 + rank 63: 25179.06 + [2026-10-08 14:26:19] iteration 4721/ 6855 | consumed samples: 604288 | elapsed time per iteration (ms): 101713.9 | throughput per GPU (TFLOP/s/GPU): 37.0 | MFU 11.84% | FLOPs estimator: workload-aware | logical tokens/s: 164945.1 | valid tokens/s: 164945.1 | physical tokens/s: 165218.0 | loss tokens/s: 117277.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.647083E-01 | lm-global-supervised-tokens: 1.192870E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.338249E+00 | lm-token-buffer-mib: 5.808467E+01 | global_load_balancing_loss: 1.001186E+00 | loss scale: 1.0 | grad norm: 0.040 | params norm: 11342.642 | params rms: 0.061417 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 12:17:37.424123 | finish at 2026-10-11 02:43:57 + [2026-10-08 14:27:52] iteration 4722/ 6855 | consumed samples: 604416 | elapsed time per iteration (ms): 92762.4 | throughput per GPU (TFLOP/s/GPU): 40.1 | MFU 12.86% | FLOPs estimator: workload-aware | logical tokens/s: 180862.1 | valid tokens/s: 180862.1 | physical tokens/s: 181332.8 | loss tokens/s: 133946.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.746446E-01 | lm-global-supervised-tokens: 1.242516E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.240066E-02 | lm-token-buffer-mib: 5.812704E+01 | global_load_balancing_loss: 9.987734E-01 | loss scale: 1.0 | grad norm: 0.075 | params norm: 11342.590 | params rms: 0.061416 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 6:57:42.169516 | finish at 2026-10-10 21:25:34 + [2026-10-08 14:29:23] iteration 4723/ 6855 | consumed samples: 604544 | elapsed time per iteration (ms): 90652.5 | throughput per GPU (TFLOP/s/GPU): 41.3 | MFU 13.23% | FLOPs estimator: workload-aware | logical tokens/s: 185071.1 | valid tokens/s: 185071.1 | physical tokens/s: 185507.3 | loss tokens/s: 135963.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.603566E-01 | lm-global-supervised-tokens: 1.232540E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.447647E-02 | lm-token-buffer-mib: 5.803891E+01 | global_load_balancing_loss: 9.985885E-01 | loss scale: 1.0 | grad norm: 0.040 | params norm: 11342.539 | params rms: 0.061416 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 5:41:11.199964 | finish at 2026-10-10 20:10:34 + [2026-10-08 14:30:45] iteration 4724/ 6855 | consumed samples: 604672 | elapsed time per iteration (ms): 82221.6 | throughput per GPU (TFLOP/s/GPU): 44.2 | MFU 14.17% | FLOPs estimator: workload-aware | logical tokens/s: 204048.5 | valid tokens/s: 204048.5 | physical tokens/s: 204558.4 | loss tokens/s: 157576.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.677363E-01 | lm-global-supervised-tokens: 1.295623E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.935270E-02 | lm-token-buffer-mib: 5.830666E+01 | global_load_balancing_loss: 9.986721E-01 | loss scale: 1.0 | grad norm: 0.050 | params norm: 11342.487 | params rms: 0.061416 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 0:40:14.250550 | finish at 2026-10-10 15:10:59 + [2026-10-08 14:32:11] iteration 4725/ 6855 | consumed samples: 604800 | elapsed time per iteration (ms): 86666.0 | throughput per GPU (TFLOP/s/GPU): 43.5 | MFU 13.96% | FLOPs estimator: workload-aware | logical tokens/s: 193583.8 | valid tokens/s: 193583.8 | physical tokens/s: 194020.7 | loss tokens/s: 142922.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.917517E-01 | lm-global-supervised-tokens: 1.238656E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.289976E-02 | lm-token-buffer-mib: 5.808141E+01 | global_load_balancing_loss: 9.992920E-01 | loss scale: 1.0 | grad norm: 0.044 | params norm: 11342.436 | params rms: 0.061416 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 3:16:38.669651 | finish at 2026-10-10 17:48:50 + [2026-10-08 14:33:35] iteration 4726/ 6855 | consumed samples: 604928 | elapsed time per iteration (ms): 83170.0 | throughput per GPU (TFLOP/s/GPU): 43.1 | MFU 13.82% | FLOPs estimator: workload-aware | logical tokens/s: 201721.1 | valid tokens/s: 201721.1 | physical tokens/s: 202655.5 | loss tokens/s: 151856.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.394878E-01 | lm-global-supervised-tokens: 1.262991E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.598409E-02 | lm-token-buffer-mib: 5.808376E+01 | global_load_balancing_loss: 1.001837E+00 | loss scale: 1.0 | grad norm: 0.084 | params norm: 11342.386 | params rms: 0.061415 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 1:11:08.988028 | finish at 2026-10-10 15:44:44 + [2026-10-08 14:35:03] iteration 4727/ 6855 | consumed samples: 605056 | elapsed time per iteration (ms): 88672.9 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.48% | FLOPs estimator: workload-aware | logical tokens/s: 189202.8 | valid tokens/s: 189202.7 | physical tokens/s: 189574.3 | loss tokens/s: 139911.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.419536E-01 | lm-global-supervised-tokens: 1.240632E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.264540E-02 | lm-token-buffer-mib: 5.808011E+01 | global_load_balancing_loss: 1.000704E+00 | loss scale: 1.0 | grad norm: 0.038 | params norm: 11342.334 | params rms: 0.061415 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 4:24:56.007221 | finish at 2026-10-10 18:59:59 + [2026-10-08 14:36:31] iteration 4728/ 6855 | consumed samples: 605184 | elapsed time per iteration (ms): 87305.7 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.56% | FLOPs estimator: workload-aware | logical tokens/s: 192165.5 | valid tokens/s: 192165.5 | physical tokens/s: 192544.9 | loss tokens/s: 135788.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.405125E-01 | lm-global-supervised-tokens: 1.185507E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.126569E-02 | lm-token-buffer-mib: 5.810942E+01 | global_load_balancing_loss: 1.001800E+00 | loss scale: 1.0 | grad norm: 0.094 | params norm: 11342.282 | params rms: 0.061415 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 3:34:59.248884 | finish at 2026-10-10 18:11:30 + [2026-10-08 14:37:57] iteration 4729/ 6855 | consumed samples: 605312 | elapsed time per iteration (ms): 85936.5 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.79% | FLOPs estimator: workload-aware | logical tokens/s: 195227.6 | valid tokens/s: 195227.6 | physical tokens/s: 195746.0 | loss tokens/s: 150841.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.678534E-01 | lm-global-supervised-tokens: 1.296277E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.140167E-02 | lm-token-buffer-mib: 5.812652E+01 | global_load_balancing_loss: 1.001159E+00 | loss scale: 1.0 | grad norm: 0.048 | params norm: 11342.228 | params rms: 0.061414 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 2:45:01.055418 | finish at 2026-10-10 17:22:58 + [2026-10-08 14:39:44] iteration 4730/ 6855 | consumed samples: 605440 | elapsed time per iteration (ms): 107212.4 | throughput per GPU (TFLOP/s/GPU): 34.3 | MFU 10.99% | FLOPs estimator: workload-aware | logical tokens/s: 156485.5 | valid tokens/s: 156485.5 | physical tokens/s: 156991.6 | loss tokens/s: 116417.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.463122E-01 | lm-global-supervised-tokens: 1.248145E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.168026E-02 | lm-token-buffer-mib: 5.814382E+01 | global_load_balancing_loss: 9.996462E-01 | loss scale: 1.0 | grad norm: 0.130 | params norm: 11342.176 | params rms: 0.061414 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 15:17:06.338261 | finish at 2026-10-11 05:56:50 + [2026-10-08 14:41:12] iteration 4731/ 6855 | consumed samples: 605568 | elapsed time per iteration (ms): 88712.3 | throughput per GPU (TFLOP/s/GPU): 41.7 | MFU 13.37% | FLOPs estimator: workload-aware | logical tokens/s: 189119.4 | valid tokens/s: 189119.4 | physical tokens/s: 189596.3 | loss tokens/s: 142924.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.851807E-01 | lm-global-supervised-tokens: 1.267916E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.795630E-02 | lm-token-buffer-mib: 5.809325E+01 | global_load_balancing_loss: 9.995897E-01 | loss scale: 1.0 | grad norm: 0.125 | params norm: 11342.123 | params rms: 0.061414 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 4:20:24.863551 | finish at 2026-10-10 19:01:37 + [2026-10-08 14:42:38] iteration 4732/ 6855 | consumed samples: 605696 | elapsed time per iteration (ms): 85644.6 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.52% | FLOPs estimator: workload-aware | logical tokens/s: 195893.0 | valid tokens/s: 195893.0 | physical tokens/s: 196342.4 | loss tokens/s: 143101.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.278778E-01 | lm-global-supervised-tokens: 1.225587E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.449745E-02 | lm-token-buffer-mib: 5.813741E+01 | global_load_balancing_loss: 9.999373E-01 | loss scale: 1.0 | grad norm: 0.081 | params norm: 11342.069 | params rms: 0.061414 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 2:30:23.479038 | finish at 2026-10-10 17:13:02 + [2026-10-08 14:44:05] iteration 4733/ 6855 | consumed samples: 605824 | elapsed time per iteration (ms): 86815.7 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.56% | FLOPs estimator: workload-aware | logical tokens/s: 193250.6 | valid tokens/s: 193250.6 | physical tokens/s: 193703.8 | loss tokens/s: 141068.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.540552E-01 | lm-global-supervised-tokens: 1.224692E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.170297E-02 | lm-token-buffer-mib: 5.809248E+01 | global_load_balancing_loss: 9.999375E-01 | loss scale: 1.0 | grad norm: 0.128 | params norm: 11342.015 | params rms: 0.061413 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 3:10:22.835579 | finish at 2026-10-10 17:54:28 + [2026-10-08 14:45:35] iteration 4734/ 6855 | consumed samples: 605952 | elapsed time per iteration (ms): 89832.5 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.43% | FLOPs estimator: workload-aware | logical tokens/s: 186760.9 | valid tokens/s: 186760.9 | physical tokens/s: 187175.4 | loss tokens/s: 134491.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.757366E-01 | lm-global-supervised-tokens: 1.208167E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.224810E-02 | lm-token-buffer-mib: 5.819407E+01 | global_load_balancing_loss: 1.000121E+00 | loss scale: 1.0 | grad norm: 0.050 | params norm: 11341.960 | params rms: 0.061413 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 4:55:34.720829 | finish at 2026-10-10 19:41:09 + [2026-10-08 14:46:58] iteration 4735/ 6855 | consumed samples: 606080 | elapsed time per iteration (ms): 83614.7 | throughput per GPU (TFLOP/s/GPU): 43.3 | MFU 13.89% | FLOPs estimator: workload-aware | logical tokens/s: 200648.4 | valid tokens/s: 200648.4 | physical tokens/s: 201238.7 | loss tokens/s: 150560.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.928436E-01 | lm-global-supervised-tokens: 1.258906E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.199313E-02 | lm-token-buffer-mib: 5.820345E+01 | global_load_balancing_loss: 9.985537E-01 | loss scale: 1.0 | grad norm: 0.143 | params norm: 11341.904 | params rms: 0.061413 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 1:14:23.175287 | finish at 2026-10-10 16:01:22 + [2026-10-08 14:48:28] iteration 4736/ 6855 | consumed samples: 606208 | elapsed time per iteration (ms): 89733.5 | throughput per GPU (TFLOP/s/GPU): 40.5 | MFU 12.97% | FLOPs estimator: workload-aware | logical tokens/s: 186967.2 | valid tokens/s: 186967.2 | physical tokens/s: 187474.5 | loss tokens/s: 141944.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.012001E-01 | lm-global-supervised-tokens: 1.273712E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.453806E-02 | lm-token-buffer-mib: 5.810577E+01 | global_load_balancing_loss: 1.000675E+00 | loss scale: 1.0 | grad norm: 0.044 | params norm: 11341.847 | params rms: 0.061412 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 4:49:05.192539 | finish at 2026-10-10 19:37:33 + [2026-10-08 14:49:57] iteration 4737/ 6855 | consumed samples: 606336 | elapsed time per iteration (ms): 89141.1 | throughput per GPU (TFLOP/s/GPU): 41.3 | MFU 13.25% | FLOPs estimator: workload-aware | logical tokens/s: 188209.3 | valid tokens/s: 188209.3 | physical tokens/s: 188626.4 | loss tokens/s: 136751.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.933460E-01 | lm-global-supervised-tokens: 1.219012E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.436948E-02 | lm-token-buffer-mib: 5.813239E+01 | global_load_balancing_loss: 1.001397E+00 | loss scale: 1.0 | grad norm: 0.088 | params norm: 11341.790 | params rms: 0.061412 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 4:26:40.749667 | finish at 2026-10-10 19:16:38 + [2026-10-08 14:51:28] iteration 4738/ 6855 | consumed samples: 606464 | elapsed time per iteration (ms): 91081.6 | throughput per GPU (TFLOP/s/GPU): 40.9 | MFU 13.11% | FLOPs estimator: workload-aware | logical tokens/s: 184199.2 | valid tokens/s: 184199.2 | physical tokens/s: 185010.3 | loss tokens/s: 134836.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.470665E-01 | lm-global-supervised-tokens: 1.228117E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.712588E-02 | lm-token-buffer-mib: 5.811991E+01 | global_load_balancing_loss: 1.001453E+00 | loss scale: 1.0 | grad norm: 0.063 | params norm: 11341.735 | params rms: 0.061412 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 5:33:39.842490 | finish at 2026-10-10 20:25:08 + [2026-10-08 14:52:57] iteration 4739/ 6855 | consumed samples: 606592 | elapsed time per iteration (ms): 88298.3 | throughput per GPU (TFLOP/s/GPU): 41.7 | MFU 13.36% | FLOPs estimator: workload-aware | logical tokens/s: 190005.7 | valid tokens/s: 190005.7 | physical tokens/s: 190401.6 | loss tokens/s: 135272.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.191463E-01 | lm-global-supervised-tokens: 1.194430E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.465950E-02 | lm-token-buffer-mib: 5.811187E+01 | global_load_balancing_loss: 1.000169E+00 | loss scale: 1.0 | grad norm: 0.087 | params norm: 11341.679 | params rms: 0.061412 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 3:53:59.285091 | finish at 2026-10-10 18:46:56 + [2026-10-08 14:54:47] iteration 4740/ 6855 | consumed samples: 606720 | elapsed time per iteration (ms): 110036.2 | throughput per GPU (TFLOP/s/GPU): 34.2 | MFU 10.95% | FLOPs estimator: workload-aware | logical tokens/s: 152469.2 | valid tokens/s: 152469.2 | physical tokens/s: 152778.0 | loss tokens/s: 112693.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.598693E-01 | lm-global-supervised-tokens: 1.240041E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.552279E-02 | lm-token-buffer-mib: 5.805714E+01 | global_load_balancing_loss: 1.000749E+00 | loss scale: 1.0 | grad norm: 0.054 | params norm: 11341.623 | params rms: 0.061411 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 16:38:46.665462 | finish at 2026-10-11 07:33:33 + [2026-10-08 14:56:11] iteration 4741/ 6855 | consumed samples: 606848 | elapsed time per iteration (ms): 84726.1 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.59% | FLOPs estimator: workload-aware | logical tokens/s: 198016.3 | valid tokens/s: 198016.3 | physical tokens/s: 198526.1 | loss tokens/s: 152083.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.263406E-01 | lm-global-supervised-tokens: 1.288545E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.144954E-02 | lm-token-buffer-mib: 5.810139E+01 | global_load_balancing_loss: 1.000172E+00 | loss scale: 1.0 | grad norm: 0.056 | params norm: 11341.568 | params rms: 0.061411 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 1:45:10.931987 | finish at 2026-10-10 16:41:22 + [2026-10-08 14:57:38] iteration 4742/ 6855 | consumed samples: 606976 | elapsed time per iteration (ms): 87090.5 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.66% | FLOPs estimator: workload-aware | logical tokens/s: 192640.8 | valid tokens/s: 192640.8 | physical tokens/s: 193053.3 | loss tokens/s: 139647.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.116008E-01 | lm-global-supervised-tokens: 1.216194E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.124625E-02 | lm-token-buffer-mib: 5.806171E+01 | global_load_balancing_loss: 9.989377E-01 | loss scale: 1.0 | grad norm: 0.040 | params norm: 11341.513 | params rms: 0.061411 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 3:07:02.183421 | finish at 2026-10-10 18:04:41 + [2026-10-08 14:59:06] iteration 4743/ 6855 | consumed samples: 607104 | elapsed time per iteration (ms): 88044.0 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.50% | FLOPs estimator: workload-aware | logical tokens/s: 190554.5 | valid tokens/s: 190554.5 | physical tokens/s: 191114.8 | loss tokens/s: 142693.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.858084E-01 | lm-global-supervised-tokens: 1.256332E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.980905E-02 | lm-token-buffer-mib: 5.833005E+01 | global_load_balancing_loss: 9.987262E-01 | loss scale: 1.0 | grad norm: 0.061 | params norm: 11341.457 | params rms: 0.061410 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 3:39:08.945435 | finish at 2026-10-10 18:38:15 + [2026-10-08 15:00:35] iteration 4744/ 6855 | consumed samples: 607232 | elapsed time per iteration (ms): 88814.7 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.58% | FLOPs estimator: workload-aware | logical tokens/s: 188901.0 | valid tokens/s: 188901.0 | physical tokens/s: 189303.8 | loss tokens/s: 134560.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.052819E-01 | lm-global-supervised-tokens: 1.195092E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.150987E-02 | lm-token-buffer-mib: 5.823640E+01 | global_load_balancing_loss: 9.999954E-01 | loss scale: 1.0 | grad norm: 0.056 | params norm: 11341.402 | params rms: 0.061410 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 4:04:47.803279 | finish at 2026-10-10 19:05:23 + [2026-10-08 15:02:03] iteration 4745/ 6855 | consumed samples: 607360 | elapsed time per iteration (ms): 88127.4 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.55% | FLOPs estimator: workload-aware | logical tokens/s: 190374.2 | valid tokens/s: 190374.2 | physical tokens/s: 190908.8 | loss tokens/s: 146242.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.933505E-01 | lm-global-supervised-tokens: 1.288793E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.349338E-02 | lm-token-buffer-mib: 5.817498E+01 | global_load_balancing_loss: 9.992394E-01 | loss scale: 1.0 | grad norm: 0.068 | params norm: 11341.344 | params rms: 0.061410 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 3:39:08.725295 | finish at 2026-10-10 18:41:12 + [2026-10-08 15:03:33] iteration 4746/ 6855 | consumed samples: 607488 | elapsed time per iteration (ms): 89907.5 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.57% | FLOPs estimator: workload-aware | logical tokens/s: 186604.8 | valid tokens/s: 186604.8 | physical tokens/s: 186914.7 | loss tokens/s: 132914.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.255810E-01 | lm-global-supervised-tokens: 1.195004E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.299343E-02 | lm-token-buffer-mib: 5.810831E+01 | global_load_balancing_loss: 1.001542E+00 | loss scale: 1.0 | grad norm: 0.071 | params norm: 11341.286 | params rms: 0.061409 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 4:40:14.976391 | finish at 2026-10-10 19:43:48 + [2026-10-08 15:05:00] iteration 4747/ 6855 | consumed samples: 607616 | elapsed time per iteration (ms): 86514.4 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.46% | FLOPs estimator: workload-aware | logical tokens/s: 193924.0 | valid tokens/s: 193924.0 | physical tokens/s: 194459.1 | loss tokens/s: 150725.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.815896E-01 | lm-global-supervised-tokens: 1.303994E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.215440E-02 | lm-token-buffer-mib: 5.811222E+01 | global_load_balancing_loss: 1.000652E+00 | loss scale: 1.0 | grad norm: 0.067 | params norm: 11341.226 | params rms: 0.061409 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 2:39:32.301435 | finish at 2026-10-10 17:44:32 + [2026-10-08 15:06:27] iteration 4748/ 6855 | consumed samples: 607744 | elapsed time per iteration (ms): 87634.4 | throughput per GPU (TFLOP/s/GPU): 41.4 | MFU 13.28% | FLOPs estimator: workload-aware | logical tokens/s: 191445.6 | valid tokens/s: 191445.6 | physical tokens/s: 191939.5 | loss tokens/s: 138843.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.199865E-01 | lm-global-supervised-tokens: 1.216746E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.158328E-02 | lm-token-buffer-mib: 5.827792E+01 | global_load_balancing_loss: 1.001406E+00 | loss scale: 1.0 | grad norm: 0.063 | params norm: 11341.168 | params rms: 0.061409 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 3:17:25.648922 | finish at 2026-10-10 18:23:53 + [2026-10-08 15:07:55] iteration 4749/ 6855 | consumed samples: 607872 | elapsed time per iteration (ms): 87647.5 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.59% | FLOPs estimator: workload-aware | logical tokens/s: 191417.0 | valid tokens/s: 191417.0 | physical tokens/s: 191827.0 | loss tokens/s: 140867.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.643643E-01 | lm-global-supervised-tokens: 1.234671E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.727820E-02 | lm-token-buffer-mib: 5.806661E+01 | global_load_balancing_loss: 1.000633E+00 | loss scale: 1.0 | grad norm: 0.126 | params norm: 11341.112 | params rms: 0.061408 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 3:16:25.554241 | finish at 2026-10-10 18:24:21 + [2026-10-08 15:09:45] iteration 4750/ 6855 | consumed samples: 608000 | elapsed time per iteration (ms): 109728.0 | throughput per GPU (TFLOP/s/GPU): 34.2 | MFU 10.96% | FLOPs estimator: workload-aware | logical tokens/s: 152898.0 | valid tokens/s: 152898.0 | physical tokens/s: 153169.0 | loss tokens/s: 115660.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.096961E-01 | lm-global-supervised-tokens: 1.269115E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.170331E-02 | lm-token-buffer-mib: 5.803829E+01 | global_load_balancing_loss: 1.000297E+00 | loss scale: 1.0 | grad norm: 0.061 | params norm: 11341.055 | params rms: 0.061408 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 16:09:37.437334 | finish at 2026-10-11 07:19:22 + [2026-10-08 15:11:12] iteration 4751/ 6855 | consumed samples: 608128 | elapsed time per iteration (ms): 87024.8 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.66% | FLOPs estimator: workload-aware | logical tokens/s: 192786.1 | valid tokens/s: 192786.1 | physical tokens/s: 193132.4 | loss tokens/s: 141888.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.496466E-01 | lm-global-supervised-tokens: 1.234784E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.317477E-02 | lm-token-buffer-mib: 5.813976E+01 | global_load_balancing_loss: 1.000458E+00 | loss scale: 1.0 | grad norm: 0.033 | params norm: 11340.998 | params rms: 0.061408 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 2:51:40.249559 | finish at 2026-10-10 18:02:52 + [2026-10-08 15:12:39] iteration 4752/ 6855 | consumed samples: 608256 | elapsed time per iteration (ms): 87339.3 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.43% | FLOPs estimator: workload-aware | logical tokens/s: 192091.7 | valid tokens/s: 192091.7 | physical tokens/s: 192443.7 | loss tokens/s: 146158.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.289569E-01 | lm-global-supervised-tokens: 1.276541E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.214814E-02 | lm-token-buffer-mib: 5.816534E+01 | global_load_balancing_loss: 9.992797E-01 | loss scale: 1.0 | grad norm: 0.043 | params norm: 11340.943 | params rms: 0.061408 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 3:01:14.614913 | finish at 2026-10-10 18:13:54 + [2026-10-08 15:14:07] iteration 4753/ 6855 | consumed samples: 608384 | elapsed time per iteration (ms): 87517.9 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.61% | FLOPs estimator: workload-aware | logical tokens/s: 191700.1 | valid tokens/s: 191700.1 | physical tokens/s: 192267.5 | loss tokens/s: 140889.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.159028E-01 | lm-global-supervised-tokens: 1.233036E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.323999E-02 | lm-token-buffer-mib: 5.808616E+01 | global_load_balancing_loss: 1.000195E+00 | loss scale: 1.0 | grad norm: 0.053 | params norm: 11340.888 | params rms: 0.061407 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 3:06:02.598216 | finish at 2026-10-10 18:20:09 + [2026-10-08 15:15:37] iteration 4754/ 6855 | consumed samples: 608512 | elapsed time per iteration (ms): 90269.1 | throughput per GPU (TFLOP/s/GPU): 41.4 | MFU 13.28% | FLOPs estimator: workload-aware | logical tokens/s: 185857.3 | valid tokens/s: 185857.3 | physical tokens/s: 186317.2 | loss tokens/s: 133724.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 5.940609E-01 | lm-global-supervised-tokens: 1.207118E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.470952E-02 | lm-token-buffer-mib: 5.814738E+01 | global_load_balancing_loss: 1.000391E+00 | loss scale: 1.0 | grad norm: 0.179 | params norm: 11340.834 | params rms: 0.061407 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 4:40:55.437103 | finish at 2026-10-10 19:56:32 + [2026-10-08 15:17:07] iteration 4755/ 6855 | consumed samples: 608640 | elapsed time per iteration (ms): 89773.1 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.58% | FLOPs estimator: workload-aware | logical tokens/s: 186884.3 | valid tokens/s: 186884.3 | physical tokens/s: 187213.6 | loss tokens/s: 134311.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.867176E-01 | lm-global-supervised-tokens: 1.205753E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.618131E-02 | lm-token-buffer-mib: 5.806065E+01 | global_load_balancing_loss: 9.998716E-01 | loss scale: 1.0 | grad norm: 0.272 | params norm: 11340.782 | params rms: 0.061407 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 4:22:03.434687 | finish at 2026-10-10 19:39:10 + [2026-10-08 15:18:34] iteration 4756/ 6855 | consumed samples: 608768 | elapsed time per iteration (ms): 86704.3 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.45% | FLOPs estimator: workload-aware | logical tokens/s: 193499.2 | valid tokens/s: 193499.2 | physical tokens/s: 194190.1 | loss tokens/s: 140332.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.595790E-01 | lm-global-supervised-tokens: 1.216746E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.158685E-02 | lm-token-buffer-mib: 5.849970E+01 | global_load_balancing_loss: 9.999829E-01 | loss scale: 1.0 | grad norm: 0.087 | params norm: 11340.729 | params rms: 0.061406 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 2:33:12.324045 | finish at 2026-10-10 17:51:46 + [2026-10-08 15:19:58] iteration 4757/ 6855 | consumed samples: 608896 | elapsed time per iteration (ms): 84279.2 | throughput per GPU (TFLOP/s/GPU): 43.5 | MFU 13.93% | FLOPs estimator: workload-aware | logical tokens/s: 199066.9 | valid tokens/s: 199066.9 | physical tokens/s: 199559.3 | loss tokens/s: 146580.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.182998E-01 | lm-global-supervised-tokens: 1.235366E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.158555E-02 | lm-token-buffer-mib: 5.818269E+01 | global_load_balancing_loss: 9.994950E-01 | loss scale: 1.0 | grad norm: 0.333 | params norm: 11340.676 | params rms: 0.061406 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 1:06:57.814283 | finish at 2026-10-10 16:26:56 + [2026-10-08 15:21:27] iteration 4758/ 6855 | consumed samples: 609024 | elapsed time per iteration (ms): 89667.0 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.60% | FLOPs estimator: workload-aware | logical tokens/s: 187104.8 | valid tokens/s: 187104.8 | physical tokens/s: 187492.7 | loss tokens/s: 142837.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.579646E-01 | lm-global-supervised-tokens: 1.280781E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.165787E-02 | lm-token-buffer-mib: 5.803029E+01 | global_load_balancing_loss: 9.986180E-01 | loss scale: 1.0 | grad norm: 0.123 | params norm: 11340.622 | params rms: 0.061406 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 4:13:51.677618 | finish at 2026-10-10 19:35:19 + [2026-10-08 15:22:54] iteration 4759/ 6855 | consumed samples: 609152 | elapsed time per iteration (ms): 86938.3 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.47% | FLOPs estimator: workload-aware | logical tokens/s: 192977.5 | valid tokens/s: 192977.5 | physical tokens/s: 193475.9 | loss tokens/s: 141028.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.501319E-01 | lm-global-supervised-tokens: 1.226077E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.269138E-02 | lm-token-buffer-mib: 5.808601E+01 | global_load_balancing_loss: 1.001615E+00 | loss scale: 1.0 | grad norm: 0.201 | params norm: 11340.567 | params rms: 0.061405 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 2:37:02.698566 | finish at 2026-10-10 17:59:57 + [2026-10-08 15:24:41] iteration 4760/ 6855 | consumed samples: 609280 | elapsed time per iteration (ms): 106351.9 | throughput per GPU (TFLOP/s/GPU): 33.8 | MFU 10.82% | FLOPs estimator: workload-aware | logical tokens/s: 157751.4 | valid tokens/s: 157751.4 | physical tokens/s: 158218.4 | loss tokens/s: 118042.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.389870E-01 | lm-global-supervised-tokens: 1.255402E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.370979E-02 | lm-token-buffer-mib: 5.809821E+01 | global_load_balancing_loss: 1.000986E+00 | loss scale: 1.0 | grad norm: 0.090 | params norm: 11340.513 | params rms: 0.061405 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 13:53:27.308131 | finish at 2026-10-11 05:18:08 +times across ranks (ms): + save-checkpoint: + rank 0: 24791.50 + rank 1: 24791.51 + rank 2: 24791.51 + rank 3: 24791.50 + rank 4: 24791.50 + rank 5: 24791.48 + rank 6: 24791.47 + rank 7: 24791.49 + rank 8: 24791.93 + rank 9: 24791.90 + rank 10: 24791.89 + rank 11: 24791.92 + rank 12: 24791.89 + rank 13: 24791.89 + rank 14: 24791.95 + rank 15: 24791.89 + rank 16: 24791.59 + rank 17: 24791.46 + rank 18: 24791.44 + rank 19: 24791.46 + rank 20: 24791.47 + rank 21: 24791.46 + rank 22: 24791.44 + rank 23: 24791.44 + rank 24: 24792.26 + rank 25: 24792.21 + rank 26: 24792.29 + rank 27: 24792.31 + rank 28: 24792.84 + rank 29: 24792.22 + rank 30: 24792.20 + rank 31: 24792.24 + rank 32: 24791.55 + rank 33: 24791.51 + rank 34: 24791.52 + rank 35: 24791.50 + rank 36: 24791.45 + rank 37: 24791.42 + rank 38: 24791.48 + rank 39: 24791.46 + rank 40: 24791.57 + rank 41: 24791.57 + rank 42: 24791.57 + rank 43: 24791.59 + rank 44: 24791.56 + rank 45: 24791.58 + rank 46: 24791.56 + rank 47: 24791.58 + rank 48: 24791.57 + rank 49: 24791.54 + rank 50: 24791.61 + rank 51: 24791.62 + rank 52: 24791.68 + rank 53: 24791.60 + rank 54: 24791.64 + rank 55: 24791.48 + rank 56: 24792.44 + rank 57: 24792.42 + rank 58: 24792.38 + rank 59: 24792.36 + rank 60: 24792.41 + rank 61: 24792.42 + rank 62: 24792.23 + rank 63: 24792.46 + [2026-10-08 15:26:32] iteration 4761/ 6855 | consumed samples: 609408 | elapsed time per iteration (ms): 86170.7 | throughput per GPU (TFLOP/s/GPU): 43.6 | MFU 13.96% | FLOPs estimator: workload-aware | logical tokens/s: 194697.5 | valid tokens/s: 194697.5 | physical tokens/s: 195098.4 | loss tokens/s: 139937.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.974465E-01 | lm-global-supervised-tokens: 1.205849E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.531953E+00 | lm-token-buffer-mib: 5.815496E+01 | global_load_balancing_loss: 1.000357E+00 | loss scale: 1.0 | grad norm: 0.152 | params norm: 11340.460 | params rms: 0.061405 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 2:07:21.355589 | finish at 2026-10-10 17:33:53 + [2026-10-08 15:28:02] iteration 4762/ 6855 | consumed samples: 609536 | elapsed time per iteration (ms): 90204.3 | throughput per GPU (TFLOP/s/GPU): 40.2 | MFU 12.89% | FLOPs estimator: workload-aware | logical tokens/s: 185991.0 | valid tokens/s: 185991.0 | physical tokens/s: 186413.5 | loss tokens/s: 143292.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.442801E-01 | lm-global-supervised-tokens: 1.292563E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.292970E-02 | lm-token-buffer-mib: 5.804337E+01 | global_load_balancing_loss: 9.997458E-01 | loss scale: 1.0 | grad norm: 0.126 | params norm: 11340.407 | params rms: 0.061405 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 4:26:37.547850 | finish at 2026-10-10 19:54:40 + [2026-10-08 15:29:26] iteration 4763/ 6855 | consumed samples: 609664 | elapsed time per iteration (ms): 84231.9 | throughput per GPU (TFLOP/s/GPU): 43.7 | MFU 14.01% | FLOPs estimator: workload-aware | logical tokens/s: 199178.4 | valid tokens/s: 199178.4 | physical tokens/s: 199726.3 | loss tokens/s: 149646.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.190094E-01 | lm-global-supervised-tokens: 1.260496E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.345093E-02 | lm-token-buffer-mib: 5.808788E+01 | global_load_balancing_loss: 1.000393E+00 | loss scale: 1.0 | grad norm: 0.069 | params norm: 11340.354 | params rms: 0.061404 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 0:56:53.118292 | finish at 2026-10-10 16:26:20 + [2026-10-08 15:30:57] iteration 4764/ 6855 | consumed samples: 609792 | elapsed time per iteration (ms): 90260.1 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.49% | FLOPs estimator: workload-aware | logical tokens/s: 185876.2 | valid tokens/s: 185876.2 | physical tokens/s: 186251.9 | loss tokens/s: 136482.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.597691E-01 | lm-global-supervised-tokens: 1.231888E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.158460E-02 | lm-token-buffer-mib: 5.803349E+01 | global_load_balancing_loss: 1.000323E+00 | loss scale: 1.0 | grad norm: 0.150 | params norm: 11340.300 | params rms: 0.061404 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 4:25:33.949129 | finish at 2026-10-10 19:56:31 + [2026-10-08 15:32:20] iteration 4765/ 6855 | consumed samples: 609920 | elapsed time per iteration (ms): 83586.7 | throughput per GPU (TFLOP/s/GPU): 44.1 | MFU 14.15% | FLOPs estimator: workload-aware | logical tokens/s: 200716.0 | valid tokens/s: 200716.0 | physical tokens/s: 201194.5 | loss tokens/s: 142208.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.789239E-01 | lm-global-supervised-tokens: 1.188674E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.730384E-02 | lm-token-buffer-mib: 5.804624E+01 | global_load_balancing_loss: 9.997048E-01 | loss scale: 1.0 | grad norm: 0.065 | params norm: 11340.244 | params rms: 0.061404 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 0:31:36.185980 | finish at 2026-10-10 16:03:57 + [2026-10-08 15:33:44] iteration 4766/ 6855 | consumed samples: 610048 | elapsed time per iteration (ms): 83519.7 | throughput per GPU (TFLOP/s/GPU): 43.3 | MFU 13.88% | FLOPs estimator: workload-aware | logical tokens/s: 200876.9 | valid tokens/s: 200876.9 | physical tokens/s: 201355.9 | loss tokens/s: 149142.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.765956E-01 | lm-global-supervised-tokens: 1.245632E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.402421E-02 | lm-token-buffer-mib: 5.804805E+01 | global_load_balancing_loss: 9.993683E-01 | loss scale: 1.0 | grad norm: 0.039 | params norm: 11340.190 | params rms: 0.061403 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 0:27:52.665359 | finish at 2026-10-10 16:01:37 + [2026-10-08 15:35:12] iteration 4767/ 6855 | consumed samples: 610176 | elapsed time per iteration (ms): 88112.1 | throughput per GPU (TFLOP/s/GPU): 42.8 | MFU 13.71% | FLOPs estimator: workload-aware | logical tokens/s: 190407.6 | valid tokens/s: 190407.6 | physical tokens/s: 190853.3 | loss tokens/s: 142480.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.000616E-01 | lm-global-supervised-tokens: 1.255423E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.314163E-02 | lm-token-buffer-mib: 5.814444E+01 | global_load_balancing_loss: 9.987140E-01 | loss scale: 1.0 | grad norm: 0.108 | params norm: 11340.138 | params rms: 0.061403 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 3:06:17.963505 | finish at 2026-10-10 18:41:30 + [2026-10-08 15:36:42] iteration 4768/ 6855 | consumed samples: 610304 | elapsed time per iteration (ms): 89643.9 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.47% | FLOPs estimator: workload-aware | logical tokens/s: 187153.3 | valid tokens/s: 187153.3 | physical tokens/s: 187607.6 | loss tokens/s: 137832.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.497105E-01 | lm-global-supervised-tokens: 1.235587E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.400306E-02 | lm-token-buffer-mib: 5.806767E+01 | global_load_balancing_loss: 1.001026E+00 | loss scale: 1.0 | grad norm: 0.058 | params norm: 11340.086 | params rms: 0.061403 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 3:58:06.792756 | finish at 2026-10-10 19:34:48 + [2026-10-08 15:38:06] iteration 4769/ 6855 | consumed samples: 610432 | elapsed time per iteration (ms): 84552.8 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.74% | FLOPs estimator: workload-aware | logical tokens/s: 198422.7 | valid tokens/s: 198422.7 | physical tokens/s: 198983.3 | loss tokens/s: 147542.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.301833E-01 | lm-global-supervised-tokens: 1.247514E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.165376E-02 | lm-token-buffer-mib: 5.804884E+01 | global_load_balancing_loss: 1.000525E+00 | loss scale: 1.0 | grad norm: 0.063 | params norm: 11340.035 | params rms: 0.061403 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 0:59:37.110337 | finish at 2026-10-10 16:37:43 + [2026-10-08 15:39:59] iteration 4770/ 6855 | consumed samples: 610560 | elapsed time per iteration (ms): 112611.6 | throughput per GPU (TFLOP/s/GPU): 33.0 | MFU 10.59% | FLOPs estimator: workload-aware | logical tokens/s: 148982.5 | valid tokens/s: 148982.5 | physical tokens/s: 149377.7 | loss tokens/s: 105211.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.749879E-01 | lm-global-supervised-tokens: 1.184804E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.171583E-02 | lm-token-buffer-mib: 5.812854E+01 | global_load_balancing_loss: 1.001956E+00 | loss scale: 1.0 | grad norm: 0.073 | params norm: 11339.985 | params rms: 0.061402 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 17:13:15.261894 | finish at 2026-10-11 08:53:14 + [2026-10-08 15:41:28] iteration 4771/ 6855 | consumed samples: 610688 | elapsed time per iteration (ms): 89563.7 | throughput per GPU (TFLOP/s/GPU): 41.6 | MFU 13.35% | FLOPs estimator: workload-aware | logical tokens/s: 187321.6 | valid tokens/s: 187321.6 | physical tokens/s: 187625.3 | loss tokens/s: 138810.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.831325E-01 | lm-global-supervised-tokens: 1.243236E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.195626E-02 | lm-token-buffer-mib: 5.802903E+01 | global_load_balancing_loss: 1.001517E+00 | loss scale: 1.0 | grad norm: 0.037 | params norm: 11339.935 | params rms: 0.061402 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 3:50:50.734818 | finish at 2026-10-10 19:32:19 + [2026-10-08 15:42:55] iteration 4772/ 6855 | consumed samples: 610816 | elapsed time per iteration (ms): 86297.1 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.42% | FLOPs estimator: workload-aware | logical tokens/s: 194411.8 | valid tokens/s: 194411.8 | physical tokens/s: 195018.2 | loss tokens/s: 141459.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.317821E-01 | lm-global-supervised-tokens: 1.220753E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.529987E-02 | lm-token-buffer-mib: 5.820089E+01 | global_load_balancing_loss: 1.001329E+00 | loss scale: 1.0 | grad norm: 0.108 | params norm: 11339.884 | params rms: 0.061402 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 1:55:56.854970 | finish at 2026-10-10 17:38:51 + [2026-10-08 15:44:22] iteration 4773/ 6855 | consumed samples: 610944 | elapsed time per iteration (ms): 87371.4 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.55% | FLOPs estimator: workload-aware | logical tokens/s: 192020.4 | valid tokens/s: 192020.4 | physical tokens/s: 192434.4 | loss tokens/s: 139968.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.403022E-01 | lm-global-supervised-tokens: 1.222920E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.193471E-02 | lm-token-buffer-mib: 5.817227E+01 | global_load_balancing_loss: 9.986143E-01 | loss scale: 1.0 | grad norm: 0.112 | params norm: 11339.832 | params rms: 0.061402 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 2:31:47.246607 | finish at 2026-10-10 18:16:09 + [2026-10-08 15:45:51] iteration 4774/ 6855 | consumed samples: 611072 | elapsed time per iteration (ms): 88533.4 | throughput per GPU (TFLOP/s/GPU): 41.0 | MFU 13.15% | FLOPs estimator: workload-aware | logical tokens/s: 189500.9 | valid tokens/s: 189500.9 | physical tokens/s: 190055.9 | loss tokens/s: 140555.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.341990E-01 | lm-global-supervised-tokens: 1.244384E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.458049E-02 | lm-token-buffer-mib: 5.828409E+01 | global_load_balancing_loss: 9.987842E-01 | loss scale: 1.0 | grad norm: 0.065 | params norm: 11339.781 | params rms: 0.061401 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 3:10:37.973769 | finish at 2026-10-10 18:56:29 + [2026-10-08 15:47:21] iteration 4775/ 6855 | consumed samples: 611200 | elapsed time per iteration (ms): 90466.4 | throughput per GPU (TFLOP/s/GPU): 40.6 | MFU 13.02% | FLOPs estimator: workload-aware | logical tokens/s: 185451.6 | valid tokens/s: 185451.6 | physical tokens/s: 185908.9 | loss tokens/s: 143603.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.441116E-01 | lm-global-supervised-tokens: 1.299128E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.460360E-02 | lm-token-buffer-mib: 5.812508E+01 | global_load_balancing_loss: 9.993402E-01 | loss scale: 1.0 | grad norm: 0.158 | params norm: 11339.729 | params rms: 0.061401 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 4:16:10.131149 | finish at 2026-10-10 20:03:31 + [2026-10-08 15:48:48] iteration 4776/ 6855 | consumed samples: 611328 | elapsed time per iteration (ms): 87273.0 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.42% | FLOPs estimator: workload-aware | logical tokens/s: 192237.8 | valid tokens/s: 192237.8 | physical tokens/s: 192763.8 | loss tokens/s: 146888.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.644412E-01 | lm-global-supervised-tokens: 1.281937E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.388470E-02 | lm-token-buffer-mib: 5.809255E+01 | global_load_balancing_loss: 1.000251E+00 | loss scale: 1.0 | grad norm: 0.057 | params norm: 11339.676 | params rms: 0.061401 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 2:24:00.559073 | finish at 2026-10-10 18:12:49 + [2026-10-08 15:50:16] iteration 4777/ 6855 | consumed samples: 611456 | elapsed time per iteration (ms): 88186.2 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.75% | FLOPs estimator: workload-aware | logical tokens/s: 190247.5 | valid tokens/s: 190247.5 | physical tokens/s: 190800.5 | loss tokens/s: 138977.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.589158E-01 | lm-global-supervised-tokens: 1.225587E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.142437E-02 | lm-token-buffer-mib: 5.827881E+01 | global_load_balancing_loss: 1.000477E+00 | loss scale: 1.0 | grad norm: 0.227 | params norm: 11339.624 | params rms: 0.061400 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 2:54:10.851562 | finish at 2026-10-10 18:44:27 + [2026-10-08 15:51:40] iteration 4778/ 6855 | consumed samples: 611584 | elapsed time per iteration (ms): 83947.7 | throughput per GPU (TFLOP/s/GPU): 42.8 | MFU 13.71% | FLOPs estimator: workload-aware | logical tokens/s: 199852.7 | valid tokens/s: 199852.7 | physical tokens/s: 200456.5 | loss tokens/s: 153602.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.286893E-01 | lm-global-supervised-tokens: 1.289454E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.153923E-02 | lm-token-buffer-mib: 5.817014E+01 | global_load_balancing_loss: 1.000396E+00 | loss scale: 1.0 | grad norm: 0.129 | params norm: 11339.569 | params rms: 0.061400 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 0:25:59.344723 | finish at 2026-10-10 16:17:40 + [2026-10-08 15:53:10] iteration 4779/ 6855 | consumed samples: 611712 | elapsed time per iteration (ms): 89368.7 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.55% | FLOPs estimator: workload-aware | logical tokens/s: 187730.1 | valid tokens/s: 187730.1 | physical tokens/s: 188076.1 | loss tokens/s: 138017.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.541094E-01 | lm-global-supervised-tokens: 1.233441E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.863447E-02 | lm-token-buffer-mib: 5.819262E+01 | global_load_balancing_loss: 1.000103E+00 | loss scale: 1.0 | grad norm: 0.098 | params norm: 11339.514 | params rms: 0.061400 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 3:32:09.409378 | finish at 2026-10-10 19:25:19 + [2026-10-08 15:54:58] iteration 4780/ 6855 | consumed samples: 611840 | elapsed time per iteration (ms): 108316.4 | throughput per GPU (TFLOP/s/GPU): 33.8 | MFU 10.84% | FLOPs estimator: workload-aware | logical tokens/s: 154889.8 | valid tokens/s: 154889.8 | physical tokens/s: 155289.9 | loss tokens/s: 111761.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.660797E-01 | lm-global-supervised-tokens: 1.210557E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.270300E-02 | lm-token-buffer-mib: 5.817393E+01 | global_load_balancing_loss: 1.001802E+00 | loss scale: 1.0 | grad norm: 0.059 | params norm: 11339.458 | params rms: 0.061399 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 14:25:56.617177 | finish at 2026-10-11 06:20:55 + [2026-10-08 15:56:24] iteration 4781/ 6855 | consumed samples: 611968 | elapsed time per iteration (ms): 85619.1 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.77% | FLOPs estimator: workload-aware | logical tokens/s: 195951.4 | valid tokens/s: 195951.4 | physical tokens/s: 196529.3 | loss tokens/s: 144914.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.516173E-01 | lm-global-supervised-tokens: 1.240746E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.051787E-02 | lm-token-buffer-mib: 5.805320E+01 | global_load_balancing_loss: 1.000745E+00 | loss scale: 1.0 | grad norm: 0.135 | params norm: 11339.401 | params rms: 0.061399 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 1:19:33.974531 | finish at 2026-10-10 17:15:58 + [2026-10-08 15:57:52] iteration 4782/ 6855 | consumed samples: 612096 | elapsed time per iteration (ms): 88541.4 | throughput per GPU (TFLOP/s/GPU): 41.4 | MFU 13.28% | FLOPs estimator: workload-aware | logical tokens/s: 189483.7 | valid tokens/s: 189483.7 | physical tokens/s: 190088.1 | loss tokens/s: 143948.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.165964E-01 | lm-global-supervised-tokens: 1.274544E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.384191E-02 | lm-token-buffer-mib: 5.811068E+01 | global_load_balancing_loss: 1.000369E+00 | loss scale: 1.0 | grad norm: 0.092 | params norm: 11339.344 | params rms: 0.061399 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 2:59:06.402670 | finish at 2026-10-10 18:56:59 + [2026-10-08 15:59:17] iteration 4783/ 6855 | consumed samples: 612224 | elapsed time per iteration (ms): 84274.2 | throughput per GPU (TFLOP/s/GPU): 43.7 | MFU 14.00% | FLOPs estimator: workload-aware | logical tokens/s: 199077.5 | valid tokens/s: 199077.5 | physical tokens/s: 199440.9 | loss tokens/s: 146057.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.821740E-01 | lm-global-supervised-tokens: 1.230892E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.282019E-02 | lm-token-buffer-mib: 5.806894E+01 | global_load_balancing_loss: 9.997703E-01 | loss scale: 1.0 | grad norm: 0.058 | params norm: 11339.287 | params rms: 0.061399 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 0:30:16.233219 | finish at 2026-10-10 16:29:33 + [2026-10-08 16:00:44] iteration 4784/ 6855 | consumed samples: 612352 | elapsed time per iteration (ms): 87101.7 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.47% | FLOPs estimator: workload-aware | logical tokens/s: 192616.1 | valid tokens/s: 192616.1 | physical tokens/s: 193052.2 | loss tokens/s: 142398.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.670752E-01 | lm-global-supervised-tokens: 1.240310E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.538793E-02 | lm-token-buffer-mib: 5.806631E+01 | global_load_balancing_loss: 9.983929E-01 | loss scale: 1.0 | grad norm: 0.101 | params norm: 11339.230 | params rms: 0.061398 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 2:06:27.534925 | finish at 2026-10-10 18:07:11 + [2026-10-08 16:02:16] iteration 4785/ 6855 | consumed samples: 612480 | elapsed time per iteration (ms): 92235.2 | throughput per GPU (TFLOP/s/GPU): 40.4 | MFU 12.95% | FLOPs estimator: workload-aware | logical tokens/s: 181896.1 | valid tokens/s: 181896.1 | physical tokens/s: 182349.2 | loss tokens/s: 132925.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.268068E-01 | lm-global-supervised-tokens: 1.226043E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.113256E-02 | lm-token-buffer-mib: 5.812751E+01 | global_load_balancing_loss: 9.994716E-01 | loss scale: 1.0 | grad norm: 0.069 | params norm: 11339.171 | params rms: 0.061398 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 5:02:06.765146 | finish at 2026-10-10 21:04:23 + [2026-10-08 16:03:47] iteration 4786/ 6855 | consumed samples: 612608 | elapsed time per iteration (ms): 90810.3 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.48% | FLOPs estimator: workload-aware | logical tokens/s: 184750.2 | valid tokens/s: 184750.2 | physical tokens/s: 185183.3 | loss tokens/s: 135107.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.928085E-01 | lm-global-supervised-tokens: 1.226915E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.354728E-02 | lm-token-buffer-mib: 5.815251E+01 | global_load_balancing_loss: 9.995328E-01 | loss scale: 1.0 | grad norm: 0.099 | params norm: 11339.113 | params rms: 0.061398 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 4:11:26.462096 | finish at 2026-10-10 20:15:13 + [2026-10-08 16:05:14] iteration 4787/ 6855 | consumed samples: 612736 | elapsed time per iteration (ms): 87747.4 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.41% | FLOPs estimator: workload-aware | logical tokens/s: 191199.0 | valid tokens/s: 191199.0 | physical tokens/s: 191760.7 | loss tokens/s: 138980.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.827850E-01 | lm-global-supervised-tokens: 1.219514E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.770436E-02 | lm-token-buffer-mib: 5.819962E+01 | global_load_balancing_loss: 1.000336E+00 | loss scale: 1.0 | grad norm: 0.036 | params norm: 11339.056 | params rms: 0.061397 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 2:24:21.569058 | finish at 2026-10-10 18:29:36 + [2026-10-08 16:06:41] iteration 4788/ 6855 | consumed samples: 612864 | elapsed time per iteration (ms): 86963.4 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.60% | FLOPs estimator: workload-aware | logical tokens/s: 192922.0 | valid tokens/s: 192922.0 | physical tokens/s: 193500.9 | loss tokens/s: 141188.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.691360E-01 | lm-global-supervised-tokens: 1.227819E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.206368E-02 | lm-token-buffer-mib: 5.816669E+01 | global_load_balancing_loss: 1.000206E+00 | loss scale: 1.0 | grad norm: 0.053 | params norm: 11338.999 | params rms: 0.061397 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 1:55:53.289908 | finish at 2026-10-10 18:02:35 + [2026-10-08 16:08:09] iteration 4789/ 6855 | consumed samples: 612992 | elapsed time per iteration (ms): 87403.4 | throughput per GPU (TFLOP/s/GPU): 41.5 | MFU 13.32% | FLOPs estimator: workload-aware | logical tokens/s: 191951.3 | valid tokens/s: 191951.3 | physical tokens/s: 192395.3 | loss tokens/s: 140718.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.072756E-01 | lm-global-supervised-tokens: 1.229931E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.148767E-02 | lm-token-buffer-mib: 5.804298E+01 | global_load_balancing_loss: 1.001016E+00 | loss scale: 1.0 | grad norm: 0.126 | params norm: 11338.942 | params rms: 0.061397 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 2:09:35.521814 | finish at 2026-10-10 18:17:44 + [2026-10-08 16:09:59] iteration 4790/ 6855 | consumed samples: 613120 | elapsed time per iteration (ms): 110485.8 | throughput per GPU (TFLOP/s/GPU): 33.4 | MFU 10.71% | FLOPs estimator: workload-aware | logical tokens/s: 151849.4 | valid tokens/s: 151849.4 | physical tokens/s: 152206.1 | loss tokens/s: 110309.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.014119E-01 | lm-global-supervised-tokens: 1.218765E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.381955E-02 | lm-token-buffer-mib: 5.813322E+01 | global_load_balancing_loss: 1.001247E+00 | loss scale: 1.0 | grad norm: 0.100 | params norm: 11338.884 | params rms: 0.061396 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 15:22:33.174596 | finish at 2026-10-11 07:32:32 + [2026-10-08 16:11:25] iteration 4791/ 6855 | consumed samples: 613248 | elapsed time per iteration (ms): 86011.4 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.80% | FLOPs estimator: workload-aware | logical tokens/s: 195057.8 | valid tokens/s: 195057.8 | physical tokens/s: 195413.7 | loss tokens/s: 147966.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.827683E-01 | lm-global-supervised-tokens: 1.272682E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.205606E-02 | lm-token-buffer-mib: 5.807214E+01 | global_load_balancing_loss: 1.000884E+00 | loss scale: 1.0 | grad norm: 0.082 | params norm: 11338.827 | params rms: 0.061396 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 1:18:47.578777 | finish at 2026-10-10 17:30:13 + [2026-10-08 16:12:51] iteration 4792/ 6855 | consumed samples: 613376 | elapsed time per iteration (ms): 85428.7 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.45% | FLOPs estimator: workload-aware | logical tokens/s: 196387.8 | valid tokens/s: 196387.8 | physical tokens/s: 197003.4 | loss tokens/s: 149624.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.458029E-01 | lm-global-supervised-tokens: 1.278222E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.315236E-02 | lm-token-buffer-mib: 5.823386E+01 | global_load_balancing_loss: 9.996623E-01 | loss scale: 1.0 | grad norm: 0.068 | params norm: 11338.771 | params rms: 0.061396 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 0:57:19.431648 | finish at 2026-10-10 17:10:10 + [2026-10-08 16:14:19] iteration 4793/ 6855 | consumed samples: 613504 | elapsed time per iteration (ms): 88127.3 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.60% | FLOPs estimator: workload-aware | logical tokens/s: 190374.7 | valid tokens/s: 190374.7 | physical tokens/s: 190741.1 | loss tokens/s: 148220.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.388042E-01 | lm-global-supervised-tokens: 1.306228E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.771775E-02 | lm-token-buffer-mib: 5.808186E+01 | global_load_balancing_loss: 9.996262E-01 | loss scale: 1.0 | grad norm: 0.086 | params norm: 11338.715 | params rms: 0.061395 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 2:28:38.407108 | finish at 2026-10-10 18:42:57 + [2026-10-08 16:15:46] iteration 4794/ 6855 | consumed samples: 613632 | elapsed time per iteration (ms): 87409.4 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.74% | FLOPs estimator: workload-aware | logical tokens/s: 191937.9 | valid tokens/s: 191937.9 | physical tokens/s: 192295.6 | loss tokens/s: 135534.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.651844E-01 | lm-global-supervised-tokens: 1.184699E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.856647E-02 | lm-token-buffer-mib: 5.806193E+01 | global_load_balancing_loss: 1.000728E+00 | loss scale: 1.0 | grad norm: 0.059 | params norm: 11338.658 | params rms: 0.061395 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 2:02:30.685906 | finish at 2026-10-10 18:18:17 + [2026-10-08 16:17:14] iteration 4795/ 6855 | consumed samples: 613760 | elapsed time per iteration (ms): 87817.0 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.62% | FLOPs estimator: workload-aware | logical tokens/s: 191046.6 | valid tokens/s: 191046.6 | physical tokens/s: 191495.5 | loss tokens/s: 145163.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.633573E-01 | lm-global-supervised-tokens: 1.274782E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.218370E-02 | lm-token-buffer-mib: 5.811245E+01 | global_load_balancing_loss: 1.000199E+00 | loss scale: 1.0 | grad norm: 0.071 | params norm: 11338.602 | params rms: 0.061395 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 2:15:03.003612 | finish at 2026-10-10 18:32:17 + [2026-10-08 16:18:43] iteration 4796/ 6855 | consumed samples: 613888 | elapsed time per iteration (ms): 88656.2 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.50% | FLOPs estimator: workload-aware | logical tokens/s: 189239.1 | valid tokens/s: 189239.1 | physical tokens/s: 189762.5 | loss tokens/s: 139954.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.353292E-01 | lm-global-supervised-tokens: 1.240786E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.109638E-02 | lm-token-buffer-mib: 5.815374E+01 | global_load_balancing_loss: 1.000098E+00 | loss scale: 1.0 | grad norm: 0.102 | params norm: 11338.546 | params rms: 0.061395 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 2:42:23.035643 | finish at 2026-10-10 19:01:06 + [2026-10-08 16:20:11] iteration 4797/ 6855 | consumed samples: 614016 | elapsed time per iteration (ms): 88073.6 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.45% | FLOPs estimator: workload-aware | logical tokens/s: 190490.6 | valid tokens/s: 190490.6 | physical tokens/s: 190913.1 | loss tokens/s: 140510.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.576575E-01 | lm-global-supervised-tokens: 1.237530E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.590875E-02 | lm-token-buffer-mib: 5.833241E+01 | global_load_balancing_loss: 1.000025E+00 | loss scale: 1.0 | grad norm: 0.126 | params norm: 11338.489 | params rms: 0.061394 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 2:20:55.470383 | finish at 2026-10-10 18:41:06 + [2026-10-08 16:21:39] iteration 4798/ 6855 | consumed samples: 614144 | elapsed time per iteration (ms): 87891.4 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.70% | FLOPs estimator: workload-aware | logical tokens/s: 190885.3 | valid tokens/s: 190885.3 | physical tokens/s: 191211.8 | loss tokens/s: 142434.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.896604E-01 | lm-global-supervised-tokens: 1.251876E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.726978E-02 | lm-token-buffer-mib: 5.808393E+01 | global_load_balancing_loss: 1.000339E+00 | loss scale: 1.0 | grad norm: 0.067 | params norm: 11338.430 | params rms: 0.061394 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 2:13:12.645314 | finish at 2026-10-10 18:34:51 + [2026-10-08 16:23:04] iteration 4799/ 6855 | consumed samples: 614272 | elapsed time per iteration (ms): 85268.4 | throughput per GPU (TFLOP/s/GPU): 43.5 | MFU 13.94% | FLOPs estimator: workload-aware | logical tokens/s: 196757.8 | valid tokens/s: 196757.8 | physical tokens/s: 197172.8 | loss tokens/s: 143919.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.028095E-01 | lm-global-supervised-tokens: 1.227174E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.153212E-02 | lm-token-buffer-mib: 5.826587E+01 | global_load_balancing_loss: 1.000774E+00 | loss scale: 1.0 | grad norm: 0.078 | params norm: 11338.372 | params rms: 0.061394 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 0:41:51.794521 | finish at 2026-10-10 17:04:56 + [2026-10-08 16:24:54] iteration 4800/ 6855 | consumed samples: 614400 | elapsed time per iteration (ms): 110178.3 | throughput per GPU (TFLOP/s/GPU): 33.9 | MFU 10.88% | FLOPs estimator: workload-aware | logical tokens/s: 152272.6 | valid tokens/s: 152272.6 | physical tokens/s: 152655.9 | loss tokens/s: 109121.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.611637E-01 | lm-global-supervised-tokens: 1.202287E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.451455E-02 | lm-token-buffer-mib: 5.841246E+01 | global_load_balancing_loss: 1.000516E+00 | loss scale: 1.0 | grad norm: 0.130 | params norm: 11338.316 | params rms: 0.061393 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 14:53:36.490574 | finish at 2026-10-11 07:18:31 +times across ranks (ms): + save-checkpoint: + rank 0: 24414.52 + rank 1: 24414.53 + rank 2: 24414.58 + rank 3: 24414.52 + rank 4: 24414.57 + rank 5: 24414.56 + rank 6: 24414.55 + rank 7: 24414.56 + rank 8: 24414.28 + rank 9: 24414.26 + rank 10: 24414.26 + rank 11: 24414.27 + rank 12: 24414.25 + rank 13: 24414.26 + rank 14: 24414.25 + rank 15: 24414.25 + rank 16: 24413.46 + rank 17: 24413.47 + rank 18: 24413.42 + rank 19: 24413.53 + rank 20: 24413.42 + rank 21: 24413.48 + rank 22: 24413.42 + rank 23: 24413.42 + rank 24: 24415.80 + rank 25: 24415.79 + rank 26: 24415.81 + rank 27: 24415.81 + rank 28: 24415.74 + rank 29: 24415.86 + rank 30: 24415.79 + rank 31: 24415.79 + rank 32: 24414.63 + rank 33: 24414.63 + rank 34: 24414.63 + rank 35: 24414.60 + rank 36: 24414.62 + rank 37: 24414.59 + rank 38: 24414.61 + rank 39: 24414.59 + rank 40: 24414.26 + rank 41: 24414.21 + rank 42: 24414.22 + rank 43: 24414.26 + rank 44: 24414.19 + rank 45: 24414.22 + rank 46: 24414.21 + rank 47: 24414.20 + rank 48: 24414.76 + rank 49: 24414.73 + rank 50: 24414.73 + rank 51: 24414.76 + rank 52: 24414.75 + rank 53: 24414.70 + rank 54: 24414.72 + rank 55: 24414.71 + rank 56: 24414.48 + rank 57: 24414.46 + rank 58: 24414.45 + rank 59: 24414.46 + rank 60: 24414.45 + rank 61: 24414.54 + rank 62: 24414.46 + rank 63: 24414.45 + [2026-10-08 16:26:47] iteration 4801/ 6855 | consumed samples: 614528 | elapsed time per iteration (ms): 88312.3 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.58% | FLOPs estimator: workload-aware | logical tokens/s: 189975.2 | valid tokens/s: 189975.2 | physical tokens/s: 190425.2 | loss tokens/s: 136597.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.703023E-01 | lm-global-supervised-tokens: 1.206326E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.109306E-02 | lm-token-buffer-mib: 5.812984E+01 | global_load_balancing_loss: 9.998871E-01 | loss scale: 1.0 | grad norm: 0.116 | params norm: 11338.260 | params rms: 0.061393 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 2:23:13.378432 | finish at 2026-10-10 18:50:01 + [2026-10-08 16:28:18] iteration 4802/ 6855 | consumed samples: 614656 | elapsed time per iteration (ms): 91205.7 | throughput per GPU (TFLOP/s/GPU): 40.3 | MFU 12.92% | FLOPs estimator: workload-aware | logical tokens/s: 183948.1 | valid tokens/s: 183948.1 | physical tokens/s: 184376.7 | loss tokens/s: 139097.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.115499E-01 | lm-global-supervised-tokens: 1.268646E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.804092E-02 | lm-token-buffer-mib: 5.804837E+01 | global_load_balancing_loss: 9.989442E-01 | loss scale: 1.0 | grad norm: 0.081 | params norm: 11338.204 | params rms: 0.061393 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 4:00:45.275016 | finish at 2026-10-10 20:29:04 + [2026-10-08 16:29:45] iteration 4803/ 6855 | consumed samples: 614784 | elapsed time per iteration (ms): 86610.7 | throughput per GPU (TFLOP/s/GPU): 43.2 | MFU 13.84% | FLOPs estimator: workload-aware | logical tokens/s: 193708.0 | valid tokens/s: 193707.9 | physical tokens/s: 194094.6 | loss tokens/s: 141819.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.034190E-01 | lm-global-supervised-tokens: 1.228306E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.415359E-02 | lm-token-buffer-mib: 5.808306E+01 | global_load_balancing_loss: 9.997508E-01 | loss scale: 1.0 | grad norm: 0.076 | params norm: 11338.149 | params rms: 0.061392 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 1:22:05.150308 | finish at 2026-10-10 17:51:50 + [2026-10-08 16:31:10] iteration 4804/ 6855 | consumed samples: 614912 | elapsed time per iteration (ms): 84795.1 | throughput per GPU (TFLOP/s/GPU): 43.4 | MFU 13.90% | FLOPs estimator: workload-aware | logical tokens/s: 197855.7 | valid tokens/s: 197855.7 | physical tokens/s: 198262.2 | loss tokens/s: 146090.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.617264E-01 | lm-global-supervised-tokens: 1.238773E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.155538E-02 | lm-token-buffer-mib: 5.813546E+01 | global_load_balancing_loss: 1.000069E+00 | loss scale: 1.0 | grad norm: 0.049 | params norm: 11338.093 | params rms: 0.061392 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 0:18:34.785254 | finish at 2026-10-10 16:49:45 + [2026-10-08 16:32:36] iteration 4805/ 6855 | consumed samples: 615040 | elapsed time per iteration (ms): 86626.3 | throughput per GPU (TFLOP/s/GPU): 43.2 | MFU 13.84% | FLOPs estimator: workload-aware | logical tokens/s: 193673.4 | valid tokens/s: 193673.3 | physical tokens/s: 194107.1 | loss tokens/s: 145073.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.608700E-01 | lm-global-supervised-tokens: 1.256716E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.207977E-02 | lm-token-buffer-mib: 5.821347E+01 | global_load_balancing_loss: 1.000997E+00 | loss scale: 1.0 | grad norm: 0.074 | params norm: 11338.038 | params rms: 0.061392 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 1:19:43.955276 | finish at 2026-10-10 17:52:20 + [2026-10-08 16:34:01] iteration 4806/ 6855 | consumed samples: 615168 | elapsed time per iteration (ms): 84313.0 | throughput per GPU (TFLOP/s/GPU): 43.2 | MFU 13.83% | FLOPs estimator: workload-aware | logical tokens/s: 198986.3 | valid tokens/s: 198986.3 | physical tokens/s: 199758.3 | loss tokens/s: 147429.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.319226E-01 | lm-global-supervised-tokens: 1.243023E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.375460E-02 | lm-token-buffer-mib: 5.831024E+01 | global_load_balancing_loss: 1.001436E+00 | loss scale: 1.0 | grad norm: 0.038 | params norm: 11337.984 | params rms: 0.061391 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 23:59:17.384312 | finish at 2026-10-10 16:33:18 + [2026-10-08 16:35:28] iteration 4807/ 6855 | consumed samples: 615296 | elapsed time per iteration (ms): 87182.3 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.68% | FLOPs estimator: workload-aware | logical tokens/s: 192437.3 | valid tokens/s: 192437.3 | physical tokens/s: 192918.6 | loss tokens/s: 145455.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.602255E-01 | lm-global-supervised-tokens: 1.268115E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.434298E-02 | lm-token-buffer-mib: 5.815685E+01 | global_load_balancing_loss: 1.001631E+00 | loss scale: 1.0 | grad norm: 0.062 | params norm: 11337.928 | params rms: 0.061391 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 1:35:49.367676 | finish at 2026-10-10 18:11:17 + [2026-10-08 16:36:55] iteration 4808/ 6855 | consumed samples: 615424 | elapsed time per iteration (ms): 86643.5 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.61% | FLOPs estimator: workload-aware | logical tokens/s: 193634.7 | valid tokens/s: 193634.7 | physical tokens/s: 194194.8 | loss tokens/s: 146419.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.565965E-01 | lm-global-supervised-tokens: 1.268630E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.145164E-02 | lm-token-buffer-mib: 5.811701E+01 | global_load_balancing_loss: 1.000270E+00 | loss scale: 1.0 | grad norm: 0.064 | params norm: 11337.872 | params rms: 0.061391 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 1:15:59.345708 | finish at 2026-10-10 17:52:54 + [2026-10-08 16:38:23] iteration 4809/ 6855 | consumed samples: 615552 | elapsed time per iteration (ms): 88382.7 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.64% | FLOPs estimator: workload-aware | logical tokens/s: 189823.6 | valid tokens/s: 189823.6 | physical tokens/s: 190529.7 | loss tokens/s: 132473.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.935930E-01 | lm-global-supervised-tokens: 1.170833E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.822372E-02 | lm-token-buffer-mib: 5.828291E+01 | global_load_balancing_loss: 9.992758E-01 | loss scale: 1.0 | grad norm: 0.046 | params norm: 11337.816 | params rms: 0.061391 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 2:13:51.053400 | finish at 2026-10-10 18:52:14 + [2026-10-08 16:40:14] iteration 4810/ 6855 | consumed samples: 615680 | elapsed time per iteration (ms): 110823.4 | throughput per GPU (TFLOP/s/GPU): 33.1 | MFU 10.61% | FLOPs estimator: workload-aware | logical tokens/s: 151385.9 | valid tokens/s: 151385.9 | physical tokens/s: 151870.1 | loss tokens/s: 112167.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.562996E-01 | lm-global-supervised-tokens: 1.243083E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.270791E-02 | lm-token-buffer-mib: 5.827836E+01 | global_load_balancing_loss: 9.992143E-01 | loss scale: 1.0 | grad norm: 0.104 | params norm: 11337.759 | params rms: 0.061390 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 14:57:13.925689 | finish at 2026-10-11 07:37:28 + [2026-10-08 16:41:44] iteration 4811/ 6855 | consumed samples: 615808 | elapsed time per iteration (ms): 90490.8 | throughput per GPU (TFLOP/s/GPU): 40.3 | MFU 12.92% | FLOPs estimator: workload-aware | logical tokens/s: 185402.1 | valid tokens/s: 185402.1 | physical tokens/s: 185975.0 | loss tokens/s: 139637.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.654025E-01 | lm-global-supervised-tokens: 1.263593E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.099105E-02 | lm-token-buffer-mib: 5.820726E+01 | global_load_balancing_loss: 9.991407E-01 | loss scale: 1.0 | grad norm: 0.112 | params norm: 11337.704 | params rms: 0.061390 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 3:22:43.195978 | finish at 2026-10-10 20:04:28 + [2026-10-08 16:43:12] iteration 4812/ 6855 | consumed samples: 615936 | elapsed time per iteration (ms): 87970.4 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.42% | FLOPs estimator: workload-aware | logical tokens/s: 190714.1 | valid tokens/s: 190714.0 | physical tokens/s: 191231.1 | loss tokens/s: 142627.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.874569E-01 | lm-global-supervised-tokens: 1.254700E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.140142E-02 | lm-token-buffer-mib: 5.806627E+01 | global_load_balancing_loss: 9.988876E-01 | loss scale: 1.0 | grad norm: 0.065 | params norm: 11337.649 | params rms: 0.061390 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 1:55:23.605815 | finish at 2026-10-10 18:38:36 + [2026-10-08 16:44:39] iteration 4813/ 6855 | consumed samples: 616064 | elapsed time per iteration (ms): 86904.8 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.54% | FLOPs estimator: workload-aware | logical tokens/s: 193051.9 | valid tokens/s: 193051.9 | physical tokens/s: 193659.7 | loss tokens/s: 144638.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.139693E-01 | lm-global-supervised-tokens: 1.256979E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.162208E-02 | lm-token-buffer-mib: 5.820148E+01 | global_load_balancing_loss: 1.000567E+00 | loss scale: 1.0 | grad norm: 0.066 | params norm: 11337.595 | params rms: 0.061389 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 1:17:39.515301 | finish at 2026-10-10 18:02:19 + [2026-10-08 16:46:03] iteration 4814/ 6855 | consumed samples: 616192 | elapsed time per iteration (ms): 83957.1 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.53% | FLOPs estimator: workload-aware | logical tokens/s: 199830.5 | valid tokens/s: 199830.5 | physical tokens/s: 200264.4 | loss tokens/s: 149919.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.267840E-01 | lm-global-supervised-tokens: 1.258676E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.452198E-02 | lm-token-buffer-mib: 5.828456E+01 | global_load_balancing_loss: 1.001378E+00 | loss scale: 1.0 | grad norm: 0.116 | params norm: 11337.539 | params rms: 0.061389 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 23:35:56.406863 | finish at 2026-10-10 16:22:00 + [2026-10-08 16:47:34] iteration 4815/ 6855 | consumed samples: 616320 | elapsed time per iteration (ms): 91267.0 | throughput per GPU (TFLOP/s/GPU): 40.8 | MFU 13.07% | FLOPs estimator: workload-aware | logical tokens/s: 183825.5 | valid tokens/s: 183825.5 | physical tokens/s: 184344.2 | loss tokens/s: 140160.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.243218E-01 | lm-global-supervised-tokens: 1.279201E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.097283E-02 | lm-token-buffer-mib: 5.811744E+01 | global_load_balancing_loss: 1.001260E+00 | loss scale: 1.0 | grad norm: 0.043 | params norm: 11337.485 | params rms: 0.061389 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 3:43:04.677486 | finish at 2026-10-10 20:30:39 + [2026-10-08 16:49:02] iteration 4816/ 6855 | consumed samples: 616448 | elapsed time per iteration (ms): 87283.8 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.78% | FLOPs estimator: workload-aware | logical tokens/s: 192213.8 | valid tokens/s: 192213.8 | physical tokens/s: 192736.5 | loss tokens/s: 140735.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.944481E-01 | lm-global-supervised-tokens: 1.228389E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.305705E-02 | lm-token-buffer-mib: 5.820427E+01 | global_load_balancing_loss: 1.001682E+00 | loss scale: 1.0 | grad norm: 0.147 | params norm: 11337.429 | params rms: 0.061388 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 1:26:11.582409 | finish at 2026-10-10 18:15:13 + [2026-10-08 16:50:30] iteration 4817/ 6855 | consumed samples: 616576 | elapsed time per iteration (ms): 88599.6 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.65% | FLOPs estimator: workload-aware | logical tokens/s: 189359.6 | valid tokens/s: 189359.6 | physical tokens/s: 189765.9 | loss tokens/s: 136321.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.042292E-01 | lm-global-supervised-tokens: 1.207806E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.564987E-02 | lm-token-buffer-mib: 5.815329E+01 | global_load_balancing_loss: 9.994624E-01 | loss scale: 1.0 | grad norm: 0.059 | params norm: 11337.371 | params rms: 0.061388 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 2:09:25.980583 | finish at 2026-10-10 18:59:56 + [2026-10-08 16:51:55] iteration 4818/ 6855 | consumed samples: 616704 | elapsed time per iteration (ms): 84220.6 | throughput per GPU (TFLOP/s/GPU): 43.9 | MFU 14.07% | FLOPs estimator: workload-aware | logical tokens/s: 199205.5 | valid tokens/s: 199205.5 | physical tokens/s: 199569.9 | loss tokens/s: 148278.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.253975E-01 | lm-global-supervised-tokens: 1.248810E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.774044E-02 | lm-token-buffer-mib: 5.810676E+01 | global_load_balancing_loss: 9.991522E-01 | loss scale: 1.0 | grad norm: 0.118 | params norm: 11337.315 | params rms: 0.061388 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 23:39:17.417341 | finish at 2026-10-10 16:31:12 + [2026-10-08 16:53:22] iteration 4819/ 6855 | consumed samples: 616832 | elapsed time per iteration (ms): 87116.0 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.70% | FLOPs estimator: workload-aware | logical tokens/s: 192584.8 | valid tokens/s: 192584.8 | physical tokens/s: 192929.5 | loss tokens/s: 141405.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.811288E-01 | lm-global-supervised-tokens: 1.231871E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.131989E-02 | lm-token-buffer-mib: 5.812740E+01 | global_load_balancing_loss: 9.986396E-01 | loss scale: 1.0 | grad norm: 0.068 | params norm: 11337.258 | params rms: 0.061388 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 1:16:08.170047 | finish at 2026-10-10 18:09:30 + [2026-10-08 16:55:11] iteration 4820/ 6855 | consumed samples: 616960 | elapsed time per iteration (ms): 109809.2 | throughput per GPU (TFLOP/s/GPU): 34.1 | MFU 10.93% | FLOPs estimator: workload-aware | logical tokens/s: 152785.1 | valid tokens/s: 152785.1 | physical tokens/s: 152992.0 | loss tokens/s: 113781.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.482556E-01 | lm-global-supervised-tokens: 1.249426E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.150578E-02 | lm-token-buffer-mib: 5.805641E+01 | global_load_balancing_loss: 1.000707E+00 | loss scale: 1.0 | grad norm: 0.081 | params norm: 11337.201 | params rms: 0.061387 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 14:04:21.765280 | finish at 2026-10-11 06:59:33 + [2026-10-08 16:56:40] iteration 4821/ 6855 | consumed samples: 617088 | elapsed time per iteration (ms): 88547.3 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.74% | FLOPs estimator: workload-aware | logical tokens/s: 189471.6 | valid tokens/s: 189471.6 | physical tokens/s: 189892.9 | loss tokens/s: 135292.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.427863E-01 | lm-global-supervised-tokens: 1.197977E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.894822E-02 | lm-token-buffer-mib: 5.814334E+01 | global_load_balancing_loss: 1.000964E+00 | loss scale: 1.0 | grad norm: 0.047 | params norm: 11337.143 | params rms: 0.061387 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 2:01:45.293269 | finish at 2026-10-10 18:58:25 + [2026-10-08 16:58:08] iteration 4822/ 6855 | consumed samples: 617216 | elapsed time per iteration (ms): 87657.9 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.77% | FLOPs estimator: workload-aware | logical tokens/s: 191393.6 | valid tokens/s: 191393.6 | physical tokens/s: 191704.3 | loss tokens/s: 134492.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.229826E-01 | lm-global-supervised-tokens: 1.178934E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.154299E-02 | lm-token-buffer-mib: 5.806382E+01 | global_load_balancing_loss: 1.001379E+00 | loss scale: 1.0 | grad norm: 0.049 | params norm: 11337.087 | params rms: 0.061387 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 1:30:08.423646 | finish at 2026-10-10 18:28:16 + [2026-10-08 16:59:36] iteration 4823/ 6855 | consumed samples: 617344 | elapsed time per iteration (ms): 88691.6 | throughput per GPU (TFLOP/s/GPU): 43.1 | MFU 13.83% | FLOPs estimator: workload-aware | logical tokens/s: 189163.5 | valid tokens/s: 189163.5 | physical tokens/s: 189479.9 | loss tokens/s: 133133.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.601864E-01 | lm-global-supervised-tokens: 1.180784E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.183932E-02 | lm-token-buffer-mib: 5.811402E+01 | global_load_balancing_loss: 1.000663E+00 | loss scale: 1.0 | grad norm: 0.067 | params norm: 11337.031 | params rms: 0.061386 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 2:03:41.367222 | finish at 2026-10-10 19:03:18 + [2026-10-08 17:01:07] iteration 4824/ 6855 | consumed samples: 617472 | elapsed time per iteration (ms): 91050.9 | throughput per GPU (TFLOP/s/GPU): 41.1 | MFU 13.16% | FLOPs estimator: workload-aware | logical tokens/s: 184261.3 | valid tokens/s: 184261.3 | physical tokens/s: 184607.8 | loss tokens/s: 139875.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.277353E-01 | lm-global-supervised-tokens: 1.273578E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.169485E-02 | lm-token-buffer-mib: 5.812762E+01 | global_load_balancing_loss: 1.000272E+00 | loss scale: 1.0 | grad norm: 0.069 | params norm: 11336.976 | params rms: 0.061386 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 3:22:04.399170 | finish at 2026-10-10 20:23:12 + [2026-10-08 17:02:39] iteration 4825/ 6855 | consumed samples: 617600 | elapsed time per iteration (ms): 91344.7 | throughput per GPU (TFLOP/s/GPU): 41.2 | MFU 13.20% | FLOPs estimator: workload-aware | logical tokens/s: 183669.2 | valid tokens/s: 183669.2 | physical tokens/s: 184124.7 | loss tokens/s: 137595.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.849849E-01 | lm-global-supervised-tokens: 1.256860E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.321138E-02 | lm-token-buffer-mib: 5.817298E+01 | global_load_balancing_loss: 9.990563E-01 | loss scale: 1.0 | grad norm: 0.055 | params norm: 11336.922 | params rms: 0.061386 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 3:30:29.711192 | finish at 2026-10-10 20:33:08 + [2026-10-08 17:04:04] iteration 4826/ 6855 | consumed samples: 617728 | elapsed time per iteration (ms): 85250.3 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.54% | FLOPs estimator: workload-aware | logical tokens/s: 196799.4 | valid tokens/s: 196799.4 | physical tokens/s: 197324.4 | loss tokens/s: 145085.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.464406E-01 | lm-global-supervised-tokens: 1.236859E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.220206E-02 | lm-token-buffer-mib: 5.819677E+01 | global_load_balancing_loss: 9.984423E-01 | loss scale: 1.0 | grad norm: 0.052 | params norm: 11336.867 | params rms: 0.061385 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 0:02:52.861945 | finish at 2026-10-10 17:06:57 + [2026-10-08 17:05:33] iteration 4827/ 6855 | consumed samples: 617856 | elapsed time per iteration (ms): 89236.1 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.41% | FLOPs estimator: workload-aware | logical tokens/s: 188008.9 | valid tokens/s: 188008.9 | physical tokens/s: 188416.5 | loss tokens/s: 138081.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.473564E-01 | lm-global-supervised-tokens: 1.232184E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.164817E-02 | lm-token-buffer-mib: 5.824563E+01 | global_load_balancing_loss: 9.997543E-01 | loss scale: 1.0 | grad norm: 0.051 | params norm: 11336.812 | params rms: 0.061385 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 2:16:10.820386 | finish at 2026-10-10 19:21:44 + [2026-10-08 17:07:02] iteration 4828/ 6855 | consumed samples: 617984 | elapsed time per iteration (ms): 88871.5 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.54% | FLOPs estimator: workload-aware | logical tokens/s: 188780.1 | valid tokens/s: 188780.1 | physical tokens/s: 189278.3 | loss tokens/s: 136954.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.411341E-01 | lm-global-supervised-tokens: 1.217136E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.857709E-02 | lm-token-buffer-mib: 5.850427E+01 | global_load_balancing_loss: 9.998933E-01 | loss scale: 1.0 | grad norm: 0.043 | params norm: 11336.758 | params rms: 0.061385 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 2:02:22.596256 | finish at 2026-10-10 19:09:25 + [2026-10-08 17:08:30] iteration 4829/ 6855 | consumed samples: 618112 | elapsed time per iteration (ms): 87521.4 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.59% | FLOPs estimator: workload-aware | logical tokens/s: 191692.8 | valid tokens/s: 191692.8 | physical tokens/s: 192161.2 | loss tokens/s: 138524.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.558480E-01 | lm-global-supervised-tokens: 1.212388E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.149572E-02 | lm-token-buffer-mib: 5.810159E+01 | global_load_balancing_loss: 1.001775E+00 | loss scale: 1.0 | grad norm: 0.044 | params norm: 11336.702 | params rms: 0.061385 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 1:15:18.286792 | finish at 2026-10-10 18:23:48 + [2026-10-08 17:10:19] iteration 4830/ 6855 | consumed samples: 618240 | elapsed time per iteration (ms): 108916.9 | throughput per GPU (TFLOP/s/GPU): 34.0 | MFU 10.91% | FLOPs estimator: workload-aware | logical tokens/s: 154036.6 | valid tokens/s: 154036.6 | physical tokens/s: 154371.3 | loss tokens/s: 116387.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.408879E-01 | lm-global-supervised-tokens: 1.267659E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.760467E-02 | lm-token-buffer-mib: 5.813653E+01 | global_load_balancing_loss: 1.001683E+00 | loss scale: 1.0 | grad norm: 0.057 | params norm: 11336.646 | params rms: 0.061384 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 13:15:56.722820 | finish at 2026-10-11 06:26:15 + [2026-10-08 17:11:49] iteration 4831/ 6855 | consumed samples: 618368 | elapsed time per iteration (ms): 90213.5 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.46% | FLOPs estimator: workload-aware | logical tokens/s: 185972.2 | valid tokens/s: 185972.2 | physical tokens/s: 186496.5 | loss tokens/s: 134983.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.192533E-01 | lm-global-supervised-tokens: 1.217731E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.149827E-02 | lm-token-buffer-mib: 5.811652E+01 | global_load_balancing_loss: 1.001170E+00 | loss scale: 1.0 | grad norm: 0.082 | params norm: 11336.591 | params rms: 0.061384 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 2:43:12.192087 | finish at 2026-10-10 19:55:01 + [2026-10-08 17:13:18] iteration 4832/ 6855 | consumed samples: 618496 | elapsed time per iteration (ms): 89265.3 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.67% | FLOPs estimator: workload-aware | logical tokens/s: 187947.7 | valid tokens/s: 187947.7 | physical tokens/s: 188357.5 | loss tokens/s: 136376.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.005683E-01 | lm-global-supervised-tokens: 1.217372E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.342245E-02 | lm-token-buffer-mib: 5.806284E+01 | global_load_balancing_loss: 1.000673E+00 | loss scale: 1.0 | grad norm: 0.055 | params norm: 11336.536 | params rms: 0.061384 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 2:09:43.764673 | finish at 2026-10-10 19:23:02 + [2026-10-08 17:14:45] iteration 4833/ 6855 | consumed samples: 618624 | elapsed time per iteration (ms): 87330.0 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.61% | FLOPs estimator: workload-aware | logical tokens/s: 192112.7 | valid tokens/s: 192112.7 | physical tokens/s: 192410.4 | loss tokens/s: 140684.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.175821E-01 | lm-global-supervised-tokens: 1.228596E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.368136E-02 | lm-token-buffer-mib: 5.814191E+01 | global_load_balancing_loss: 9.997296E-01 | loss scale: 1.0 | grad norm: 0.044 | params norm: 11336.480 | params rms: 0.061383 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 1:03:01.237670 | finish at 2026-10-10 18:17:47 + [2026-10-08 17:16:13] iteration 4834/ 6855 | consumed samples: 618752 | elapsed time per iteration (ms): 87406.1 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.46% | FLOPs estimator: workload-aware | logical tokens/s: 191945.5 | valid tokens/s: 191945.5 | physical tokens/s: 192470.7 | loss tokens/s: 137120.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.529406E-01 | lm-global-supervised-tokens: 1.198514E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.110172E-02 | lm-token-buffer-mib: 5.807687E+01 | global_load_balancing_loss: 9.992280E-01 | loss scale: 1.0 | grad norm: 0.071 | params norm: 11336.426 | params rms: 0.061383 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 1:04:07.671313 | finish at 2026-10-10 18:20:20 + [2026-10-08 17:17:40] iteration 4835/ 6855 | consumed samples: 618880 | elapsed time per iteration (ms): 87263.5 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.61% | FLOPs estimator: workload-aware | logical tokens/s: 192258.9 | valid tokens/s: 192258.9 | physical tokens/s: 192717.0 | loss tokens/s: 138661.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.743893E-01 | lm-global-supervised-tokens: 1.210005E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.030298E-02 | lm-token-buffer-mib: 5.822377E+01 | global_load_balancing_loss: 9.995549E-01 | loss scale: 1.0 | grad norm: 0.179 | params norm: 11336.374 | params rms: 0.061383 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 0:57:52.217455 | finish at 2026-10-10 18:15:32 + [2026-10-08 17:19:04] iteration 4836/ 6855 | consumed samples: 619008 | elapsed time per iteration (ms): 84359.2 | throughput per GPU (TFLOP/s/GPU): 43.2 | MFU 13.83% | FLOPs estimator: workload-aware | logical tokens/s: 198877.8 | valid tokens/s: 198877.7 | physical tokens/s: 199335.5 | loss tokens/s: 152005.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.392729E-01 | lm-global-supervised-tokens: 1.282307E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.324033E-02 | lm-token-buffer-mib: 5.824412E+01 | global_load_balancing_loss: 9.996317E-01 | loss scale: 1.0 | grad norm: 0.440 | params norm: 11336.321 | params rms: 0.061382 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 23:18:41.284972 | finish at 2026-10-10 16:37:46 + [2026-10-08 17:20:30] iteration 4837/ 6855 | consumed samples: 619136 | elapsed time per iteration (ms): 85249.2 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.70% | FLOPs estimator: workload-aware | logical tokens/s: 196801.0 | valid tokens/s: 196800.9 | physical tokens/s: 197189.1 | loss tokens/s: 146790.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.129251E-01 | lm-global-supervised-tokens: 1.251374E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.119652E-02 | lm-token-buffer-mib: 5.803874E+01 | global_load_balancing_loss: 9.996505E-01 | loss scale: 1.0 | grad norm: 0.316 | params norm: 11336.267 | params rms: 0.061382 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 23:47:12.812201 | finish at 2026-10-10 17:07:42 + [2026-10-08 17:21:56] iteration 4838/ 6855 | consumed samples: 619264 | elapsed time per iteration (ms): 86841.3 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.68% | FLOPs estimator: workload-aware | logical tokens/s: 193193.2 | valid tokens/s: 193193.2 | physical tokens/s: 193574.2 | loss tokens/s: 141241.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.932567E-01 | lm-global-supervised-tokens: 1.226560E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.167179E-02 | lm-token-buffer-mib: 5.809739E+01 | global_load_balancing_loss: 1.001277E+00 | loss scale: 1.0 | grad norm: 0.154 | params norm: 11336.213 | params rms: 0.061382 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 0:39:18.808348 | finish at 2026-10-10 18:01:15 + [2026-10-08 17:23:24] iteration 4839/ 6855 | consumed samples: 619392 | elapsed time per iteration (ms): 87720.6 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.41% | FLOPs estimator: workload-aware | logical tokens/s: 191257.4 | valid tokens/s: 191257.3 | physical tokens/s: 191688.0 | loss tokens/s: 140680.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.478741E-01 | lm-global-supervised-tokens: 1.234062E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.157521E-02 | lm-token-buffer-mib: 5.817330E+01 | global_load_balancing_loss: 1.001272E+00 | loss scale: 1.0 | grad norm: 0.219 | params norm: 11336.159 | params rms: 0.061382 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 1:07:24.772636 | finish at 2026-10-10 18:30:49 + [2026-10-08 17:25:16] iteration 4840/ 6855 | consumed samples: 619520 | elapsed time per iteration (ms): 112216.2 | throughput per GPU (TFLOP/s/GPU): 34.6 | MFU 11.10% | FLOPs estimator: workload-aware | logical tokens/s: 149507.8 | valid tokens/s: 149507.8 | physical tokens/s: 149804.6 | loss tokens/s: 101013.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.318386E-01 | lm-global-supervised-tokens: 1.133532E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.155624E-02 | lm-token-buffer-mib: 5.802576E+01 | global_load_balancing_loss: 1.000607E+00 | loss scale: 1.0 | grad norm: 0.143 | params norm: 11336.105 | params rms: 0.061381 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 14:48:35.699437 | finish at 2026-10-11 08:13:52 +times across ranks (ms): + save-checkpoint: + rank 0: 25222.47 + rank 1: 25222.49 + rank 2: 25222.48 + rank 3: 25222.46 + rank 4: 25222.47 + rank 5: 25222.46 + rank 6: 25222.44 + rank 7: 25222.48 + rank 8: 25223.88 + rank 9: 25223.85 + rank 10: 25223.84 + rank 11: 25223.85 + rank 12: 25223.88 + rank 13: 25223.85 + rank 14: 25223.85 + rank 15: 25223.84 + rank 16: 25225.01 + rank 17: 25224.92 + rank 18: 25224.93 + rank 19: 25224.96 + rank 20: 25224.92 + rank 21: 25224.99 + rank 22: 25224.93 + rank 23: 25224.92 + rank 24: 25223.90 + rank 25: 25224.07 + rank 26: 25223.97 + rank 27: 25223.94 + rank 28: 25223.83 + rank 29: 25223.88 + rank 30: 25223.82 + rank 31: 25223.91 + rank 32: 25224.50 + rank 33: 25224.54 + rank 34: 25224.61 + rank 35: 25224.45 + rank 36: 25224.55 + rank 37: 25224.44 + rank 38: 25224.49 + rank 39: 25224.43 + rank 40: 25224.11 + rank 41: 25224.07 + rank 42: 25224.08 + rank 43: 25224.10 + rank 44: 25224.09 + rank 45: 25224.07 + rank 46: 25224.08 + rank 47: 25224.06 + rank 48: 25223.71 + rank 49: 25223.89 + rank 50: 25223.91 + rank 51: 25223.89 + rank 52: 25224.01 + rank 53: 25223.92 + rank 54: 25223.94 + rank 55: 25223.87 + rank 56: 25223.91 + rank 57: 25223.89 + rank 58: 25223.89 + rank 59: 25223.88 + rank 60: 25223.88 + rank 61: 25223.89 + rank 62: 25223.88 + rank 63: 25223.96 + [2026-10-08 17:27:06] iteration 4841/ 6855 | consumed samples: 619648 | elapsed time per iteration (ms): 84553.7 | throughput per GPU (TFLOP/s/GPU): 42.8 | MFU 13.73% | FLOPs estimator: workload-aware | logical tokens/s: 198420.6 | valid tokens/s: 198420.6 | physical tokens/s: 198763.6 | loss tokens/s: 150857.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.725071E-01 | lm-global-supervised-tokens: 1.275556E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 7.532084E-02 | lm-token-buffer-mib: 5.812860E+01 | global_load_balancing_loss: 1.000166E+00 | loss scale: 1.0 | grad norm: 0.103 | params norm: 11336.049 | params rms: 0.061381 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 23:18:11.213683 | finish at 2026-10-10 16:45:18 + [2026-10-08 17:28:40] iteration 4842/ 6855 | consumed samples: 619776 | elapsed time per iteration (ms): 93256.3 | throughput per GPU (TFLOP/s/GPU): 40.0 | MFU 12.83% | FLOPs estimator: workload-aware | logical tokens/s: 179903.8 | valid tokens/s: 179903.7 | physical tokens/s: 180212.6 | loss tokens/s: 134430.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.054701E-01 | lm-global-supervised-tokens: 1.253646E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.428230E-02 | lm-token-buffer-mib: 5.808091E+01 | global_load_balancing_loss: 9.996461E-01 | loss scale: 1.0 | grad norm: 0.065 | params norm: 11335.993 | params rms: 0.061381 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 4:08:45.015354 | finish at 2026-10-10 21:37:25 + [2026-10-08 17:30:08] iteration 4843/ 6855 | consumed samples: 619904 | elapsed time per iteration (ms): 88080.5 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.51% | FLOPs estimator: workload-aware | logical tokens/s: 190475.4 | valid tokens/s: 190475.4 | physical tokens/s: 191072.2 | loss tokens/s: 141388.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.465149E-01 | lm-global-supervised-tokens: 1.245360E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.108944E-02 | lm-token-buffer-mib: 5.811694E+01 | global_load_balancing_loss: 9.994823E-01 | loss scale: 1.0 | grad norm: 0.138 | params norm: 11335.938 | params rms: 0.061380 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 1:13:37.924999 | finish at 2026-10-10 18:43:46 + [2026-10-08 17:31:32] iteration 4844/ 6855 | consumed samples: 620032 | elapsed time per iteration (ms): 84404.5 | throughput per GPU (TFLOP/s/GPU): 42.8 | MFU 13.70% | FLOPs estimator: workload-aware | logical tokens/s: 198770.8 | valid tokens/s: 198770.8 | physical tokens/s: 199293.3 | loss tokens/s: 150162.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.575772E-01 | lm-global-supervised-tokens: 1.267435E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.145718E-02 | lm-token-buffer-mib: 5.807285E+01 | global_load_balancing_loss: 9.989464E-01 | loss scale: 1.0 | grad norm: 0.053 | params norm: 11335.885 | params rms: 0.061380 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 23:08:57.451433 | finish at 2026-10-10 16:40:30 + [2026-10-08 17:33:00] iteration 4845/ 6855 | consumed samples: 620160 | elapsed time per iteration (ms): 87651.2 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.63% | FLOPs estimator: workload-aware | logical tokens/s: 191408.2 | valid tokens/s: 191407.8 | physical tokens/s: 191962.3 | loss tokens/s: 138149.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.691326E-01 | lm-global-supervised-tokens: 1.210898E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.186079E-02 | lm-token-buffer-mib: 5.803248E+01 | global_load_balancing_loss: 1.001026E+00 | loss scale: 1.0 | grad norm: 0.137 | params norm: 11335.831 | params rms: 0.061380 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 0:56:18.908758 | finish at 2026-10-10 18:29:19 + [2026-10-08 17:34:22] iteration 4846/ 6855 | consumed samples: 620288 | elapsed time per iteration (ms): 82332.1 | throughput per GPU (TFLOP/s/GPU): 44.0 | MFU 14.10% | FLOPs estimator: workload-aware | logical tokens/s: 203774.9 | valid tokens/s: 203774.9 | physical tokens/s: 204111.3 | loss tokens/s: 153919.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.356176E-01 | lm-global-supervised-tokens: 1.267250E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.168633E-02 | lm-token-buffer-mib: 5.815479E+01 | global_load_balancing_loss: 1.000401E+00 | loss scale: 1.0 | grad norm: 0.037 | params norm: 11335.776 | params rms: 0.061380 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 21:56:45.214593 | finish at 2026-10-10 15:31:07 + [2026-10-08 17:35:50] iteration 4847/ 6855 | consumed samples: 620416 | elapsed time per iteration (ms): 88078.4 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.64% | FLOPs estimator: workload-aware | logical tokens/s: 190479.8 | valid tokens/s: 190479.7 | physical tokens/s: 190824.5 | loss tokens/s: 137552.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.149576E-01 | lm-global-supervised-tokens: 1.211543E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.477261E-02 | lm-token-buffer-mib: 5.803437E+01 | global_load_balancing_loss: 1.000889E+00 | loss scale: 1.0 | grad norm: 0.096 | params norm: 11335.719 | params rms: 0.061379 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 1:07:41.423162 | finish at 2026-10-10 18:43:32 + [2026-10-08 17:37:20] iteration 4848/ 6855 | consumed samples: 620544 | elapsed time per iteration (ms): 89581.9 | throughput per GPU (TFLOP/s/GPU): 41.0 | MFU 13.16% | FLOPs estimator: workload-aware | logical tokens/s: 187283.5 | valid tokens/s: 187283.5 | physical tokens/s: 187829.9 | loss tokens/s: 143947.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.978765E-01 | lm-global-supervised-tokens: 1.289507E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.169102E-02 | lm-token-buffer-mib: 5.817164E+01 | global_load_balancing_loss: 1.000372E+00 | loss scale: 1.0 | grad norm: 0.065 | params norm: 11335.665 | params rms: 0.061379 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 1:56:30.850094 | finish at 2026-10-10 19:33:51 + [2026-10-08 17:38:49] iteration 4849/ 6855 | consumed samples: 620672 | elapsed time per iteration (ms): 88880.0 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.57% | FLOPs estimator: workload-aware | logical tokens/s: 188762.1 | valid tokens/s: 188762.0 | physical tokens/s: 189169.8 | loss tokens/s: 136680.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.627566E-01 | lm-global-supervised-tokens: 1.214816E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.878093E-02 | lm-token-buffer-mib: 5.811275E+01 | global_load_balancing_loss: 9.994097E-01 | loss scale: 1.0 | grad norm: 0.071 | params norm: 11335.612 | params rms: 0.061379 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 1:31:33.236229 | finish at 2026-10-10 19:10:22 + [2026-10-08 17:40:39] iteration 4850/ 6855 | consumed samples: 620800 | elapsed time per iteration (ms): 110706.1 | throughput per GPU (TFLOP/s/GPU): 33.6 | MFU 10.76% | FLOPs estimator: workload-aware | logical tokens/s: 151546.8 | valid tokens/s: 151546.8 | physical tokens/s: 151795.1 | loss tokens/s: 111849.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.062669E-01 | lm-global-supervised-tokens: 1.238241E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.395726E-02 | lm-token-buffer-mib: 5.808164E+01 | global_load_balancing_loss: 1.000316E+00 | loss scale: 1.0 | grad norm: 0.040 | params norm: 11335.559 | params rms: 0.061378 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 13:39:25.664506 | finish at 2026-10-11 07:20:05 + [2026-10-08 17:42:08] iteration 4851/ 6855 | consumed samples: 620928 | elapsed time per iteration (ms): 88253.1 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.56% | FLOPs estimator: workload-aware | logical tokens/s: 190102.6 | valid tokens/s: 190102.6 | physical tokens/s: 190423.9 | loss tokens/s: 139693.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.662946E-01 | lm-global-supervised-tokens: 1.232843E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.101108E-02 | lm-token-buffer-mib: 5.817525E+01 | global_load_balancing_loss: 9.999360E-01 | loss scale: 1.0 | grad norm: 0.060 | params norm: 11335.506 | params rms: 0.061378 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 1:07:39.248549 | finish at 2026-10-10 18:49:47 + [2026-10-08 17:43:39] iteration 4852/ 6855 | consumed samples: 621056 | elapsed time per iteration (ms): 91759.9 | throughput per GPU (TFLOP/s/GPU): 40.4 | MFU 12.94% | FLOPs estimator: workload-aware | logical tokens/s: 182837.9 | valid tokens/s: 182837.8 | physical tokens/s: 183235.8 | loss tokens/s: 132942.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.137393E-01 | lm-global-supervised-tokens: 1.219878E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.207148E-02 | lm-token-buffer-mib: 5.813625E+01 | global_load_balancing_loss: 9.999022E-01 | loss scale: 1.0 | grad norm: 0.065 | params norm: 11335.452 | params rms: 0.061378 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 3:03:15.074633 | finish at 2026-10-10 20:46:54 + [2026-10-08 17:45:06] iteration 4853/ 6855 | consumed samples: 621184 | elapsed time per iteration (ms): 86554.7 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.66% | FLOPs estimator: workload-aware | logical tokens/s: 193833.2 | valid tokens/s: 193833.2 | physical tokens/s: 194233.6 | loss tokens/s: 144798.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.972241E-01 | lm-global-supervised-tokens: 1.253294E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.236409E-02 | lm-token-buffer-mib: 5.805989E+01 | global_load_balancing_loss: 9.992803E-01 | loss scale: 1.0 | grad norm: 0.102 | params norm: 11335.400 | params rms: 0.061378 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 0:08:02.439507 | finish at 2026-10-10 17:53:08 + [2026-10-08 17:46:32] iteration 4854/ 6855 | consumed samples: 621312 | elapsed time per iteration (ms): 85602.6 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.65% | FLOPs estimator: workload-aware | logical tokens/s: 195988.9 | valid tokens/s: 195988.9 | physical tokens/s: 196382.7 | loss tokens/s: 147178.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.200483E-01 | lm-global-supervised-tokens: 1.259885E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.161723E-02 | lm-token-buffer-mib: 5.807278E+01 | global_load_balancing_loss: 1.000576E+00 | loss scale: 1.0 | grad norm: 0.049 | params norm: 11335.346 | params rms: 0.061377 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 23:34:50.721215 | finish at 2026-10-10 17:21:22 + [2026-10-08 17:48:01] iteration 4855/ 6855 | consumed samples: 621440 | elapsed time per iteration (ms): 89217.7 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.62% | FLOPs estimator: workload-aware | logical tokens/s: 188046.8 | valid tokens/s: 188046.8 | physical tokens/s: 188566.8 | loss tokens/s: 132769.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.674275E-01 | lm-global-supervised-tokens: 1.184539E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.548956E-02 | lm-token-buffer-mib: 5.813541E+01 | global_load_balancing_loss: 1.001786E+00 | loss scale: 1.0 | grad norm: 0.110 | params norm: 11335.293 | params rms: 0.061377 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 1:33:55.467243 | finish at 2026-10-10 19:21:56 + [2026-10-08 17:49:30] iteration 4856/ 6855 | consumed samples: 621568 | elapsed time per iteration (ms): 89340.4 | throughput per GPU (TFLOP/s/GPU): 41.7 | MFU 13.36% | FLOPs estimator: workload-aware | logical tokens/s: 187789.8 | valid tokens/s: 187789.7 | physical tokens/s: 188287.4 | loss tokens/s: 140824.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.722388E-01 | lm-global-supervised-tokens: 1.258130E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.142308E-02 | lm-token-buffer-mib: 5.826238E+01 | global_load_balancing_loss: 1.000433E+00 | loss scale: 1.0 | grad norm: 0.100 | params norm: 11335.239 | params rms: 0.061377 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 1:36:31.425246 | finish at 2026-10-10 19:26:02 + [2026-10-08 17:50:58] iteration 4857/ 6855 | consumed samples: 621696 | elapsed time per iteration (ms): 87643.8 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.67% | FLOPs estimator: workload-aware | logical tokens/s: 191424.9 | valid tokens/s: 191424.8 | physical tokens/s: 191833.7 | loss tokens/s: 138453.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.839659E-01 | lm-global-supervised-tokens: 1.213456E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.155934E-02 | lm-token-buffer-mib: 5.825490E+01 | global_load_balancing_loss: 1.001441E+00 | loss scale: 1.0 | grad norm: 0.035 | params norm: 11335.186 | params rms: 0.061376 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 0:38:32.400567 | finish at 2026-10-10 18:29:30 + [2026-10-08 17:52:23] iteration 4858/ 6855 | consumed samples: 621824 | elapsed time per iteration (ms): 84925.4 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.62% | FLOPs estimator: workload-aware | logical tokens/s: 197552.2 | valid tokens/s: 197552.2 | physical tokens/s: 198123.6 | loss tokens/s: 147396.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.421596E-01 | lm-global-supervised-tokens: 1.251768E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.587372E-02 | lm-token-buffer-mib: 5.810815E+01 | global_load_balancing_loss: 1.000736E+00 | loss scale: 1.0 | grad norm: 0.118 | params norm: 11335.133 | params rms: 0.061376 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 23:06:36.075258 | finish at 2026-10-10 16:58:59 + [2026-10-08 17:53:48] iteration 4859/ 6855 | consumed samples: 621952 | elapsed time per iteration (ms): 85336.9 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.67% | FLOPs estimator: workload-aware | logical tokens/s: 196599.6 | valid tokens/s: 196599.6 | physical tokens/s: 197003.9 | loss tokens/s: 146058.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.344560E-01 | lm-global-supervised-tokens: 1.246421E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.138328E-02 | lm-token-buffer-mib: 5.808513E+01 | global_load_balancing_loss: 9.999194E-01 | loss scale: 1.0 | grad norm: 0.145 | params norm: 11335.080 | params rms: 0.061376 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 23:18:52.396237 | finish at 2026-10-10 17:12:40 + [2026-10-08 17:55:38] iteration 4860/ 6855 | consumed samples: 622080 | elapsed time per iteration (ms): 109762.4 | throughput per GPU (TFLOP/s/GPU): 33.2 | MFU 10.65% | FLOPs estimator: workload-aware | logical tokens/s: 152850.3 | valid tokens/s: 152850.3 | physical tokens/s: 153122.6 | loss tokens/s: 116431.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.809738E-01 | lm-global-supervised-tokens: 1.277978E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.149812E-02 | lm-token-buffer-mib: 5.813367E+01 | global_load_balancing_loss: 9.993132E-01 | loss scale: 1.0 | grad norm: 0.068 | params norm: 11335.027 | params rms: 0.061375 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 12:49:35.949773 | finish at 2026-10-11 06:45:14 + [2026-10-08 17:57:04] iteration 4861/ 6855 | consumed samples: 622208 | elapsed time per iteration (ms): 86207.2 | throughput per GPU (TFLOP/s/GPU): 43.2 | MFU 13.86% | FLOPs estimator: workload-aware | logical tokens/s: 194614.3 | valid tokens/s: 194614.3 | physical tokens/s: 194962.4 | loss tokens/s: 141872.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.777464E-01 | lm-global-supervised-tokens: 1.223041E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.531446E-02 | lm-token-buffer-mib: 5.821071E+01 | global_load_balancing_loss: 9.973176E-01 | loss scale: 1.0 | grad norm: 0.144 | params norm: 11334.975 | params rms: 0.061375 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 23:44:57.074180 | finish at 2026-10-10 17:42:01 + [2026-10-08 17:58:30] iteration 4862/ 6855 | consumed samples: 622336 | elapsed time per iteration (ms): 86285.2 | throughput per GPU (TFLOP/s/GPU): 43.3 | MFU 13.87% | FLOPs estimator: workload-aware | logical tokens/s: 194437.5 | valid tokens/s: 194437.5 | physical tokens/s: 195006.5 | loss tokens/s: 142191.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.671531E-01 | lm-global-supervised-tokens: 1.226904E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.352960E-02 | lm-token-buffer-mib: 5.823547E+01 | global_load_balancing_loss: 9.993972E-01 | loss scale: 1.0 | grad norm: 0.067 | params norm: 11334.922 | params rms: 0.061375 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 23:46:06.436149 | finish at 2026-10-10 17:44:37 + [2026-10-08 18:00:00] iteration 4863/ 6855 | consumed samples: 622464 | elapsed time per iteration (ms): 89868.0 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.48% | FLOPs estimator: workload-aware | logical tokens/s: 186686.9 | valid tokens/s: 186686.9 | physical tokens/s: 187203.2 | loss tokens/s: 132951.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.588156E-01 | lm-global-supervised-tokens: 1.194809E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.521751E-02 | lm-token-buffer-mib: 5.814834E+01 | global_load_balancing_loss: 9.997002E-01 | loss scale: 1.0 | grad norm: 0.184 | params norm: 11334.869 | params rms: 0.061375 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 1:43:37.136324 | finish at 2026-10-10 19:43:37 + [2026-10-08 18:01:26] iteration 4864/ 6855 | consumed samples: 622592 | elapsed time per iteration (ms): 85974.8 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.51% | FLOPs estimator: workload-aware | logical tokens/s: 195140.4 | valid tokens/s: 195140.4 | physical tokens/s: 195545.1 | loss tokens/s: 150910.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.840192E-01 | lm-global-supervised-tokens: 1.297451E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.127250E-02 | lm-token-buffer-mib: 5.804597E+01 | global_load_balancing_loss: 1.000755E+00 | loss scale: 1.0 | grad norm: 0.086 | params norm: 11334.815 | params rms: 0.061374 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 23:32:55.819898 | finish at 2026-10-10 17:34:22 + [2026-10-08 18:02:54] iteration 4865/ 6855 | consumed samples: 622720 | elapsed time per iteration (ms): 88362.4 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.43% | FLOPs estimator: workload-aware | logical tokens/s: 189868.3 | valid tokens/s: 189868.1 | physical tokens/s: 190319.2 | loss tokens/s: 141425.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.395948E-01 | lm-global-supervised-tokens: 1.249671E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.160243E-02 | lm-token-buffer-mib: 5.815419E+01 | global_load_balancing_loss: 1.002380E+00 | loss scale: 1.0 | grad norm: 0.170 | params norm: 11334.763 | params rms: 0.061374 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 0:50:41.112533 | finish at 2026-10-10 18:53:36 + [2026-10-08 18:04:25] iteration 4866/ 6855 | consumed samples: 622848 | elapsed time per iteration (ms): 90263.8 | throughput per GPU (TFLOP/s/GPU): 41.2 | MFU 13.20% | FLOPs estimator: workload-aware | logical tokens/s: 185868.1 | valid tokens/s: 185868.1 | physical tokens/s: 186342.3 | loss tokens/s: 137530.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.571661E-01 | lm-global-supervised-tokens: 1.241404E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.153622E-02 | lm-token-buffer-mib: 5.810056E+01 | global_load_balancing_loss: 1.002783E+00 | loss scale: 1.0 | grad norm: 0.116 | params norm: 11334.710 | params rms: 0.061374 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 1:52:14.610757 | finish at 2026-10-10 19:56:39 + [2026-10-08 18:05:52] iteration 4867/ 6855 | consumed samples: 622976 | elapsed time per iteration (ms): 87426.4 | throughput per GPU (TFLOP/s/GPU): 41.1 | MFU 13.19% | FLOPs estimator: workload-aware | logical tokens/s: 191900.8 | valid tokens/s: 191900.8 | physical tokens/s: 192351.9 | loss tokens/s: 141403.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.222894E-01 | lm-global-supervised-tokens: 1.236242E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.483880E-02 | lm-token-buffer-mib: 5.811482E+01 | global_load_balancing_loss: 1.001496E+00 | loss scale: 1.0 | grad norm: 0.184 | params norm: 11334.659 | params rms: 0.061373 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 0:16:43.588298 | finish at 2026-10-10 18:22:36 + [2026-10-08 18:07:17] iteration 4868/ 6855 | consumed samples: 623104 | elapsed time per iteration (ms): 84464.3 | throughput per GPU (TFLOP/s/GPU): 43.1 | MFU 13.81% | FLOPs estimator: workload-aware | logical tokens/s: 198630.4 | valid tokens/s: 198630.4 | physical tokens/s: 199112.5 | loss tokens/s: 147795.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.452051E-01 | lm-global-supervised-tokens: 1.248347E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.357917E-02 | lm-token-buffer-mib: 5.816566E+01 | global_load_balancing_loss: 1.000243E+00 | loss scale: 1.0 | grad norm: 0.073 | params norm: 11334.609 | params rms: 0.061373 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 22:37:10.611309 | finish at 2026-10-10 16:44:27 + [2026-10-08 18:08:41] iteration 4869/ 6855 | consumed samples: 623232 | elapsed time per iteration (ms): 84263.0 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.63% | FLOPs estimator: workload-aware | logical tokens/s: 199105.2 | valid tokens/s: 199105.2 | physical tokens/s: 199623.3 | loss tokens/s: 158777.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.573888E-01 | lm-global-supervised-tokens: 1.337904E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.352043E-02 | lm-token-buffer-mib: 5.831876E+01 | global_load_balancing_loss: 9.983475E-01 | loss scale: 1.0 | grad norm: 0.144 | params norm: 11334.559 | params rms: 0.061373 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 22:29:06.230899 | finish at 2026-10-10 16:37:47 + [2026-10-08 18:10:34] iteration 4870/ 6855 | consumed samples: 623360 | elapsed time per iteration (ms): 112631.2 | throughput per GPU (TFLOP/s/GPU): 33.2 | MFU 10.63% | FLOPs estimator: workload-aware | logical tokens/s: 148956.3 | valid tokens/s: 148956.3 | physical tokens/s: 149490.8 | loss tokens/s: 106283.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.249306E-01 | lm-global-supervised-tokens: 1.197083E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.304730E-02 | lm-token-buffer-mib: 5.805331E+01 | global_load_balancing_loss: 1.000059E+00 | loss scale: 1.0 | grad norm: 0.038 | params norm: 11334.508 | params rms: 0.061373 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 14:06:12.871572 | finish at 2026-10-11 08:16:46 + [2026-10-08 18:11:58] iteration 4871/ 6855 | consumed samples: 623488 | elapsed time per iteration (ms): 84218.6 | throughput per GPU (TFLOP/s/GPU): 43.1 | MFU 13.83% | FLOPs estimator: workload-aware | logical tokens/s: 199210.4 | valid tokens/s: 199210.4 | physical tokens/s: 199627.4 | loss tokens/s: 154528.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.251556E-01 | lm-global-supervised-tokens: 1.301420E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.379573E-02 | lm-token-buffer-mib: 5.814452E+01 | global_load_balancing_loss: 9.986041E-01 | loss scale: 1.0 | grad norm: 0.087 | params norm: 11334.456 | params rms: 0.061372 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 22:24:49.630569 | finish at 2026-10-10 16:36:47 + [2026-10-08 18:13:26] iteration 4872/ 6855 | consumed samples: 623616 | elapsed time per iteration (ms): 88373.6 | throughput per GPU (TFLOP/s/GPU): 41.4 | MFU 13.26% | FLOPs estimator: workload-aware | logical tokens/s: 189843.8 | valid tokens/s: 189843.8 | physical tokens/s: 190302.0 | loss tokens/s: 141037.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.778089E-01 | lm-global-supervised-tokens: 1.246403E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.133173E-02 | lm-token-buffer-mib: 5.827049E+01 | global_load_balancing_loss: 9.986877E-01 | loss scale: 1.0 | grad norm: 0.063 | params norm: 11334.406 | params rms: 0.061372 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 0:40:44.895618 | finish at 2026-10-10 18:54:11 + [2026-10-08 18:14:54] iteration 4873/ 6855 | consumed samples: 623744 | elapsed time per iteration (ms): 87393.4 | throughput per GPU (TFLOP/s/GPU): 43.3 | MFU 13.88% | FLOPs estimator: workload-aware | logical tokens/s: 191973.1 | valid tokens/s: 191973.1 | physical tokens/s: 192524.4 | loss tokens/s: 143103.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.643968E-01 | lm-global-supervised-tokens: 1.250631E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.339103E-02 | lm-token-buffer-mib: 5.847618E+01 | global_load_balancing_loss: 1.000046E+00 | loss scale: 1.0 | grad norm: 0.066 | params norm: 11334.356 | params rms: 0.061372 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 0:06:53.737138 | finish at 2026-10-10 18:21:47 + [2026-10-08 18:16:19] iteration 4874/ 6855 | consumed samples: 623872 | elapsed time per iteration (ms): 85971.1 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.64% | FLOPs estimator: workload-aware | logical tokens/s: 195148.5 | valid tokens/s: 195148.5 | physical tokens/s: 195653.2 | loss tokens/s: 144189.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.072845E-01 | lm-global-supervised-tokens: 1.239612E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.116588E-02 | lm-token-buffer-mib: 5.801735E+01 | global_load_balancing_loss: 1.000182E+00 | loss scale: 1.0 | grad norm: 0.070 | params norm: 11334.302 | params rms: 0.061372 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 23:18:28.779510 | finish at 2026-10-10 17:34:48 + [2026-10-08 18:17:48] iteration 4875/ 6855 | consumed samples: 624000 | elapsed time per iteration (ms): 88753.4 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.59% | FLOPs estimator: workload-aware | logical tokens/s: 189031.7 | valid tokens/s: 189031.7 | physical tokens/s: 189357.0 | loss tokens/s: 138199.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.595507E-01 | lm-global-supervised-tokens: 1.226567E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.146822E-02 | lm-token-buffer-mib: 5.814877E+01 | global_load_balancing_loss: 1.000313E+00 | loss scale: 1.0 | grad norm: 0.066 | params norm: 11334.248 | params rms: 0.061371 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 0:48:51.764746 | finish at 2026-10-10 19:06:40 + [2026-10-08 18:19:14] iteration 4876/ 6855 | consumed samples: 624128 | elapsed time per iteration (ms): 85721.3 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.77% | FLOPs estimator: workload-aware | logical tokens/s: 195717.5 | valid tokens/s: 195717.5 | physical tokens/s: 196361.2 | loss tokens/s: 147561.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.565254E-01 | lm-global-supervised-tokens: 1.264919E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.658016E-02 | lm-token-buffer-mib: 5.831345E+01 | global_load_balancing_loss: 1.002219E+00 | loss scale: 1.0 | grad norm: 0.067 | params norm: 11334.195 | params rms: 0.061371 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 23:07:22.442567 | finish at 2026-10-10 17:26:36 + [2026-10-08 18:20:41] iteration 4877/ 6855 | consumed samples: 624256 | elapsed time per iteration (ms): 87070.6 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.62% | FLOPs estimator: workload-aware | logical tokens/s: 192684.8 | valid tokens/s: 192684.8 | physical tokens/s: 193019.8 | loss tokens/s: 144529.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.568242E-01 | lm-global-supervised-tokens: 1.258422E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.854982E-02 | lm-token-buffer-mib: 5.801689E+01 | global_load_balancing_loss: 1.001430E+00 | loss scale: 1.0 | grad norm: 0.065 | params norm: 11334.141 | params rms: 0.061371 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 23:50:25.674217 | finish at 2026-10-10 18:11:07 + [2026-10-08 18:22:09] iteration 4878/ 6855 | consumed samples: 624384 | elapsed time per iteration (ms): 87708.9 | throughput per GPU (TFLOP/s/GPU): 43.9 | MFU 14.08% | FLOPs estimator: workload-aware | logical tokens/s: 191283.0 | valid tokens/s: 191283.0 | physical tokens/s: 191663.0 | loss tokens/s: 141515.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.505020E-01 | lm-global-supervised-tokens: 1.241213E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.097205E-02 | lm-token-buffer-mib: 5.803205E+01 | global_load_balancing_loss: 9.995259E-01 | loss scale: 1.0 | grad norm: 0.076 | params norm: 11334.087 | params rms: 0.061370 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 0:10:00.396737 | finish at 2026-10-10 18:32:09 + [2026-10-08 18:23:37] iteration 4879/ 6855 | consumed samples: 624512 | elapsed time per iteration (ms): 87859.4 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.58% | FLOPs estimator: workload-aware | logical tokens/s: 190955.1 | valid tokens/s: 190955.1 | physical tokens/s: 191426.5 | loss tokens/s: 138940.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.104188E-01 | lm-global-supervised-tokens: 1.220721E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.175671E-02 | lm-token-buffer-mib: 5.819901E+01 | global_load_balancing_loss: 1.000614E+00 | loss scale: 1.0 | grad norm: 0.099 | params norm: 11334.030 | params rms: 0.061370 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 0:13:30.120760 | finish at 2026-10-10 18:37:07 + [2026-10-08 18:25:24] iteration 4880/ 6855 | consumed samples: 624640 | elapsed time per iteration (ms): 107696.8 | throughput per GPU (TFLOP/s/GPU): 33.9 | MFU 10.86% | FLOPs estimator: workload-aware | logical tokens/s: 155781.7 | valid tokens/s: 155781.7 | physical tokens/s: 156277.5 | loss tokens/s: 117974.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.883074E-01 | lm-global-supervised-tokens: 1.270546E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.150440E-02 | lm-token-buffer-mib: 5.819829E+01 | global_load_balancing_loss: 9.998825E-01 | loss scale: 1.0 | grad norm: 0.051 | params norm: 11333.973 | params rms: 0.061370 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 11:05:01.250148 | finish at 2026-10-11 05:30:26 +times across ranks (ms): + save-checkpoint: + rank 0: 24021.86 + rank 1: 24021.90 + rank 2: 24021.91 + rank 3: 24021.88 + rank 4: 24021.91 + rank 5: 24021.87 + rank 6: 24021.86 + rank 7: 24021.89 + rank 8: 24023.01 + rank 9: 24022.99 + rank 10: 24023.00 + rank 11: 24022.98 + rank 12: 24023.01 + rank 13: 24022.99 + rank 14: 24022.99 + rank 15: 24022.98 + rank 16: 24021.69 + rank 17: 24021.68 + rank 18: 24021.66 + rank 19: 24021.68 + rank 20: 24021.66 + rank 21: 24021.69 + rank 22: 24021.65 + rank 23: 24021.63 + rank 24: 24022.21 + rank 25: 24022.18 + rank 26: 24022.17 + rank 27: 24022.20 + rank 28: 24022.09 + rank 29: 24022.13 + rank 30: 24022.10 + rank 31: 24022.11 + rank 32: 24021.14 + rank 33: 24021.14 + rank 34: 24021.08 + rank 35: 24021.06 + rank 36: 24021.05 + rank 37: 24021.05 + rank 38: 24021.06 + rank 39: 24021.07 + rank 40: 24022.11 + rank 41: 24022.11 + rank 42: 24022.10 + rank 43: 24022.11 + rank 44: 24022.12 + rank 45: 24022.08 + rank 46: 24022.10 + rank 47: 24022.08 + rank 48: 24021.55 + rank 49: 24021.51 + rank 50: 24021.51 + rank 51: 24021.49 + rank 52: 24021.53 + rank 53: 24021.49 + rank 54: 24021.51 + rank 55: 24021.51 + rank 56: 24021.66 + rank 57: 24021.65 + rank 58: 24021.66 + rank 59: 24021.64 + rank 60: 24021.65 + rank 61: 24021.66 + rank 62: 24021.65 + rank 63: 24021.63 + [2026-10-08 18:27:18] iteration 4881/ 6855 | consumed samples: 624768 | elapsed time per iteration (ms): 89233.0 | throughput per GPU (TFLOP/s/GPU): 41.5 | MFU 13.29% | FLOPs estimator: workload-aware | logical tokens/s: 188015.4 | valid tokens/s: 188015.3 | physical tokens/s: 188526.4 | loss tokens/s: 142872.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.519379E-01 | lm-global-supervised-tokens: 1.274892E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 5.582418E-01 | lm-token-buffer-mib: 5.827945E+01 | global_load_balancing_loss: 1.000229E+00 | loss scale: 1.0 | grad norm: 0.100 | params norm: 11333.917 | params rms: 0.061369 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 0:55:45.979259 | finish at 2026-10-10 19:23:04 + [2026-10-08 18:28:50] iteration 4882/ 6855 | consumed samples: 624896 | elapsed time per iteration (ms): 92484.9 | throughput per GPU (TFLOP/s/GPU): 39.7 | MFU 12.73% | FLOPs estimator: workload-aware | logical tokens/s: 181404.5 | valid tokens/s: 181404.5 | physical tokens/s: 181837.2 | loss tokens/s: 139194.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.585077E-01 | lm-global-supervised-tokens: 1.287343E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.422485E-02 | lm-token-buffer-mib: 5.806934E+01 | global_load_balancing_loss: 9.994813E-01 | loss scale: 1.0 | grad norm: 0.068 | params norm: 11333.863 | params rms: 0.061369 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 2:41:12.754735 | finish at 2026-10-10 21:10:03 + [2026-10-08 18:30:16] iteration 4883/ 6855 | consumed samples: 625024 | elapsed time per iteration (ms): 85461.5 | throughput per GPU (TFLOP/s/GPU): 43.3 | MFU 13.89% | FLOPs estimator: workload-aware | logical tokens/s: 196312.1 | valid tokens/s: 196312.1 | physical tokens/s: 196997.2 | loss tokens/s: 143868.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.626846E-01 | lm-global-supervised-tokens: 1.229519E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.893521E-02 | lm-token-buffer-mib: 5.813560E+01 | global_load_balancing_loss: 1.000298E+00 | loss scale: 1.0 | grad norm: 0.063 | params norm: 11333.808 | params rms: 0.061369 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 22:48:50.120646 | finish at 2026-10-10 17:19:06 + [2026-10-08 18:31:44] iteration 4884/ 6855 | consumed samples: 625152 | elapsed time per iteration (ms): 88631.4 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.59% | FLOPs estimator: workload-aware | logical tokens/s: 189291.5 | valid tokens/s: 189291.5 | physical tokens/s: 189655.6 | loss tokens/s: 138699.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.903293E-01 | lm-global-supervised-tokens: 1.229312E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.298788E-02 | lm-token-buffer-mib: 5.811099E+01 | global_load_balancing_loss: 9.997891E-01 | loss scale: 1.0 | grad norm: 0.061 | params norm: 11333.753 | params rms: 0.061369 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 0:31:32.521098 | finish at 2026-10-10 19:03:17 + [2026-10-08 18:33:15] iteration 4885/ 6855 | consumed samples: 625280 | elapsed time per iteration (ms): 90598.1 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.46% | FLOPs estimator: workload-aware | logical tokens/s: 185182.5 | valid tokens/s: 185182.5 | physical tokens/s: 185684.4 | loss tokens/s: 134493.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.992607E-01 | lm-global-supervised-tokens: 1.218485E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.867356E-02 | lm-token-buffer-mib: 5.808871E+01 | global_load_balancing_loss: 9.992372E-01 | loss scale: 1.0 | grad norm: 0.051 | params norm: 11333.698 | params rms: 0.061368 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 1:34:38.247502 | finish at 2026-10-10 20:07:53 + [2026-10-08 18:34:41] iteration 4886/ 6855 | consumed samples: 625408 | elapsed time per iteration (ms): 86058.9 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.52% | FLOPs estimator: workload-aware | logical tokens/s: 194949.2 | valid tokens/s: 194949.2 | physical tokens/s: 195528.3 | loss tokens/s: 143265.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.572787E-01 | lm-global-supervised-tokens: 1.232926E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.206974E-02 | lm-token-buffer-mib: 5.811444E+01 | global_load_balancing_loss: 1.001046E+00 | loss scale: 1.0 | grad norm: 0.068 | params norm: 11333.644 | params rms: 0.061368 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 23:04:10.070569 | finish at 2026-10-10 17:38:51 + [2026-10-08 18:36:10] iteration 4887/ 6855 | consumed samples: 625536 | elapsed time per iteration (ms): 88537.3 | throughput per GPU (TFLOP/s/GPU): 41.3 | MFU 13.24% | FLOPs estimator: workload-aware | logical tokens/s: 189493.3 | valid tokens/s: 189493.2 | physical tokens/s: 190130.1 | loss tokens/s: 145434.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.859304E-01 | lm-global-supervised-tokens: 1.287634E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.457735E-02 | lm-token-buffer-mib: 5.822324E+01 | global_load_balancing_loss: 9.998719E-01 | loss scale: 1.0 | grad norm: 0.121 | params norm: 11333.591 | params rms: 0.061368 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 0:24:01.328259 | finish at 2026-10-10 19:00:11 + [2026-10-08 18:37:32] iteration 4888/ 6855 | consumed samples: 625664 | elapsed time per iteration (ms): 82564.7 | throughput per GPU (TFLOP/s/GPU): 43.3 | MFU 13.88% | FLOPs estimator: workload-aware | logical tokens/s: 203200.2 | valid tokens/s: 203200.2 | physical tokens/s: 203633.3 | loss tokens/s: 159570.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.444576E-01 | lm-global-supervised-tokens: 1.317493E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.899828E-02 | lm-token-buffer-mib: 5.807921E+01 | global_load_balancing_loss: 9.995341E-01 | loss scale: 1.0 | grad norm: 0.072 | params norm: 11333.537 | params rms: 0.061367 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 21:06:44.850971 | finish at 2026-10-10 15:44:17 + [2026-10-08 18:39:00] iteration 4889/ 6855 | consumed samples: 625792 | elapsed time per iteration (ms): 87615.7 | throughput per GPU (TFLOP/s/GPU): 41.7 | MFU 13.35% | FLOPs estimator: workload-aware | logical tokens/s: 191485.8 | valid tokens/s: 191485.8 | physical tokens/s: 191934.3 | loss tokens/s: 143907.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.646003E-01 | lm-global-supervised-tokens: 1.260853E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.476411E-02 | lm-token-buffer-mib: 5.806736E+01 | global_load_balancing_loss: 1.000250E+00 | loss scale: 1.0 | grad norm: 0.062 | params norm: 11333.482 | params rms: 0.061367 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 23:50:52.382779 | finish at 2026-10-10 18:29:52 + [2026-10-08 18:40:52] iteration 4890/ 6855 | consumed samples: 625920 | elapsed time per iteration (ms): 112279.7 | throughput per GPU (TFLOP/s/GPU): 32.5 | MFU 10.42% | FLOPs estimator: workload-aware | logical tokens/s: 149422.7 | valid tokens/s: 149422.5 | physical tokens/s: 149884.1 | loss tokens/s: 111495.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.413752E-01 | lm-global-supervised-tokens: 1.251865E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.171619E-02 | lm-token-buffer-mib: 5.816916E+01 | global_load_balancing_loss: 1.000669E+00 | loss scale: 1.0 | grad norm: 0.079 | params norm: 11333.429 | params rms: 0.061367 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 13:17:09.630725 | finish at 2026-10-11 07:58:02 + [2026-10-08 18:42:23] iteration 4891/ 6855 | consumed samples: 626048 | elapsed time per iteration (ms): 90644.9 | throughput per GPU (TFLOP/s/GPU): 41.2 | MFU 13.20% | FLOPs estimator: workload-aware | logical tokens/s: 185087.3 | valid tokens/s: 185087.3 | physical tokens/s: 185480.8 | loss tokens/s: 132455.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.359560E-01 | lm-global-supervised-tokens: 1.200642E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.155008E-02 | lm-token-buffer-mib: 5.808334E+01 | global_load_balancing_loss: 1.001698E+00 | loss scale: 1.0 | grad norm: 0.131 | params norm: 11333.376 | params rms: 0.061367 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 1:27:06.526169 | finish at 2026-10-10 20:09:29 + [2026-10-08 18:43:50] iteration 4892/ 6855 | consumed samples: 626176 | elapsed time per iteration (ms): 86957.9 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.56% | FLOPs estimator: workload-aware | logical tokens/s: 192933.9 | valid tokens/s: 192933.9 | physical tokens/s: 193320.1 | loss tokens/s: 141767.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.242419E-01 | lm-global-supervised-tokens: 1.232779E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.504106E-02 | lm-token-buffer-mib: 5.812558E+01 | global_load_balancing_loss: 1.000707E+00 | loss scale: 1.0 | grad norm: 0.104 | params norm: 11333.324 | params rms: 0.061366 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 23:24:58.447184 | finish at 2026-10-10 18:08:48 + [2026-10-08 18:45:19] iteration 4893/ 6855 | consumed samples: 626304 | elapsed time per iteration (ms): 89203.7 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.41% | FLOPs estimator: workload-aware | logical tokens/s: 188077.2 | valid tokens/s: 188077.2 | physical tokens/s: 188602.2 | loss tokens/s: 138827.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.458445E-01 | lm-global-supervised-tokens: 1.238393E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.179366E-02 | lm-token-buffer-mib: 5.815498E+01 | global_load_balancing_loss: 1.000466E+00 | loss scale: 1.0 | grad norm: 0.146 | params norm: 11333.273 | params rms: 0.061366 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 0:36:57.665142 | finish at 2026-10-10 19:22:17 + [2026-10-08 18:46:47] iteration 4894/ 6855 | consumed samples: 626432 | elapsed time per iteration (ms): 87797.3 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.62% | FLOPs estimator: workload-aware | logical tokens/s: 191090.3 | valid tokens/s: 191090.2 | physical tokens/s: 191470.6 | loss tokens/s: 135379.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.554178E-01 | lm-global-supervised-tokens: 1.188597E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.158059E-02 | lm-token-buffer-mib: 5.816595E+01 | global_load_balancing_loss: 9.999306E-01 | loss scale: 1.0 | grad norm: 0.283 | params norm: 11333.223 | params rms: 0.061366 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 23:49:30.568614 | finish at 2026-10-10 18:36:17 + [2026-10-08 18:48:14] iteration 4895/ 6855 | consumed samples: 626560 | elapsed time per iteration (ms): 87277.8 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.63% | FLOPs estimator: workload-aware | logical tokens/s: 192227.6 | valid tokens/s: 192227.5 | physical tokens/s: 193025.7 | loss tokens/s: 143279.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.345299E-01 | lm-global-supervised-tokens: 1.250512E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.105114E-02 | lm-token-buffer-mib: 5.811978E+01 | global_load_balancing_loss: 9.992939E-01 | loss scale: 1.0 | grad norm: 0.099 | params norm: 11333.171 | params rms: 0.061365 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 23:31:04.540501 | finish at 2026-10-10 18:19:19 + [2026-10-08 18:49:39] iteration 4896/ 6855 | consumed samples: 626688 | elapsed time per iteration (ms): 84508.6 | throughput per GPU (TFLOP/s/GPU): 43.5 | MFU 13.93% | FLOPs estimator: workload-aware | logical tokens/s: 198526.7 | valid tokens/s: 198526.7 | physical tokens/s: 198908.4 | loss tokens/s: 147278.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.746694E-01 | lm-global-supervised-tokens: 1.244626E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.557037E-02 | lm-token-buffer-mib: 5.820560E+01 | global_load_balancing_loss: 9.990544E-01 | loss scale: 1.0 | grad norm: 0.171 | params norm: 11333.119 | params rms: 0.061365 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 21:59:12.399704 | finish at 2026-10-10 16:48:51 + [2026-10-08 18:51:06] iteration 4897/ 6855 | consumed samples: 626816 | elapsed time per iteration (ms): 87564.3 | throughput per GPU (TFLOP/s/GPU): 41.7 | MFU 13.36% | FLOPs estimator: workload-aware | logical tokens/s: 191598.1 | valid tokens/s: 191598.1 | physical tokens/s: 192211.2 | loss tokens/s: 145506.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.091416E-01 | lm-global-supervised-tokens: 1.274118E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.195491E-02 | lm-token-buffer-mib: 5.834692E+01 | global_load_balancing_loss: 9.986849E-01 | loss scale: 1.0 | grad norm: 0.163 | params norm: 11333.068 | params rms: 0.061365 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 23:37:30.982613 | finish at 2026-10-10 18:28:37 + [2026-10-08 18:52:29] iteration 4898/ 6855 | consumed samples: 626944 | elapsed time per iteration (ms): 82819.6 | throughput per GPU (TFLOP/s/GPU): 43.2 | MFU 13.83% | FLOPs estimator: workload-aware | logical tokens/s: 202574.6 | valid tokens/s: 202574.6 | physical tokens/s: 203099.7 | loss tokens/s: 153936.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.235293E-01 | lm-global-supervised-tokens: 1.274899E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.297291E-02 | lm-token-buffer-mib: 5.818749E+01 | global_load_balancing_loss: 9.998283E-01 | loss scale: 1.0 | grad norm: 0.196 | params norm: 11333.020 | params rms: 0.061365 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 21:01:18.014796 | finish at 2026-10-10 15:53:47 + [2026-10-08 18:53:58] iteration 4899/ 6855 | consumed samples: 627072 | elapsed time per iteration (ms): 89229.7 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.59% | FLOPs estimator: workload-aware | logical tokens/s: 188021.8 | valid tokens/s: 188021.8 | physical tokens/s: 188464.6 | loss tokens/s: 136459.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.752934E-01 | lm-global-supervised-tokens: 1.217626E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.119507E-02 | lm-token-buffer-mib: 5.822121E+01 | global_load_balancing_loss: 1.000859E+00 | loss scale: 1.0 | grad norm: 0.085 | params norm: 11332.971 | params rms: 0.061364 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 0:28:53.263527 | finish at 2026-10-10 19:22:51 + [2026-10-08 18:55:46] iteration 4900/ 6855 | consumed samples: 627200 | elapsed time per iteration (ms): 108136.7 | throughput per GPU (TFLOP/s/GPU): 34.9 | MFU 11.18% | FLOPs estimator: workload-aware | logical tokens/s: 155148.1 | valid tokens/s: 155148.1 | physical tokens/s: 155424.2 | loss tokens/s: 116142.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.217673E-01 | lm-global-supervised-tokens: 1.255931E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.190891E-02 | lm-token-buffer-mib: 5.804485E+01 | global_load_balancing_loss: 1.001965E+00 | loss scale: 1.0 | grad norm: 0.063 | params norm: 11332.921 | params rms: 0.061364 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 10:43:27.338759 | finish at 2026-10-11 05:39:14 + [2026-10-08 18:57:12] iteration 4901/ 6855 | consumed samples: 627328 | elapsed time per iteration (ms): 85947.7 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.69% | FLOPs estimator: workload-aware | logical tokens/s: 195202.5 | valid tokens/s: 195202.5 | physical tokens/s: 195571.7 | loss tokens/s: 146960.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.706750E-01 | lm-global-supervised-tokens: 1.263091E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.733474E-02 | lm-token-buffer-mib: 5.806499E+01 | global_load_balancing_loss: 1.002175E+00 | loss scale: 1.0 | grad norm: 0.141 | params norm: 11332.872 | params rms: 0.061364 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 22:39:01.724785 | finish at 2026-10-10 17:36:14 + [2026-10-08 18:58:38] iteration 4902/ 6855 | consumed samples: 627456 | elapsed time per iteration (ms): 85310.8 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.67% | FLOPs estimator: workload-aware | logical tokens/s: 196659.3 | valid tokens/s: 196659.2 | physical tokens/s: 197064.6 | loss tokens/s: 148033.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.616818E-01 | lm-global-supervised-tokens: 1.262884E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.277039E-02 | lm-token-buffer-mib: 5.805199E+01 | global_load_balancing_loss: 1.000761E+00 | loss scale: 1.0 | grad norm: 0.097 | params norm: 11332.823 | params rms: 0.061364 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 22:16:52.050751 | finish at 2026-10-10 17:15:30 + [2026-10-08 19:00:06] iteration 4903/ 6855 | consumed samples: 627584 | elapsed time per iteration (ms): 88213.9 | throughput per GPU (TFLOP/s/GPU): 41.7 | MFU 13.38% | FLOPs estimator: workload-aware | logical tokens/s: 190187.7 | valid tokens/s: 190187.7 | physical tokens/s: 190758.5 | loss tokens/s: 139058.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.589853E-01 | lm-global-supervised-tokens: 1.226690E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.870916E-02 | lm-token-buffer-mib: 5.819145E+01 | global_load_balancing_loss: 1.000376E+00 | loss scale: 1.0 | grad norm: 0.077 | params norm: 11332.772 | params rms: 0.061363 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 23:49:53.559502 | finish at 2026-10-10 18:49:59 + [2026-10-08 19:01:34] iteration 4904/ 6855 | consumed samples: 627712 | elapsed time per iteration (ms): 88615.4 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.52% | FLOPs estimator: workload-aware | logical tokens/s: 189326.0 | valid tokens/s: 189326.0 | physical tokens/s: 189597.3 | loss tokens/s: 138550.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.284030E-01 | lm-global-supervised-tokens: 1.227775E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.151979E-02 | lm-token-buffer-mib: 5.810572E+01 | global_load_balancing_loss: 9.994065E-01 | loss scale: 1.0 | grad norm: 0.051 | params norm: 11332.722 | params rms: 0.061363 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 0:01:28.718112 | finish at 2026-10-10 19:03:03 + [2026-10-08 19:03:00] iteration 4905/ 6855 | consumed samples: 627840 | elapsed time per iteration (ms): 86029.9 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.57% | FLOPs estimator: workload-aware | logical tokens/s: 195015.3 | valid tokens/s: 195015.3 | physical tokens/s: 195364.5 | loss tokens/s: 147291.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.222397E-01 | lm-global-supervised-tokens: 1.267149E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.251249E-02 | lm-token-buffer-mib: 5.803784E+01 | global_load_balancing_loss: 9.983183E-01 | loss scale: 1.0 | grad norm: 0.095 | params norm: 11332.671 | params rms: 0.061363 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 22:35:58.397663 | finish at 2026-10-10 17:38:59 + [2026-10-08 19:04:28] iteration 4906/ 6855 | consumed samples: 627968 | elapsed time per iteration (ms): 87925.7 | throughput per GPU (TFLOP/s/GPU): 41.0 | MFU 13.14% | FLOPs estimator: workload-aware | logical tokens/s: 190810.6 | valid tokens/s: 190810.6 | physical tokens/s: 191298.3 | loss tokens/s: 142336.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.430161E-01 | lm-global-supervised-tokens: 1.251505E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.150861E-02 | lm-token-buffer-mib: 5.804753E+01 | global_load_balancing_loss: 1.000023E+00 | loss scale: 1.0 | grad norm: 0.203 | params norm: 11332.620 | params rms: 0.061362 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 23:36:07.151098 | finish at 2026-10-10 18:40:35 + [2026-10-08 19:05:55] iteration 4907/ 6855 | consumed samples: 628096 | elapsed time per iteration (ms): 86824.7 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.66% | FLOPs estimator: workload-aware | logical tokens/s: 193230.1 | valid tokens/s: 193230.1 | physical tokens/s: 193848.0 | loss tokens/s: 144326.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.394584E-01 | lm-global-supervised-tokens: 1.253111E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.152193E-02 | lm-token-buffer-mib: 5.812006E+01 | global_load_balancing_loss: 9.996698E-01 | loss scale: 1.0 | grad norm: 0.370 | params norm: 11332.569 | params rms: 0.061362 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 22:58:54.606394 | finish at 2026-10-10 18:04:50 + [2026-10-08 19:07:27] iteration 4908/ 6855 | consumed samples: 628224 | elapsed time per iteration (ms): 91408.9 | throughput per GPU (TFLOP/s/GPU): 41.1 | MFU 13.16% | FLOPs estimator: workload-aware | logical tokens/s: 183540.2 | valid tokens/s: 183540.2 | physical tokens/s: 183978.7 | loss tokens/s: 139559.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.875660E-01 | lm-global-supervised-tokens: 1.275695E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.172682E-02 | lm-token-buffer-mib: 5.814777E+01 | global_load_balancing_loss: 1.000065E+00 | loss scale: 1.0 | grad norm: 0.202 | params norm: 11332.518 | params rms: 0.061362 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 1:26:13.142734 | finish at 2026-10-10 20:33:40 + [2026-10-08 19:08:53] iteration 4909/ 6855 | consumed samples: 628352 | elapsed time per iteration (ms): 86870.1 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.58% | FLOPs estimator: workload-aware | logical tokens/s: 193129.7 | valid tokens/s: 193129.7 | physical tokens/s: 193535.5 | loss tokens/s: 149838.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.594444E-01 | lm-global-supervised-tokens: 1.301646E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.215489E-02 | lm-token-buffer-mib: 5.808185E+01 | global_load_balancing_loss: 1.001130E+00 | loss scale: 1.0 | grad norm: 0.056 | params norm: 11332.466 | params rms: 0.061362 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 22:57:29.195619 | finish at 2026-10-10 18:06:23 + [2026-10-08 19:10:43] iteration 4910/ 6855 | consumed samples: 628480 | elapsed time per iteration (ms): 109558.1 | throughput per GPU (TFLOP/s/GPU): 33.8 | MFU 10.83% | FLOPs estimator: workload-aware | logical tokens/s: 153134.9 | valid tokens/s: 153134.9 | physical tokens/s: 153500.6 | loss tokens/s: 110903.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.509048E-01 | lm-global-supervised-tokens: 1.215037E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.649005E-02 | lm-token-buffer-mib: 5.822723E+01 | global_load_balancing_loss: 1.001280E+00 | loss scale: 1.0 | grad norm: 0.067 | params norm: 11332.413 | params rms: 0.061361 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 11:11:30.513746 | finish at 2026-10-11 06:22:14 + [2026-10-08 19:12:11] iteration 4911/ 6855 | consumed samples: 628608 | elapsed time per iteration (ms): 87799.3 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.58% | FLOPs estimator: workload-aware | logical tokens/s: 191085.6 | valid tokens/s: 191085.6 | physical tokens/s: 191349.3 | loss tokens/s: 146773.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.543920E-01 | lm-global-supervised-tokens: 1.288660E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.407650E-02 | lm-token-buffer-mib: 5.805978E+01 | global_load_balancing_loss: 1.000296E+00 | loss scale: 1.0 | grad norm: 0.037 | params norm: 11332.361 | params rms: 0.061361 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 23:24:41.872948 | finish at 2026-10-10 18:36:53 + [2026-10-08 19:13:40] iteration 4912/ 6855 | consumed samples: 628736 | elapsed time per iteration (ms): 89615.2 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.45% | FLOPs estimator: workload-aware | logical tokens/s: 187214.0 | valid tokens/s: 187214.0 | physical tokens/s: 187608.8 | loss tokens/s: 141048.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.326447E-01 | lm-global-supervised-tokens: 1.264006E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.445104E-02 | lm-token-buffer-mib: 5.808521E+01 | global_load_balancing_loss: 9.992767E-01 | loss scale: 1.0 | grad norm: 0.072 | params norm: 11332.310 | params rms: 0.061361 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 0:22:02.278093 | finish at 2026-10-10 19:35:43 + [2026-10-08 19:15:08] iteration 4913/ 6855 | consumed samples: 628864 | elapsed time per iteration (ms): 87872.5 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.56% | FLOPs estimator: workload-aware | logical tokens/s: 190925.7 | valid tokens/s: 190925.6 | physical tokens/s: 191465.9 | loss tokens/s: 142345.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.849165E-01 | lm-global-supervised-tokens: 1.250826E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.399818E-02 | lm-token-buffer-mib: 5.822056E+01 | global_load_balancing_loss: 1.000054E+00 | loss scale: 1.0 | grad norm: 0.079 | params norm: 11332.255 | params rms: 0.061360 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 23:24:08.470822 | finish at 2026-10-10 18:39:17 + [2026-10-08 19:16:34] iteration 4914/ 6855 | consumed samples: 628992 | elapsed time per iteration (ms): 85668.0 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.39% | FLOPs estimator: workload-aware | logical tokens/s: 195839.7 | valid tokens/s: 195839.7 | physical tokens/s: 196527.5 | loss tokens/s: 149263.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.104692E-01 | lm-global-supervised-tokens: 1.278706E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.171897E-02 | lm-token-buffer-mib: 5.819332E+01 | global_load_balancing_loss: 9.999439E-01 | loss scale: 1.0 | grad norm: 0.136 | params norm: 11332.201 | params rms: 0.061360 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 22:11:21.499115 | finish at 2026-10-10 17:27:55 + [2026-10-08 19:18:06] iteration 4915/ 6855 | consumed samples: 629120 | elapsed time per iteration (ms): 92421.6 | throughput per GPU (TFLOP/s/GPU): 41.1 | MFU 13.17% | FLOPs estimator: workload-aware | logical tokens/s: 181529.2 | valid tokens/s: 181529.2 | physical tokens/s: 181913.9 | loss tokens/s: 132141.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.034558E-01 | lm-global-supervised-tokens: 1.221269E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.138786E-02 | lm-token-buffer-mib: 5.805820E+01 | global_load_balancing_loss: 1.000364E+00 | loss scale: 1.0 | grad norm: 0.185 | params norm: 11332.148 | params rms: 0.061360 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 1:48:17.884312 | finish at 2026-10-10 21:06:24 + [2026-10-08 19:19:35] iteration 4916/ 6855 | consumed samples: 629248 | elapsed time per iteration (ms): 88751.3 | throughput per GPU (TFLOP/s/GPU): 41.2 | MFU 13.21% | FLOPs estimator: workload-aware | logical tokens/s: 189036.0 | valid tokens/s: 189036.0 | physical tokens/s: 189536.9 | loss tokens/s: 142328.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.511185E-01 | lm-global-supervised-tokens: 1.263185E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.153170E-02 | lm-token-buffer-mib: 5.804811E+01 | global_load_balancing_loss: 1.000836E+00 | loss scale: 1.0 | grad norm: 0.061 | params norm: 11332.096 | params rms: 0.061360 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 23:48:08.782907 | finish at 2026-10-10 19:07:44 + [2026-10-08 19:21:01] iteration 4917/ 6855 | consumed samples: 629376 | elapsed time per iteration (ms): 85459.4 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.74% | FLOPs estimator: workload-aware | logical tokens/s: 196317.3 | valid tokens/s: 196317.3 | physical tokens/s: 196795.8 | loss tokens/s: 147510.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.552525E-01 | lm-global-supervised-tokens: 1.260615E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.814776E-02 | lm-token-buffer-mib: 5.821226E+01 | global_load_balancing_loss: 1.001335E+00 | loss scale: 1.0 | grad norm: 0.188 | params norm: 11332.043 | params rms: 0.061359 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 22:00:20.298137 | finish at 2026-10-10 17:21:21 + [2026-10-08 19:22:31] iteration 4918/ 6855 | consumed samples: 629504 | elapsed time per iteration (ms): 90454.9 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.46% | FLOPs estimator: workload-aware | logical tokens/s: 185475.0 | valid tokens/s: 185475.0 | physical tokens/s: 185868.4 | loss tokens/s: 137031.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.903709E-01 | lm-global-supervised-tokens: 1.239520E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.963920E-02 | lm-token-buffer-mib: 5.809386E+01 | global_load_balancing_loss: 1.000134E+00 | loss scale: 1.0 | grad norm: 0.216 | params norm: 11331.989 | params rms: 0.061359 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 0:40:11.234638 | finish at 2026-10-10 20:02:42 + [2026-10-08 19:23:58] iteration 4919/ 6855 | consumed samples: 629632 | elapsed time per iteration (ms): 87245.7 | throughput per GPU (TFLOP/s/GPU): 41.6 | MFU 13.33% | FLOPs estimator: workload-aware | logical tokens/s: 192298.4 | valid tokens/s: 192298.4 | physical tokens/s: 192736.3 | loss tokens/s: 151185.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.611271E-01 | lm-global-supervised-tokens: 1.319025E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.165156E-02 | lm-token-buffer-mib: 5.823129E+01 | global_load_balancing_loss: 9.988499E-01 | loss scale: 1.0 | grad norm: 0.059 | params norm: 11331.933 | params rms: 0.061359 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 22:55:07.653278 | finish at 2026-10-10 18:19:06 + [2026-10-08 19:25:48] iteration 4920/ 6855 | consumed samples: 629760 | elapsed time per iteration (ms): 109999.0 | throughput per GPU (TFLOP/s/GPU): 33.3 | MFU 10.66% | FLOPs estimator: workload-aware | logical tokens/s: 152521.5 | valid tokens/s: 152521.5 | physical tokens/s: 153011.4 | loss tokens/s: 113100.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.018299E-01 | lm-global-supervised-tokens: 1.244088E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.394683E-02 | lm-token-buffer-mib: 5.827736E+01 | global_load_balancing_loss: 9.995665E-01 | loss scale: 1.0 | grad norm: 0.196 | params norm: 11331.879 | params rms: 0.061358 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 11:07:28.090053 | finish at 2026-10-11 06:33:16 +times across ranks (ms): + save-checkpoint: + rank 0: 23831.73 + rank 1: 23831.76 + rank 2: 23831.74 + rank 3: 23831.72 + rank 4: 23831.77 + rank 5: 23831.73 + rank 6: 23831.71 + rank 7: 23831.76 + rank 8: 23831.87 + rank 9: 23831.87 + rank 10: 23831.85 + rank 11: 23831.86 + rank 12: 23831.85 + rank 13: 23831.86 + rank 14: 23831.86 + rank 15: 23831.85 + rank 16: 23831.59 + rank 17: 23831.59 + rank 18: 23831.60 + rank 19: 23831.60 + rank 20: 23831.57 + rank 21: 23831.61 + rank 22: 23831.58 + rank 23: 23831.55 + rank 24: 23831.96 + rank 25: 23831.96 + rank 26: 23831.96 + rank 27: 23831.98 + rank 28: 23831.96 + rank 29: 23831.95 + rank 30: 23831.96 + rank 31: 23831.94 + rank 32: 23831.88 + rank 33: 23831.85 + rank 34: 23831.82 + rank 35: 23831.88 + rank 36: 23832.21 + rank 37: 23831.69 + rank 38: 23831.80 + rank 39: 23831.69 + rank 40: 23831.72 + rank 41: 23831.70 + rank 42: 23831.72 + rank 43: 23831.73 + rank 44: 23831.72 + rank 45: 23831.73 + rank 46: 23831.72 + rank 47: 23831.74 + rank 48: 23832.73 + rank 49: 23832.72 + rank 50: 23832.70 + rank 51: 23832.71 + rank 52: 23832.70 + rank 53: 23832.71 + rank 54: 23832.73 + rank 55: 23832.74 + rank 56: 23831.04 + rank 57: 23831.05 + rank 58: 23831.03 + rank 59: 23831.03 + rank 60: 23831.01 + rank 61: 23831.04 + rank 62: 23831.04 + rank 63: 23831.04 + [2026-10-08 19:27:46] iteration 4921/ 6855 | consumed samples: 629888 | elapsed time per iteration (ms): 93710.0 | throughput per GPU (TFLOP/s/GPU): 40.4 | MFU 12.95% | FLOPs estimator: workload-aware | logical tokens/s: 179033.2 | valid tokens/s: 179033.2 | physical tokens/s: 179370.5 | loss tokens/s: 129166.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.144278E-01 | lm-global-supervised-tokens: 1.210416E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.179979E-02 | lm-token-buffer-mib: 5.804619E+01 | global_load_balancing_loss: 9.994394E-01 | loss scale: 1.0 | grad norm: 0.054 | params norm: 11331.825 | params rms: 0.061358 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 2:20:35.078747 | finish at 2026-10-10 21:48:21 + [2026-10-08 19:29:15] iteration 4922/ 6855 | consumed samples: 630016 | elapsed time per iteration (ms): 89540.7 | throughput per GPU (TFLOP/s/GPU): 40.5 | MFU 12.98% | FLOPs estimator: workload-aware | logical tokens/s: 187369.2 | valid tokens/s: 187369.2 | physical tokens/s: 187854.7 | loss tokens/s: 137419.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.332908E-01 | lm-global-supervised-tokens: 1.230466E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.312029E-02 | lm-token-buffer-mib: 5.821826E+01 | global_load_balancing_loss: 1.000897E+00 | loss scale: 1.0 | grad norm: 0.168 | params norm: 11331.772 | params rms: 0.061358 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 0:04:42.166107 | finish at 2026-10-10 19:33:58 + [2026-10-08 19:30:44] iteration 4923/ 6855 | consumed samples: 630144 | elapsed time per iteration (ms): 88663.0 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.44% | FLOPs estimator: workload-aware | logical tokens/s: 189223.7 | valid tokens/s: 189223.7 | physical tokens/s: 189546.4 | loss tokens/s: 142000.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.128461E-01 | lm-global-supervised-tokens: 1.259020E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.305202E-02 | lm-token-buffer-mib: 5.806295E+01 | global_load_balancing_loss: 1.000095E+00 | loss scale: 1.0 | grad norm: 0.040 | params norm: 11331.721 | params rms: 0.061358 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 23:34:56.833755 | finish at 2026-10-10 19:05:41 + [2026-10-08 19:32:10] iteration 4924/ 6855 | consumed samples: 630272 | elapsed time per iteration (ms): 85720.3 | throughput per GPU (TFLOP/s/GPU): 43.8 | MFU 14.05% | FLOPs estimator: workload-aware | logical tokens/s: 195720.0 | valid tokens/s: 195720.0 | physical tokens/s: 195937.3 | loss tokens/s: 148034.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.164111E-01 | lm-global-supervised-tokens: 1.268955E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.890377E-02 | lm-token-buffer-mib: 5.801951E+01 | global_load_balancing_loss: 1.001107E+00 | loss scale: 1.0 | grad norm: 0.136 | params norm: 11331.668 | params rms: 0.061357 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 21:58:45.906127 | finish at 2026-10-10 17:30:56 + [2026-10-08 19:33:36] iteration 4925/ 6855 | consumed samples: 630400 | elapsed time per iteration (ms): 86131.5 | throughput per GPU (TFLOP/s/GPU): 42.8 | MFU 13.71% | FLOPs estimator: workload-aware | logical tokens/s: 194785.7 | valid tokens/s: 194785.7 | physical tokens/s: 195392.4 | loss tokens/s: 138038.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.811495E-01 | lm-global-supervised-tokens: 1.188947E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.245192E-02 | lm-token-buffer-mib: 5.806088E+01 | global_load_balancing_loss: 1.000251E+00 | loss scale: 1.0 | grad norm: 0.060 | params norm: 11331.615 | params rms: 0.061357 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 22:10:33.799152 | finish at 2026-10-10 17:44:10 + [2026-10-08 19:35:05] iteration 4926/ 6855 | consumed samples: 630528 | elapsed time per iteration (ms): 88525.3 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.65% | FLOPs estimator: workload-aware | logical tokens/s: 189518.5 | valid tokens/s: 189518.5 | physical tokens/s: 190019.2 | loss tokens/s: 142679.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.132546E-01 | lm-global-supervised-tokens: 1.263072E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.125896E-02 | lm-token-buffer-mib: 5.820190E+01 | global_load_balancing_loss: 1.000498E+00 | loss scale: 1.0 | grad norm: 0.074 | params norm: 11331.562 | params rms: 0.061357 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 23:26:05.292712 | finish at 2026-10-10 19:01:10 + [2026-10-08 19:36:30] iteration 4927/ 6855 | consumed samples: 630656 | elapsed time per iteration (ms): 85271.1 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.52% | FLOPs estimator: workload-aware | logical tokens/s: 196751.2 | valid tokens/s: 196751.2 | physical tokens/s: 197440.2 | loss tokens/s: 146936.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.845415E-01 | lm-global-supervised-tokens: 1.252944E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.278770E-02 | lm-token-buffer-mib: 5.867942E+01 | global_load_balancing_loss: 1.000553E+00 | loss scale: 1.0 | grad norm: 0.127 | params norm: 11331.508 | params rms: 0.061356 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 21:40:02.589411 | finish at 2026-10-10 17:16:32 + [2026-10-08 19:38:00] iteration 4928/ 6855 | consumed samples: 630784 | elapsed time per iteration (ms): 90411.0 | throughput per GPU (TFLOP/s/GPU): 41.5 | MFU 13.29% | FLOPs estimator: workload-aware | logical tokens/s: 185564.7 | valid tokens/s: 185564.7 | physical tokens/s: 185907.5 | loss tokens/s: 145724.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.183765E-01 | lm-global-supervised-tokens: 1.317505E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.400503E-02 | lm-token-buffer-mib: 5.812595E+01 | global_load_balancing_loss: 1.000456E+00 | loss scale: 1.0 | grad norm: 0.060 | params norm: 11331.457 | params rms: 0.061356 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 0:23:42.006214 | finish at 2026-10-10 20:01:42 + [2026-10-08 19:39:32] iteration 4929/ 6855 | consumed samples: 630912 | elapsed time per iteration (ms): 91695.5 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.60% | FLOPs estimator: workload-aware | logical tokens/s: 182966.3 | valid tokens/s: 182966.3 | physical tokens/s: 183354.1 | loss tokens/s: 129511.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.040920E-01 | lm-global-supervised-tokens: 1.187562E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.346600E-02 | lm-token-buffer-mib: 5.807771E+01 | global_load_balancing_loss: 9.997307E-01 | loss scale: 1.0 | grad norm: 0.162 | params norm: 11331.407 | params rms: 0.061356 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 1:03:25.463463 | finish at 2026-10-10 20:42:57 + [2026-10-08 19:41:27] iteration 4930/ 6855 | consumed samples: 631040 | elapsed time per iteration (ms): 115063.8 | throughput per GPU (TFLOP/s/GPU): 32.8 | MFU 10.52% | FLOPs estimator: workload-aware | logical tokens/s: 145807.3 | valid tokens/s: 145807.3 | physical tokens/s: 146104.6 | loss tokens/s: 98203.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.257902E-01 | lm-global-supervised-tokens: 1.129971E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.137283E-02 | lm-token-buffer-mib: 5.813299E+01 | global_load_balancing_loss: 1.000795E+00 | loss scale: 1.0 | grad norm: 0.054 | params norm: 11331.357 | params rms: 0.061356 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 13:31:37.867507 | finish at 2026-10-11 09:13:05 + [2026-10-08 19:42:57] iteration 4931/ 6855 | consumed samples: 631168 | elapsed time per iteration (ms): 89772.6 | throughput per GPU (TFLOP/s/GPU): 41.3 | MFU 13.22% | FLOPs estimator: workload-aware | logical tokens/s: 186885.1 | valid tokens/s: 186885.1 | physical tokens/s: 187368.0 | loss tokens/s: 137930.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.799381E-01 | lm-global-supervised-tokens: 1.238242E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.293810E-02 | lm-token-buffer-mib: 5.818732E+01 | global_load_balancing_loss: 1.000254E+00 | loss scale: 1.0 | grad norm: 0.087 | params norm: 11331.308 | params rms: 0.061355 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 23:58:42.496955 | finish at 2026-10-10 19:41:39 + [2026-10-08 19:44:25] iteration 4932/ 6855 | consumed samples: 631296 | elapsed time per iteration (ms): 88108.5 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.47% | FLOPs estimator: workload-aware | logical tokens/s: 190414.1 | valid tokens/s: 190414.1 | physical tokens/s: 190845.5 | loss tokens/s: 137974.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.343519E-01 | lm-global-supervised-tokens: 1.215672E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.793502E-02 | lm-token-buffer-mib: 5.817034E+01 | global_load_balancing_loss: 9.989914E-01 | loss scale: 1.0 | grad norm: 0.121 | params norm: 11331.260 | params rms: 0.061355 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 23:03:52.558855 | finish at 2026-10-10 18:48:17 + [2026-10-08 19:45:53] iteration 4933/ 6855 | consumed samples: 631424 | elapsed time per iteration (ms): 87832.8 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.41% | FLOPs estimator: workload-aware | logical tokens/s: 191012.4 | valid tokens/s: 191012.4 | physical tokens/s: 191365.8 | loss tokens/s: 139092.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.695829E-01 | lm-global-supervised-tokens: 1.221686E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.217088E-02 | lm-token-buffer-mib: 5.810559E+01 | global_load_balancing_loss: 9.990565E-01 | loss scale: 1.0 | grad norm: 0.108 | params norm: 11331.211 | params rms: 0.061355 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 22:53:34.603854 | finish at 2026-10-10 18:39:27 + [2026-10-08 19:47:21] iteration 4934/ 6855 | consumed samples: 631552 | elapsed time per iteration (ms): 88495.1 | throughput per GPU (TFLOP/s/GPU): 41.7 | MFU 13.37% | FLOPs estimator: workload-aware | logical tokens/s: 189583.3 | valid tokens/s: 189583.3 | physical tokens/s: 189954.7 | loss tokens/s: 141937.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.052427E-01 | lm-global-supervised-tokens: 1.256076E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.157283E-02 | lm-token-buffer-mib: 5.809740E+01 | global_load_balancing_loss: 1.000465E+00 | loss scale: 1.0 | grad norm: 0.097 | params norm: 11331.162 | params rms: 0.061355 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 23:13:19.129735 | finish at 2026-10-10 19:00:40 + [2026-10-08 19:48:46] iteration 4935/ 6855 | consumed samples: 631680 | elapsed time per iteration (ms): 85267.8 | throughput per GPU (TFLOP/s/GPU): 43.5 | MFU 13.95% | FLOPs estimator: workload-aware | logical tokens/s: 196758.7 | valid tokens/s: 196758.7 | physical tokens/s: 197348.3 | loss tokens/s: 147923.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.103621E-01 | lm-global-supervised-tokens: 1.261313E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.569333E-02 | lm-token-buffer-mib: 5.816902E+01 | global_load_balancing_loss: 1.000721E+00 | loss scale: 1.0 | grad norm: 0.083 | params norm: 11331.113 | params rms: 0.061354 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 21:28:34.147797 | finish at 2026-10-10 17:17:21 + [2026-10-08 19:50:14] iteration 4936/ 6855 | consumed samples: 631808 | elapsed time per iteration (ms): 87834.5 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.50% | FLOPs estimator: workload-aware | logical tokens/s: 191008.8 | valid tokens/s: 191008.8 | physical tokens/s: 191443.6 | loss tokens/s: 146242.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.752179E-01 | lm-global-supervised-tokens: 1.284509E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.156103E-02 | lm-token-buffer-mib: 5.810987E+01 | global_load_balancing_loss: 1.001300E+00 | loss scale: 1.0 | grad norm: 0.098 | params norm: 11331.061 | params rms: 0.061354 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 22:49:14.437212 | finish at 2026-10-10 18:39:29 + [2026-10-08 19:51:43] iteration 4937/ 6855 | consumed samples: 631936 | elapsed time per iteration (ms): 88546.1 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.46% | FLOPs estimator: workload-aware | logical tokens/s: 189474.0 | valid tokens/s: 189474.0 | physical tokens/s: 189833.2 | loss tokens/s: 145165.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.979331E-01 | lm-global-supervised-tokens: 1.285383E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.221610E-02 | lm-token-buffer-mib: 5.826741E+01 | global_load_balancing_loss: 9.994363E-01 | loss scale: 1.0 | grad norm: 0.112 | params norm: 11331.009 | params rms: 0.061354 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 23:10:31.421897 | finish at 2026-10-10 19:02:14 + [2026-10-08 19:53:12] iteration 4938/ 6855 | consumed samples: 632064 | elapsed time per iteration (ms): 89398.3 | throughput per GPU (TFLOP/s/GPU): 42.8 | MFU 13.73% | FLOPs estimator: workload-aware | logical tokens/s: 187668.2 | valid tokens/s: 187668.2 | physical tokens/s: 187967.6 | loss tokens/s: 141275.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.742749E-01 | lm-global-supervised-tokens: 1.262975E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.453526E-02 | lm-token-buffer-mib: 5.819350E+01 | global_load_balancing_loss: 1.000882E+00 | loss scale: 1.0 | grad norm: 0.039 | params norm: 11330.956 | params rms: 0.061353 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 23:36:16.519032 | finish at 2026-10-10 19:29:29 + [2026-10-08 19:54:38] iteration 4939/ 6855 | consumed samples: 632192 | elapsed time per iteration (ms): 86239.4 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.53% | FLOPs estimator: workload-aware | logical tokens/s: 194541.6 | valid tokens/s: 194541.6 | physical tokens/s: 195174.9 | loss tokens/s: 150489.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.550837E-01 | lm-global-supervised-tokens: 1.297810E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.287261E-02 | lm-token-buffer-mib: 5.818661E+01 | global_load_balancing_loss: 1.000854E+00 | loss scale: 1.0 | grad norm: 0.130 | params norm: 11330.904 | params rms: 0.061353 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 21:53:54.766515 | finish at 2026-10-10 17:48:33 + [2026-10-08 19:56:28] iteration 4940/ 6855 | consumed samples: 632320 | elapsed time per iteration (ms): 109236.6 | throughput per GPU (TFLOP/s/GPU): 33.8 | MFU 10.83% | FLOPs estimator: workload-aware | logical tokens/s: 153585.7 | valid tokens/s: 153585.7 | physical tokens/s: 153970.2 | loss tokens/s: 116954.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.872244E-01 | lm-global-supervised-tokens: 1.277575E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.176598E-02 | lm-token-buffer-mib: 5.814770E+01 | global_load_balancing_loss: 1.001159E+00 | loss scale: 1.0 | grad norm: 0.084 | params norm: 11330.854 | params rms: 0.061353 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 10:06:28.101178 | finish at 2026-10-11 06:02:56 + [2026-10-08 19:57:56] iteration 4941/ 6855 | consumed samples: 632448 | elapsed time per iteration (ms): 87938.3 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.64% | FLOPs estimator: workload-aware | logical tokens/s: 190783.5 | valid tokens/s: 190783.5 | physical tokens/s: 191181.6 | loss tokens/s: 143070.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.564666E-01 | lm-global-supervised-tokens: 1.258135E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.236540E-02 | lm-token-buffer-mib: 5.805164E+01 | global_load_balancing_loss: 1.000796E+00 | loss scale: 1.0 | grad norm: 0.086 | params norm: 11330.804 | params rms: 0.061353 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 22:45:13.946155 | finish at 2026-10-10 18:43:10 + [2026-10-08 19:59:20] iteration 4942/ 6855 | consumed samples: 632576 | elapsed time per iteration (ms): 84315.2 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.69% | FLOPs estimator: workload-aware | logical tokens/s: 198981.4 | valid tokens/s: 198981.4 | physical tokens/s: 199456.5 | loss tokens/s: 146284.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.491363E-01 | lm-global-supervised-tokens: 1.233401E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.261877E-02 | lm-token-buffer-mib: 5.811300E+01 | global_load_balancing_loss: 9.997280E-01 | loss scale: 1.0 | grad norm: 0.090 | params norm: 11330.753 | params rms: 0.061352 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 20:48:15.016997 | finish at 2026-10-10 16:47:35 + [2026-10-08 20:00:50] iteration 4943/ 6855 | consumed samples: 632704 | elapsed time per iteration (ms): 89645.8 | throughput per GPU (TFLOP/s/GPU): 40.2 | MFU 12.90% | FLOPs estimator: workload-aware | logical tokens/s: 187149.2 | valid tokens/s: 187149.2 | physical tokens/s: 187711.8 | loss tokens/s: 135841.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.208039E-01 | lm-global-supervised-tokens: 1.217758E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.244073E-02 | lm-token-buffer-mib: 5.824860E+01 | global_load_balancing_loss: 9.993558E-01 | loss scale: 1.0 | grad norm: 0.140 | params norm: 11330.702 | params rms: 0.061352 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 23:36:42.746113 | finish at 2026-10-10 19:37:32 + [2026-10-08 20:02:15] iteration 4944/ 6855 | consumed samples: 632832 | elapsed time per iteration (ms): 85309.2 | throughput per GPU (TFLOP/s/GPU): 43.3 | MFU 13.88% | FLOPs estimator: workload-aware | logical tokens/s: 196663.0 | valid tokens/s: 196663.0 | physical tokens/s: 197115.4 | loss tokens/s: 147629.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.223220E-01 | lm-global-supervised-tokens: 1.259416E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.623726E-02 | lm-token-buffer-mib: 5.823537E+01 | global_load_balancing_loss: 9.993010E-01 | loss scale: 1.0 | grad norm: 0.327 | params norm: 11330.651 | params rms: 0.061352 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 21:17:05.948268 | finish at 2026-10-10 17:19:21 + [2026-10-08 20:03:45] iteration 4945/ 6855 | consumed samples: 632960 | elapsed time per iteration (ms): 90067.7 | throughput per GPU (TFLOP/s/GPU): 40.7 | MFU 13.05% | FLOPs estimator: workload-aware | logical tokens/s: 186273.2 | valid tokens/s: 186273.1 | physical tokens/s: 186732.5 | loss tokens/s: 137056.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.618121E-01 | lm-global-supervised-tokens: 1.234436E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.571804E-02 | lm-token-buffer-mib: 5.812784E+01 | global_load_balancing_loss: 9.997333E-01 | loss scale: 1.0 | grad norm: 0.077 | params norm: 11330.598 | params rms: 0.061352 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 23:47:09.248083 | finish at 2026-10-10 19:50:54 + [2026-10-08 20:05:10] iteration 4946/ 6855 | consumed samples: 633088 | elapsed time per iteration (ms): 84569.7 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.46% | FLOPs estimator: workload-aware | logical tokens/s: 198381.7 | valid tokens/s: 198381.6 | physical tokens/s: 199025.2 | loss tokens/s: 144584.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.192729E-01 | lm-global-supervised-tokens: 1.222750E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.125794E-02 | lm-token-buffer-mib: 5.823473E+01 | global_load_balancing_loss: 1.000123E+00 | loss scale: 1.0 | grad norm: 0.530 | params norm: 11330.544 | params rms: 0.061351 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 20:50:43.564587 | finish at 2026-10-10 16:55:53 + [2026-10-08 20:06:36] iteration 4947/ 6855 | consumed samples: 633216 | elapsed time per iteration (ms): 86496.5 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.66% | FLOPs estimator: workload-aware | logical tokens/s: 193963.3 | valid tokens/s: 193963.3 | physical tokens/s: 194391.1 | loss tokens/s: 139679.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.897734E-01 | lm-global-supervised-tokens: 1.208182E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.148961E-02 | lm-token-buffer-mib: 5.806264E+01 | global_load_balancing_loss: 1.000552E+00 | loss scale: 1.0 | grad norm: 0.178 | params norm: 11330.490 | params rms: 0.061351 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 21:50:35.408461 | finish at 2026-10-10 17:57:11 + [2026-10-08 20:08:05] iteration 4948/ 6855 | consumed samples: 633344 | elapsed time per iteration (ms): 89394.0 | throughput per GPU (TFLOP/s/GPU): 40.7 | MFU 13.04% | FLOPs estimator: workload-aware | logical tokens/s: 187676.5 | valid tokens/s: 187676.5 | physical tokens/s: 188203.1 | loss tokens/s: 141846.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.949671E-01 | lm-global-supervised-tokens: 1.268025E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.161725E-02 | lm-token-buffer-mib: 5.814852E+01 | global_load_balancing_loss: 1.000364E+00 | loss scale: 1.0 | grad norm: 0.376 | params norm: 11330.436 | params rms: 0.061351 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 23:21:14.292630 | finish at 2026-10-10 19:29:20 + [2026-10-08 20:09:37] iteration 4949/ 6855 | consumed samples: 633472 | elapsed time per iteration (ms): 91694.9 | throughput per GPU (TFLOP/s/GPU): 41.0 | MFU 13.13% | FLOPs estimator: workload-aware | logical tokens/s: 182967.7 | valid tokens/s: 182967.7 | physical tokens/s: 183372.1 | loss tokens/s: 135899.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.040055E-01 | lm-global-supervised-tokens: 1.246131E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.169613E-02 | lm-token-buffer-mib: 5.823890E+01 | global_load_balancing_loss: 1.000161E+00 | loss scale: 1.0 | grad norm: 0.217 | params norm: 11330.380 | params rms: 0.061350 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 0:32:50.535817 | finish at 2026-10-10 20:42:28 + [2026-10-08 20:11:28] iteration 4950/ 6855 | consumed samples: 633600 | elapsed time per iteration (ms): 110961.9 | throughput per GPU (TFLOP/s/GPU): 33.3 | MFU 10.67% | FLOPs estimator: workload-aware | logical tokens/s: 151197.8 | valid tokens/s: 151197.8 | physical tokens/s: 151674.7 | loss tokens/s: 111788.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.001715E-01 | lm-global-supervised-tokens: 1.240422E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.128077E-02 | lm-token-buffer-mib: 5.807937E+01 | global_load_balancing_loss: 1.000400E+00 | loss scale: 1.0 | grad norm: 0.192 | params norm: 11330.322 | params rms: 0.061350 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 10:43:02.372257 | finish at 2026-10-11 06:54:30 + [2026-10-08 20:12:55] iteration 4951/ 6855 | consumed samples: 633728 | elapsed time per iteration (ms): 86653.2 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.50% | FLOPs estimator: workload-aware | logical tokens/s: 193612.7 | valid tokens/s: 193612.7 | physical tokens/s: 194007.9 | loss tokens/s: 143166.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.149162E-01 | lm-global-supervised-tokens: 1.240579E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.121912E-02 | lm-token-buffer-mib: 5.818600E+01 | global_load_balancing_loss: 1.000556E+00 | loss scale: 1.0 | grad norm: 0.176 | params norm: 11330.263 | params rms: 0.061350 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 21:49:47.708065 | finish at 2026-10-10 18:02:42 + [2026-10-08 20:14:22] iteration 4952/ 6855 | consumed samples: 633856 | elapsed time per iteration (ms): 87363.6 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.69% | FLOPs estimator: workload-aware | logical tokens/s: 192038.7 | valid tokens/s: 192038.7 | physical tokens/s: 192508.8 | loss tokens/s: 139112.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.762001E-01 | lm-global-supervised-tokens: 1.215334E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.532595E-02 | lm-token-buffer-mib: 5.825242E+01 | global_load_balancing_loss: 1.000022E+00 | loss scale: 1.0 | grad norm: 0.241 | params norm: 11330.204 | params rms: 0.061349 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 22:10:52.837365 | finish at 2026-10-10 18:25:15 + [2026-10-08 20:15:48] iteration 4953/ 6855 | consumed samples: 633984 | elapsed time per iteration (ms): 85952.0 | throughput per GPU (TFLOP/s/GPU): 43.9 | MFU 14.06% | FLOPs estimator: workload-aware | logical tokens/s: 195192.5 | valid tokens/s: 195192.5 | physical tokens/s: 195842.9 | loss tokens/s: 143612.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.811438E-01 | lm-global-supervised-tokens: 1.234375E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.272071E-02 | lm-token-buffer-mib: 5.808016E+01 | global_load_balancing_loss: 1.000543E+00 | loss scale: 1.0 | grad norm: 0.056 | params norm: 11330.145 | params rms: 0.061349 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 21:24:40.695199 | finish at 2026-10-10 17:40:29 + [2026-10-08 20:17:15] iteration 4954/ 6855 | consumed samples: 634112 | elapsed time per iteration (ms): 86802.2 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.59% | FLOPs estimator: workload-aware | logical tokens/s: 193281.0 | valid tokens/s: 193281.0 | physical tokens/s: 193712.4 | loss tokens/s: 148568.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.183847E-01 | lm-global-supervised-tokens: 1.289610E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.162534E-02 | lm-token-buffer-mib: 5.815241E+01 | global_load_balancing_loss: 9.993970E-01 | loss scale: 1.0 | grad norm: 0.181 | params norm: 11330.087 | params rms: 0.061349 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 21:50:10.911192 | finish at 2026-10-10 18:07:26 + [2026-10-08 20:18:37] iteration 4955/ 6855 | consumed samples: 634240 | elapsed time per iteration (ms): 82447.8 | throughput per GPU (TFLOP/s/GPU): 43.7 | MFU 14.02% | FLOPs estimator: workload-aware | logical tokens/s: 203488.8 | valid tokens/s: 203488.8 | physical tokens/s: 203980.2 | loss tokens/s: 150806.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.219017E-01 | lm-global-supervised-tokens: 1.243366E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.506902E-02 | lm-token-buffer-mib: 5.820753E+01 | global_load_balancing_loss: 1.000832E+00 | loss scale: 1.0 | grad norm: 0.191 | params norm: 11330.029 | params rms: 0.061348 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 19:30:50.902748 | finish at 2026-10-10 15:49:28 + [2026-10-08 20:20:00] iteration 4956/ 6855 | consumed samples: 634368 | elapsed time per iteration (ms): 83009.0 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.67% | FLOPs estimator: workload-aware | logical tokens/s: 202111.9 | valid tokens/s: 202111.9 | physical tokens/s: 202706.6 | loss tokens/s: 152491.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.528325E-01 | lm-global-supervised-tokens: 1.265812E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.407875E-02 | lm-token-buffer-mib: 5.815398E+01 | global_load_balancing_loss: 1.000735E+00 | loss scale: 1.0 | grad norm: 0.127 | params norm: 11329.971 | params rms: 0.061348 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 19:47:14.028639 | finish at 2026-10-10 16:07:14 + [2026-10-08 20:21:27] iteration 4957/ 6855 | consumed samples: 634496 | elapsed time per iteration (ms): 86235.3 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.60% | FLOPs estimator: workload-aware | logical tokens/s: 194551.1 | valid tokens/s: 194551.1 | physical tokens/s: 195029.9 | loss tokens/s: 146096.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.626565E-01 | lm-global-supervised-tokens: 1.259867E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.516812E-02 | lm-token-buffer-mib: 5.808720E+01 | global_load_balancing_loss: 1.000853E+00 | loss scale: 1.0 | grad norm: 0.057 | params norm: 11329.915 | params rms: 0.061348 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 21:27:54.620790 | finish at 2026-10-10 17:49:21 + [2026-10-08 20:22:55] iteration 4958/ 6855 | consumed samples: 634624 | elapsed time per iteration (ms): 88578.3 | throughput per GPU (TFLOP/s/GPU): 43.2 | MFU 13.85% | FLOPs estimator: workload-aware | logical tokens/s: 189405.1 | valid tokens/s: 189405.1 | physical tokens/s: 189798.0 | loss tokens/s: 141400.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.467511E-01 | lm-global-supervised-tokens: 1.252503E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.144294E-02 | lm-token-buffer-mib: 5.804540E+01 | global_load_balancing_loss: 9.991383E-01 | loss scale: 1.0 | grad norm: 0.232 | params norm: 11329.859 | params rms: 0.061348 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 22:40:33.029671 | finish at 2026-10-10 19:03:28 + [2026-10-08 20:24:23] iteration 4959/ 6855 | consumed samples: 634752 | elapsed time per iteration (ms): 87789.5 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.53% | FLOPs estimator: workload-aware | logical tokens/s: 191106.9 | valid tokens/s: 191106.9 | physical tokens/s: 191596.2 | loss tokens/s: 145145.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.308803E-01 | lm-global-supervised-tokens: 1.274222E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.538223E-02 | lm-token-buffer-mib: 5.818200E+01 | global_load_balancing_loss: 1.000174E+00 | loss scale: 1.0 | grad norm: 0.212 | params norm: 11329.804 | params rms: 0.061347 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 22:14:08.980597 | finish at 2026-10-10 18:38:32 + [2026-10-08 20:26:13] iteration 4960/ 6855 | consumed samples: 634880 | elapsed time per iteration (ms): 110514.7 | throughput per GPU (TFLOP/s/GPU): 33.5 | MFU 10.72% | FLOPs estimator: workload-aware | logical tokens/s: 151809.8 | valid tokens/s: 151809.8 | physical tokens/s: 152138.3 | loss tokens/s: 110634.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.644302E-01 | lm-global-supervised-tokens: 1.222674E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.273169E-02 | lm-token-buffer-mib: 5.810431E+01 | global_load_balancing_loss: 1.000290E+00 | loss scale: 1.0 | grad norm: 0.101 | params norm: 11329.746 | params rms: 0.061347 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 10:10:25.312102 | finish at 2026-10-11 06:36:39 +times across ranks (ms): + save-checkpoint: + rank 0: 24786.17 + rank 1: 24786.22 + rank 2: 24786.25 + rank 3: 24786.17 + rank 4: 24786.18 + rank 5: 24786.16 + rank 6: 24786.16 + rank 7: 24786.18 + rank 8: 24786.24 + rank 9: 24786.20 + rank 10: 24786.25 + rank 11: 24786.19 + rank 12: 24786.20 + rank 13: 24786.19 + rank 14: 24786.21 + rank 15: 24786.21 + rank 16: 24786.25 + rank 17: 24786.25 + rank 18: 24786.26 + rank 19: 24786.29 + rank 20: 24786.26 + rank 21: 24786.28 + rank 22: 24786.26 + rank 23: 24786.25 + rank 24: 24787.05 + rank 25: 24787.05 + rank 26: 24787.08 + rank 27: 24787.08 + rank 28: 24787.02 + rank 29: 24787.05 + rank 30: 24787.04 + rank 31: 24787.04 + rank 32: 24786.14 + rank 33: 24786.13 + rank 34: 24786.13 + rank 35: 24786.11 + rank 36: 24786.09 + rank 37: 24786.11 + rank 38: 24786.13 + rank 39: 24786.09 + rank 40: 24785.97 + rank 41: 24785.97 + rank 42: 24785.96 + rank 43: 24785.97 + rank 44: 24785.95 + rank 45: 24785.95 + rank 46: 24785.96 + rank 47: 24785.95 + rank 48: 24784.84 + rank 49: 24784.83 + rank 50: 24784.83 + rank 51: 24784.87 + rank 52: 24784.85 + rank 53: 24784.82 + rank 54: 24784.83 + rank 55: 24784.84 + rank 56: 24786.15 + rank 57: 24786.16 + rank 58: 24786.13 + rank 59: 24786.13 + rank 60: 24786.15 + rank 61: 24786.13 + rank 62: 24786.14 + rank 63: 24786.12 + [2026-10-08 20:28:32] iteration 4961/ 6855 | consumed samples: 635008 | elapsed time per iteration (ms): 113929.3 | throughput per GPU (TFLOP/s/GPU): 32.6 | MFU 10.44% | FLOPs estimator: workload-aware | logical tokens/s: 147259.6 | valid tokens/s: 147259.6 | physical tokens/s: 147603.8 | loss tokens/s: 106047.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.485507E-01 | lm-global-supervised-tokens: 1.208192E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 6.507939E-01 | lm-token-buffer-mib: 5.806512E+01 | global_load_balancing_loss: 9.999629E-01 | loss scale: 1.0 | grad norm: 0.128 | params norm: 11329.690 | params rms: 0.061347 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 11:56:22.147617 | finish at 2026-10-11 08:24:55 + [2026-10-08 20:30:07] iteration 4962/ 6855 | consumed samples: 635136 | elapsed time per iteration (ms): 94263.6 | throughput per GPU (TFLOP/s/GPU): 40.2 | MFU 12.89% | FLOPs estimator: workload-aware | logical tokens/s: 177981.4 | valid tokens/s: 177981.4 | physical tokens/s: 178382.9 | loss tokens/s: 129404.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.728094E-01 | lm-global-supervised-tokens: 1.219816E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.362461E-02 | lm-token-buffer-mib: 5.812907E+01 | global_load_balancing_loss: 1.000737E+00 | loss scale: 1.0 | grad norm: 0.107 | params norm: 11329.636 | params rms: 0.061346 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 1:34:01.080311 | finish at 2026-10-10 22:04:08 + [2026-10-08 20:31:32] iteration 4963/ 6855 | consumed samples: 635264 | elapsed time per iteration (ms): 85171.4 | throughput per GPU (TFLOP/s/GPU): 43.5 | MFU 13.94% | FLOPs estimator: workload-aware | logical tokens/s: 196981.4 | valid tokens/s: 196981.4 | physical tokens/s: 197446.7 | loss tokens/s: 145156.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.595518E-01 | lm-global-supervised-tokens: 1.236322E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.318822E-02 | lm-token-buffer-mib: 5.824567E+01 | global_load_balancing_loss: 1.000469E+00 | loss scale: 1.0 | grad norm: 0.050 | params norm: 11329.580 | params rms: 0.061346 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 20:45:44.346672 | finish at 2026-10-10 17:17:16 + [2026-10-08 20:32:59] iteration 4964/ 6855 | consumed samples: 635392 | elapsed time per iteration (ms): 87515.1 | throughput per GPU (TFLOP/s/GPU): 43.2 | MFU 13.86% | FLOPs estimator: workload-aware | logical tokens/s: 191705.5 | valid tokens/s: 191705.5 | physical tokens/s: 192185.8 | loss tokens/s: 137171.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.059153E-01 | lm-global-supervised-tokens: 1.200460E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.156897E-02 | lm-token-buffer-mib: 5.803062E+01 | global_load_balancing_loss: 9.990753E-01 | loss scale: 1.0 | grad norm: 0.072 | params norm: 11329.526 | params rms: 0.061346 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 21:58:10.989633 | finish at 2026-10-10 18:31:10 + [2026-10-08 20:34:27] iteration 4965/ 6855 | consumed samples: 635520 | elapsed time per iteration (ms): 87322.1 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.59% | FLOPs estimator: workload-aware | logical tokens/s: 192129.5 | valid tokens/s: 192129.5 | physical tokens/s: 192545.6 | loss tokens/s: 143558.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.371888E-01 | lm-global-supervised-tokens: 1.253578E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.226974E-02 | lm-token-buffer-mib: 5.805353E+01 | global_load_balancing_loss: 9.994007E-01 | loss scale: 1.0 | grad norm: 0.041 | params norm: 11329.472 | params rms: 0.061345 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 21:50:38.794992 | finish at 2026-10-10 18:25:06 + [2026-10-08 20:35:54] iteration 4966/ 6855 | consumed samples: 635648 | elapsed time per iteration (ms): 87547.3 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.46% | FLOPs estimator: workload-aware | logical tokens/s: 191634.9 | valid tokens/s: 191634.9 | physical tokens/s: 192061.6 | loss tokens/s: 138611.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.582938E-01 | lm-global-supervised-tokens: 1.213504E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.200983E-02 | lm-token-buffer-mib: 5.812913E+01 | global_load_balancing_loss: 9.999533E-01 | loss scale: 1.0 | grad norm: 0.091 | params norm: 11329.419 | params rms: 0.061345 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 21:56:16.940414 | finish at 2026-10-10 18:32:11 + [2026-10-08 20:37:20] iteration 4967/ 6855 | consumed samples: 635776 | elapsed time per iteration (ms): 86110.3 | throughput per GPU (TFLOP/s/GPU): 43.2 | MFU 13.84% | FLOPs estimator: workload-aware | logical tokens/s: 194833.2 | valid tokens/s: 194833.2 | physical tokens/s: 195196.8 | loss tokens/s: 142483.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.278423E-01 | lm-global-supervised-tokens: 1.226930E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.667495E-02 | lm-token-buffer-mib: 5.810051E+01 | global_load_balancing_loss: 1.000471E+00 | loss scale: 1.0 | grad norm: 0.093 | params norm: 11329.364 | params rms: 0.061345 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 21:09:36.202408 | finish at 2026-10-10 17:46:57 + [2026-10-08 20:38:46] iteration 4968/ 6855 | consumed samples: 635904 | elapsed time per iteration (ms): 85588.3 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.64% | FLOPs estimator: workload-aware | logical tokens/s: 196022.3 | valid tokens/s: 196022.3 | physical tokens/s: 196517.9 | loss tokens/s: 146944.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.293053E-01 | lm-global-supervised-tokens: 1.257676E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.440816E-02 | lm-token-buffer-mib: 5.812496E+01 | global_load_balancing_loss: 1.001735E+00 | loss scale: 1.0 | grad norm: 0.105 | params norm: 11329.311 | params rms: 0.061345 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 20:51:45.150424 | finish at 2026-10-10 17:30:31 + [2026-10-08 20:40:14] iteration 4969/ 6855 | consumed samples: 636032 | elapsed time per iteration (ms): 87837.9 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.41% | FLOPs estimator: workload-aware | logical tokens/s: 191001.5 | valid tokens/s: 191001.5 | physical tokens/s: 191516.0 | loss tokens/s: 149099.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.894322E-01 | lm-global-supervised-tokens: 1.309662E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.141579E-02 | lm-token-buffer-mib: 5.807325E+01 | global_load_balancing_loss: 1.000509E+00 | loss scale: 1.0 | grad norm: 0.159 | params norm: 11329.258 | params rms: 0.061344 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 22:01:02.358395 | finish at 2026-10-10 18:41:16 + [2026-10-08 20:42:00] iteration 4970/ 6855 | consumed samples: 636160 | elapsed time per iteration (ms): 106501.1 | throughput per GPU (TFLOP/s/GPU): 33.9 | MFU 10.87% | FLOPs estimator: workload-aware | logical tokens/s: 157530.4 | valid tokens/s: 157530.3 | physical tokens/s: 158057.8 | loss tokens/s: 114156.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.473558E-01 | lm-global-supervised-tokens: 1.215784E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.970663E-02 | lm-token-buffer-mib: 5.817486E+01 | global_load_balancing_loss: 1.001932E+00 | loss scale: 1.0 | grad norm: 0.051 | params norm: 11329.207 | params rms: 0.061344 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 7:45:54.618112 | finish at 2026-10-11 04:27:55 + [2026-10-08 20:43:25] iteration 4971/ 6855 | consumed samples: 636288 | elapsed time per iteration (ms): 84893.8 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.52% | FLOPs estimator: workload-aware | logical tokens/s: 197625.8 | valid tokens/s: 197625.8 | physical tokens/s: 198218.2 | loss tokens/s: 153595.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.381660E-01 | lm-global-supervised-tokens: 1.303928E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.095950E-02 | lm-token-buffer-mib: 5.834673E+01 | global_load_balancing_loss: 1.000003E+00 | loss scale: 1.0 | grad norm: 0.121 | params norm: 11329.154 | params rms: 0.061344 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 20:25:39.997845 | finish at 2026-10-10 17:09:05 + [2026-10-08 20:44:52] iteration 4972/ 6855 | consumed samples: 636416 | elapsed time per iteration (ms): 86529.8 | throughput per GPU (TFLOP/s/GPU): 43.1 | MFU 13.82% | FLOPs estimator: workload-aware | logical tokens/s: 193888.9 | valid tokens/s: 193888.9 | physical tokens/s: 194627.2 | loss tokens/s: 143812.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.522148E-01 | lm-global-supervised-tokens: 1.244402E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.142625E-02 | lm-token-buffer-mib: 5.847578E+01 | global_load_balancing_loss: 9.996516E-01 | loss scale: 1.0 | grad norm: 0.253 | params norm: 11329.102 | params rms: 0.061343 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 21:15:35.525740 | finish at 2026-10-10 18:00:27 + [2026-10-08 20:46:20] iteration 4973/ 6855 | consumed samples: 636544 | elapsed time per iteration (ms): 87785.1 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.80% | FLOPs estimator: workload-aware | logical tokens/s: 191116.5 | valid tokens/s: 191116.5 | physical tokens/s: 191498.0 | loss tokens/s: 144237.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.733605E-01 | lm-global-supervised-tokens: 1.266194E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.979389E-02 | lm-token-buffer-mib: 5.809779E+01 | global_load_balancing_loss: 9.987847E-01 | loss scale: 1.0 | grad norm: 0.160 | params norm: 11329.051 | params rms: 0.061343 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 21:53:31.538874 | finish at 2026-10-10 18:39:51 + [2026-10-08 20:47:46] iteration 4974/ 6855 | consumed samples: 636672 | elapsed time per iteration (ms): 86517.0 | throughput per GPU (TFLOP/s/GPU): 42.8 | MFU 13.73% | FLOPs estimator: workload-aware | logical tokens/s: 193917.5 | valid tokens/s: 193917.5 | physical tokens/s: 194449.4 | loss tokens/s: 141360.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.463041E-01 | lm-global-supervised-tokens: 1.223004E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.362502E-02 | lm-token-buffer-mib: 5.814508E+01 | global_load_balancing_loss: 1.000061E+00 | loss scale: 1.0 | grad norm: 0.193 | params norm: 11328.998 | params rms: 0.061343 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 21:12:18.397098 | finish at 2026-10-10 18:00:04 + [2026-10-08 20:49:14] iteration 4975/ 6855 | consumed samples: 636800 | elapsed time per iteration (ms): 87722.3 | throughput per GPU (TFLOP/s/GPU): 41.4 | MFU 13.26% | FLOPs estimator: workload-aware | logical tokens/s: 191253.3 | valid tokens/s: 191253.3 | physical tokens/s: 191682.5 | loss tokens/s: 146112.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.758247E-01 | lm-global-supervised-tokens: 1.281736E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.126916E-02 | lm-token-buffer-mib: 5.813898E+01 | global_load_balancing_loss: 9.994113E-01 | loss scale: 1.0 | grad norm: 0.192 | params norm: 11328.947 | params rms: 0.061343 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 21:48:37.963095 | finish at 2026-10-10 18:37:52 + [2026-10-08 20:50:39] iteration 4976/ 6855 | consumed samples: 636928 | elapsed time per iteration (ms): 85726.5 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.54% | FLOPs estimator: workload-aware | logical tokens/s: 195705.9 | valid tokens/s: 195705.9 | physical tokens/s: 196250.6 | loss tokens/s: 148794.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.379393E-01 | lm-global-supervised-tokens: 1.275564E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.310316E-02 | lm-token-buffer-mib: 5.843552E+01 | global_load_balancing_loss: 1.001167E+00 | loss scale: 1.0 | grad norm: 0.211 | params norm: 11328.892 | params rms: 0.061342 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 20:44:40.028158 | finish at 2026-10-10 17:35:19 + [2026-10-08 20:52:08] iteration 4977/ 6855 | consumed samples: 637056 | elapsed time per iteration (ms): 88234.1 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.44% | FLOPs estimator: workload-aware | logical tokens/s: 190144.0 | valid tokens/s: 190144.0 | physical tokens/s: 190519.0 | loss tokens/s: 142782.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.881498E-01 | lm-global-supervised-tokens: 1.259824E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.120302E-02 | lm-token-buffer-mib: 5.814727E+01 | global_load_balancing_loss: 9.995444E-01 | loss scale: 1.0 | grad norm: 0.242 | params norm: 11328.838 | params rms: 0.061342 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 22:01:43.654770 | finish at 2026-10-10 18:53:51 + [2026-10-08 20:53:34] iteration 4978/ 6855 | consumed samples: 637184 | elapsed time per iteration (ms): 86315.1 | throughput per GPU (TFLOP/s/GPU): 41.7 | MFU 13.37% | FLOPs estimator: workload-aware | logical tokens/s: 194370.5 | valid tokens/s: 194370.5 | physical tokens/s: 195053.3 | loss tokens/s: 144069.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.798978E-01 | lm-global-supervised-tokens: 1.243540E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.107148E-02 | lm-token-buffer-mib: 5.840380E+01 | global_load_balancing_loss: 1.000134E+00 | loss scale: 1.0 | grad norm: 0.164 | params norm: 11328.785 | params rms: 0.061342 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 21:00:13.456935 | finish at 2026-10-10 17:53:47 + [2026-10-08 20:54:58] iteration 4979/ 6855 | consumed samples: 637312 | elapsed time per iteration (ms): 84124.0 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.69% | FLOPs estimator: workload-aware | logical tokens/s: 199434.1 | valid tokens/s: 199434.1 | physical tokens/s: 200062.9 | loss tokens/s: 150227.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.308246E-01 | lm-global-supervised-tokens: 1.263772E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.158665E-02 | lm-token-buffer-mib: 5.814593E+01 | global_load_balancing_loss: 1.000223E+00 | loss scale: 1.0 | grad norm: 0.153 | params norm: 11328.733 | params rms: 0.061341 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 19:50:16.660365 | finish at 2026-10-10 16:45:15 + [2026-10-08 20:56:45] iteration 4980/ 6855 | consumed samples: 637440 | elapsed time per iteration (ms): 106501.8 | throughput per GPU (TFLOP/s/GPU): 33.9 | MFU 10.88% | FLOPs estimator: workload-aware | logical tokens/s: 157529.7 | valid tokens/s: 157529.6 | physical tokens/s: 157900.2 | loss tokens/s: 117409.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.218421E-01 | lm-global-supervised-tokens: 1.250429E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.412725E-02 | lm-token-buffer-mib: 5.806448E+01 | global_load_balancing_loss: 1.000604E+00 | loss scale: 1.0 | grad norm: 0.144 | params norm: 11328.680 | params rms: 0.061341 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 7:28:10.877795 | finish at 2026-10-11 04:24:56 + [2026-10-08 20:58:12] iteration 4981/ 6855 | consumed samples: 637568 | elapsed time per iteration (ms): 87598.5 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.58% | FLOPs estimator: workload-aware | logical tokens/s: 191523.6 | valid tokens/s: 191523.5 | physical tokens/s: 191942.9 | loss tokens/s: 136501.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.233518E-01 | lm-global-supervised-tokens: 1.195736E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.307750E-02 | lm-token-buffer-mib: 5.821427E+01 | global_load_balancing_loss: 1.002104E+00 | loss scale: 1.0 | grad norm: 0.045 | params norm: 11328.627 | params rms: 0.061341 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 21:35:59.674810 | finish at 2026-10-10 18:34:12 + [2026-10-08 20:59:39] iteration 4982/ 6855 | consumed samples: 637696 | elapsed time per iteration (ms): 86509.8 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.50% | FLOPs estimator: workload-aware | logical tokens/s: 193934.2 | valid tokens/s: 193934.2 | physical tokens/s: 194533.0 | loss tokens/s: 147672.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.249201E-01 | lm-global-supervised-tokens: 1.277508E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.764942E-02 | lm-token-buffer-mib: 5.818745E+01 | global_load_balancing_loss: 1.000037E+00 | loss scale: 1.0 | grad norm: 0.128 | params norm: 11328.573 | params rms: 0.061341 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 21:00:32.852641 | finish at 2026-10-10 18:00:12 + [2026-10-08 21:01:08] iteration 4983/ 6855 | consumed samples: 637824 | elapsed time per iteration (ms): 89026.0 | throughput per GPU (TFLOP/s/GPU): 41.2 | MFU 13.19% | FLOPs estimator: workload-aware | logical tokens/s: 188452.8 | valid tokens/s: 188452.8 | physical tokens/s: 188953.8 | loss tokens/s: 130784.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.296803E-01 | lm-global-supervised-tokens: 1.164323E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.142401E-02 | lm-token-buffer-mib: 5.802715E+01 | global_load_balancing_loss: 1.000031E+00 | loss scale: 1.0 | grad norm: 0.070 | params norm: 11328.518 | params rms: 0.061340 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 22:17:36.763538 | finish at 2026-10-10 19:18:45 + [2026-10-08 21:02:34] iteration 4984/ 6855 | consumed samples: 637952 | elapsed time per iteration (ms): 85872.1 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.41% | FLOPs estimator: workload-aware | logical tokens/s: 195372.8 | valid tokens/s: 195372.8 | physical tokens/s: 195995.2 | loss tokens/s: 150632.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.712563E-01 | lm-global-supervised-tokens: 1.293511E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.452464E-02 | lm-token-buffer-mib: 5.821317E+01 | global_load_balancing_loss: 9.995770E-01 | loss scale: 1.0 | grad norm: 0.043 | params norm: 11328.462 | params rms: 0.061340 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 20:37:46.647123 | finish at 2026-10-10 17:40:20 + [2026-10-08 21:04:01] iteration 4985/ 6855 | consumed samples: 638080 | elapsed time per iteration (ms): 87389.9 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.54% | FLOPs estimator: workload-aware | logical tokens/s: 191979.7 | valid tokens/s: 191979.7 | physical tokens/s: 192559.3 | loss tokens/s: 142783.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.620664E-01 | lm-global-supervised-tokens: 1.247779E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.108219E-02 | lm-token-buffer-mib: 5.818102E+01 | global_load_balancing_loss: 9.999217E-01 | loss scale: 1.0 | grad norm: 0.097 | params norm: 11328.407 | params rms: 0.061340 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 21:23:39.165998 | finish at 2026-10-10 18:27:40 + [2026-10-08 21:05:29] iteration 4986/ 6855 | consumed samples: 638208 | elapsed time per iteration (ms): 88414.5 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.65% | FLOPs estimator: workload-aware | logical tokens/s: 189756.2 | valid tokens/s: 189756.1 | physical tokens/s: 190236.0 | loss tokens/s: 140399.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.922349E-01 | lm-global-supervised-tokens: 1.241332E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.953423E-02 | lm-token-buffer-mib: 5.810027E+01 | global_load_balancing_loss: 9.981362E-01 | loss scale: 1.0 | grad norm: 0.092 | params norm: 11328.352 | params rms: 0.061339 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 21:54:06.665739 | finish at 2026-10-10 18:59:36 + [2026-10-08 21:07:01] iteration 4987/ 6855 | consumed samples: 638336 | elapsed time per iteration (ms): 91302.9 | throughput per GPU (TFLOP/s/GPU): 41.1 | MFU 13.16% | FLOPs estimator: workload-aware | logical tokens/s: 183753.3 | valid tokens/s: 183753.3 | physical tokens/s: 184150.4 | loss tokens/s: 133849.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.368271E-01 | lm-global-supervised-tokens: 1.222089E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.219587E-02 | lm-token-buffer-mib: 5.809609E+01 | global_load_balancing_loss: 9.993446E-01 | loss scale: 1.0 | grad norm: 0.057 | params norm: 11328.296 | params rms: 0.061339 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 23:22:33.903743 | finish at 2026-10-10 20:29:35 + [2026-10-08 21:08:27] iteration 4988/ 6855 | consumed samples: 638464 | elapsed time per iteration (ms): 85975.8 | throughput per GPU (TFLOP/s/GPU): 43.4 | MFU 13.92% | FLOPs estimator: workload-aware | logical tokens/s: 195138.5 | valid tokens/s: 195138.5 | physical tokens/s: 195591.1 | loss tokens/s: 146948.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.091683E-01 | lm-global-supervised-tokens: 1.263400E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.308920E-02 | lm-token-buffer-mib: 5.815126E+01 | global_load_balancing_loss: 1.000011E+00 | loss scale: 1.0 | grad norm: 0.072 | params norm: 11328.241 | params rms: 0.061339 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 20:35:16.779944 | finish at 2026-10-10 17:43:44 + [2026-10-08 21:10:51] iteration 4989/ 6855 | consumed samples: 638592 | elapsed time per iteration (ms): 143881.9 | throughput per GPU (TFLOP/s/GPU): 26.6 | MFU 8.53% | FLOPs estimator: workload-aware | logical tokens/s: 116604.0 | valid tokens/s: 116604.0 | physical tokens/s: 117044.1 | loss tokens/s: 82916.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.029431E-01 | lm-global-supervised-tokens: 1.193012E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.889318E-02 | lm-token-buffer-mib: 5.809290E+01 | global_load_balancing_loss: 1.003389E+00 | loss scale: 1.0 | grad norm: 0.044 | params norm: 11328.185 | params rms: 0.061338 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 2:34:43.628204 | finish at 2026-10-11 23:45:34 + [2026-10-08 21:12:39] iteration 4990/ 6855 | consumed samples: 638720 | elapsed time per iteration (ms): 108058.5 | throughput per GPU (TFLOP/s/GPU): 33.4 | MFU 10.71% | FLOPs estimator: workload-aware | logical tokens/s: 155260.0 | valid tokens/s: 155260.0 | physical tokens/s: 155684.1 | loss tokens/s: 120744.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.691409E-01 | lm-global-supervised-tokens: 1.304748E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.332733E-02 | lm-token-buffer-mib: 5.823182E+01 | global_load_balancing_loss: 1.001304E+00 | loss scale: 1.0 | grad norm: 0.056 | params norm: 11328.130 | params rms: 0.061338 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 7:58:49.139947 | finish at 2026-10-11 05:11:28 + [2026-10-08 21:14:06] iteration 4991/ 6855 | consumed samples: 638848 | elapsed time per iteration (ms): 87014.1 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.44% | FLOPs estimator: workload-aware | logical tokens/s: 192809.7 | valid tokens/s: 192809.7 | physical tokens/s: 193305.5 | loss tokens/s: 144375.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.605043E-01 | lm-global-supervised-tokens: 1.256268E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.934366E-02 | lm-token-buffer-mib: 5.809348E+01 | global_load_balancing_loss: 1.000242E+00 | loss scale: 1.0 | grad norm: 0.052 | params norm: 11328.075 | params rms: 0.061338 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 21:03:14.293205 | finish at 2026-10-10 18:17:20 + [2026-10-08 21:15:30] iteration 4992/ 6855 | consumed samples: 638976 | elapsed time per iteration (ms): 84108.9 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.60% | FLOPs estimator: workload-aware | logical tokens/s: 199469.0 | valid tokens/s: 199469.0 | physical tokens/s: 199932.3 | loss tokens/s: 151780.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.343150E-01 | lm-global-supervised-tokens: 1.276611E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.147229E-02 | lm-token-buffer-mib: 5.829338E+01 | global_load_balancing_loss: 9.994329E-01 | loss scale: 1.0 | grad norm: 0.038 | params norm: 11328.021 | params rms: 0.061338 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 19:31:34.820423 | finish at 2026-10-10 16:47:05 + [2026-10-08 21:16:56] iteration 4993/ 6855 | consumed samples: 639104 | elapsed time per iteration (ms): 86636.2 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.44% | FLOPs estimator: workload-aware | logical tokens/s: 193651.0 | valid tokens/s: 193651.0 | physical tokens/s: 194100.5 | loss tokens/s: 148553.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.648552E-01 | lm-global-supervised-tokens: 1.287011E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.306466E-02 | lm-token-buffer-mib: 5.820912E+01 | global_load_balancing_loss: 9.992614E-01 | loss scale: 1.0 | grad norm: 0.047 | params norm: 11327.967 | params rms: 0.061337 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 20:48:36.581224 | finish at 2026-10-10 18:05:33 + [2026-10-08 21:18:20] iteration 4994/ 6855 | consumed samples: 639232 | elapsed time per iteration (ms): 83270.3 | throughput per GPU (TFLOP/s/GPU): 43.6 | MFU 13.96% | FLOPs estimator: workload-aware | logical tokens/s: 201478.0 | valid tokens/s: 201478.0 | physical tokens/s: 201979.4 | loss tokens/s: 150308.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.645640E-01 | lm-global-supervised-tokens: 1.251627E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.124293E-02 | lm-token-buffer-mib: 5.807317E+01 | global_load_balancing_loss: 1.000581E+00 | loss scale: 1.0 | grad norm: 0.045 | params norm: 11327.914 | params rms: 0.061337 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 19:02:45.962468 | finish at 2026-10-10 16:21:06 + [2026-10-08 21:19:45] iteration 4995/ 6855 | consumed samples: 639360 | elapsed time per iteration (ms): 85250.6 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.59% | FLOPs estimator: workload-aware | logical tokens/s: 196798.3 | valid tokens/s: 196798.3 | physical tokens/s: 197246.8 | loss tokens/s: 150406.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.779975E-01 | lm-global-supervised-tokens: 1.282227E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.329302E-02 | lm-token-buffer-mib: 5.813974E+01 | global_load_balancing_loss: 9.995692E-01 | loss scale: 1.0 | grad norm: 0.039 | params norm: 11327.862 | params rms: 0.061337 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 20:02:46.102438 | finish at 2026-10-10 17:22:31 + [2026-10-08 21:21:11] iteration 4996/ 6855 | consumed samples: 639488 | elapsed time per iteration (ms): 85568.4 | throughput per GPU (TFLOP/s/GPU): 43.5 | MFU 13.94% | FLOPs estimator: workload-aware | logical tokens/s: 196067.6 | valid tokens/s: 196067.6 | physical tokens/s: 196504.1 | loss tokens/s: 141980.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.063384E-01 | lm-global-supervised-tokens: 1.214900E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.466453E-02 | lm-token-buffer-mib: 5.819963E+01 | global_load_balancing_loss: 1.000617E+00 | loss scale: 1.0 | grad norm: 0.033 | params norm: 11327.809 | params rms: 0.061336 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 20:11:11.665176 | finish at 2026-10-10 17:32:22 + [2026-10-08 21:22:36] iteration 4997/ 6855 | consumed samples: 639616 | elapsed time per iteration (ms): 84965.8 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.68% | FLOPs estimator: workload-aware | logical tokens/s: 197458.5 | valid tokens/s: 197458.5 | physical tokens/s: 197909.9 | loss tokens/s: 149841.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.801277E-01 | lm-global-supervised-tokens: 1.273143E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.228901E-02 | lm-token-buffer-mib: 5.813684E+01 | global_load_balancing_loss: 1.000155E+00 | loss scale: 1.0 | grad norm: 0.049 | params norm: 11327.757 | params rms: 0.061336 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 19:51:06.423707 | finish at 2026-10-10 17:13:42 + [2026-10-08 21:24:03] iteration 4998/ 6855 | consumed samples: 639744 | elapsed time per iteration (ms): 87040.7 | throughput per GPU (TFLOP/s/GPU): 43.2 | MFU 13.84% | FLOPs estimator: workload-aware | logical tokens/s: 192751.3 | valid tokens/s: 192751.3 | physical tokens/s: 193093.1 | loss tokens/s: 142740.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.070721E-01 | lm-global-supervised-tokens: 1.242426E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.143695E-02 | lm-token-buffer-mib: 5.814299E+01 | global_load_balancing_loss: 9.997406E-01 | loss scale: 1.0 | grad norm: 0.033 | params norm: 11327.704 | params rms: 0.061336 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 20:53:54.634724 | finish at 2026-10-10 18:17:57 + [2026-10-08 21:25:31] iteration 4999/ 6855 | consumed samples: 639872 | elapsed time per iteration (ms): 88222.2 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.78% | FLOPs estimator: workload-aware | logical tokens/s: 190169.5 | valid tokens/s: 190169.5 | physical tokens/s: 190562.4 | loss tokens/s: 143626.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.651846E-01 | lm-global-supervised-tokens: 1.267106E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.334580E-02 | lm-token-buffer-mib: 5.815499E+01 | global_load_balancing_loss: 1.001592E+00 | loss scale: 1.0 | grad norm: 0.046 | params norm: 11327.651 | params rms: 0.061336 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 21:29:00.324280 | finish at 2026-10-10 18:54:31 + [2026-10-08 21:27:23] iteration 5000/ 6855 | consumed samples: 640000 | elapsed time per iteration (ms): 111930.3 | throughput per GPU (TFLOP/s/GPU): 33.1 | MFU 10.60% | FLOPs estimator: workload-aware | logical tokens/s: 149889.7 | valid tokens/s: 149889.7 | physical tokens/s: 150257.7 | loss tokens/s: 109306.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.518704E-01 | lm-global-supervised-tokens: 1.223473E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.138092E-02 | lm-token-buffer-mib: 5.813724E+01 | global_load_balancing_loss: 1.000553E+00 | loss scale: 1.0 | grad norm: 0.038 | params norm: 11327.598 | params rms: 0.061335 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 9:40:30.686593 | finish at 2026-10-11 07:07:53 +times across ranks (ms): + save-checkpoint: + rank 0: 25731.55 + rank 1: 25731.57 + rank 2: 25731.55 + rank 3: 25731.55 + rank 4: 25731.51 + rank 5: 25731.50 + rank 6: 25731.49 + rank 7: 25731.51 + rank 8: 25732.66 + rank 9: 25732.64 + rank 10: 25732.65 + rank 11: 25732.65 + rank 12: 25732.68 + rank 13: 25732.64 + rank 14: 25732.62 + rank 15: 25732.63 + rank 16: 25734.43 + rank 17: 25734.42 + rank 18: 25734.40 + rank 19: 25734.42 + rank 20: 25734.40 + rank 21: 25734.42 + rank 22: 25734.40 + rank 23: 25734.43 + rank 24: 25732.30 + rank 25: 25732.29 + rank 26: 25732.29 + rank 27: 25732.30 + rank 28: 25732.27 + rank 29: 25732.25 + rank 30: 25732.27 + rank 31: 25732.29 + rank 32: 25732.15 + rank 33: 25732.14 + rank 34: 25732.11 + rank 35: 25732.11 + rank 36: 25732.10 + rank 37: 25732.03 + rank 38: 25732.05 + rank 39: 25732.05 + rank 40: 25732.36 + rank 41: 25732.30 + rank 42: 25732.36 + rank 43: 25732.36 + rank 44: 25732.20 + rank 45: 25732.34 + rank 46: 25732.34 + rank 47: 25732.33 + rank 48: 25733.09 + rank 49: 25732.96 + rank 50: 25732.98 + rank 51: 25732.84 + rank 52: 25733.01 + rank 53: 25732.98 + rank 54: 25732.99 + rank 55: 25733.05 + rank 56: 25732.65 + rank 57: 25732.62 + rank 58: 25732.63 + rank 59: 25732.61 + rank 60: 25732.61 + rank 61: 25732.62 + rank 62: 25732.62 + rank 63: 25732.62 + [2026-10-08 21:29:22] iteration 5001/ 6855 | consumed samples: 640128 | elapsed time per iteration (ms): 93023.8 | throughput per GPU (TFLOP/s/GPU): 40.4 | MFU 12.96% | FLOPs estimator: workload-aware | logical tokens/s: 180353.7 | valid tokens/s: 180353.7 | physical tokens/s: 180681.0 | loss tokens/s: 131055.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.622528E-01 | lm-global-supervised-tokens: 1.219123E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.350613E-01 | lm-token-buffer-mib: 5.808604E+01 | global_load_balancing_loss: 1.000341E+00 | loss scale: 1.0 | grad norm: 0.045 | params norm: 11327.546 | params rms: 0.061335 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 23:54:26.094950 | finish at 2026-10-10 21:23:48 +times across ranks (ms): + forward-backward: + rank 0: 92490.02 + rank 1: 92493.61 + rank 2: 92494.41 + rank 3: 92496.19 + rank 4: 92492.30 + rank 5: 92495.56 + rank 6: 92488.20 + rank 7: 92492.66 + rank 8: 92489.86 + rank 9: 92493.55 + rank 10: 92493.41 + rank 11: 92496.82 + rank 12: 92492.83 + rank 13: 92495.70 + rank 14: 92487.70 + rank 15: 92491.67 + rank 16: 92492.18 + rank 17: 92495.82 + rank 18: 92495.59 + rank 19: 92498.62 + rank 20: 92494.88 + rank 21: 92498.17 + rank 22: 92490.27 + rank 23: 92494.37 + rank 24: 92482.96 + rank 25: 92486.71 + rank 26: 92487.43 + rank 27: 92489.70 + rank 28: 92485.78 + rank 29: 92488.85 + rank 30: 92482.29 + rank 31: 92486.72 + rank 32: 92479.95 + rank 33: 92483.26 + rank 34: 92481.94 + rank 35: 92485.45 + rank 36: 92480.97 + rank 37: 92484.35 + rank 38: 92479.21 + rank 39: 92483.13 + rank 40: 92484.30 + rank 41: 92486.36 + rank 42: 92486.86 + rank 43: 92487.14 + rank 44: 92483.06 + rank 45: 92485.89 + rank 46: 92483.89 + rank 47: 92486.92 + rank 48: 92493.58 + rank 49: 92494.73 + rank 50: 92493.35 + rank 51: 92496.42 + rank 52: 92490.95 + rank 53: 92493.65 + rank 54: 92492.08 + rank 55: 92495.86 + rank 56: 92493.65 + rank 57: 92496.34 + rank 58: 92495.59 + rank 59: 92497.06 + rank 60: 92493.57 + rank 61: 92494.76 + rank 62: 92493.55 + rank 63: 92496.39 + forward-compute: + rank 0: 35080.05 + rank 1: 35086.15 + rank 2: 35087.03 + rank 3: 35086.24 + rank 4: 35084.02 + rank 5: 35086.84 + rank 6: 35080.60 + rank 7: 35087.81 + rank 8: 38014.63 + rank 9: 38012.11 + rank 10: 38012.12 + rank 11: 38013.26 + rank 12: 38011.80 + rank 13: 38018.83 + rank 14: 38005.92 + rank 15: 38014.06 + rank 16: 40444.70 + rank 17: 40453.37 + rank 18: 40454.81 + rank 19: 40457.89 + rank 20: 40452.59 + rank 21: 40456.03 + rank 22: 40450.84 + rank 23: 40455.49 + rank 24: 32791.82 + rank 25: 32800.09 + rank 26: 32806.85 + rank 27: 32806.91 + rank 28: 32811.07 + rank 29: 32809.82 + rank 30: 32821.01 + rank 31: 32804.00 + rank 32: 40318.40 + rank 33: 40325.31 + rank 34: 40325.48 + rank 35: 40327.79 + rank 36: 40326.13 + rank 37: 40332.12 + rank 38: 40325.83 + rank 39: 40325.94 + rank 40: 38023.79 + rank 41: 38029.21 + rank 42: 38030.91 + rank 43: 38031.91 + rank 44: 38028.62 + rank 45: 38030.63 + rank 46: 38026.64 + rank 47: 38029.42 + rank 48: 37540.12 + rank 49: 37549.65 + rank 50: 37551.73 + rank 51: 37554.83 + rank 52: 37554.80 + rank 53: 37561.52 + rank 54: 37553.40 + rank 55: 37556.80 + rank 56: 39354.46 + rank 57: 39357.04 + rank 58: 39358.11 + rank 59: 39358.76 + rank 60: 39355.68 + rank 61: 39358.50 + rank 62: 39353.27 + rank 63: 39356.85 + backward-compute: + rank 0: 57373.57 + rank 1: 57373.36 + rank 2: 57371.54 + rank 3: 57370.75 + rank 4: 57374.37 + rank 5: 57372.86 + rank 6: 57372.46 + rank 7: 57369.37 + rank 8: 54441.54 + rank 9: 54445.71 + rank 10: 54444.28 + rank 11: 54447.37 + rank 12: 54445.92 + rank 13: 54441.91 + rank 14: 54446.87 + rank 15: 54445.25 + rank 16: 52011.86 + rank 17: 52005.79 + rank 18: 52005.74 + rank 19: 52003.18 + rank 20: 52006.38 + rank 21: 52006.70 + rank 22: 51998.36 + rank 23: 52001.43 + rank 24: 59651.59 + rank 25: 59645.76 + rank 26: 59641.45 + rank 27: 59640.83 + rank 28: 59635.90 + rank 29: 59637.82 + rank 30: 59617.84 + rank 31: 59642.11 + rank 32: 52117.53 + rank 33: 52117.22 + rank 34: 52113.15 + rank 35: 52116.55 + rank 36: 52114.04 + rank 37: 52110.12 + rank 38: 52113.89 + rank 39: 52114.77 + rank 40: 54421.21 + rank 41: 54424.81 + rank 42: 54418.09 + rank 43: 54418.25 + rank 44: 54418.89 + rank 45: 54418.73 + rank 46: 54421.35 + rank 47: 54422.82 + rank 48: 54907.36 + rank 49: 54907.15 + rank 50: 54903.57 + rank 51: 54905.92 + rank 52: 54895.75 + rank 53: 54892.89 + rank 54: 54900.15 + rank 55: 54902.59 + rank 56: 53103.34 + rank 57: 53101.16 + rank 58: 53102.69 + rank 59: 53101.39 + rank 60: 53098.41 + rank 61: 53100.69 + rank 62: 53104.29 + rank 63: 53103.04 + batch-generator: + rank 0: 26.34 + rank 1: 24.79 + rank 2: 26.37 + rank 3: 26.23 + rank 4: 24.62 + rank 5: 25.05 + rank 6: 26.82 + rank 7: 24.20 + rank 8: 23.39 + rank 9: 24.54 + rank 10: 23.26 + rank 11: 24.91 + rank 12: 24.65 + rank 13: 24.55 + rank 14: 25.45 + rank 15: 24.40 + rank 16: 26.77 + rank 17: 24.56 + rank 18: 23.57 + rank 19: 24.38 + rank 20: 24.83 + rank 21: 24.26 + rank 22: 23.96 + rank 23: 24.44 + rank 24: 33.80 + rank 25: 31.95 + rank 26: 36.85 + rank 27: 35.24 + rank 28: 32.95 + rank 29: 34.12 + rank 30: 35.11 + rank 31: 32.27 + rank 32: 37.32 + rank 33: 33.36 + rank 34: 33.96 + rank 35: 32.24 + rank 36: 26.73 + rank 37: 31.14 + rank 38: 32.75 + rank 39: 33.34 + rank 40: 23.32 + rank 41: 21.79 + rank 42: 24.13 + rank 43: 23.77 + rank 44: 24.75 + rank 45: 24.40 + rank 46: 23.16 + rank 47: 24.10 + rank 48: 35.70 + rank 49: 30.30 + rank 50: 29.66 + rank 51: 24.15 + rank 52: 27.12 + rank 53: 29.16 + rank 54: 25.82 + rank 55: 22.73 + rank 56: 25.38 + rank 57: 24.94 + rank 58: 25.58 + rank 59: 24.47 + rank 60: 24.95 + rank 61: 25.30 + rank 62: 25.20 + rank 63: 26.08 + embedding-grads-all-reduce: + rank 0: 0.03 + rank 1: 0.02 + rank 2: 0.03 + rank 3: 0.03 + rank 4: 0.03 + rank 5: 0.03 + rank 6: 0.03 + rank 7: 0.03 + rank 8: 0.02 + rank 9: 0.03 + rank 10: 0.03 + rank 11: 0.03 + rank 12: 0.03 + rank 13: 0.02 + rank 14: 0.03 + rank 15: 0.03 + rank 16: 0.03 + rank 17: 0.02 + rank 18: 0.02 + rank 19: 0.03 + rank 20: 0.03 + rank 21: 0.04 + rank 22: 0.03 + rank 23: 0.03 + rank 24: 0.04 + rank 25: 0.04 + rank 26: 0.04 + rank 27: 0.04 + rank 28: 0.03 + rank 29: 0.04 + rank 30: 0.04 + rank 31: 0.04 + rank 32: 0.03 + rank 33: 0.04 + rank 34: 0.04 + rank 35: 0.03 + rank 36: 0.03 + rank 37: 0.03 + rank 38: 0.02 + rank 39: 0.03 + rank 40: 0.03 + rank 41: 0.03 + rank 42: 0.03 + rank 43: 0.03 + rank 44: 0.03 + rank 45: 0.03 + rank 46: 0.03 + rank 47: 0.03 + rank 48: 0.06 + rank 49: 0.03 + rank 50: 0.03 + rank 51: 0.03 + rank 52: 0.03 + rank 53: 0.03 + rank 54: 0.03 + rank 55: 0.03 + rank 56: 0.03 + rank 57: 0.03 + rank 58: 0.03 + rank 59: 0.02 + rank 60: 0.03 + rank 61: 0.03 + rank 62: 0.03 + rank 63: 0.03 + all-grads-sync: + rank 0: 0.86 + rank 1: 0.76 + rank 2: 0.84 + rank 3: 0.78 + rank 4: 0.81 + rank 5: 1.01 + rank 6: 0.89 + rank 7: 0.93 + rank 8: 0.76 + rank 9: 0.77 + rank 10: 0.75 + rank 11: 0.80 + rank 12: 0.86 + rank 13: 0.78 + rank 14: 0.77 + rank 15: 0.78 + rank 16: 0.85 + rank 17: 0.77 + rank 18: 0.82 + rank 19: 0.80 + rank 20: 0.80 + rank 21: 0.82 + rank 22: 0.78 + rank 23: 0.95 + rank 24: 1.28 + rank 25: 1.17 + rank 26: 1.39 + rank 27: 1.82 + rank 28: 1.27 + rank 29: 1.27 + rank 30: 1.22 + rank 31: 1.75 + rank 32: 1.22 + rank 33: 1.15 + rank 34: 1.23 + rank 35: 1.18 + rank 36: 0.95 + rank 37: 1.43 + rank 38: 1.18 + rank 39: 1.19 + rank 40: 0.78 + rank 41: 0.79 + rank 42: 1.42 + rank 43: 0.77 + rank 44: 1.09 + rank 45: 1.31 + rank 46: 0.82 + rank 47: 0.95 + rank 48: 1.28 + rank 49: 1.23 + rank 50: 1.05 + rank 51: 1.02 + rank 52: 1.21 + rank 53: 0.96 + rank 54: 1.22 + rank 55: 1.10 + rank 56: 0.76 + rank 57: 0.91 + rank 58: 0.76 + rank 59: 0.87 + rank 60: 1.09 + rank 61: 0.76 + rank 62: 0.84 + rank 63: 0.78 + params-all-gather: + rank 0: 0.02 + rank 1: 0.02 + rank 2: 0.02 + rank 3: 0.02 + rank 4: 0.02 + rank 5: 0.02 + rank 6: 0.02 + rank 7: 0.02 + rank 8: 0.02 + rank 9: 0.02 + rank 10: 0.02 + rank 11: 0.02 + rank 12: 0.02 + rank 13: 0.02 + rank 14: 0.02 + rank 15: 0.02 + rank 16: 0.02 + rank 17: 0.02 + rank 18: 0.02 + rank 19: 0.02 + rank 20: 0.02 + rank 21: 0.02 + rank 22: 0.02 + rank 23: 0.02 + rank 24: 0.03 + rank 25: 0.03 + rank 26: 0.04 + rank 27: 0.03 + rank 28: 0.03 + rank 29: 0.03 + rank 30: 0.04 + rank 31: 0.04 + rank 32: 0.02 + rank 33: 0.02 + rank 34: 0.02 + rank 35: 0.02 + rank 36: 0.02 + rank 37: 0.02 + rank 38: 0.02 + rank 39: 0.03 + rank 40: 0.02 + rank 41: 0.02 + rank 42: 0.02 + rank 43: 0.02 + rank 44: 0.02 + rank 45: 0.02 + rank 46: 0.02 + rank 47: 0.02 + rank 48: 0.03 + rank 49: 0.03 + rank 50: 0.02 + rank 51: 0.03 + rank 52: 0.03 + rank 53: 0.02 + rank 54: 0.03 + rank 55: 0.02 + rank 56: 0.02 + rank 57: 0.03 + rank 58: 0.02 + rank 59: 0.02 + rank 60: 0.02 + rank 61: 0.02 + rank 62: 0.02 + rank 63: 0.02 + optimizer-copy-to-main-grad: + rank 0: 2.28 + rank 1: 1.84 + rank 2: 1.82 + rank 3: 2.19 + rank 4: 1.77 + rank 5: 1.82 + rank 6: 1.85 + rank 7: 2.04 + rank 8: 1.70 + rank 9: 2.17 + rank 10: 1.73 + rank 11: 1.72 + rank 12: 1.79 + rank 13: 1.75 + rank 14: 1.95 + rank 15: 2.00 + rank 16: 2.60 + rank 17: 2.09 + rank 18: 2.19 + rank 19: 2.09 + rank 20: 2.21 + rank 21: 2.27 + rank 22: 2.72 + rank 23: 2.36 + rank 24: 1.94 + rank 25: 2.27 + rank 26: 1.98 + rank 27: 2.05 + rank 28: 2.25 + rank 29: 2.40 + rank 30: 2.22 + rank 31: 2.06 + rank 32: 2.08 + rank 33: 1.89 + rank 34: 1.96 + rank 35: 2.25 + rank 36: 2.02 + rank 37: 2.00 + rank 38: 2.29 + rank 39: 1.92 + rank 40: 1.74 + rank 41: 1.81 + rank 42: 1.86 + rank 43: 1.81 + rank 44: 1.82 + rank 45: 2.04 + rank 46: 1.79 + rank 47: 1.70 + rank 48: 2.13 + rank 49: 1.85 + rank 50: 2.30 + rank 51: 1.84 + rank 52: 1.96 + rank 53: 1.74 + rank 54: 1.80 + rank 55: 1.86 + rank 56: 1.73 + rank 57: 2.39 + rank 58: 1.83 + rank 59: 1.72 + rank 60: 1.73 + rank 61: 1.66 + rank 62: 1.93 + rank 63: 1.73 + optimizer-inner-step: + rank 0: 45.32 + rank 1: 45.18 + rank 2: 44.96 + rank 3: 45.14 + rank 4: 44.87 + rank 5: 45.01 + rank 6: 45.41 + rank 7: 45.37 + rank 8: 45.18 + rank 9: 45.18 + rank 10: 45.04 + rank 11: 45.16 + rank 12: 45.05 + rank 13: 45.37 + rank 14: 45.11 + rank 15: 45.01 + rank 16: 45.50 + rank 17: 45.20 + rank 18: 45.26 + rank 19: 45.19 + rank 20: 45.13 + rank 21: 45.09 + rank 22: 45.54 + rank 23: 45.22 + rank 24: 44.80 + rank 25: 45.30 + rank 26: 45.11 + rank 27: 45.26 + rank 28: 46.42 + rank 29: 45.04 + rank 30: 45.11 + rank 31: 45.34 + rank 32: 45.25 + rank 33: 44.96 + rank 34: 45.12 + rank 35: 47.86 + rank 36: 45.19 + rank 37: 45.17 + rank 38: 45.08 + rank 39: 45.06 + rank 40: 45.11 + rank 41: 45.21 + rank 42: 45.31 + rank 43: 45.33 + rank 44: 44.88 + rank 45: 45.46 + rank 46: 45.18 + rank 47: 44.79 + rank 48: 45.14 + rank 49: 45.03 + rank 50: 53.34 + rank 51: 44.83 + rank 52: 45.14 + rank 53: 45.00 + rank 54: 45.20 + rank 55: 45.11 + rank 56: 44.75 + rank 57: 54.60 + rank 58: 45.07 + rank 59: 44.91 + rank 60: 45.29 + rank 61: 44.90 + rank 62: 45.02 + rank 63: 44.86 + optimizer-copy-main-to-model-params: + rank 0: 4.47 + rank 1: 3.92 + rank 2: 3.84 + rank 3: 4.39 + rank 4: 3.82 + rank 5: 3.98 + rank 6: 3.92 + rank 7: 4.07 + rank 8: 3.88 + rank 9: 4.45 + rank 10: 3.78 + rank 11: 3.81 + rank 12: 3.90 + rank 13: 3.79 + rank 14: 4.05 + rank 15: 4.25 + rank 16: 4.44 + rank 17: 4.08 + rank 18: 4.05 + rank 19: 4.11 + rank 20: 4.07 + rank 21: 4.27 + rank 22: 4.59 + rank 23: 4.23 + rank 24: 3.89 + rank 25: 4.24 + rank 26: 4.02 + rank 27: 4.13 + rank 28: 4.36 + rank 29: 4.42 + rank 30: 4.38 + rank 31: 4.05 + rank 32: 4.12 + rank 33: 3.89 + rank 34: 4.02 + rank 35: 4.52 + rank 36: 4.26 + rank 37: 4.07 + rank 38: 4.51 + rank 39: 4.01 + rank 40: 3.80 + rank 41: 3.97 + rank 42: 3.92 + rank 43: 3.98 + rank 44: 3.99 + rank 45: 4.26 + rank 46: 3.73 + rank 47: 3.76 + rank 48: 4.22 + rank 49: 3.91 + rank 50: 4.92 + rank 51: 3.97 + rank 52: 4.22 + rank 53: 3.87 + rank 54: 3.87 + rank 55: 3.87 + rank 56: 3.83 + rank 57: 4.85 + rank 58: 3.90 + rank 59: 3.77 + rank 60: 3.78 + rank 61: 3.73 + rank 62: 3.94 + rank 63: 3.70 + optimizer: + rank 0: 58.39 + rank 1: 57.51 + rank 2: 57.15 + rank 3: 58.11 + rank 4: 57.01 + rank 5: 57.41 + rank 6: 57.78 + rank 7: 57.89 + rank 8: 57.45 + rank 9: 58.24 + rank 10: 57.18 + rank 11: 57.33 + rank 12: 57.35 + rank 13: 57.52 + rank 14: 57.59 + rank 15: 57.78 + rank 16: 58.75 + rank 17: 57.94 + rank 18: 57.99 + rank 19: 57.97 + rank 20: 57.94 + rank 21: 58.14 + rank 22: 59.03 + rank 23: 58.21 + rank 24: 57.08 + rank 25: 58.01 + rank 26: 57.52 + rank 27: 57.81 + rank 28: 59.32 + rank 29: 57.99 + rank 30: 58.00 + rank 31: 57.81 + rank 32: 57.75 + rank 33: 57.17 + rank 34: 57.49 + rank 35: 60.92 + rank 36: 57.88 + rank 37: 57.65 + rank 38: 58.12 + rank 39: 57.39 + rank 40: 57.26 + rank 41: 57.58 + rank 42: 57.64 + rank 43: 57.74 + rank 44: 57.26 + rank 45: 58.28 + rank 46: 57.22 + rank 47: 56.90 + rank 48: 57.88 + rank 49: 57.30 + rank 50: 67.01 + rank 51: 57.18 + rank 52: 57.83 + rank 53: 57.22 + rank 54: 57.45 + rank 55: 57.32 + rank 56: 56.94 + rank 57: 68.25 + rank 58: 57.37 + rank 59: 57.00 + rank 60: 57.42 + rank 61: 56.95 + rank 62: 57.35 + rank 63: 56.88 + interval-time: + rank 0: 0.81 + rank 1: 0.84 + rank 2: 0.84 + rank 3: 0.73 + rank 4: 0.83 + rank 5: 0.80 + rank 6: 0.81 + rank 7: 0.74 + rank 8: 0.82 + rank 9: 0.76 + rank 10: 0.81 + rank 11: 0.81 + rank 12: 0.81 + rank 13: 0.79 + rank 14: 0.73 + rank 15: 0.72 + rank 16: 0.75 + rank 17: 0.76 + rank 18: 0.82 + rank 19: 0.86 + rank 20: 0.73 + rank 21: 0.76 + rank 22: 0.73 + rank 23: 0.73 + rank 24: 1.02 + rank 25: 0.87 + rank 26: 1.09 + rank 27: 0.95 + rank 28: 0.84 + rank 29: 0.96 + rank 30: 0.85 + rank 31: 0.89 + rank 32: 0.87 + rank 33: 0.90 + rank 34: 0.84 + rank 35: 0.93 + rank 36: 0.89 + rank 37: 0.89 + rank 38: 0.89 + rank 39: 0.81 + rank 40: 0.82 + rank 41: 0.73 + rank 42: 0.82 + rank 43: 0.83 + rank 44: 0.81 + rank 45: 0.73 + rank 46: 0.74 + rank 47: 0.73 + rank 48: 1.07 + rank 49: 0.83 + rank 50: 0.91 + rank 51: 0.82 + rank 52: 0.93 + rank 53: 0.82 + rank 54: 0.82 + rank 55: 0.88 + rank 56: 0.82 + rank 57: 0.73 + rank 58: 0.82 + rank 59: 0.77 + rank 60: 0.81 + rank 61: 0.81 + rank 62: 0.85 + rank 63: 3.18 + [2026-10-08 21:30:51] iteration 5002/ 6855 | consumed samples: 640256 | elapsed time per iteration (ms): 89863.8 | throughput per GPU (TFLOP/s/GPU): 40.6 | MFU 13.03% | FLOPs estimator: workload-aware | logical tokens/s: 186695.6 | valid tokens/s: 186695.6 | physical tokens/s: 187218.1 | loss tokens/s: 140043.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 5.979326E-01 | lm-global-supervised-tokens: 1.258484E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.408021E-02 | lm-token-buffer-mib: 5.803157E+01 | global_load_balancing_loss: 9.993962E-01 | loss scale: 1.0 | grad norm: 0.042 | params norm: 11327.491 | params rms: 0.061335 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 22:15:17.625025 | finish at 2026-10-10 19:46:09 +times across ranks (ms): + forward-backward: + rank 0: 84732.70 + rank 1: 84812.90 + rank 2: 84813.10 + rank 3: 84814.45 + rank 4: 84814.98 + rank 5: 84814.72 + rank 6: 84812.43 + rank 7: 84811.84 + rank 8: 84732.13 + rank 9: 84811.70 + rank 10: 84811.64 + rank 11: 84814.01 + rank 12: 84815.18 + rank 13: 84814.14 + rank 14: 84810.84 + rank 15: 84810.30 + rank 16: 84732.49 + rank 17: 84812.50 + rank 18: 84811.97 + rank 19: 84813.75 + rank 20: 84826.46 + rank 21: 84815.49 + rank 22: 84823.78 + rank 23: 84812.04 + rank 24: 84724.95 + rank 25: 84805.91 + rank 26: 84805.36 + rank 27: 84805.91 + rank 28: 84808.59 + rank 29: 84808.33 + rank 30: 84805.05 + rank 31: 84803.75 + rank 32: 84721.89 + rank 33: 84801.33 + rank 34: 84800.94 + rank 35: 84802.27 + rank 36: 84802.98 + rank 37: 84802.41 + rank 38: 84801.98 + rank 39: 84801.70 + rank 40: 84725.95 + rank 41: 84806.16 + rank 42: 84803.73 + rank 43: 84805.27 + rank 44: 84805.13 + rank 45: 84804.38 + rank 46: 84806.61 + rank 47: 84805.74 + rank 48: 84732.76 + rank 49: 84813.47 + rank 50: 84810.99 + rank 51: 84812.11 + rank 52: 84812.70 + rank 53: 84811.89 + rank 54: 84814.93 + rank 55: 84813.84 + rank 56: 84735.62 + rank 57: 84815.77 + rank 58: 84813.41 + rank 59: 84814.05 + rank 60: 84814.39 + rank 61: 84813.78 + rank 62: 84816.44 + rank 63: 84815.38 + forward-compute: + rank 0: 29405.04 + rank 1: 29488.93 + rank 2: 29491.65 + rank 3: 29487.55 + rank 4: 29490.98 + rank 5: 29490.19 + rank 6: 29488.68 + rank 7: 29499.40 + rank 8: 33433.04 + rank 9: 33515.72 + rank 10: 33515.46 + rank 11: 33518.07 + rank 12: 33519.77 + rank 13: 33518.84 + rank 14: 33515.88 + rank 15: 33516.82 + rank 16: 31431.96 + rank 17: 31516.15 + rank 18: 31512.18 + rank 19: 31516.07 + rank 20: 31516.41 + rank 21: 31518.72 + rank 22: 31516.66 + rank 23: 31518.75 + rank 24: 29513.06 + rank 25: 29603.68 + rank 26: 29607.08 + rank 27: 29604.33 + rank 28: 29619.14 + rank 29: 29613.72 + rank 30: 29611.48 + rank 31: 29603.65 + rank 32: 32466.70 + rank 33: 32552.56 + rank 34: 32552.23 + rank 35: 32552.32 + rank 36: 32558.35 + rank 37: 32558.85 + rank 38: 32559.70 + rank 39: 32553.13 + rank 40: 32938.61 + rank 41: 33019.83 + rank 42: 33018.90 + rank 43: 33019.28 + rank 44: 33018.88 + rank 45: 33020.52 + rank 46: 33020.80 + rank 47: 33019.06 + rank 48: 35459.82 + rank 49: 35539.14 + rank 50: 35538.21 + rank 51: 35538.56 + rank 52: 35542.93 + rank 53: 35540.75 + rank 54: 35539.71 + rank 55: 35541.00 + rank 56: 30432.21 + rank 57: 30510.90 + rank 58: 30511.99 + rank 59: 30511.33 + rank 60: 30513.66 + rank 61: 30508.52 + rank 62: 30507.65 + rank 63: 30508.95 + backward-compute: + rank 0: 55294.48 + rank 1: 55292.37 + rank 2: 55287.36 + rank 3: 55291.03 + rank 4: 55291.26 + rank 5: 55292.07 + rank 6: 55289.07 + rank 7: 55278.11 + rank 8: 51265.39 + rank 9: 51263.63 + rank 10: 51262.25 + rank 11: 51260.98 + rank 12: 51259.90 + rank 13: 51261.04 + rank 14: 51259.44 + rank 15: 51258.86 + rank 16: 53267.14 + rank 17: 53263.93 + rank 18: 53267.25 + rank 19: 53263.18 + rank 20: 53264.97 + rank 21: 53261.82 + rank 22: 53260.81 + rank 23: 53258.34 + rank 24: 55173.37 + rank 25: 55161.38 + rank 26: 55158.12 + rank 27: 55162.64 + rank 28: 55144.81 + rank 29: 55156.12 + rank 30: 55153.63 + rank 31: 55160.07 + rank 32: 52214.25 + rank 33: 52210.91 + rank 34: 52211.05 + rank 35: 52213.13 + rank 36: 52207.69 + rank 37: 52205.70 + rank 38: 52205.68 + rank 39: 52208.25 + rank 40: 51750.91 + rank 41: 51752.40 + rank 42: 51750.62 + rank 43: 51749.14 + rank 44: 51752.02 + rank 45: 51749.29 + rank 46: 51751.57 + rank 47: 51755.04 + rank 48: 49231.75 + rank 49: 49236.99 + rank 50: 49237.64 + rank 51: 49236.61 + rank 52: 49228.13 + rank 53: 49234.33 + rank 54: 49235.90 + rank 55: 49235.18 + rank 56: 54269.21 + rank 57: 54267.56 + rank 58: 54267.97 + rank 59: 54270.77 + rank 60: 54267.67 + rank 61: 54273.11 + rank 62: 54276.11 + rank 63: 54272.40 + batch-generator: + rank 0: 23.55 + rank 1: 22.18 + rank 2: 23.25 + rank 3: 23.55 + rank 4: 23.68 + rank 5: 22.66 + rank 6: 22.72 + rank 7: 22.86 + rank 8: 22.84 + rank 9: 21.75 + rank 10: 21.30 + rank 11: 23.93 + rank 12: 23.27 + rank 13: 21.75 + rank 14: 23.58 + rank 15: 21.79 + rank 16: 23.25 + rank 17: 23.05 + rank 18: 22.33 + rank 19: 24.42 + rank 20: 22.94 + rank 21: 22.73 + rank 22: 22.28 + rank 23: 23.35 + rank 24: 32.18 + rank 25: 32.80 + rank 26: 34.64 + rank 27: 33.12 + rank 28: 33.48 + rank 29: 32.66 + rank 30: 31.39 + rank 31: 32.16 + rank 32: 31.43 + rank 33: 34.90 + rank 34: 35.39 + rank 35: 32.83 + rank 36: 29.51 + rank 37: 31.53 + rank 38: 32.66 + rank 39: 33.45 + rank 40: 22.38 + rank 41: 21.89 + rank 42: 22.94 + rank 43: 22.94 + rank 44: 22.64 + rank 45: 22.02 + rank 46: 21.67 + rank 47: 23.07 + rank 48: 30.30 + rank 49: 29.08 + rank 50: 27.32 + rank 51: 30.02 + rank 52: 32.83 + rank 53: 30.26 + rank 54: 24.00 + rank 55: 25.14 + rank 56: 24.32 + rank 57: 22.90 + rank 58: 23.77 + rank 59: 23.60 + rank 60: 25.74 + rank 61: 23.65 + rank 62: 23.28 + rank 63: 25.28 + embedding-grads-all-reduce: + rank 0: 0.03 + rank 1: 0.02 + rank 2: 0.03 + rank 3: 0.02 + rank 4: 0.03 + rank 5: 0.03 + rank 6: 0.03 + rank 7: 0.03 + rank 8: 0.02 + rank 9: 0.03 + rank 10: 0.03 + rank 11: 0.03 + rank 12: 0.03 + rank 13: 0.02 + rank 14: 0.02 + rank 15: 0.02 + rank 16: 0.03 + rank 17: 0.02 + rank 18: 0.02 + rank 19: 0.03 + rank 20: 0.04 + rank 21: 0.03 + rank 22: 0.04 + rank 23: 0.03 + rank 24: 0.04 + rank 25: 0.04 + rank 26: 0.04 + rank 27: 0.04 + rank 28: 0.04 + rank 29: 0.03 + rank 30: 0.04 + rank 31: 0.04 + rank 32: 0.03 + rank 33: 0.03 + rank 34: 0.04 + rank 35: 0.03 + rank 36: 0.03 + rank 37: 0.03 + rank 38: 0.03 + rank 39: 0.03 + rank 40: 0.02 + rank 41: 0.03 + rank 42: 0.02 + rank 43: 0.03 + rank 44: 0.03 + rank 45: 0.03 + rank 46: 0.03 + rank 47: 0.03 + rank 48: 0.04 + rank 49: 0.03 + rank 50: 0.03 + rank 51: 0.03 + rank 52: 0.03 + rank 53: 0.03 + rank 54: 0.03 + rank 55: 0.03 + rank 56: 0.03 + rank 57: 0.03 + rank 58: 0.02 + rank 59: 0.03 + rank 60: 0.03 + rank 61: 0.02 + rank 62: 0.02 + rank 63: 0.03 + all-grads-sync: + rank 0: 0.80 + rank 1: 0.78 + rank 2: 0.86 + rank 3: 1.11 + rank 4: 0.86 + rank 5: 0.90 + rank 6: 1.12 + rank 7: 1.14 + rank 8: 0.78 + rank 9: 0.74 + rank 10: 1.12 + rank 11: 1.27 + rank 12: 1.22 + rank 13: 0.79 + rank 14: 0.84 + rank 15: 0.82 + rank 16: 0.77 + rank 17: 0.78 + rank 18: 0.86 + rank 19: 0.92 + rank 20: 1.17 + rank 21: 0.80 + rank 22: 1.09 + rank 23: 0.84 + rank 24: 1.41 + rank 25: 1.22 + rank 26: 1.22 + rank 27: 1.25 + rank 28: 1.18 + rank 29: 1.26 + rank 30: 1.27 + rank 31: 1.25 + rank 32: 1.24 + rank 33: 1.24 + rank 34: 1.22 + rank 35: 1.22 + rank 36: 1.28 + rank 37: 1.24 + rank 38: 1.22 + rank 39: 1.20 + rank 40: 0.79 + rank 41: 0.75 + rank 42: 0.80 + rank 43: 0.75 + rank 44: 0.89 + rank 45: 1.28 + rank 46: 0.76 + rank 47: 0.79 + rank 48: 0.98 + rank 49: 1.06 + rank 50: 1.25 + rank 51: 1.06 + rank 52: 1.27 + rank 53: 1.05 + rank 54: 0.99 + rank 55: 0.99 + rank 56: 0.86 + rank 57: 1.15 + rank 58: 0.80 + rank 59: 0.80 + rank 60: 0.84 + rank 61: 0.99 + rank 62: 0.86 + rank 63: 0.81 + params-all-gather: + rank 0: 0.02 + rank 1: 0.02 + rank 2: 0.02 + rank 3: 0.03 + rank 4: 0.02 + rank 5: 0.02 + rank 6: 0.02 + rank 7: 0.02 + rank 8: 0.02 + rank 9: 0.02 + rank 10: 0.02 + rank 11: 0.02 + rank 12: 0.02 + rank 13: 0.02 + rank 14: 0.02 + rank 15: 0.02 + rank 16: 0.02 + rank 17: 0.02 + rank 18: 0.02 + rank 19: 0.02 + rank 20: 0.04 + rank 21: 0.02 + rank 22: 0.04 + rank 23: 0.02 + rank 24: 0.04 + rank 25: 0.03 + rank 26: 0.04 + rank 27: 0.04 + rank 28: 0.03 + rank 29: 0.03 + rank 30: 0.03 + rank 31: 0.04 + rank 32: 0.04 + rank 33: 0.03 + rank 34: 0.02 + rank 35: 0.02 + rank 36: 0.02 + rank 37: 0.02 + rank 38: 0.02 + rank 39: 0.02 + rank 40: 0.02 + rank 41: 0.02 + rank 42: 0.02 + rank 43: 0.02 + rank 44: 0.02 + rank 45: 0.02 + rank 46: 0.02 + rank 47: 0.02 + rank 48: 0.03 + rank 49: 0.03 + rank 50: 0.03 + rank 51: 0.03 + rank 52: 0.03 + rank 53: 0.03 + rank 54: 0.03 + rank 55: 0.03 + rank 56: 0.03 + rank 57: 0.02 + rank 58: 0.02 + rank 59: 0.02 + rank 60: 0.02 + rank 61: 0.02 + rank 62: 0.02 + rank 63: 0.02 + optimizer-copy-to-main-grad: + rank 0: 2.15 + rank 1: 1.81 + rank 2: 1.71 + rank 3: 2.10 + rank 4: 1.71 + rank 5: 1.82 + rank 6: 1.86 + rank 7: 1.87 + rank 8: 1.78 + rank 9: 2.11 + rank 10: 1.74 + rank 11: 1.88 + rank 12: 1.77 + rank 13: 1.73 + rank 14: 1.81 + rank 15: 2.00 + rank 16: 2.40 + rank 17: 2.04 + rank 18: 2.11 + rank 19: 2.21 + rank 20: 3.54 + rank 21: 2.36 + rank 22: 4.08 + rank 23: 2.35 + rank 24: 2.01 + rank 25: 2.31 + rank 26: 2.13 + rank 27: 2.23 + rank 28: 2.33 + rank 29: 2.43 + rank 30: 2.26 + rank 31: 2.01 + rank 32: 2.24 + rank 33: 1.88 + rank 34: 1.88 + rank 35: 2.20 + rank 36: 2.10 + rank 37: 2.04 + rank 38: 2.26 + rank 39: 1.91 + rank 40: 1.66 + rank 41: 1.83 + rank 42: 1.78 + rank 43: 1.84 + rank 44: 1.84 + rank 45: 2.02 + rank 46: 1.75 + rank 47: 1.62 + rank 48: 2.02 + rank 49: 1.79 + rank 50: 2.31 + rank 51: 1.81 + rank 52: 1.98 + rank 53: 1.77 + rank 54: 1.79 + rank 55: 1.81 + rank 56: 1.79 + rank 57: 2.37 + rank 58: 1.84 + rank 59: 1.74 + rank 60: 1.71 + rank 61: 1.68 + rank 62: 1.84 + rank 63: 1.72 + optimizer-inner-step: + rank 0: 51.34 + rank 1: 51.17 + rank 2: 50.97 + rank 3: 51.04 + rank 4: 50.89 + rank 5: 50.96 + rank 6: 51.40 + rank 7: 51.35 + rank 8: 51.09 + rank 9: 51.10 + rank 10: 50.93 + rank 11: 51.10 + rank 12: 51.11 + rank 13: 51.34 + rank 14: 51.04 + rank 15: 50.95 + rank 16: 51.48 + rank 17: 51.26 + rank 18: 51.23 + rank 19: 51.20 + rank 20: 52.30 + rank 21: 51.10 + rank 22: 63.64 + rank 23: 51.26 + rank 24: 50.77 + rank 25: 51.17 + rank 26: 51.04 + rank 27: 51.18 + rank 28: 51.49 + rank 29: 51.01 + rank 30: 51.00 + rank 31: 51.23 + rank 32: 51.20 + rank 33: 50.95 + rank 34: 51.08 + rank 35: 51.46 + rank 36: 51.26 + rank 37: 51.18 + rank 38: 51.01 + rank 39: 51.01 + rank 40: 51.12 + rank 41: 51.20 + rank 42: 51.29 + rank 43: 51.29 + rank 44: 50.88 + rank 45: 51.48 + rank 46: 51.13 + rank 47: 50.81 + rank 48: 51.18 + rank 49: 51.03 + rank 50: 53.41 + rank 51: 50.82 + rank 52: 51.14 + rank 53: 50.97 + rank 54: 51.19 + rank 55: 51.02 + rank 56: 50.69 + rank 57: 52.06 + rank 58: 51.12 + rank 59: 50.95 + rank 60: 51.21 + rank 61: 50.89 + rank 62: 51.04 + rank 63: 50.87 + optimizer-copy-main-to-model-params: + rank 0: 4.44 + rank 1: 3.94 + rank 2: 3.87 + rank 3: 4.44 + rank 4: 3.74 + rank 5: 3.92 + rank 6: 3.91 + rank 7: 4.01 + rank 8: 3.88 + rank 9: 4.38 + rank 10: 3.71 + rank 11: 3.79 + rank 12: 3.91 + rank 13: 3.77 + rank 14: 4.04 + rank 15: 4.24 + rank 16: 4.44 + rank 17: 4.07 + rank 18: 4.05 + rank 19: 4.08 + rank 20: 6.17 + rank 21: 4.27 + rank 22: 6.24 + rank 23: 4.27 + rank 24: 3.88 + rank 25: 4.45 + rank 26: 3.97 + rank 27: 4.12 + rank 28: 4.58 + rank 29: 4.34 + rank 30: 4.51 + rank 31: 4.04 + rank 32: 4.32 + rank 33: 3.90 + rank 34: 4.06 + rank 35: 4.65 + rank 36: 4.20 + rank 37: 4.08 + rank 38: 4.50 + rank 39: 3.95 + rank 40: 3.79 + rank 41: 3.96 + rank 42: 3.86 + rank 43: 3.99 + rank 44: 3.93 + rank 45: 4.21 + rank 46: 3.78 + rank 47: 3.72 + rank 48: 4.20 + rank 49: 3.91 + rank 50: 4.83 + rank 51: 3.98 + rank 52: 4.23 + rank 53: 3.92 + rank 54: 3.86 + rank 55: 3.91 + rank 56: 3.75 + rank 57: 4.68 + rank 58: 3.86 + rank 59: 3.74 + rank 60: 3.77 + rank 61: 3.64 + rank 62: 3.90 + rank 63: 3.68 + optimizer: + rank 0: 65.66 + rank 1: 64.85 + rank 2: 64.48 + rank 3: 65.44 + rank 4: 64.26 + rank 5: 64.67 + rank 6: 65.05 + rank 7: 65.12 + rank 8: 64.64 + rank 9: 65.40 + rank 10: 64.41 + rank 11: 64.56 + rank 12: 64.70 + rank 13: 64.75 + rank 14: 64.81 + rank 15: 64.99 + rank 16: 65.99 + rank 17: 65.24 + rank 18: 65.29 + rank 19: 65.25 + rank 20: 69.35 + rank 21: 65.39 + rank 22: 81.14 + rank 23: 65.58 + rank 24: 64.31 + rank 25: 65.37 + rank 26: 64.68 + rank 27: 65.01 + rank 28: 65.90 + rank 29: 65.16 + rank 30: 65.30 + rank 31: 64.95 + rank 32: 65.25 + rank 33: 64.45 + rank 34: 64.76 + rank 35: 65.92 + rank 36: 65.19 + rank 37: 64.93 + rank 38: 65.32 + rank 39: 64.55 + rank 40: 64.54 + rank 41: 64.87 + rank 42: 64.84 + rank 43: 65.01 + rank 44: 64.51 + rank 45: 65.53 + rank 46: 64.50 + rank 47: 64.13 + rank 48: 65.14 + rank 49: 64.59 + rank 50: 68.26 + rank 51: 64.48 + rank 52: 65.12 + rank 53: 64.52 + rank 54: 64.66 + rank 55: 64.57 + rank 56: 64.07 + rank 57: 66.77 + rank 58: 64.65 + rank 59: 64.30 + rank 60: 64.63 + rank 61: 64.13 + rank 62: 64.62 + rank 63: 64.11 + interval-time: + rank 0: 0.82 + rank 1: 0.84 + rank 2: 0.84 + rank 3: 0.75 + rank 4: 0.85 + rank 5: 0.81 + rank 6: 0.80 + rank 7: 0.74 + rank 8: 0.84 + rank 9: 0.73 + rank 10: 0.81 + rank 11: 0.83 + rank 12: 0.82 + rank 13: 0.79 + rank 14: 0.76 + rank 15: 0.76 + rank 16: 0.74 + rank 17: 0.74 + rank 18: 0.74 + rank 19: 0.85 + rank 20: 1.10 + rank 21: 0.83 + rank 22: 1.10 + rank 23: 0.75 + rank 24: 0.87 + rank 25: 0.86 + rank 26: 0.97 + rank 27: 0.87 + rank 28: 0.84 + rank 29: 0.84 + rank 30: 0.84 + rank 31: 0.88 + rank 32: 0.83 + rank 33: 0.89 + rank 34: 0.84 + rank 35: 0.80 + rank 36: 0.89 + rank 37: 0.89 + rank 38: 0.89 + rank 39: 0.79 + rank 40: 0.83 + rank 41: 0.74 + rank 42: 0.80 + rank 43: 0.83 + rank 44: 0.80 + rank 45: 0.83 + rank 46: 0.74 + rank 47: 0.74 + rank 48: 0.98 + rank 49: 0.83 + rank 50: 0.90 + rank 51: 0.81 + rank 52: 0.93 + rank 53: 0.82 + rank 54: 0.82 + rank 55: 0.84 + rank 56: 0.83 + rank 57: 0.72 + rank 58: 0.83 + rank 59: 0.73 + rank 60: 0.82 + rank 61: 0.80 + rank 62: 0.81 + rank 63: 3.65 + [2026-10-08 21:32:16] iteration 5003/ 6855 | consumed samples: 640384 | elapsed time per iteration (ms): 84876.9 | throughput per GPU (TFLOP/s/GPU): 43.9 | MFU 14.07% | FLOPs estimator: workload-aware | logical tokens/s: 197664.7 | valid tokens/s: 197664.7 | physical tokens/s: 198048.1 | loss tokens/s: 148137.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.953822E-01 | lm-global-supervised-tokens: 1.257347E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.935873E-02 | lm-token-buffer-mib: 5.804194E+01 | global_load_balancing_loss: 9.993464E-01 | loss scale: 1.0 | grad norm: 0.046 | params norm: 11327.438 | params rms: 0.061334 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 19:39:52.015189 | finish at 2026-10-10 17:12:08 +times across ranks (ms): + forward-backward: + rank 0: 84659.55 + rank 1: 84659.49 + rank 2: 84660.05 + rank 3: 84661.40 + rank 4: 84660.55 + rank 5: 84660.45 + rank 6: 84658.30 + rank 7: 84658.51 + rank 8: 84658.24 + rank 9: 84658.55 + rank 10: 84659.14 + rank 11: 84660.52 + rank 12: 84659.57 + rank 13: 84660.38 + rank 14: 84657.12 + rank 15: 84657.16 + rank 16: 84657.81 + rank 17: 84658.92 + rank 18: 84659.09 + rank 19: 84659.70 + rank 20: 84659.66 + rank 21: 84661.42 + rank 22: 84657.58 + rank 23: 84657.70 + rank 24: 84655.20 + rank 25: 84653.30 + rank 26: 84653.24 + rank 27: 84654.83 + rank 28: 84655.06 + rank 29: 84654.75 + rank 30: 84653.86 + rank 31: 84650.94 + rank 32: 84649.45 + rank 33: 84648.67 + rank 34: 84648.55 + rank 35: 84648.60 + rank 36: 84649.62 + rank 37: 84648.81 + rank 38: 84648.68 + rank 39: 84649.41 + rank 40: 84652.56 + rank 41: 84651.63 + rank 42: 84651.40 + rank 43: 84650.58 + rank 44: 84650.86 + rank 45: 84650.23 + rank 46: 84653.34 + rank 47: 84651.85 + rank 48: 84661.62 + rank 49: 84659.00 + rank 50: 84657.93 + rank 51: 84659.05 + rank 52: 84658.96 + rank 53: 84658.16 + rank 54: 84660.18 + rank 55: 84660.29 + rank 56: 84662.18 + rank 57: 84661.08 + rank 58: 84662.03 + rank 59: 84661.01 + rank 60: 84660.33 + rank 61: 84658.94 + rank 62: 84662.79 + rank 63: 84663.01 + forward-compute: + rank 0: 34166.27 + rank 1: 34164.82 + rank 2: 34166.68 + rank 3: 34166.12 + rank 4: 34166.94 + rank 5: 34166.53 + rank 6: 34164.73 + rank 7: 34167.09 + rank 8: 33085.06 + rank 9: 33088.20 + rank 10: 33086.73 + rank 11: 33085.91 + rank 12: 33086.36 + rank 13: 33090.77 + rank 14: 33084.15 + rank 15: 33084.14 + rank 16: 32502.08 + rank 17: 32502.39 + rank 18: 32503.13 + rank 19: 32504.10 + rank 20: 32503.85 + rank 21: 32506.40 + rank 22: 32504.21 + rank 23: 32505.69 + rank 24: 28910.09 + rank 25: 28918.27 + rank 26: 28922.83 + rank 27: 28915.36 + rank 28: 28923.88 + rank 29: 28919.23 + rank 30: 28924.11 + rank 31: 28925.50 + rank 32: 31501.74 + rank 33: 31504.27 + rank 34: 31503.94 + rank 35: 31506.63 + rank 36: 31511.98 + rank 37: 31516.52 + rank 38: 31515.61 + rank 39: 31506.13 + rank 40: 28375.45 + rank 41: 28364.02 + rank 42: 28373.87 + rank 43: 28375.41 + rank 44: 28372.89 + rank 45: 28378.56 + rank 46: 28374.11 + rank 47: 28379.02 + rank 48: 32353.79 + rank 49: 32352.86 + rank 50: 32349.00 + rank 51: 32347.62 + rank 52: 32378.63 + rank 53: 32353.85 + rank 54: 32348.70 + rank 55: 32351.42 + rank 56: 32214.81 + rank 57: 32213.94 + rank 58: 32219.60 + rank 59: 32218.88 + rank 60: 32217.74 + rank 61: 32217.40 + rank 62: 32218.00 + rank 63: 32217.65 + backward-compute: + rank 0: 50458.25 + rank 1: 50460.70 + rank 2: 50456.62 + rank 3: 50458.99 + rank 4: 50458.45 + rank 5: 50461.73 + rank 6: 50459.24 + rank 7: 50457.98 + rank 8: 51537.47 + rank 9: 51537.48 + rank 10: 51538.04 + rank 11: 51538.43 + rank 12: 51537.84 + rank 13: 51537.05 + rank 14: 51538.37 + rank 15: 51540.30 + rank 16: 52123.15 + rank 17: 52123.03 + rank 18: 52123.18 + rank 19: 52120.43 + rank 20: 52123.05 + rank 21: 52119.32 + rank 22: 52120.41 + rank 23: 52117.20 + rank 24: 55701.49 + rank 25: 55694.57 + rank 26: 55691.32 + rank 27: 55698.81 + rank 28: 55690.77 + rank 29: 55692.96 + rank 30: 55687.31 + rank 31: 55685.80 + rank 32: 53107.32 + rank 33: 53104.67 + rank 34: 53106.25 + rank 35: 53104.04 + rank 36: 53100.24 + rank 37: 53095.73 + rank 38: 53096.52 + rank 39: 53106.63 + rank 40: 56244.11 + rank 41: 56255.77 + rank 42: 56245.61 + rank 43: 56242.34 + rank 44: 56246.19 + rank 45: 56240.30 + rank 46: 56247.97 + rank 47: 56241.41 + rank 48: 52267.06 + rank 49: 52270.70 + rank 50: 52275.63 + rank 51: 52274.51 + rank 52: 52241.05 + rank 53: 52268.46 + rank 54: 52276.28 + rank 55: 52274.89 + rank 56: 52411.66 + rank 57: 52414.08 + rank 58: 52408.18 + rank 59: 52408.07 + rank 60: 52408.55 + rank 61: 52409.34 + rank 62: 52409.73 + rank 63: 52406.27 + batch-generator: + rank 0: 22.33 + rank 1: 23.04 + rank 2: 23.15 + rank 3: 22.09 + rank 4: 22.59 + rank 5: 21.22 + rank 6: 22.83 + rank 7: 22.98 + rank 8: 23.74 + rank 9: 21.13 + rank 10: 21.10 + rank 11: 22.80 + rank 12: 23.22 + rank 13: 23.37 + rank 14: 23.89 + rank 15: 22.72 + rank 16: 22.94 + rank 17: 22.67 + rank 18: 21.96 + rank 19: 22.88 + rank 20: 21.71 + rank 21: 23.96 + rank 22: 23.71 + rank 23: 22.02 + rank 24: 30.82 + rank 25: 34.29 + rank 26: 32.64 + rank 27: 34.44 + rank 28: 33.86 + rank 29: 35.34 + rank 30: 34.42 + rank 31: 32.94 + rank 32: 35.51 + rank 33: 33.85 + rank 34: 34.31 + rank 35: 31.86 + rank 36: 31.22 + rank 37: 30.34 + rank 38: 32.73 + rank 39: 31.43 + rank 40: 22.86 + rank 41: 22.00 + rank 42: 22.14 + rank 43: 22.03 + rank 44: 21.15 + rank 45: 21.65 + rank 46: 20.64 + rank 47: 22.04 + rank 48: 36.73 + rank 49: 29.73 + rank 50: 27.24 + rank 51: 28.75 + rank 52: 32.41 + rank 53: 29.05 + rank 54: 21.65 + rank 55: 25.89 + rank 56: 24.80 + rank 57: 23.61 + rank 58: 22.62 + rank 59: 21.90 + rank 60: 23.40 + rank 61: 23.06 + rank 62: 22.31 + rank 63: 22.51 + embedding-grads-all-reduce: + rank 0: 0.03 + rank 1: 0.02 + rank 2: 0.02 + rank 3: 0.02 + rank 4: 0.02 + rank 5: 0.03 + rank 6: 0.02 + rank 7: 0.03 + rank 8: 0.02 + rank 9: 0.03 + rank 10: 0.02 + rank 11: 0.03 + rank 12: 0.03 + rank 13: 0.02 + rank 14: 0.03 + rank 15: 0.02 + rank 16: 0.03 + rank 17: 0.03 + rank 18: 0.03 + rank 19: 0.03 + rank 20: 0.03 + rank 21: 0.03 + rank 22: 0.03 + rank 23: 0.03 + rank 24: 0.04 + rank 25: 0.04 + rank 26: 0.04 + rank 27: 0.03 + rank 28: 0.04 + rank 29: 0.04 + rank 30: 0.04 + rank 31: 0.04 + rank 32: 0.03 + rank 33: 0.03 + rank 34: 0.04 + rank 35: 0.03 + rank 36: 0.03 + rank 37: 0.03 + rank 38: 0.03 + rank 39: 0.03 + rank 40: 0.03 + rank 41: 0.02 + rank 42: 0.02 + rank 43: 0.03 + rank 44: 0.03 + rank 45: 0.03 + rank 46: 0.02 + rank 47: 0.03 + rank 48: 0.04 + rank 49: 0.03 + rank 50: 0.03 + rank 51: 0.03 + rank 52: 0.03 + rank 53: 0.03 + rank 54: 0.03 + rank 55: 0.03 + rank 56: 0.03 + rank 57: 0.03 + rank 58: 0.03 + rank 59: 0.02 + rank 60: 0.02 + rank 61: 0.03 + rank 62: 0.03 + rank 63: 0.03 + all-grads-sync: + rank 0: 1.15 + rank 1: 0.75 + rank 2: 0.81 + rank 3: 1.24 + rank 4: 0.77 + rank 5: 0.81 + rank 6: 0.76 + rank 7: 0.83 + rank 8: 0.78 + rank 9: 0.74 + rank 10: 0.97 + rank 11: 1.10 + rank 12: 0.79 + rank 13: 0.79 + rank 14: 0.94 + rank 15: 0.78 + rank 16: 0.88 + rank 17: 0.86 + rank 18: 0.94 + rank 19: 0.77 + rank 20: 0.87 + rank 21: 0.79 + rank 22: 1.05 + rank 23: 0.98 + rank 24: 1.28 + rank 25: 1.18 + rank 26: 1.33 + rank 27: 1.34 + rank 28: 1.22 + rank 29: 1.24 + rank 30: 1.26 + rank 31: 1.29 + rank 32: 1.57 + rank 33: 1.23 + rank 34: 1.29 + rank 35: 1.20 + rank 36: 1.14 + rank 37: 1.27 + rank 38: 1.16 + rank 39: 1.22 + rank 40: 0.86 + rank 41: 0.76 + rank 42: 0.79 + rank 43: 0.75 + rank 44: 1.11 + rank 45: 0.74 + rank 46: 0.85 + rank 47: 0.90 + rank 48: 1.61 + rank 49: 1.02 + rank 50: 1.03 + rank 51: 1.08 + rank 52: 1.50 + rank 53: 1.09 + rank 54: 0.98 + rank 55: 1.01 + rank 56: 0.81 + rank 57: 0.76 + rank 58: 1.11 + rank 59: 1.12 + rank 60: 0.80 + rank 61: 0.79 + rank 62: 0.86 + rank 63: 1.44 + params-all-gather: + rank 0: 0.02 + rank 1: 0.02 + rank 2: 0.02 + rank 3: 0.02 + rank 4: 0.02 + rank 5: 0.03 + rank 6: 0.02 + rank 7: 0.02 + rank 8: 0.02 + rank 9: 0.02 + rank 10: 0.02 + rank 11: 0.02 + rank 12: 0.02 + rank 13: 0.02 + rank 14: 0.02 + rank 15: 0.02 + rank 16: 0.02 + rank 17: 0.02 + rank 18: 0.02 + rank 19: 0.02 + rank 20: 0.02 + rank 21: 0.02 + rank 22: 0.02 + rank 23: 0.02 + rank 24: 0.03 + rank 25: 0.03 + rank 26: 0.03 + rank 27: 0.03 + rank 28: 0.04 + rank 29: 0.03 + rank 30: 0.03 + rank 31: 0.03 + rank 32: 0.02 + rank 33: 0.02 + rank 34: 0.02 + rank 35: 0.02 + rank 36: 0.02 + rank 37: 0.03 + rank 38: 0.03 + rank 39: 0.02 + rank 40: 0.02 + rank 41: 0.02 + rank 42: 0.02 + rank 43: 0.02 + rank 44: 0.02 + rank 45: 0.02 + rank 46: 0.02 + rank 47: 0.02 + rank 48: 0.03 + rank 49: 0.03 + rank 50: 0.03 + rank 51: 0.03 + rank 52: 0.03 + rank 53: 0.02 + rank 54: 0.03 + rank 55: 0.03 + rank 56: 0.02 + rank 57: 0.02 + rank 58: 0.02 + rank 59: 0.02 + rank 60: 0.02 + rank 61: 0.02 + rank 62: 0.02 + rank 63: 0.02 + optimizer-copy-to-main-grad: + rank 0: 2.12 + rank 1: 1.80 + rank 2: 1.65 + rank 3: 2.10 + rank 4: 1.67 + rank 5: 1.76 + rank 6: 1.84 + rank 7: 1.90 + rank 8: 1.78 + rank 9: 2.08 + rank 10: 1.72 + rank 11: 1.72 + rank 12: 1.78 + rank 13: 1.75 + rank 14: 1.83 + rank 15: 1.97 + rank 16: 2.36 + rank 17: 2.07 + rank 18: 2.12 + rank 19: 2.10 + rank 20: 2.25 + rank 21: 2.31 + rank 22: 2.56 + rank 23: 2.34 + rank 24: 2.03 + rank 25: 2.19 + rank 26: 1.84 + rank 27: 2.06 + rank 28: 2.37 + rank 29: 2.41 + rank 30: 2.30 + rank 31: 1.94 + rank 32: 2.13 + rank 33: 1.90 + rank 34: 1.88 + rank 35: 2.23 + rank 36: 2.09 + rank 37: 1.99 + rank 38: 2.24 + rank 39: 1.89 + rank 40: 1.72 + rank 41: 1.82 + rank 42: 1.78 + rank 43: 1.79 + rank 44: 1.82 + rank 45: 2.05 + rank 46: 1.74 + rank 47: 1.64 + rank 48: 2.19 + rank 49: 1.76 + rank 50: 2.28 + rank 51: 1.80 + rank 52: 1.98 + rank 53: 1.76 + rank 54: 1.78 + rank 55: 1.77 + rank 56: 1.76 + rank 57: 2.33 + rank 58: 1.89 + rank 59: 1.74 + rank 60: 1.70 + rank 61: 1.66 + rank 62: 1.89 + rank 63: 1.75 + optimizer-inner-step: + rank 0: 42.03 + rank 1: 41.91 + rank 2: 41.72 + rank 3: 41.89 + rank 4: 41.67 + rank 5: 41.82 + rank 6: 42.20 + rank 7: 42.18 + rank 8: 41.91 + rank 9: 41.94 + rank 10: 41.73 + rank 11: 41.92 + rank 12: 41.90 + rank 13: 42.15 + rank 14: 41.88 + rank 15: 41.78 + rank 16: 42.30 + rank 17: 42.05 + rank 18: 41.96 + rank 19: 42.00 + rank 20: 42.03 + rank 21: 41.92 + rank 22: 42.37 + rank 23: 42.01 + rank 24: 41.68 + rank 25: 41.92 + rank 26: 41.86 + rank 27: 41.98 + rank 28: 45.95 + rank 29: 41.80 + rank 30: 41.81 + rank 31: 42.06 + rank 32: 41.93 + rank 33: 41.70 + rank 34: 41.89 + rank 35: 47.16 + rank 36: 42.02 + rank 37: 42.02 + rank 38: 41.86 + rank 39: 41.84 + rank 40: 41.90 + rank 41: 41.93 + rank 42: 42.04 + rank 43: 42.09 + rank 44: 41.65 + rank 45: 42.29 + rank 46: 41.87 + rank 47: 41.55 + rank 48: 43.16 + rank 49: 41.83 + rank 50: 53.58 + rank 51: 41.60 + rank 52: 41.93 + rank 53: 41.85 + rank 54: 41.97 + rank 55: 41.91 + rank 56: 41.56 + rank 57: 47.25 + rank 58: 41.88 + rank 59: 41.76 + rank 60: 42.10 + rank 61: 41.64 + rank 62: 41.80 + rank 63: 41.67 + optimizer-copy-main-to-model-params: + rank 0: 4.39 + rank 1: 3.90 + rank 2: 3.72 + rank 3: 4.33 + rank 4: 3.70 + rank 5: 3.88 + rank 6: 3.92 + rank 7: 3.98 + rank 8: 3.97 + rank 9: 4.42 + rank 10: 3.76 + rank 11: 3.84 + rank 12: 4.04 + rank 13: 3.77 + rank 14: 4.07 + rank 15: 4.24 + rank 16: 4.40 + rank 17: 4.00 + rank 18: 3.99 + rank 19: 3.99 + rank 20: 4.05 + rank 21: 4.22 + rank 22: 4.60 + rank 23: 4.18 + rank 24: 3.91 + rank 25: 4.23 + rank 26: 3.94 + rank 27: 4.06 + rank 28: 4.40 + rank 29: 4.36 + rank 30: 4.37 + rank 31: 3.99 + rank 32: 4.13 + rank 33: 3.95 + rank 34: 3.96 + rank 35: 4.54 + rank 36: 4.22 + rank 37: 4.05 + rank 38: 4.45 + rank 39: 3.97 + rank 40: 3.75 + rank 41: 4.01 + rank 42: 3.92 + rank 43: 3.96 + rank 44: 3.98 + rank 45: 4.23 + rank 46: 3.74 + rank 47: 3.68 + rank 48: 4.26 + rank 49: 3.89 + rank 50: 4.86 + rank 51: 3.94 + rank 52: 4.23 + rank 53: 3.93 + rank 54: 3.82 + rank 55: 3.86 + rank 56: 3.75 + rank 57: 4.69 + rank 58: 3.88 + rank 59: 3.73 + rank 60: 3.76 + rank 61: 3.67 + rank 62: 3.93 + rank 63: 3.70 + optimizer: + rank 0: 53.73 + rank 1: 52.90 + rank 2: 52.45 + rank 3: 53.45 + rank 4: 52.39 + rank 5: 52.81 + rank 6: 53.23 + rank 7: 53.25 + rank 8: 52.92 + rank 9: 53.62 + rank 10: 52.52 + rank 11: 52.73 + rank 12: 52.97 + rank 13: 52.93 + rank 14: 53.03 + rank 15: 53.19 + rank 16: 54.13 + rank 17: 53.31 + rank 18: 53.33 + rank 19: 53.28 + rank 20: 53.32 + rank 21: 53.53 + rank 22: 54.54 + rank 23: 53.62 + rank 24: 52.62 + rank 25: 53.25 + rank 26: 52.83 + rank 27: 53.13 + rank 28: 57.53 + rank 29: 53.32 + rank 30: 53.31 + rank 31: 53.09 + rank 32: 53.11 + rank 33: 52.65 + rank 34: 52.84 + rank 35: 58.89 + rank 36: 53.32 + rank 37: 53.11 + rank 38: 53.50 + rank 39: 52.78 + rank 40: 52.68 + rank 41: 53.08 + rank 42: 53.07 + rank 43: 53.15 + rank 44: 52.74 + rank 45: 53.75 + rank 46: 52.61 + rank 47: 52.26 + rank 48: 54.54 + rank 49: 52.75 + rank 50: 65.82 + rank 51: 52.54 + rank 52: 53.29 + rank 53: 52.75 + rank 54: 52.79 + rank 55: 52.76 + rank 56: 52.30 + rank 57: 59.33 + rank 58: 52.80 + rank 59: 52.45 + rank 60: 52.86 + rank 61: 52.26 + rank 62: 52.76 + rank 63: 52.31 + interval-time: + rank 0: 0.95 + rank 1: 0.97 + rank 2: 1.03 + rank 3: 0.92 + rank 4: 1.03 + rank 5: 0.99 + rank 6: 1.00 + rank 7: 0.92 + rank 8: 0.85 + rank 9: 0.74 + rank 10: 0.80 + rank 11: 0.83 + rank 12: 0.84 + rank 13: 0.78 + rank 14: 0.74 + rank 15: 0.73 + rank 16: 0.73 + rank 17: 0.73 + rank 18: 0.73 + rank 19: 0.82 + rank 20: 0.73 + rank 21: 0.83 + rank 22: 0.72 + rank 23: 0.73 + rank 24: 0.89 + rank 25: 0.87 + rank 26: 0.98 + rank 27: 0.88 + rank 28: 0.83 + rank 29: 0.85 + rank 30: 0.83 + rank 31: 0.88 + rank 32: 0.84 + rank 33: 0.84 + rank 34: 0.86 + rank 35: 0.90 + rank 36: 0.88 + rank 37: 0.88 + rank 38: 0.89 + rank 39: 0.80 + rank 40: 0.84 + rank 41: 0.78 + rank 42: 0.80 + rank 43: 0.84 + rank 44: 0.81 + rank 45: 0.73 + rank 46: 0.76 + rank 47: 0.74 + rank 48: 1.00 + rank 49: 0.83 + rank 50: 0.90 + rank 51: 0.82 + rank 52: 0.95 + rank 53: 0.82 + rank 54: 0.83 + rank 55: 0.83 + rank 56: 0.81 + rank 57: 0.74 + rank 58: 0.82 + rank 59: 0.74 + rank 60: 0.81 + rank 61: 0.81 + rank 62: 0.81 + rank 63: 2.94 + [2026-10-08 21:33:43] iteration 5004/ 6855 | consumed samples: 640512 | elapsed time per iteration (ms): 86784.1 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.76% | FLOPs estimator: workload-aware | logical tokens/s: 193320.5 | valid tokens/s: 193320.5 | physical tokens/s: 193929.5 | loss tokens/s: 139744.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.687118E-01 | lm-global-supervised-tokens: 1.212761E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.409341E-02 | lm-token-buffer-mib: 5.804935E+01 | global_load_balancing_loss: 1.000209E+00 | loss scale: 1.0 | grad norm: 0.070 | params norm: 11327.383 | params rms: 0.061334 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 20:37:17.433635 | finish at 2026-10-10 18:11:01 +times across ranks (ms): + forward-backward: + rank 0: 86566.83 + rank 1: 86566.42 + rank 2: 86568.21 + rank 3: 86567.16 + rank 4: 86568.23 + rank 5: 86567.65 + rank 6: 86565.09 + rank 7: 86564.91 + rank 8: 86565.02 + rank 9: 86564.33 + rank 10: 86565.40 + rank 11: 86565.98 + rank 12: 86566.36 + rank 13: 86566.56 + rank 14: 86563.15 + rank 15: 86563.88 + rank 16: 86564.81 + rank 17: 86564.62 + rank 18: 86564.39 + rank 19: 86565.37 + rank 20: 86565.27 + rank 21: 86566.80 + rank 22: 86563.45 + rank 23: 86563.41 + rank 24: 86559.05 + rank 25: 86559.48 + rank 26: 86559.69 + rank 27: 86559.82 + rank 28: 86560.59 + rank 29: 86561.11 + rank 30: 86559.34 + rank 31: 86559.23 + rank 32: 86554.34 + rank 33: 86553.94 + rank 34: 86554.25 + rank 35: 86554.65 + rank 36: 86555.94 + rank 37: 86555.27 + rank 38: 86554.77 + rank 39: 86556.03 + rank 40: 86558.56 + rank 41: 86558.52 + rank 42: 86558.02 + rank 43: 86558.05 + rank 44: 86557.12 + rank 45: 86556.53 + rank 46: 86560.46 + rank 47: 86559.27 + rank 48: 86566.74 + rank 49: 86564.96 + rank 50: 86564.74 + rank 51: 86566.00 + rank 52: 86564.32 + rank 53: 86564.54 + rank 54: 86566.62 + rank 55: 86566.20 + rank 56: 86568.50 + rank 57: 86567.64 + rank 58: 86567.05 + rank 59: 86567.09 + rank 60: 86566.95 + rank 61: 86566.29 + rank 62: 86569.66 + rank 63: 86568.77 + forward-compute: + rank 0: 32394.72 + rank 1: 32402.22 + rank 2: 32405.14 + rank 3: 32403.20 + rank 4: 32406.97 + rank 5: 32402.31 + rank 6: 32401.70 + rank 7: 32410.05 + rank 8: 33912.34 + rank 9: 33909.07 + rank 10: 33911.25 + rank 11: 33910.59 + rank 12: 33910.54 + rank 13: 33912.99 + rank 14: 33911.84 + rank 15: 33912.21 + rank 16: 31442.17 + rank 17: 31434.51 + rank 18: 31445.98 + rank 19: 31439.03 + rank 20: 31445.27 + rank 21: 31450.08 + rank 22: 31443.54 + rank 23: 31449.11 + rank 24: 29728.81 + rank 25: 29724.23 + rank 26: 29723.27 + rank 27: 29724.38 + rank 28: 29736.69 + rank 29: 29735.91 + rank 30: 29733.65 + rank 31: 29730.23 + rank 32: 33842.32 + rank 33: 33844.92 + rank 34: 33850.56 + rank 35: 33853.48 + rank 36: 33853.41 + rank 37: 33855.15 + rank 38: 33853.48 + rank 39: 33847.77 + rank 40: 33777.94 + rank 41: 33768.58 + rank 42: 33772.62 + rank 43: 33772.05 + rank 44: 33771.51 + rank 45: 33776.30 + rank 46: 33770.15 + rank 47: 33768.80 + rank 48: 35769.43 + rank 49: 35778.06 + rank 50: 35778.90 + rank 51: 35781.46 + rank 52: 35786.22 + rank 53: 35788.21 + rank 54: 35784.78 + rank 55: 35785.43 + rank 56: 30195.22 + rank 57: 30193.84 + rank 58: 30187.97 + rank 59: 30188.37 + rank 60: 30192.04 + rank 61: 30195.97 + rank 62: 30188.05 + rank 63: 30195.72 + backward-compute: + rank 0: 54137.96 + rank 1: 54129.81 + rank 2: 54127.93 + rank 3: 54131.31 + rank 4: 54124.87 + rank 5: 54133.11 + rank 6: 54128.49 + rank 7: 54121.54 + rank 8: 52617.18 + rank 9: 52621.62 + rank 10: 52620.95 + rank 11: 52620.05 + rank 12: 52620.22 + rank 13: 52619.99 + rank 14: 52616.00 + rank 15: 52617.74 + rank 16: 55087.14 + rank 17: 55096.95 + rank 18: 55084.93 + rank 19: 55092.98 + rank 20: 55088.12 + rank 21: 55082.01 + rank 22: 55083.21 + rank 23: 55081.25 + rank 24: 56783.94 + rank 25: 56788.44 + rank 26: 56788.05 + rank 27: 56791.70 + rank 28: 56779.62 + rank 29: 56775.50 + rank 30: 56779.94 + rank 31: 56783.89 + rank 32: 52671.85 + rank 33: 52667.48 + rank 34: 52664.00 + rank 35: 52662.55 + rank 36: 52664.33 + rank 37: 52661.28 + rank 38: 52665.08 + rank 39: 52669.92 + rank 40: 52744.64 + rank 41: 52755.14 + rank 42: 52750.80 + rank 43: 52749.96 + rank 44: 52750.66 + rank 45: 52744.33 + rank 46: 52755.71 + rank 47: 52755.45 + rank 48: 50753.58 + rank 49: 50747.98 + rank 50: 50750.47 + rank 51: 50747.52 + rank 52: 50739.95 + rank 53: 50739.47 + rank 54: 50746.57 + rank 55: 50746.00 + rank 56: 56339.57 + rank 57: 56341.12 + rank 58: 56347.20 + rank 59: 56347.81 + rank 60: 56342.12 + rank 61: 56337.91 + rank 62: 56348.80 + rank 63: 56339.00 + batch-generator: + rank 0: 23.24 + rank 1: 22.43 + rank 2: 24.39 + rank 3: 23.80 + rank 4: 22.31 + rank 5: 23.76 + rank 6: 22.88 + rank 7: 23.25 + rank 8: 23.54 + rank 9: 21.13 + rank 10: 21.20 + rank 11: 21.56 + rank 12: 22.83 + rank 13: 22.27 + rank 14: 21.90 + rank 15: 21.49 + rank 16: 23.12 + rank 17: 22.51 + rank 18: 21.84 + rank 19: 22.26 + rank 20: 22.96 + rank 21: 22.93 + rank 22: 21.61 + rank 23: 23.37 + rank 24: 33.35 + rank 25: 32.01 + rank 26: 34.06 + rank 27: 35.67 + rank 28: 31.41 + rank 29: 41.32 + rank 30: 30.70 + rank 31: 30.14 + rank 32: 26.74 + rank 33: 32.46 + rank 34: 35.12 + rank 35: 30.92 + rank 36: 30.56 + rank 37: 32.32 + rank 38: 30.66 + rank 39: 32.91 + rank 40: 21.93 + rank 41: 21.75 + rank 42: 22.03 + rank 43: 22.36 + rank 44: 22.44 + rank 45: 21.67 + rank 46: 20.86 + rank 47: 22.78 + rank 48: 36.31 + rank 49: 28.16 + rank 50: 28.18 + rank 51: 31.20 + rank 52: 28.31 + rank 53: 30.78 + rank 54: 23.88 + rank 55: 29.17 + rank 56: 22.68 + rank 57: 22.57 + rank 58: 22.56 + rank 59: 22.48 + rank 60: 23.21 + rank 61: 22.18 + rank 62: 22.03 + rank 63: 22.65 + embedding-grads-all-reduce: + rank 0: 0.03 + rank 1: 0.03 + rank 2: 0.02 + rank 3: 0.03 + rank 4: 0.02 + rank 5: 0.02 + rank 6: 0.03 + rank 7: 0.03 + rank 8: 0.02 + rank 9: 0.02 + rank 10: 0.03 + rank 11: 0.03 + rank 12: 0.03 + rank 13: 0.02 + rank 14: 0.02 + rank 15: 0.02 + rank 16: 0.03 + rank 17: 0.03 + rank 18: 0.03 + rank 19: 0.03 + rank 20: 0.03 + rank 21: 0.03 + rank 22: 0.03 + rank 23: 0.03 + rank 24: 0.04 + rank 25: 0.04 + rank 26: 0.04 + rank 27: 0.04 + rank 28: 0.04 + rank 29: 0.03 + rank 30: 0.04 + rank 31: 0.04 + rank 32: 0.03 + rank 33: 0.03 + rank 34: 0.03 + rank 35: 0.03 + rank 36: 0.03 + rank 37: 0.03 + rank 38: 0.03 + rank 39: 0.03 + rank 40: 0.03 + rank 41: 0.03 + rank 42: 0.02 + rank 43: 0.03 + rank 44: 0.03 + rank 45: 0.02 + rank 46: 0.03 + rank 47: 0.03 + rank 48: 0.04 + rank 49: 0.03 + rank 50: 0.03 + rank 51: 0.03 + rank 52: 0.03 + rank 53: 0.03 + rank 54: 0.03 + rank 55: 0.03 + rank 56: 0.03 + rank 57: 0.03 + rank 58: 0.02 + rank 59: 0.03 + rank 60: 0.03 + rank 61: 0.03 + rank 62: 0.03 + rank 63: 0.03 + all-grads-sync: + rank 0: 1.45 + rank 1: 0.77 + rank 2: 1.37 + rank 3: 0.78 + rank 4: 0.77 + rank 5: 0.85 + rank 6: 0.77 + rank 7: 0.79 + rank 8: 1.22 + rank 9: 0.82 + rank 10: 1.28 + rank 11: 0.82 + rank 12: 0.83 + rank 13: 0.81 + rank 14: 0.78 + rank 15: 0.90 + rank 16: 1.22 + rank 17: 0.85 + rank 18: 0.77 + rank 19: 0.84 + rank 20: 0.78 + rank 21: 0.90 + rank 22: 0.86 + rank 23: 0.78 + rank 24: 1.89 + rank 25: 1.55 + rank 26: 1.36 + rank 27: 1.29 + rank 28: 1.42 + rank 29: 1.33 + rank 30: 2.66 + rank 31: 3.20 + rank 32: 1.19 + rank 33: 1.19 + rank 34: 1.16 + rank 35: 1.10 + rank 36: 1.25 + rank 37: 1.41 + rank 38: 1.51 + rank 39: 1.19 + rank 40: 0.86 + rank 41: 0.78 + rank 42: 0.81 + rank 43: 0.79 + rank 44: 1.08 + rank 45: 0.75 + rank 46: 1.20 + rank 47: 0.80 + rank 48: 1.32 + rank 49: 0.83 + rank 50: 1.12 + rank 51: 1.35 + rank 52: 1.12 + rank 53: 1.19 + rank 54: 0.86 + rank 55: 0.84 + rank 56: 1.06 + rank 57: 0.82 + rank 58: 0.79 + rank 59: 0.80 + rank 60: 0.81 + rank 61: 0.81 + rank 62: 1.23 + rank 63: 0.84 + params-all-gather: + rank 0: 0.02 + rank 1: 0.02 + rank 2: 0.02 + rank 3: 0.02 + rank 4: 0.02 + rank 5: 0.02 + rank 6: 0.02 + rank 7: 0.02 + rank 8: 0.02 + rank 9: 0.02 + rank 10: 0.02 + rank 11: 0.02 + rank 12: 0.03 + rank 13: 0.02 + rank 14: 0.02 + rank 15: 0.03 + rank 16: 0.02 + rank 17: 0.02 + rank 18: 0.02 + rank 19: 0.03 + rank 20: 0.02 + rank 21: 0.02 + rank 22: 0.02 + rank 23: 0.02 + rank 24: 0.04 + rank 25: 0.04 + rank 26: 0.03 + rank 27: 0.04 + rank 28: 0.03 + rank 29: 0.04 + rank 30: 0.03 + rank 31: 0.04 + rank 32: 0.02 + rank 33: 0.02 + rank 34: 0.02 + rank 35: 0.02 + rank 36: 0.02 + rank 37: 0.02 + rank 38: 0.02 + rank 39: 0.02 + rank 40: 0.02 + rank 41: 0.02 + rank 42: 0.02 + rank 43: 0.02 + rank 44: 0.02 + rank 45: 0.02 + rank 46: 0.02 + rank 47: 0.02 + rank 48: 0.03 + rank 49: 0.03 + rank 50: 0.03 + rank 51: 0.03 + rank 52: 0.03 + rank 53: 0.03 + rank 54: 0.03 + rank 55: 0.03 + rank 56: 0.02 + rank 57: 0.02 + rank 58: 0.02 + rank 59: 0.02 + rank 60: 0.02 + rank 61: 0.02 + rank 62: 0.02 + rank 63: 0.02 + optimizer-copy-to-main-grad: + rank 0: 2.19 + rank 1: 1.81 + rank 2: 1.71 + rank 3: 2.10 + rank 4: 1.69 + rank 5: 1.77 + rank 6: 1.84 + rank 7: 1.84 + rank 8: 1.82 + rank 9: 2.15 + rank 10: 1.76 + rank 11: 1.70 + rank 12: 1.82 + rank 13: 1.77 + rank 14: 1.80 + rank 15: 2.02 + rank 16: 2.49 + rank 17: 2.13 + rank 18: 2.11 + rank 19: 2.15 + rank 20: 2.15 + rank 21: 2.38 + rank 22: 2.50 + rank 23: 2.30 + rank 24: 2.05 + rank 25: 2.32 + rank 26: 1.98 + rank 27: 2.12 + rank 28: 2.30 + rank 29: 2.40 + rank 30: 2.35 + rank 31: 2.06 + rank 32: 2.00 + rank 33: 1.89 + rank 34: 1.99 + rank 35: 2.21 + rank 36: 2.08 + rank 37: 2.00 + rank 38: 2.26 + rank 39: 1.90 + rank 40: 1.70 + rank 41: 1.89 + rank 42: 1.84 + rank 43: 1.85 + rank 44: 1.87 + rank 45: 1.99 + rank 46: 1.73 + rank 47: 1.67 + rank 48: 1.93 + rank 49: 1.75 + rank 50: 2.35 + rank 51: 1.83 + rank 52: 1.92 + rank 53: 1.76 + rank 54: 1.74 + rank 55: 1.74 + rank 56: 1.81 + rank 57: 2.35 + rank 58: 1.83 + rank 59: 1.72 + rank 60: 1.77 + rank 61: 1.70 + rank 62: 1.87 + rank 63: 1.70 + optimizer-inner-step: + rank 0: 48.79 + rank 1: 48.62 + rank 2: 48.39 + rank 3: 48.58 + rank 4: 48.37 + rank 5: 48.45 + rank 6: 48.84 + rank 7: 48.76 + rank 8: 48.47 + rank 9: 48.55 + rank 10: 48.44 + rank 11: 48.54 + rank 12: 48.50 + rank 13: 48.75 + rank 14: 48.54 + rank 15: 48.37 + rank 16: 48.93 + rank 17: 48.62 + rank 18: 48.63 + rank 19: 48.75 + rank 20: 48.72 + rank 21: 48.59 + rank 22: 49.06 + rank 23: 48.72 + rank 24: 45.39 + rank 25: 50.51 + rank 26: 47.24 + rank 27: 51.21 + rank 28: 54.51 + rank 29: 49.63 + rank 30: 48.45 + rank 31: 50.65 + rank 32: 48.64 + rank 33: 48.39 + rank 34: 48.63 + rank 35: 48.92 + rank 36: 48.76 + rank 37: 48.78 + rank 38: 48.57 + rank 39: 48.59 + rank 40: 48.52 + rank 41: 48.63 + rank 42: 48.71 + rank 43: 48.78 + rank 44: 48.34 + rank 45: 48.89 + rank 46: 48.59 + rank 47: 48.26 + rank 48: 48.65 + rank 49: 48.46 + rank 50: 53.29 + rank 51: 48.26 + rank 52: 48.62 + rank 53: 48.45 + rank 54: 48.69 + rank 55: 48.53 + rank 56: 48.19 + rank 57: 49.51 + rank 58: 48.55 + rank 59: 48.34 + rank 60: 48.70 + rank 61: 48.34 + rank 62: 48.46 + rank 63: 48.36 + optimizer-copy-main-to-model-params: + rank 0: 4.38 + rank 1: 3.84 + rank 2: 3.73 + rank 3: 4.32 + rank 4: 3.69 + rank 5: 3.88 + rank 6: 3.94 + rank 7: 4.01 + rank 8: 3.88 + rank 9: 4.43 + rank 10: 3.78 + rank 11: 3.74 + rank 12: 3.92 + rank 13: 3.77 + rank 14: 3.95 + rank 15: 4.27 + rank 16: 4.74 + rank 17: 4.16 + rank 18: 4.09 + rank 19: 5.59 + rank 20: 4.01 + rank 21: 4.66 + rank 22: 4.60 + rank 23: 4.20 + rank 24: 3.88 + rank 25: 4.21 + rank 26: 3.93 + rank 27: 4.03 + rank 28: 4.40 + rank 29: 4.35 + rank 30: 4.33 + rank 31: 4.00 + rank 32: 4.18 + rank 33: 3.93 + rank 34: 4.04 + rank 35: 4.72 + rank 36: 4.19 + rank 37: 4.05 + rank 38: 4.70 + rank 39: 3.91 + rank 40: 3.72 + rank 41: 3.98 + rank 42: 3.96 + rank 43: 4.03 + rank 44: 3.97 + rank 45: 4.25 + rank 46: 3.77 + rank 47: 3.72 + rank 48: 4.12 + rank 49: 3.92 + rank 50: 4.96 + rank 51: 3.95 + rank 52: 4.21 + rank 53: 3.90 + rank 54: 3.86 + rank 55: 3.85 + rank 56: 3.81 + rank 57: 4.75 + rank 58: 3.91 + rank 59: 3.74 + rank 60: 3.74 + rank 61: 3.67 + rank 62: 3.84 + rank 63: 3.71 + optimizer: + rank 0: 60.41 + rank 1: 59.52 + rank 2: 59.09 + rank 3: 60.11 + rank 4: 59.04 + rank 5: 59.44 + rank 6: 59.89 + rank 7: 59.87 + rank 8: 59.45 + rank 9: 60.29 + rank 10: 59.26 + rank 11: 59.30 + rank 12: 59.53 + rank 13: 59.56 + rank 14: 59.60 + rank 15: 59.87 + rank 16: 61.16 + rank 17: 60.17 + rank 18: 60.13 + rank 19: 61.77 + rank 20: 60.01 + rank 21: 60.71 + rank 22: 61.24 + rank 23: 60.35 + rank 24: 59.48 + rank 25: 65.14 + rank 26: 61.45 + rank 27: 65.57 + rank 28: 69.30 + rank 29: 64.31 + rank 30: 63.11 + rank 31: 64.92 + rank 32: 59.90 + rank 33: 59.32 + rank 34: 59.70 + rank 35: 60.86 + rank 36: 60.06 + rank 37: 59.89 + rank 38: 60.49 + rank 39: 59.50 + rank 40: 59.26 + rank 41: 59.73 + rank 42: 59.80 + rank 43: 59.91 + rank 44: 59.42 + rank 45: 60.38 + rank 46: 59.36 + rank 47: 59.00 + rank 48: 59.90 + rank 49: 59.43 + rank 50: 65.66 + rank 51: 59.26 + rank 52: 59.95 + rank 53: 59.37 + rank 54: 59.56 + rank 55: 59.41 + rank 56: 59.05 + rank 57: 61.69 + rank 58: 59.54 + rank 59: 59.07 + rank 60: 59.48 + rank 61: 59.01 + rank 62: 59.34 + rank 63: 59.05 + interval-time: + rank 0: 0.81 + rank 1: 0.83 + rank 2: 0.82 + rank 3: 0.73 + rank 4: 0.82 + rank 5: 0.82 + rank 6: 0.81 + rank 7: 0.74 + rank 8: 0.84 + rank 9: 0.75 + rank 10: 0.82 + rank 11: 0.87 + rank 12: 0.83 + rank 13: 0.81 + rank 14: 0.73 + rank 15: 0.73 + rank 16: 0.75 + rank 17: 0.77 + rank 18: 0.73 + rank 19: 0.86 + rank 20: 0.73 + rank 21: 0.88 + rank 22: 0.82 + rank 23: 0.84 + rank 24: 0.90 + rank 25: 0.87 + rank 26: 0.97 + rank 27: 0.89 + rank 28: 0.85 + rank 29: 0.86 + rank 30: 0.84 + rank 31: 0.87 + rank 32: 0.84 + rank 33: 0.84 + rank 34: 0.84 + rank 35: 0.89 + rank 36: 0.88 + rank 37: 0.89 + rank 38: 0.92 + rank 39: 0.79 + rank 40: 0.82 + rank 41: 0.75 + rank 42: 0.80 + rank 43: 0.82 + rank 44: 0.80 + rank 45: 0.81 + rank 46: 0.73 + rank 47: 0.73 + rank 48: 0.97 + rank 49: 0.84 + rank 50: 0.93 + rank 51: 0.81 + rank 52: 0.93 + rank 53: 0.82 + rank 54: 0.90 + rank 55: 0.87 + rank 56: 0.83 + rank 57: 0.74 + rank 58: 0.83 + rank 59: 0.74 + rank 60: 0.79 + rank 61: 0.82 + rank 62: 0.80 + rank 63: 3.10 + [2026-10-08 21:35:10] iteration 5005/ 6855 | consumed samples: 640640 | elapsed time per iteration (ms): 86785.5 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.77% | FLOPs estimator: workload-aware | logical tokens/s: 193318.1 | valid tokens/s: 193318.1 | physical tokens/s: 193724.7 | loss tokens/s: 139801.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.293238E-01 | lm-global-supervised-tokens: 1.213276E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.541881E-02 | lm-token-buffer-mib: 5.834113E+01 | global_load_balancing_loss: 1.000415E+00 | loss scale: 1.0 | grad norm: 0.063 | params norm: 11327.330 | params rms: 0.061334 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 20:35:53.163624 | finish at 2026-10-10 18:11:03 +times across ranks (ms): + forward-backward: + rank 0: 86568.72 + rank 1: 86569.40 + rank 2: 86569.21 + rank 3: 86568.77 + rank 4: 86569.40 + rank 5: 86568.94 + rank 6: 86569.02 + rank 7: 86569.07 + rank 8: 86557.98 + rank 9: 86557.91 + rank 10: 86557.70 + rank 11: 86559.21 + rank 12: 86559.42 + rank 13: 86559.77 + rank 14: 86556.59 + rank 15: 86557.52 + rank 16: 86557.44 + rank 17: 86558.39 + rank 18: 86557.76 + rank 19: 86558.21 + rank 20: 86558.42 + rank 21: 86559.68 + rank 22: 86556.20 + rank 23: 86556.81 + rank 24: 86549.41 + rank 25: 86552.80 + rank 26: 86552.73 + rank 27: 86553.55 + rank 28: 86552.88 + rank 29: 86553.81 + rank 30: 86550.98 + rank 31: 86549.24 + rank 32: 86549.91 + rank 33: 86550.50 + rank 34: 86550.15 + rank 35: 86551.30 + rank 36: 86550.39 + rank 37: 86551.40 + rank 38: 86550.40 + rank 39: 86549.46 + rank 40: 86553.75 + rank 41: 86553.42 + rank 42: 86554.25 + rank 43: 86552.69 + rank 44: 86553.12 + rank 45: 86551.61 + rank 46: 86554.58 + rank 47: 86554.21 + rank 48: 86560.24 + rank 49: 86558.53 + rank 50: 86557.72 + rank 51: 86558.98 + rank 52: 86558.16 + rank 53: 86558.12 + rank 54: 86560.92 + rank 55: 86560.07 + rank 56: 86562.09 + rank 57: 86561.29 + rank 58: 86561.12 + rank 59: 86559.96 + rank 60: 86560.52 + rank 61: 86559.59 + rank 62: 86563.77 + rank 63: 86561.91 + forward-compute: + rank 0: 32173.23 + rank 1: 32175.85 + rank 2: 32177.90 + rank 3: 32176.90 + rank 4: 32178.32 + rank 5: 32176.63 + rank 6: 32177.26 + rank 7: 32179.16 + rank 8: 29519.92 + rank 9: 29513.75 + rank 10: 29515.79 + rank 11: 29519.68 + rank 12: 29520.61 + rank 13: 29524.38 + rank 14: 29511.79 + rank 15: 29513.60 + rank 16: 32737.92 + rank 17: 32739.54 + rank 18: 32739.70 + rank 19: 32736.67 + rank 20: 32738.26 + rank 21: 32739.16 + rank 22: 32734.73 + rank 23: 32739.04 + rank 24: 32787.31 + rank 25: 32793.67 + rank 26: 32795.41 + rank 27: 32796.84 + rank 28: 32800.17 + rank 29: 32798.26 + rank 30: 32798.46 + rank 31: 32798.46 + rank 32: 31510.47 + rank 33: 31515.55 + rank 34: 31520.38 + rank 35: 31519.81 + rank 36: 31523.53 + rank 37: 31524.20 + rank 38: 31526.23 + rank 39: 31520.99 + rank 40: 33305.96 + rank 41: 33306.00 + rank 42: 33306.71 + rank 43: 33307.48 + rank 44: 33308.70 + rank 45: 33311.85 + rank 46: 33312.25 + rank 47: 33308.30 + rank 48: 32263.97 + rank 49: 32271.05 + rank 50: 32267.53 + rank 51: 32265.52 + rank 52: 32274.35 + rank 53: 32275.88 + rank 54: 32269.00 + rank 55: 32270.98 + rank 56: 33475.55 + rank 57: 33481.84 + rank 58: 33519.30 + rank 59: 33484.68 + rank 60: 33489.36 + rank 61: 33489.62 + rank 62: 33487.73 + rank 63: 33490.07 + backward-compute: + rank 0: 54324.21 + rank 1: 54321.77 + rank 2: 54321.46 + rank 3: 54322.25 + rank 4: 54321.71 + rank 5: 54324.15 + rank 6: 54319.11 + rank 7: 54318.40 + rank 8: 57002.88 + rank 9: 57012.46 + rank 10: 57010.49 + rank 11: 57005.48 + rank 12: 57003.90 + rank 13: 57002.79 + rank 14: 57012.15 + rank 15: 57010.27 + rank 16: 53785.50 + rank 17: 53784.91 + rank 18: 53783.84 + rank 19: 53787.42 + rank 20: 53787.14 + rank 21: 53783.63 + rank 22: 53787.29 + rank 23: 53784.41 + rank 24: 53725.02 + rank 25: 53715.30 + rank 26: 53718.46 + rank 27: 53711.28 + rank 28: 53716.95 + rank 29: 53712.26 + rank 30: 53711.13 + rank 31: 53711.08 + rank 32: 54998.03 + rank 33: 54994.14 + rank 34: 54988.90 + rank 35: 54989.51 + rank 36: 54986.94 + rank 37: 54986.02 + rank 38: 54981.80 + rank 39: 54989.55 + rank 40: 53211.12 + rank 41: 53210.70 + rank 42: 53211.64 + rank 43: 53209.38 + rank 44: 53209.44 + rank 45: 53203.72 + rank 46: 53206.60 + rank 47: 53211.11 + rank 48: 54259.31 + rank 49: 54253.45 + rank 50: 54253.20 + rank 51: 54257.95 + rank 52: 54246.82 + rank 53: 54246.14 + rank 54: 54254.31 + rank 55: 54256.31 + rank 56: 53051.46 + rank 57: 53042.06 + rank 58: 53005.10 + rank 59: 53040.13 + rank 60: 53035.17 + rank 61: 53033.78 + rank 62: 53041.28 + rank 63: 53036.16 + batch-generator: + rank 0: 30.55 + rank 1: 30.46 + rank 2: 31.73 + rank 3: 29.92 + rank 4: 30.03 + rank 5: 30.18 + rank 6: 30.25 + rank 7: 31.14 + rank 8: 22.54 + rank 9: 22.22 + rank 10: 20.26 + rank 11: 21.36 + rank 12: 22.86 + rank 13: 22.26 + rank 14: 23.15 + rank 15: 21.28 + rank 16: 22.62 + rank 17: 22.23 + rank 18: 21.71 + rank 19: 25.12 + rank 20: 21.63 + rank 21: 24.38 + rank 22: 23.67 + rank 23: 24.09 + rank 24: 31.31 + rank 25: 35.63 + rank 26: 35.51 + rank 27: 35.75 + rank 28: 25.52 + rank 29: 38.13 + rank 30: 28.76 + rank 31: 29.51 + rank 32: 35.24 + rank 33: 32.42 + rank 34: 32.16 + rank 35: 31.14 + rank 36: 29.85 + rank 37: 28.56 + rank 38: 33.30 + rank 39: 31.44 + rank 40: 22.82 + rank 41: 23.11 + rank 42: 21.85 + rank 43: 23.22 + rank 44: 21.96 + rank 45: 21.99 + rank 46: 20.62 + rank 47: 21.81 + rank 48: 30.62 + rank 49: 21.86 + rank 50: 31.29 + rank 51: 29.70 + rank 52: 34.36 + rank 53: 29.65 + rank 54: 25.52 + rank 55: 21.26 + rank 56: 24.02 + rank 57: 23.56 + rank 58: 21.77 + rank 59: 21.83 + rank 60: 23.15 + rank 61: 22.30 + rank 62: 22.72 + rank 63: 24.49 + embedding-grads-all-reduce: + rank 0: 0.03 + rank 1: 0.02 + rank 2: 0.03 + rank 3: 0.03 + rank 4: 0.03 + rank 5: 0.03 + rank 6: 0.02 + rank 7: 0.03 + rank 8: 0.02 + rank 9: 0.02 + rank 10: 0.03 + rank 11: 0.03 + rank 12: 0.02 + rank 13: 0.02 + rank 14: 0.02 + rank 15: 0.02 + rank 16: 0.03 + rank 17: 0.03 + rank 18: 0.03 + rank 19: 0.03 + rank 20: 0.03 + rank 21: 0.03 + rank 22: 0.03 + rank 23: 0.03 + rank 24: 0.04 + rank 25: 0.04 + rank 26: 0.04 + rank 27: 0.04 + rank 28: 0.03 + rank 29: 0.04 + rank 30: 0.04 + rank 31: 0.04 + rank 32: 0.03 + rank 33: 0.03 + rank 34: 0.03 + rank 35: 0.04 + rank 36: 0.03 + rank 37: 0.03 + rank 38: 0.03 + rank 39: 0.03 + rank 40: 0.03 + rank 41: 0.02 + rank 42: 0.03 + rank 43: 0.03 + rank 44: 0.03 + rank 45: 0.03 + rank 46: 0.02 + rank 47: 0.03 + rank 48: 0.03 + rank 49: 0.03 + rank 50: 0.03 + rank 51: 0.03 + rank 52: 0.03 + rank 53: 0.03 + rank 54: 0.03 + rank 55: 0.03 + rank 56: 0.03 + rank 57: 0.03 + rank 58: 0.02 + rank 59: 0.03 + rank 60: 0.02 + rank 61: 0.02 + rank 62: 0.03 + rank 63: 0.03 + all-grads-sync: + rank 0: 0.98 + rank 1: 0.81 + rank 2: 0.84 + rank 3: 0.77 + rank 4: 0.77 + rank 5: 0.81 + rank 6: 1.02 + rank 7: 0.81 + rank 8: 0.83 + rank 9: 0.77 + rank 10: 0.75 + rank 11: 0.84 + rank 12: 0.86 + rank 13: 0.79 + rank 14: 0.81 + rank 15: 0.83 + rank 16: 0.77 + rank 17: 0.77 + rank 18: 0.80 + rank 19: 0.76 + rank 20: 0.85 + rank 21: 0.78 + rank 22: 0.79 + rank 23: 0.80 + rank 24: 1.16 + rank 25: 1.24 + rank 26: 1.43 + rank 27: 1.64 + rank 28: 0.97 + rank 29: 1.26 + rank 30: 1.56 + rank 31: 1.21 + rank 32: 1.25 + rank 33: 1.37 + rank 34: 1.31 + rank 35: 1.56 + rank 36: 1.20 + rank 37: 1.65 + rank 38: 1.20 + rank 39: 1.23 + rank 40: 1.26 + rank 41: 0.81 + rank 42: 1.44 + rank 43: 0.80 + rank 44: 0.76 + rank 45: 0.78 + rank 46: 1.23 + rank 47: 1.15 + rank 48: 1.25 + rank 49: 0.89 + rank 50: 1.12 + rank 51: 1.31 + rank 52: 1.24 + rank 53: 1.19 + rank 54: 1.41 + rank 55: 0.81 + rank 56: 0.83 + rank 57: 0.84 + rank 58: 1.02 + rank 59: 0.78 + rank 60: 0.82 + rank 61: 1.17 + rank 62: 1.29 + rank 63: 0.88 + params-all-gather: + rank 0: 0.02 + rank 1: 0.02 + rank 2: 0.02 + rank 3: 0.02 + rank 4: 0.02 + rank 5: 0.02 + rank 6: 0.02 + rank 7: 0.02 + rank 8: 0.02 + rank 9: 0.02 + rank 10: 0.02 + rank 11: 0.02 + rank 12: 0.03 + rank 13: 0.02 + rank 14: 0.02 + rank 15: 0.02 + rank 16: 0.02 + rank 17: 0.02 + rank 18: 0.02 + rank 19: 0.02 + rank 20: 0.02 + rank 21: 0.02 + rank 22: 0.02 + rank 23: 0.02 + rank 24: 0.04 + rank 25: 0.04 + rank 26: 0.04 + rank 27: 0.04 + rank 28: 0.03 + rank 29: 0.03 + rank 30: 0.03 + rank 31: 0.04 + rank 32: 0.02 + rank 33: 0.02 + rank 34: 0.02 + rank 35: 0.02 + rank 36: 0.03 + rank 37: 0.02 + rank 38: 0.02 + rank 39: 0.03 + rank 40: 0.02 + rank 41: 0.02 + rank 42: 0.02 + rank 43: 0.02 + rank 44: 0.02 + rank 45: 0.02 + rank 46: 0.02 + rank 47: 0.02 + rank 48: 0.03 + rank 49: 0.03 + rank 50: 0.03 + rank 51: 0.03 + rank 52: 0.03 + rank 53: 0.03 + rank 54: 0.03 + rank 55: 0.03 + rank 56: 0.03 + rank 57: 0.02 + rank 58: 0.02 + rank 59: 0.02 + rank 60: 0.02 + rank 61: 0.02 + rank 62: 0.02 + rank 63: 0.02 + optimizer-copy-to-main-grad: + rank 0: 2.12 + rank 1: 1.96 + rank 2: 1.76 + rank 3: 2.10 + rank 4: 1.80 + rank 5: 1.81 + rank 6: 1.87 + rank 7: 1.93 + rank 8: 1.84 + rank 9: 2.12 + rank 10: 1.69 + rank 11: 1.69 + rank 12: 1.73 + rank 13: 1.72 + rank 14: 1.83 + rank 15: 2.02 + rank 16: 2.42 + rank 17: 2.10 + rank 18: 2.17 + rank 19: 2.11 + rank 20: 2.14 + rank 21: 2.32 + rank 22: 2.54 + rank 23: 2.30 + rank 24: 2.02 + rank 25: 2.29 + rank 26: 1.86 + rank 27: 2.07 + rank 28: 2.23 + rank 29: 2.39 + rank 30: 2.33 + rank 31: 2.00 + rank 32: 2.04 + rank 33: 1.89 + rank 34: 1.97 + rank 35: 2.19 + rank 36: 2.04 + rank 37: 2.08 + rank 38: 2.26 + rank 39: 1.77 + rank 40: 1.76 + rank 41: 1.82 + rank 42: 1.82 + rank 43: 1.82 + rank 44: 1.87 + rank 45: 2.07 + rank 46: 1.73 + rank 47: 1.68 + rank 48: 1.93 + rank 49: 1.77 + rank 50: 2.30 + rank 51: 1.92 + rank 52: 1.96 + rank 53: 1.77 + rank 54: 1.79 + rank 55: 1.77 + rank 56: 1.96 + rank 57: 2.36 + rank 58: 1.84 + rank 59: 1.74 + rank 60: 1.72 + rank 61: 1.73 + rank 62: 1.99 + rank 63: 1.75 + optimizer-inner-step: + rank 0: 44.42 + rank 1: 42.30 + rank 2: 42.02 + rank 3: 42.20 + rank 4: 42.01 + rank 5: 42.13 + rank 6: 42.55 + rank 7: 42.50 + rank 8: 42.28 + rank 9: 42.25 + rank 10: 42.16 + rank 11: 42.25 + rank 12: 42.18 + rank 13: 42.49 + rank 14: 42.27 + rank 15: 42.16 + rank 16: 42.63 + rank 17: 42.25 + rank 18: 42.39 + rank 19: 42.37 + rank 20: 42.38 + rank 21: 42.21 + rank 22: 42.72 + rank 23: 42.42 + rank 24: 41.99 + rank 25: 42.29 + rank 26: 42.21 + rank 27: 42.38 + rank 28: 45.80 + rank 29: 42.22 + rank 30: 42.10 + rank 31: 42.47 + rank 32: 42.26 + rank 33: 42.06 + rank 34: 42.24 + rank 35: 47.47 + rank 36: 42.36 + rank 37: 42.35 + rank 38: 42.16 + rank 39: 42.14 + rank 40: 42.25 + rank 41: 42.35 + rank 42: 42.43 + rank 43: 42.42 + rank 44: 42.01 + rank 45: 42.57 + rank 46: 42.29 + rank 47: 41.94 + rank 48: 42.31 + rank 49: 42.14 + rank 50: 54.05 + rank 51: 41.99 + rank 52: 42.26 + rank 53: 42.22 + rank 54: 42.39 + rank 55: 42.24 + rank 56: 41.88 + rank 57: 47.46 + rank 58: 42.24 + rank 59: 42.04 + rank 60: 42.29 + rank 61: 42.02 + rank 62: 42.18 + rank 63: 42.04 + optimizer-copy-main-to-model-params: + rank 0: 4.38 + rank 1: 3.89 + rank 2: 3.75 + rank 3: 4.29 + rank 4: 3.76 + rank 5: 3.95 + rank 6: 3.98 + rank 7: 4.09 + rank 8: 3.96 + rank 9: 4.45 + rank 10: 3.74 + rank 11: 3.78 + rank 12: 3.95 + rank 13: 3.74 + rank 14: 4.02 + rank 15: 4.22 + rank 16: 4.43 + rank 17: 4.04 + rank 18: 4.03 + rank 19: 4.00 + rank 20: 3.97 + rank 21: 4.25 + rank 22: 4.56 + rank 23: 4.24 + rank 24: 3.90 + rank 25: 4.23 + rank 26: 3.90 + rank 27: 4.08 + rank 28: 4.41 + rank 29: 4.34 + rank 30: 4.52 + rank 31: 4.04 + rank 32: 4.13 + rank 33: 3.83 + rank 34: 3.98 + rank 35: 4.51 + rank 36: 4.20 + rank 37: 4.27 + rank 38: 4.50 + rank 39: 3.87 + rank 40: 3.76 + rank 41: 3.99 + rank 42: 3.87 + rank 43: 3.97 + rank 44: 4.01 + rank 45: 4.29 + rank 46: 3.75 + rank 47: 3.73 + rank 48: 4.11 + rank 49: 3.89 + rank 50: 5.09 + rank 51: 4.01 + rank 52: 4.22 + rank 53: 4.00 + rank 54: 3.87 + rank 55: 3.86 + rank 56: 3.94 + rank 57: 4.85 + rank 58: 3.88 + rank 59: 3.75 + rank 60: 3.77 + rank 61: 3.76 + rank 62: 3.90 + rank 63: 3.83 + optimizer: + rank 0: 56.81 + rank 1: 53.98 + rank 2: 53.52 + rank 3: 54.49 + rank 4: 53.49 + rank 5: 53.91 + rank 6: 54.38 + rank 7: 54.42 + rank 8: 54.04 + rank 9: 54.70 + rank 10: 53.66 + rank 11: 53.79 + rank 12: 53.95 + rank 13: 53.98 + rank 14: 54.15 + rank 15: 54.28 + rank 16: 55.25 + rank 17: 54.40 + rank 18: 54.51 + rank 19: 54.41 + rank 20: 54.36 + rank 21: 54.61 + rank 22: 55.56 + rank 23: 54.80 + rank 24: 53.68 + rank 25: 54.38 + rank 26: 53.90 + rank 27: 54.31 + rank 28: 58.14 + rank 29: 54.47 + rank 30: 54.57 + rank 31: 54.33 + rank 32: 54.19 + rank 33: 53.62 + rank 34: 53.97 + rank 35: 59.93 + rank 36: 54.39 + rank 37: 54.42 + rank 38: 54.61 + rank 39: 53.73 + rank 40: 53.76 + rank 41: 54.17 + rank 42: 54.11 + rank 43: 54.22 + rank 44: 53.85 + rank 45: 54.82 + rank 46: 53.76 + rank 47: 53.41 + rank 48: 54.27 + rank 49: 53.80 + rank 50: 67.28 + rank 51: 53.76 + rank 52: 54.38 + rank 53: 53.99 + rank 54: 53.99 + rank 55: 53.85 + rank 56: 53.61 + rank 57: 60.50 + rank 58: 53.92 + rank 59: 53.53 + rank 60: 53.93 + rank 61: 53.52 + rank 62: 53.87 + rank 63: 53.60 + interval-time: + rank 0: 0.82 + rank 1: 0.84 + rank 2: 0.83 + rank 3: 0.71 + rank 4: 0.82 + rank 5: 0.80 + rank 6: 0.81 + rank 7: 0.74 + rank 8: 1.13 + rank 9: 0.75 + rank 10: 0.81 + rank 11: 0.83 + rank 12: 0.84 + rank 13: 0.81 + rank 14: 0.75 + rank 15: 0.74 + rank 16: 0.72 + rank 17: 0.73 + rank 18: 0.74 + rank 19: 0.82 + rank 20: 0.71 + rank 21: 0.83 + rank 22: 0.73 + rank 23: 0.81 + rank 24: 0.86 + rank 25: 0.85 + rank 26: 0.97 + rank 27: 0.88 + rank 28: 0.82 + rank 29: 0.84 + rank 30: 0.84 + rank 31: 0.86 + rank 32: 0.84 + rank 33: 0.84 + rank 34: 0.83 + rank 35: 0.89 + rank 36: 0.90 + rank 37: 0.93 + rank 38: 0.90 + rank 39: 0.81 + rank 40: 0.81 + rank 41: 0.74 + rank 42: 0.79 + rank 43: 0.83 + rank 44: 0.81 + rank 45: 0.72 + rank 46: 0.74 + rank 47: 0.73 + rank 48: 0.96 + rank 49: 0.83 + rank 50: 1.01 + rank 51: 0.81 + rank 52: 0.94 + rank 53: 0.84 + rank 54: 0.83 + rank 55: 0.84 + rank 56: 0.85 + rank 57: 0.74 + rank 58: 0.83 + rank 59: 0.73 + rank 60: 0.82 + rank 61: 0.82 + rank 62: 0.80 + rank 63: 2.91 + [2026-10-08 21:36:37] iteration 5006/ 6855 | consumed samples: 640768 | elapsed time per iteration (ms): 87458.8 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.51% | FLOPs estimator: workload-aware | logical tokens/s: 191829.1 | valid tokens/s: 191829.1 | physical tokens/s: 192344.3 | loss tokens/s: 143327.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.038438E-01 | lm-global-supervised-tokens: 1.253522E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.322884E-02 | lm-token-buffer-mib: 5.808716E+01 | global_load_balancing_loss: 9.998475E-01 | loss scale: 1.0 | grad norm: 0.034 | params norm: 11327.275 | params rms: 0.061334 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 20:55:11.402457 | finish at 2026-10-10 18:31:49 + [2026-10-08 21:38:03] iteration 5007/ 6855 | consumed samples: 640896 | elapsed time per iteration (ms): 85428.2 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.65% | FLOPs estimator: workload-aware | logical tokens/s: 196389.3 | valid tokens/s: 196389.3 | physical tokens/s: 196878.6 | loss tokens/s: 149253.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.077396E-01 | lm-global-supervised-tokens: 1.275047E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.119858E-02 | lm-token-buffer-mib: 5.806146E+01 | global_load_balancing_loss: 1.000655E+00 | loss scale: 1.0 | grad norm: 0.064 | params norm: 11327.221 | params rms: 0.061333 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 19:51:11.243557 | finish at 2026-10-10 17:29:14 + [2026-10-08 21:39:30] iteration 5008/ 6855 | consumed samples: 641024 | elapsed time per iteration (ms): 87468.2 | throughput per GPU (TFLOP/s/GPU): 41.5 | MFU 13.29% | FLOPs estimator: workload-aware | logical tokens/s: 191809.3 | valid tokens/s: 191809.2 | physical tokens/s: 192461.4 | loss tokens/s: 138634.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.192420E-01 | lm-global-supervised-tokens: 1.212611E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.174112E-02 | lm-token-buffer-mib: 5.807790E+01 | global_load_balancing_loss: 1.000900E+00 | loss scale: 1.0 | grad norm: 0.153 | params norm: 11327.169 | params rms: 0.061333 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 20:52:33.789121 | finish at 2026-10-10 18:32:04 + [2026-10-08 21:40:58] iteration 5009/ 6855 | consumed samples: 641152 | elapsed time per iteration (ms): 87756.0 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.49% | FLOPs estimator: workload-aware | logical tokens/s: 191180.0 | valid tokens/s: 191179.9 | physical tokens/s: 191644.5 | loss tokens/s: 145748.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.798273E-01 | lm-global-supervised-tokens: 1.279031E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.311253E-02 | lm-token-buffer-mib: 5.810045E+01 | global_load_balancing_loss: 1.000492E+00 | loss scale: 1.0 | grad norm: 0.237 | params norm: 11327.117 | params rms: 0.061333 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 20:59:57.542188 | finish at 2026-10-10 18:40:56 + [2026-10-08 21:42:47] iteration 5010/ 6855 | consumed samples: 641280 | elapsed time per iteration (ms): 109442.2 | throughput per GPU (TFLOP/s/GPU): 34.8 | MFU 11.16% | FLOPs estimator: workload-aware | logical tokens/s: 153297.3 | valid tokens/s: 153297.3 | physical tokens/s: 153540.6 | loss tokens/s: 111046.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.769969E-01 | lm-global-supervised-tokens: 1.215315E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.913992E-02 | lm-token-buffer-mib: 5.814156E+01 | global_load_balancing_loss: 1.000484E+00 | loss scale: 1.0 | grad norm: 0.078 | params norm: 11327.062 | params rms: 0.061332 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 8:05:20.900248 | finish at 2026-10-11 05:48:08 + [2026-10-08 21:44:15] iteration 5011/ 6855 | consumed samples: 641408 | elapsed time per iteration (ms): 87803.7 | throughput per GPU (TFLOP/s/GPU): 41.7 | MFU 13.35% | FLOPs estimator: workload-aware | logical tokens/s: 191075.9 | valid tokens/s: 191075.9 | physical tokens/s: 191653.4 | loss tokens/s: 142768.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.348153E-01 | lm-global-supervised-tokens: 1.253560E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.170264E-02 | lm-token-buffer-mib: 5.817273E+01 | global_load_balancing_loss: 1.001531E+00 | loss scale: 1.0 | grad norm: 0.320 | params norm: 11327.009 | params rms: 0.061332 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 20:58:29.955919 | finish at 2026-10-10 18:42:45 + [2026-10-08 21:45:42] iteration 5012/ 6855 | consumed samples: 641536 | elapsed time per iteration (ms): 86384.3 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.58% | FLOPs estimator: workload-aware | logical tokens/s: 194215.6 | valid tokens/s: 194215.6 | physical tokens/s: 194688.0 | loss tokens/s: 147460.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.537056E-01 | lm-global-supervised-tokens: 1.273823E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.579177E-02 | lm-token-buffer-mib: 5.813148E+01 | global_load_balancing_loss: 9.994654E-01 | loss scale: 1.0 | grad norm: 0.354 | params norm: 11326.957 | params rms: 0.061332 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 20:13:26.214356 | finish at 2026-10-10 17:59:08 + [2026-10-08 21:47:08] iteration 5013/ 6855 | consumed samples: 641664 | elapsed time per iteration (ms): 85845.0 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.62% | FLOPs estimator: workload-aware | logical tokens/s: 195435.8 | valid tokens/s: 195435.8 | physical tokens/s: 196010.7 | loss tokens/s: 143500.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.503634E-01 | lm-global-supervised-tokens: 1.231881E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.153781E-02 | lm-token-buffer-mib: 5.806719E+01 | global_load_balancing_loss: 9.983908E-01 | loss scale: 1.0 | grad norm: 0.097 | params norm: 11326.905 | params rms: 0.061332 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 19:55:26.555049 | finish at 2026-10-10 17:42:34 + [2026-10-08 21:48:35] iteration 5014/ 6855 | consumed samples: 641792 | elapsed time per iteration (ms): 87434.4 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.75% | FLOPs estimator: workload-aware | logical tokens/s: 191882.9 | valid tokens/s: 191882.9 | physical tokens/s: 192364.1 | loss tokens/s: 135436.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.556567E-01 | lm-global-supervised-tokens: 1.184177E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.147014E-02 | lm-token-buffer-mib: 5.803766E+01 | global_load_balancing_loss: 1.000961E+00 | loss scale: 1.0 | grad norm: 0.147 | params norm: 11326.851 | params rms: 0.061331 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 20:42:46.682707 | finish at 2026-10-10 18:31:22 + [2026-10-08 21:50:04] iteration 5015/ 6855 | consumed samples: 641920 | elapsed time per iteration (ms): 89307.9 | throughput per GPU (TFLOP/s/GPU): 41.6 | MFU 13.34% | FLOPs estimator: workload-aware | logical tokens/s: 187858.0 | valid tokens/s: 187858.0 | physical tokens/s: 188344.6 | loss tokens/s: 130631.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.619864E-01 | lm-global-supervised-tokens: 1.166646E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.346597E-02 | lm-token-buffer-mib: 5.806850E+01 | global_load_balancing_loss: 9.986911E-01 | loss scale: 1.0 | grad norm: 0.288 | params norm: 11326.798 | params rms: 0.061331 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 21:38:46.618824 | finish at 2026-10-10 19:28:51 + [2026-10-08 21:51:34] iteration 5016/ 6855 | consumed samples: 642048 | elapsed time per iteration (ms): 89434.8 | throughput per GPU (TFLOP/s/GPU): 41.0 | MFU 13.13% | FLOPs estimator: workload-aware | logical tokens/s: 187591.4 | valid tokens/s: 187591.4 | physical tokens/s: 188046.9 | loss tokens/s: 145820.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.912637E-01 | lm-global-supervised-tokens: 1.304144E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.827221E-02 | lm-token-buffer-mib: 5.812123E+01 | global_load_balancing_loss: 1.000114E+00 | loss scale: 1.0 | grad norm: 0.065 | params norm: 11326.747 | params rms: 0.061331 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 21:41:10.586072 | finish at 2026-10-10 19:32:44 + [2026-10-08 21:53:00] iteration 5017/ 6855 | consumed samples: 642176 | elapsed time per iteration (ms): 86715.2 | throughput per GPU (TFLOP/s/GPU): 43.1 | MFU 13.83% | FLOPs estimator: workload-aware | logical tokens/s: 193474.6 | valid tokens/s: 193474.6 | physical tokens/s: 193929.9 | loss tokens/s: 138503.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.516179E-01 | lm-global-supervised-tokens: 1.201035E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.470347E-02 | lm-token-buffer-mib: 5.808602E+01 | global_load_balancing_loss: 1.001282E+00 | loss scale: 1.0 | grad norm: 0.202 | params norm: 11326.696 | params rms: 0.061330 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 20:16:22.594909 | finish at 2026-10-10 18:09:23 + [2026-10-08 21:54:28] iteration 5018/ 6855 | consumed samples: 642304 | elapsed time per iteration (ms): 87699.4 | throughput per GPU (TFLOP/s/GPU): 41.5 | MFU 13.32% | FLOPs estimator: workload-aware | logical tokens/s: 191303.7 | valid tokens/s: 191303.6 | physical tokens/s: 191961.6 | loss tokens/s: 144028.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.494644E-01 | lm-global-supervised-tokens: 1.263117E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.156852E-02 | lm-token-buffer-mib: 5.805803E+01 | global_load_balancing_loss: 1.001202E+00 | loss scale: 1.0 | grad norm: 0.190 | params norm: 11326.645 | params rms: 0.061330 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 20:45:03.720621 | finish at 2026-10-10 18:39:32 + [2026-10-08 21:55:57] iteration 5019/ 6855 | consumed samples: 642432 | elapsed time per iteration (ms): 88424.3 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.50% | FLOPs estimator: workload-aware | logical tokens/s: 189734.6 | valid tokens/s: 189734.6 | physical tokens/s: 190005.9 | loss tokens/s: 137087.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.345399E-01 | lm-global-supervised-tokens: 1.212186E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.843743E-02 | lm-token-buffer-mib: 5.804486E+01 | global_load_balancing_loss: 1.000638E+00 | loss scale: 1.0 | grad norm: 0.069 | params norm: 11326.593 | params rms: 0.061330 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 21:05:47.019971 | finish at 2026-10-10 19:01:44 + [2026-10-08 21:57:48] iteration 5020/ 6855 | consumed samples: 642560 | elapsed time per iteration (ms): 111974.2 | throughput per GPU (TFLOP/s/GPU): 33.7 | MFU 10.81% | FLOPs estimator: workload-aware | logical tokens/s: 149830.6 | valid tokens/s: 149830.5 | physical tokens/s: 150194.9 | loss tokens/s: 110204.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.548393E-01 | lm-global-supervised-tokens: 1.234008E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.145640E-02 | lm-token-buffer-mib: 5.814658E+01 | global_load_balancing_loss: 1.000459E+00 | loss scale: 1.0 | grad norm: 0.337 | params norm: 11326.542 | params rms: 0.061330 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 9:04:32.685894 | finish at 2026-10-11 07:02:21 + [2026-10-08 21:59:14] iteration 5021/ 6855 | consumed samples: 642688 | elapsed time per iteration (ms): 85391.8 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.74% | FLOPs estimator: workload-aware | logical tokens/s: 196473.0 | valid tokens/s: 196473.0 | physical tokens/s: 196966.8 | loss tokens/s: 147676.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.914020E-01 | lm-global-supervised-tokens: 1.261034E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.314755E-02 | lm-token-buffer-mib: 5.810783E+01 | global_load_balancing_loss: 9.989343E-01 | loss scale: 1.0 | grad norm: 0.084 | params norm: 11326.491 | params rms: 0.061329 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 19:30:08.482359 | finish at 2026-10-10 17:29:22 + [2026-10-08 22:00:42] iteration 5022/ 6855 | consumed samples: 642816 | elapsed time per iteration (ms): 88052.8 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.41% | FLOPs estimator: workload-aware | logical tokens/s: 190535.8 | valid tokens/s: 190535.8 | physical tokens/s: 191054.8 | loss tokens/s: 141789.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.340240E-01 | lm-global-supervised-tokens: 1.248492E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.335942E-02 | lm-token-buffer-mib: 5.809588E+01 | global_load_balancing_loss: 9.994370E-01 | loss scale: 1.0 | grad norm: 0.166 | params norm: 11326.440 | params rms: 0.061329 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 20:50:00.797149 | finish at 2026-10-10 18:50:43 + [2026-10-08 22:02:08] iteration 5023/ 6855 | consumed samples: 642944 | elapsed time per iteration (ms): 85572.3 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.74% | FLOPs estimator: workload-aware | logical tokens/s: 196058.6 | valid tokens/s: 196058.6 | physical tokens/s: 196598.7 | loss tokens/s: 145259.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.499262E-01 | lm-global-supervised-tokens: 1.243022E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.772467E-02 | lm-token-buffer-mib: 5.811697E+01 | global_load_balancing_loss: 9.998954E-01 | loss scale: 1.0 | grad norm: 0.116 | params norm: 11326.389 | params rms: 0.061329 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 19:32:48.374901 | finish at 2026-10-10 17:34:56 + [2026-10-08 22:03:34] iteration 5024/ 6855 | consumed samples: 643072 | elapsed time per iteration (ms): 86632.8 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.78% | FLOPs estimator: workload-aware | logical tokens/s: 193658.4 | valid tokens/s: 193658.4 | physical tokens/s: 194139.3 | loss tokens/s: 142406.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.159997E-01 | lm-global-supervised-tokens: 1.233704E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.166158E-02 | lm-token-buffer-mib: 5.804010E+01 | global_load_balancing_loss: 9.992116E-01 | loss scale: 1.0 | grad norm: 0.145 | params norm: 11326.339 | params rms: 0.061328 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 20:03:44.656114 | finish at 2026-10-10 18:07:19 + [2026-10-08 22:05:01] iteration 5025/ 6855 | consumed samples: 643200 | elapsed time per iteration (ms): 86951.7 | throughput per GPU (TFLOP/s/GPU): 41.7 | MFU 13.35% | FLOPs estimator: workload-aware | logical tokens/s: 192947.8 | valid tokens/s: 192947.8 | physical tokens/s: 193285.6 | loss tokens/s: 145068.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.281106E-01 | lm-global-supervised-tokens: 1.261397E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.177904E-02 | lm-token-buffer-mib: 5.820587E+01 | global_load_balancing_loss: 1.000525E+00 | loss scale: 1.0 | grad norm: 0.038 | params norm: 11326.287 | params rms: 0.061328 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 20:12:01.641490 | finish at 2026-10-10 18:17:03 + [2026-10-08 22:06:32] iteration 5026/ 6855 | consumed samples: 643328 | elapsed time per iteration (ms): 91095.7 | throughput per GPU (TFLOP/s/GPU): 41.3 | MFU 13.25% | FLOPs estimator: workload-aware | logical tokens/s: 184170.9 | valid tokens/s: 184170.9 | physical tokens/s: 184578.6 | loss tokens/s: 136115.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.273051E-01 | lm-global-supervised-tokens: 1.239956E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.161930E-02 | lm-token-buffer-mib: 5.818844E+01 | global_load_balancing_loss: 1.002141E+00 | loss scale: 1.0 | grad norm: 0.121 | params norm: 11326.237 | params rms: 0.061328 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 22:16:54.029242 | finish at 2026-10-10 20:23:26 + [2026-10-08 22:08:00] iteration 5027/ 6855 | consumed samples: 643456 | elapsed time per iteration (ms): 87537.4 | throughput per GPU (TFLOP/s/GPU): 40.8 | MFU 13.07% | FLOPs estimator: workload-aware | logical tokens/s: 191657.5 | valid tokens/s: 191657.5 | physical tokens/s: 192271.0 | loss tokens/s: 149116.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.418021E-01 | lm-global-supervised-tokens: 1.305330E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.239181E-02 | lm-token-buffer-mib: 5.827775E+01 | global_load_balancing_loss: 1.000260E+00 | loss scale: 1.0 | grad norm: 0.076 | params norm: 11326.186 | params rms: 0.061328 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 20:26:58.288328 | finish at 2026-10-10 18:34:58 + [2026-10-08 22:09:29] iteration 5028/ 6855 | consumed samples: 643584 | elapsed time per iteration (ms): 89168.0 | throughput per GPU (TFLOP/s/GPU): 40.6 | MFU 13.00% | FLOPs estimator: workload-aware | logical tokens/s: 188152.8 | valid tokens/s: 188152.8 | physical tokens/s: 188529.4 | loss tokens/s: 142364.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.453056E-01 | lm-global-supervised-tokens: 1.269432E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.176183E-02 | lm-token-buffer-mib: 5.805948E+01 | global_load_balancing_loss: 1.000411E+00 | loss scale: 1.0 | grad norm: 0.113 | params norm: 11326.136 | params rms: 0.061327 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 21:15:09.919416 | finish at 2026-10-10 19:24:39 + [2026-10-08 22:10:58] iteration 5029/ 6855 | consumed samples: 643712 | elapsed time per iteration (ms): 89472.2 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.45% | FLOPs estimator: workload-aware | logical tokens/s: 187512.4 | valid tokens/s: 187512.4 | physical tokens/s: 187938.6 | loss tokens/s: 136000.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.468861E-01 | lm-global-supervised-tokens: 1.216827E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.355010E-02 | lm-token-buffer-mib: 5.829211E+01 | global_load_balancing_loss: 1.001201E+00 | loss scale: 1.0 | grad norm: 0.053 | params norm: 11326.086 | params rms: 0.061327 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 21:22:56.268394 | finish at 2026-10-10 19:33:55 + [2026-10-08 22:12:50] iteration 5030/ 6855 | consumed samples: 643840 | elapsed time per iteration (ms): 111362.7 | throughput per GPU (TFLOP/s/GPU): 34.5 | MFU 11.07% | FLOPs estimator: workload-aware | logical tokens/s: 150653.4 | valid tokens/s: 150653.4 | physical tokens/s: 150946.9 | loss tokens/s: 108305.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.496245E-01 | lm-global-supervised-tokens: 1.206122E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.203970E-02 | lm-token-buffer-mib: 5.822219E+01 | global_load_balancing_loss: 9.989189E-01 | loss scale: 1.0 | grad norm: 0.089 | params norm: 11326.035 | params rms: 0.061327 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 8:27:17.014061 | finish at 2026-10-11 06:40:07 + [2026-10-08 22:14:15] iteration 5031/ 6855 | consumed samples: 643968 | elapsed time per iteration (ms): 85320.1 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.56% | FLOPs estimator: workload-aware | logical tokens/s: 196638.3 | valid tokens/s: 196638.2 | physical tokens/s: 197286.9 | loss tokens/s: 150092.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.901779E-01 | lm-global-supervised-tokens: 1.280586E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.538349E-02 | lm-token-buffer-mib: 5.820021E+01 | global_load_balancing_loss: 9.991034E-01 | loss scale: 1.0 | grad norm: 0.129 | params norm: 11325.981 | params rms: 0.061327 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 19:13:43.885139 | finish at 2026-10-10 17:27:59 + [2026-10-08 22:15:42] iteration 5032/ 6855 | consumed samples: 644096 | elapsed time per iteration (ms): 86971.8 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.62% | FLOPs estimator: workload-aware | logical tokens/s: 192903.1 | valid tokens/s: 192903.1 | physical tokens/s: 193422.6 | loss tokens/s: 142696.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.855861E-01 | lm-global-supervised-tokens: 1.241055E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.118790E-02 | lm-token-buffer-mib: 5.811780E+01 | global_load_balancing_loss: 9.995341E-01 | loss scale: 1.0 | grad norm: 0.212 | params norm: 11325.928 | params rms: 0.061326 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 20:02:29.675447 | finish at 2026-10-10 18:18:12 + [2026-10-08 22:17:04] iteration 5033/ 6855 | consumed samples: 644224 | elapsed time per iteration (ms): 81990.6 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.74% | FLOPs estimator: workload-aware | logical tokens/s: 204622.8 | valid tokens/s: 204622.8 | physical tokens/s: 205425.5 | loss tokens/s: 153078.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.248879E-01 | lm-global-supervised-tokens: 1.255102E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.139317E-02 | lm-token-buffer-mib: 5.837278E+01 | global_load_balancing_loss: 1.001199E+00 | loss scale: 1.0 | grad norm: 0.088 | params norm: 11325.875 | params rms: 0.061326 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 17:29:46.869055 | finish at 2026-10-10 15:46:51 + [2026-10-08 22:18:32] iteration 5034/ 6855 | consumed samples: 644352 | elapsed time per iteration (ms): 87769.8 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.65% | FLOPs estimator: workload-aware | logical tokens/s: 191149.8 | valid tokens/s: 191149.8 | physical tokens/s: 191621.4 | loss tokens/s: 139719.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.338081E-01 | lm-global-supervised-tokens: 1.226312E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.229677E-02 | lm-token-buffer-mib: 5.812928E+01 | global_load_balancing_loss: 1.000671E+00 | loss scale: 1.0 | grad norm: 0.124 | params norm: 11325.824 | params rms: 0.061326 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 20:23:48.855199 | finish at 2026-10-10 18:42:21 + [2026-10-08 22:19:59] iteration 5035/ 6855 | consumed samples: 644480 | elapsed time per iteration (ms): 86740.0 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.66% | FLOPs estimator: workload-aware | logical tokens/s: 193418.9 | valid tokens/s: 193418.9 | physical tokens/s: 193823.5 | loss tokens/s: 141967.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.951845E-01 | lm-global-supervised-tokens: 1.231426E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.218119E-02 | lm-token-buffer-mib: 5.819821E+01 | global_load_balancing_loss: 1.000666E+00 | loss scale: 1.0 | grad norm: 0.074 | params norm: 11325.773 | params rms: 0.061325 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 19:51:06.865973 | finish at 2026-10-10 18:11:05 + [2026-10-08 22:21:23] iteration 5036/ 6855 | consumed samples: 644608 | elapsed time per iteration (ms): 84059.9 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.51% | FLOPs estimator: workload-aware | logical tokens/s: 199586.1 | valid tokens/s: 199586.1 | physical tokens/s: 200167.2 | loss tokens/s: 149933.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.671100E-01 | lm-global-supervised-tokens: 1.260338E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.580712E-02 | lm-token-buffer-mib: 5.812122E+01 | global_load_balancing_loss: 1.000192E+00 | loss scale: 1.0 | grad norm: 0.123 | params norm: 11325.724 | params rms: 0.061325 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 18:28:24.943437 | finish at 2026-10-10 16:49:48 + [2026-10-08 22:22:50] iteration 5037/ 6855 | consumed samples: 644736 | elapsed time per iteration (ms): 87015.8 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.54% | FLOPs estimator: workload-aware | logical tokens/s: 192805.9 | valid tokens/s: 192805.9 | physical tokens/s: 193516.4 | loss tokens/s: 143744.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.001080E-01 | lm-global-supervised-tokens: 1.250808E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.547474E-02 | lm-token-buffer-mib: 5.819713E+01 | global_load_balancing_loss: 9.991011E-01 | loss scale: 1.0 | grad norm: 0.072 | params norm: 11325.676 | params rms: 0.061325 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 19:56:34.638143 | finish at 2026-10-10 18:19:24 + [2026-10-08 22:24:20] iteration 5038/ 6855 | consumed samples: 644864 | elapsed time per iteration (ms): 90678.2 | throughput per GPU (TFLOP/s/GPU): 40.9 | MFU 13.11% | FLOPs estimator: workload-aware | logical tokens/s: 185018.5 | valid tokens/s: 185018.5 | physical tokens/s: 185344.4 | loss tokens/s: 134220.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.248302E-01 | lm-global-supervised-tokens: 1.217083E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.315058E-02 | lm-token-buffer-mib: 5.811354E+01 | global_load_balancing_loss: 9.989234E-01 | loss scale: 1.0 | grad norm: 0.044 | params norm: 11325.626 | params rms: 0.061325 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 21:46:02.328400 | finish at 2026-10-10 20:10:23 + [2026-10-08 22:25:43] iteration 5039/ 6855 | consumed samples: 644992 | elapsed time per iteration (ms): 82628.9 | throughput per GPU (TFLOP/s/GPU): 43.4 | MFU 13.91% | FLOPs estimator: workload-aware | logical tokens/s: 203042.6 | valid tokens/s: 203042.6 | physical tokens/s: 203605.0 | loss tokens/s: 156659.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.637655E-01 | lm-global-supervised-tokens: 1.294460E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.143994E-02 | lm-token-buffer-mib: 5.841019E+01 | global_load_balancing_loss: 9.996044E-01 | loss scale: 1.0 | grad norm: 0.060 | params norm: 11325.577 | params rms: 0.061324 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 17:40:54.016249 | finish at 2026-10-10 16:06:37 + [2026-10-08 22:27:32] iteration 5040/ 6855 | consumed samples: 645120 | elapsed time per iteration (ms): 108670.7 | throughput per GPU (TFLOP/s/GPU): 33.6 | MFU 10.77% | FLOPs estimator: workload-aware | logical tokens/s: 154384.9 | valid tokens/s: 154384.8 | physical tokens/s: 154752.7 | loss tokens/s: 118602.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.502459E-01 | lm-global-supervised-tokens: 1.288862E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.467430E-02 | lm-token-buffer-mib: 5.812645E+01 | global_load_balancing_loss: 1.000777E+00 | loss scale: 1.0 | grad norm: 0.044 | params norm: 11325.527 | params rms: 0.061324 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 6:47:17.387273 | finish at 2026-10-11 05:14:49 +times across ranks (ms): + save-checkpoint: + rank 0: 24133.62 + rank 1: 24133.67 + rank 2: 24133.65 + rank 3: 24133.65 + rank 4: 24133.66 + rank 5: 24133.62 + rank 6: 24133.64 + rank 7: 24133.67 + rank 8: 24134.21 + rank 9: 24134.23 + rank 10: 24134.22 + rank 11: 24134.18 + rank 12: 24134.20 + rank 13: 24134.19 + rank 14: 24134.19 + rank 15: 24134.19 + rank 16: 24133.43 + rank 17: 24133.39 + rank 18: 24133.41 + rank 19: 24133.44 + rank 20: 24133.43 + rank 21: 24133.41 + rank 22: 24133.40 + rank 23: 24133.39 + rank 24: 24133.94 + rank 25: 24134.00 + rank 26: 24133.99 + rank 27: 24133.99 + rank 28: 24133.94 + rank 29: 24133.95 + rank 30: 24133.93 + rank 31: 24133.96 + rank 32: 24133.01 + rank 33: 24133.01 + rank 34: 24133.03 + rank 35: 24132.99 + rank 36: 24133.01 + rank 37: 24132.97 + rank 38: 24132.98 + rank 39: 24132.99 + rank 40: 24133.41 + rank 41: 24133.43 + rank 42: 24133.40 + rank 43: 24133.40 + rank 44: 24133.43 + rank 45: 24133.40 + rank 46: 24133.42 + rank 47: 24133.40 + rank 48: 24133.18 + rank 49: 24133.17 + rank 50: 24133.17 + rank 51: 24133.16 + rank 52: 24133.19 + rank 53: 24133.18 + rank 54: 24133.25 + rank 55: 24133.18 + rank 56: 24133.61 + rank 57: 24133.60 + rank 58: 24133.60 + rank 59: 24133.59 + rank 60: 24133.59 + rank 61: 24133.61 + rank 62: 24133.60 + rank 63: 24133.59 + [2026-10-08 22:29:24] iteration 5041/ 6855 | consumed samples: 645248 | elapsed time per iteration (ms): 88597.6 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.45% | FLOPs estimator: workload-aware | logical tokens/s: 189364.0 | valid tokens/s: 189364.0 | physical tokens/s: 189984.9 | loss tokens/s: 142257.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.498457E-01 | lm-global-supervised-tokens: 1.260369E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.533828E-02 | lm-token-buffer-mib: 5.826991E+01 | global_load_balancing_loss: 1.001264E+00 | loss scale: 1.0 | grad norm: 0.054 | params norm: 11325.478 | params rms: 0.061324 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 20:38:36.054587 | finish at 2026-10-10 19:08:01 + [2026-10-08 22:30:57] iteration 5042/ 6855 | consumed samples: 645376 | elapsed time per iteration (ms): 92902.3 | throughput per GPU (TFLOP/s/GPU): 39.7 | MFU 12.74% | FLOPs estimator: workload-aware | logical tokens/s: 180589.6 | valid tokens/s: 180589.6 | physical tokens/s: 181106.7 | loss tokens/s: 131683.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.149497E-01 | lm-global-supervised-tokens: 1.223368E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.690785E-02 | lm-token-buffer-mib: 5.816657E+01 | global_load_balancing_loss: 1.001194E+00 | loss scale: 1.0 | grad norm: 0.064 | params norm: 11325.430 | params rms: 0.061324 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 22:47:11.782801 | finish at 2026-10-10 21:18:09 + [2026-10-08 22:32:21] iteration 5043/ 6855 | consumed samples: 645504 | elapsed time per iteration (ms): 83991.0 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.67% | FLOPs estimator: workload-aware | logical tokens/s: 199749.3 | valid tokens/s: 199749.3 | physical tokens/s: 200154.9 | loss tokens/s: 152403.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.312864E-01 | lm-global-supervised-tokens: 1.280048E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.190405E-02 | lm-token-buffer-mib: 5.817876E+01 | global_load_balancing_loss: 1.000598E+00 | loss scale: 1.0 | grad norm: 0.031 | params norm: 11325.381 | params rms: 0.061323 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 18:16:31.727311 | finish at 2026-10-10 16:48:53 + [2026-10-08 22:33:50] iteration 5044/ 6855 | consumed samples: 645632 | elapsed time per iteration (ms): 88410.2 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.39% | FLOPs estimator: workload-aware | logical tokens/s: 189764.1 | valid tokens/s: 189764.1 | physical tokens/s: 190201.1 | loss tokens/s: 140723.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.702705E-01 | lm-global-supervised-tokens: 1.244137E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.922438E-02 | lm-token-buffer-mib: 5.807386E+01 | global_load_balancing_loss: 1.000376E+00 | loss scale: 1.0 | grad norm: 0.042 | params norm: 11325.332 | params rms: 0.061323 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 20:28:30.961793 | finish at 2026-10-10 19:02:21 + [2026-10-08 22:35:17] iteration 5045/ 6855 | consumed samples: 645760 | elapsed time per iteration (ms): 87297.8 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.56% | FLOPs estimator: workload-aware | logical tokens/s: 192183.0 | valid tokens/s: 192183.0 | physical tokens/s: 192705.0 | loss tokens/s: 143370.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.221094E-01 | lm-global-supervised-tokens: 1.251589E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.135949E-02 | lm-token-buffer-mib: 5.806924E+01 | global_load_balancing_loss: 9.991878E-01 | loss scale: 1.0 | grad norm: 0.079 | params norm: 11325.282 | params rms: 0.061323 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 19:53:29.019842 | finish at 2026-10-10 18:28:46 + [2026-10-08 22:36:42] iteration 5046/ 6855 | consumed samples: 645888 | elapsed time per iteration (ms): 85319.5 | throughput per GPU (TFLOP/s/GPU): 43.4 | MFU 13.90% | FLOPs estimator: workload-aware | logical tokens/s: 196639.4 | valid tokens/s: 196639.4 | physical tokens/s: 197088.2 | loss tokens/s: 146261.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.446202E-01 | lm-global-supervised-tokens: 1.247892E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.294074E-02 | lm-token-buffer-mib: 5.812506E+01 | global_load_balancing_loss: 1.000269E+00 | loss scale: 1.0 | grad norm: 0.070 | params norm: 11325.233 | params rms: 0.061322 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 18:52:22.902987 | finish at 2026-10-10 17:29:05 + [2026-10-08 22:38:10] iteration 5047/ 6855 | consumed samples: 646016 | elapsed time per iteration (ms): 87252.1 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.61% | FLOPs estimator: workload-aware | logical tokens/s: 192284.2 | valid tokens/s: 192284.2 | physical tokens/s: 192728.2 | loss tokens/s: 144247.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.609971E-01 | lm-global-supervised-tokens: 1.258591E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.860176E-02 | lm-token-buffer-mib: 5.809893E+01 | global_load_balancing_loss: 9.988190E-01 | loss scale: 1.0 | grad norm: 0.080 | params norm: 11325.182 | params rms: 0.061322 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 19:49:11.790749 | finish at 2026-10-10 18:27:21 + [2026-10-08 22:39:37] iteration 5048/ 6855 | consumed samples: 646144 | elapsed time per iteration (ms): 87057.0 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.64% | FLOPs estimator: workload-aware | logical tokens/s: 192715.0 | valid tokens/s: 192715.0 | physical tokens/s: 193088.0 | loss tokens/s: 143862.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.534932E-01 | lm-global-supervised-tokens: 1.252425E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.280326E-02 | lm-token-buffer-mib: 5.806249E+01 | global_load_balancing_loss: 1.000715E+00 | loss scale: 1.0 | grad norm: 0.044 | params norm: 11325.132 | params rms: 0.061322 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 19:41:52.027724 | finish at 2026-10-10 18:21:29 + [2026-10-08 22:41:04] iteration 5049/ 6855 | consumed samples: 646272 | elapsed time per iteration (ms): 87236.5 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.56% | FLOPs estimator: workload-aware | logical tokens/s: 192318.1 | valid tokens/s: 192318.0 | physical tokens/s: 192697.0 | loss tokens/s: 143302.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.013046E-01 | lm-global-supervised-tokens: 1.250121E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.244110E-02 | lm-token-buffer-mib: 5.816851E+01 | global_load_balancing_loss: 1.000455E+00 | loss scale: 1.0 | grad norm: 0.110 | params norm: 11325.082 | params rms: 0.061322 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 19:45:49.066083 | finish at 2026-10-10 18:26:53 + [2026-10-08 22:42:56] iteration 5050/ 6855 | consumed samples: 646400 | elapsed time per iteration (ms): 111616.2 | throughput per GPU (TFLOP/s/GPU): 33.1 | MFU 10.62% | FLOPs estimator: workload-aware | logical tokens/s: 150311.1 | valid tokens/s: 150311.1 | physical tokens/s: 150732.4 | loss tokens/s: 108459.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.513804E-01 | lm-global-supervised-tokens: 1.210584E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.857972E-02 | lm-token-buffer-mib: 5.814555E+01 | global_load_balancing_loss: 1.000916E+00 | loss scale: 1.0 | grad norm: 0.081 | params norm: 11325.031 | params rms: 0.061321 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 7:57:47.252996 | finish at 2026-10-11 06:40:43 + [2026-10-08 22:44:25] iteration 5051/ 6855 | consumed samples: 646528 | elapsed time per iteration (ms): 89578.7 | throughput per GPU (TFLOP/s/GPU): 41.2 | MFU 13.21% | FLOPs estimator: workload-aware | logical tokens/s: 187290.0 | valid tokens/s: 187290.0 | physical tokens/s: 187693.0 | loss tokens/s: 136860.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.399277E-01 | lm-global-supervised-tokens: 1.225976E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.460791E-02 | lm-token-buffer-mib: 5.804068E+01 | global_load_balancing_loss: 9.998260E-01 | loss scale: 1.0 | grad norm: 0.059 | params norm: 11324.980 | params rms: 0.061321 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 20:53:19.986961 | finish at 2026-10-10 19:37:45 + [2026-10-08 22:45:53] iteration 5052/ 6855 | consumed samples: 646656 | elapsed time per iteration (ms): 88014.4 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.49% | FLOPs estimator: workload-aware | logical tokens/s: 190619.0 | valid tokens/s: 190619.0 | physical tokens/s: 190987.8 | loss tokens/s: 135968.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.063172E-01 | lm-global-supervised-tokens: 1.196718E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.244683E-02 | lm-token-buffer-mib: 5.804274E+01 | global_load_balancing_loss: 9.990816E-01 | loss scale: 1.0 | grad norm: 0.072 | params norm: 11324.929 | params rms: 0.061321 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 20:04:49.990721 | finish at 2026-10-10 18:50:43 + [2026-10-08 22:47:23] iteration 5053/ 6855 | consumed samples: 646784 | elapsed time per iteration (ms): 89487.1 | throughput per GPU (TFLOP/s/GPU): 40.9 | MFU 13.11% | FLOPs estimator: workload-aware | logical tokens/s: 187480.4 | valid tokens/s: 187480.3 | physical tokens/s: 188036.3 | loss tokens/s: 143429.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.603395E-01 | lm-global-supervised-tokens: 1.283511E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.455833E-02 | lm-token-buffer-mib: 5.819054E+01 | global_load_balancing_loss: 1.000108E+00 | loss scale: 1.0 | grad norm: 0.239 | params norm: 11324.876 | params rms: 0.061321 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 20:47:35.813713 | finish at 2026-10-10 19:34:58 + [2026-10-08 22:48:52] iteration 5054/ 6855 | consumed samples: 646912 | elapsed time per iteration (ms): 88923.8 | throughput per GPU (TFLOP/s/GPU): 43.1 | MFU 13.80% | FLOPs estimator: workload-aware | logical tokens/s: 188669.3 | valid tokens/s: 188669.3 | physical tokens/s: 189003.2 | loss tokens/s: 138531.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.887412E-01 | lm-global-supervised-tokens: 1.231871E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.233068E-02 | lm-token-buffer-mib: 5.809275E+01 | global_load_balancing_loss: 1.000571E+00 | loss scale: 1.0 | grad norm: 0.344 | params norm: 11324.823 | params rms: 0.061320 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 20:29:11.757344 | finish at 2026-10-10 19:18:03 + [2026-10-08 22:50:22] iteration 5055/ 6855 | consumed samples: 647040 | elapsed time per iteration (ms): 90521.9 | throughput per GPU (TFLOP/s/GPU): 41.1 | MFU 13.17% | FLOPs estimator: workload-aware | logical tokens/s: 185337.8 | valid tokens/s: 185337.8 | physical tokens/s: 185734.4 | loss tokens/s: 138117.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.818554E-01 | lm-global-supervised-tokens: 1.250266E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.207365E-02 | lm-token-buffer-mib: 5.814213E+01 | global_load_balancing_loss: 1.000999E+00 | loss scale: 1.0 | grad norm: 0.172 | params norm: 11324.770 | params rms: 0.061320 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 21:15:39.488554 | finish at 2026-10-10 20:06:02 + [2026-10-08 22:51:48] iteration 5056/ 6855 | consumed samples: 647168 | elapsed time per iteration (ms): 85562.6 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.58% | FLOPs estimator: workload-aware | logical tokens/s: 196080.7 | valid tokens/s: 196080.6 | physical tokens/s: 196600.2 | loss tokens/s: 148685.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.869408E-01 | lm-global-supervised-tokens: 1.272190E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.408774E-02 | lm-token-buffer-mib: 5.812556E+01 | global_load_balancing_loss: 1.001452E+00 | loss scale: 1.0 | grad norm: 0.541 | params norm: 11324.716 | params rms: 0.061320 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 18:45:27.203856 | finish at 2026-10-10 17:37:15 + [2026-10-08 22:53:17] iteration 5057/ 6855 | consumed samples: 647296 | elapsed time per iteration (ms): 89551.7 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.43% | FLOPs estimator: workload-aware | logical tokens/s: 187346.5 | valid tokens/s: 187346.5 | physical tokens/s: 187702.2 | loss tokens/s: 142269.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.214431E-01 | lm-global-supervised-tokens: 1.274043E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.791101E-02 | lm-token-buffer-mib: 5.816786E+01 | global_load_balancing_loss: 1.000931E+00 | loss scale: 1.0 | grad norm: 0.098 | params norm: 11324.662 | params rms: 0.061319 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 20:43:33.963237 | finish at 2026-10-10 19:36:51 + [2026-10-08 22:54:46] iteration 5058/ 6855 | consumed samples: 647424 | elapsed time per iteration (ms): 88875.0 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.43% | FLOPs estimator: workload-aware | logical tokens/s: 188772.8 | valid tokens/s: 188772.8 | physical tokens/s: 189300.8 | loss tokens/s: 142997.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.481720E-01 | lm-global-supervised-tokens: 1.270892E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.369931E-02 | lm-token-buffer-mib: 5.808288E+01 | global_load_balancing_loss: 9.998456E-01 | loss scale: 1.0 | grad norm: 0.459 | params norm: 11324.609 | params rms: 0.061319 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 20:21:48.389138 | finish at 2026-10-10 19:16:34 + [2026-10-08 22:56:14] iteration 5059/ 6855 | consumed samples: 647552 | elapsed time per iteration (ms): 88025.6 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.79% | FLOPs estimator: workload-aware | logical tokens/s: 190594.4 | valid tokens/s: 190594.4 | physical tokens/s: 191000.3 | loss tokens/s: 131361.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.900658E-01 | lm-global-supervised-tokens: 1.156322E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.547711E-02 | lm-token-buffer-mib: 5.805482E+01 | global_load_balancing_loss: 9.994958E-01 | loss scale: 1.0 | grad norm: 0.082 | params norm: 11324.554 | params rms: 0.061319 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 19:54:54.064875 | finish at 2026-10-10 18:51:08 + [2026-10-08 22:58:03] iteration 5060/ 6855 | consumed samples: 647680 | elapsed time per iteration (ms): 108525.1 | throughput per GPU (TFLOP/s/GPU): 34.0 | MFU 10.89% | FLOPs estimator: workload-aware | logical tokens/s: 154592.1 | valid tokens/s: 154592.1 | physical tokens/s: 154960.3 | loss tokens/s: 116725.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.982384E-01 | lm-global-supervised-tokens: 1.266765E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.556977E-02 | lm-token-buffer-mib: 5.823081E+01 | global_load_balancing_loss: 9.993834E-01 | loss scale: 1.0 | grad norm: 0.262 | params norm: 11324.499 | params rms: 0.061318 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 6:06:42.496284 | finish at 2026-10-11 05:04:45 + [2026-10-08 22:59:28] iteration 5061/ 6855 | consumed samples: 647808 | elapsed time per iteration (ms): 85762.6 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.66% | FLOPs estimator: workload-aware | logical tokens/s: 195622.2 | valid tokens/s: 195622.2 | physical tokens/s: 196139.0 | loss tokens/s: 147585.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.582934E-01 | lm-global-supervised-tokens: 1.265727E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.218612E-02 | lm-token-buffer-mib: 5.828671E+01 | global_load_balancing_loss: 9.997405E-01 | loss scale: 1.0 | grad norm: 0.189 | params norm: 11324.445 | params rms: 0.061318 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 18:44:18.144935 | finish at 2026-10-10 17:43:47 + [2026-10-08 23:00:55] iteration 5062/ 6855 | consumed samples: 647936 | elapsed time per iteration (ms): 86116.3 | throughput per GPU (TFLOP/s/GPU): 44.0 | MFU 14.11% | FLOPs estimator: workload-aware | logical tokens/s: 194820.0 | valid tokens/s: 194820.0 | physical tokens/s: 195190.3 | loss tokens/s: 142172.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.058675E-01 | lm-global-supervised-tokens: 1.224335E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.140352E-02 | lm-token-buffer-mib: 5.817586E+01 | global_load_balancing_loss: 9.998294E-01 | loss scale: 1.0 | grad norm: 0.083 | params norm: 11324.391 | params rms: 0.061318 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 18:53:26.612870 | finish at 2026-10-10 17:54:21 + [2026-10-08 23:02:21] iteration 5063/ 6855 | consumed samples: 648064 | elapsed time per iteration (ms): 86077.6 | throughput per GPU (TFLOP/s/GPU): 43.4 | MFU 13.90% | FLOPs estimator: workload-aware | logical tokens/s: 194907.3 | valid tokens/s: 194907.3 | physical tokens/s: 195346.0 | loss tokens/s: 147574.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.713685E-01 | lm-global-supervised-tokens: 1.270287E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.179816E-02 | lm-token-buffer-mib: 5.826086E+01 | global_load_balancing_loss: 1.000185E+00 | loss scale: 1.0 | grad norm: 0.130 | params norm: 11324.336 | params rms: 0.061318 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 18:50:51.043823 | finish at 2026-10-10 17:53:12 + [2026-10-08 23:03:52] iteration 5064/ 6855 | consumed samples: 648192 | elapsed time per iteration (ms): 91093.4 | throughput per GPU (TFLOP/s/GPU): 41.2 | MFU 13.20% | FLOPs estimator: workload-aware | logical tokens/s: 184175.0 | valid tokens/s: 184175.0 | physical tokens/s: 184538.1 | loss tokens/s: 129237.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.569491E-01 | lm-global-supervised-tokens: 1.177268E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.493803E-02 | lm-token-buffer-mib: 5.813988E+01 | global_load_balancing_loss: 1.000783E+00 | loss scale: 1.0 | grad norm: 0.190 | params norm: 11324.281 | params rms: 0.061317 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 21:19:08.356264 | finish at 2026-10-10 20:23:00 + [2026-10-08 23:05:21] iteration 5065/ 6855 | consumed samples: 648320 | elapsed time per iteration (ms): 89069.4 | throughput per GPU (TFLOP/s/GPU): 42.8 | MFU 13.72% | FLOPs estimator: workload-aware | logical tokens/s: 188361.1 | valid tokens/s: 188361.1 | physical tokens/s: 188914.7 | loss tokens/s: 139632.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.890400E-01 | lm-global-supervised-tokens: 1.243695E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.148511E-02 | lm-token-buffer-mib: 5.809842E+01 | global_load_balancing_loss: 1.000535E+00 | loss scale: 1.0 | grad norm: 0.064 | params norm: 11324.224 | params rms: 0.061317 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 20:17:14.160407 | finish at 2026-10-10 19:22:35 + [2026-10-08 23:06:50] iteration 5066/ 6855 | consumed samples: 648448 | elapsed time per iteration (ms): 89687.9 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.42% | FLOPs estimator: workload-aware | logical tokens/s: 187061.6 | valid tokens/s: 187061.5 | physical tokens/s: 187514.2 | loss tokens/s: 140316.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.065215E-01 | lm-global-supervised-tokens: 1.258469E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.696683E-02 | lm-token-buffer-mib: 5.816513E+01 | global_load_balancing_loss: 1.000273E+00 | loss scale: 1.0 | grad norm: 0.062 | params norm: 11324.170 | params rms: 0.061317 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 20:34:11.580709 | finish at 2026-10-10 19:41:02 + [2026-10-08 23:08:18] iteration 5067/ 6855 | consumed samples: 648576 | elapsed time per iteration (ms): 87069.3 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.56% | FLOPs estimator: workload-aware | logical tokens/s: 192687.8 | valid tokens/s: 192687.8 | physical tokens/s: 193197.6 | loss tokens/s: 141625.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.728260E-01 | lm-global-supervised-tokens: 1.233120E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.300229E-02 | lm-token-buffer-mib: 5.807507E+01 | global_load_balancing_loss: 1.000791E+00 | loss scale: 1.0 | grad norm: 0.205 | params norm: 11324.116 | params rms: 0.061316 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 19:14:39.908286 | finish at 2026-10-10 18:22:57 + [2026-10-08 23:09:45] iteration 5068/ 6855 | consumed samples: 648704 | elapsed time per iteration (ms): 87716.5 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.43% | FLOPs estimator: workload-aware | logical tokens/s: 191266.0 | valid tokens/s: 191266.0 | physical tokens/s: 191746.6 | loss tokens/s: 143535.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.251132E-01 | lm-global-supervised-tokens: 1.259040E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.290245E-02 | lm-token-buffer-mib: 5.812788E+01 | global_load_balancing_loss: 9.994968E-01 | loss scale: 1.0 | grad norm: 0.121 | params norm: 11324.063 | params rms: 0.061316 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 19:32:29.403473 | finish at 2026-10-10 18:42:15 + [2026-10-08 23:11:12] iteration 5069/ 6855 | consumed samples: 648832 | elapsed time per iteration (ms): 86387.6 | throughput per GPU (TFLOP/s/GPU): 43.1 | MFU 13.81% | FLOPs estimator: workload-aware | logical tokens/s: 194207.9 | valid tokens/s: 194207.9 | physical tokens/s: 194738.7 | loss tokens/s: 138348.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.356005E-01 | lm-global-supervised-tokens: 1.195155E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.195231E-02 | lm-token-buffer-mib: 5.814597E+01 | global_load_balancing_loss: 1.000804E+00 | loss scale: 1.0 | grad norm: 0.098 | params norm: 11324.011 | params rms: 0.061316 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 18:51:28.250521 | finish at 2026-10-10 18:02:40 + [2026-10-08 23:13:00] iteration 5070/ 6855 | consumed samples: 648960 | elapsed time per iteration (ms): 108216.3 | throughput per GPU (TFLOP/s/GPU): 33.9 | MFU 10.85% | FLOPs estimator: workload-aware | logical tokens/s: 155033.7 | valid tokens/s: 155033.7 | physical tokens/s: 155266.5 | loss tokens/s: 112635.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.203908E-01 | lm-global-supervised-tokens: 1.218905E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.561696E-02 | lm-token-buffer-mib: 5.817015E+01 | global_load_balancing_loss: 9.996830E-01 | loss scale: 1.0 | grad norm: 0.140 | params norm: 11323.960 | params rms: 0.061316 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 5:39:26.151695 | finish at 2026-10-11 04:52:26 + [2026-10-08 23:14:26] iteration 5071/ 6855 | consumed samples: 649088 | elapsed time per iteration (ms): 85726.8 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.57% | FLOPs estimator: workload-aware | logical tokens/s: 195704.3 | valid tokens/s: 195704.3 | physical tokens/s: 196226.8 | loss tokens/s: 152026.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.378548E-01 | lm-global-supervised-tokens: 1.303270E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.503047E-02 | lm-token-buffer-mib: 5.825157E+01 | global_load_balancing_loss: 9.988772E-01 | loss scale: 1.0 | grad norm: 0.140 | params norm: 11323.908 | params rms: 0.061315 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 18:28:56.653671 | finish at 2026-10-10 17:43:22 + [2026-10-08 23:15:53] iteration 5072/ 6855 | consumed samples: 649216 | elapsed time per iteration (ms): 87068.6 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.64% | FLOPs estimator: workload-aware | logical tokens/s: 192688.8 | valid tokens/s: 192688.8 | physical tokens/s: 193029.7 | loss tokens/s: 140136.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.988197E-01 | lm-global-supervised-tokens: 1.220150E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.254942E-02 | lm-token-buffer-mib: 5.804817E+01 | global_load_balancing_loss: 1.001051E+00 | loss scale: 1.0 | grad norm: 0.044 | params norm: 11323.857 | params rms: 0.061315 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 19:07:23.377032 | finish at 2026-10-10 18:23:16 + [2026-10-08 23:17:17] iteration 5073/ 6855 | consumed samples: 649344 | elapsed time per iteration (ms): 84610.6 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.78% | FLOPs estimator: workload-aware | logical tokens/s: 198287.2 | valid tokens/s: 198287.2 | physical tokens/s: 198788.1 | loss tokens/s: 146974.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 5.885756E-01 | lm-global-supervised-tokens: 1.243561E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.160313E-02 | lm-token-buffer-mib: 5.810400E+01 | global_load_balancing_loss: 1.001049E+00 | loss scale: 1.0 | grad norm: 0.086 | params norm: 11323.805 | params rms: 0.061315 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 17:52:56.085367 | finish at 2026-10-10 17:10:13 + [2026-10-08 23:18:49] iteration 5074/ 6855 | consumed samples: 649472 | elapsed time per iteration (ms): 92091.0 | throughput per GPU (TFLOP/s/GPU): 41.7 | MFU 13.37% | FLOPs estimator: workload-aware | logical tokens/s: 182179.8 | valid tokens/s: 182179.8 | physical tokens/s: 182688.1 | loss tokens/s: 132222.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.705730E-01 | lm-global-supervised-tokens: 1.217651E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.158347E-02 | lm-token-buffer-mib: 5.828796E+01 | global_load_balancing_loss: 1.000373E+00 | loss scale: 1.0 | grad norm: 0.137 | params norm: 11323.754 | params rms: 0.061314 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 21:33:34.005751 | finish at 2026-10-10 20:52:23 + [2026-10-08 23:20:17] iteration 5075/ 6855 | consumed samples: 649600 | elapsed time per iteration (ms): 87382.6 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.57% | FLOPs estimator: workload-aware | logical tokens/s: 191997.0 | valid tokens/s: 191997.0 | physical tokens/s: 192358.9 | loss tokens/s: 142244.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.516772E-01 | lm-global-supervised-tokens: 1.242970E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.130568E-02 | lm-token-buffer-mib: 5.820995E+01 | global_load_balancing_loss: 1.000096E+00 | loss scale: 1.0 | grad norm: 0.084 | params norm: 11323.702 | params rms: 0.061314 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 19:12:20.965314 | finish at 2026-10-10 18:32:38 + [2026-10-08 23:21:45] iteration 5076/ 6855 | consumed samples: 649728 | elapsed time per iteration (ms): 88683.4 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.42% | FLOPs estimator: workload-aware | logical tokens/s: 189180.0 | valid tokens/s: 189179.9 | physical tokens/s: 189617.0 | loss tokens/s: 136218.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.298769E-01 | lm-global-supervised-tokens: 1.208028E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.175337E-02 | lm-token-buffer-mib: 5.825294E+01 | global_load_balancing_loss: 1.000480E+00 | loss scale: 1.0 | grad norm: 0.092 | params norm: 11323.650 | params rms: 0.061314 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 19:49:27.842252 | finish at 2026-10-10 19:11:13 + [2026-10-08 23:23:13] iteration 5077/ 6855 | consumed samples: 649856 | elapsed time per iteration (ms): 87292.6 | throughput per GPU (TFLOP/s/GPU): 43.4 | MFU 13.92% | FLOPs estimator: workload-aware | logical tokens/s: 192194.6 | valid tokens/s: 192194.6 | physical tokens/s: 192680.6 | loss tokens/s: 137501.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.208223E-01 | lm-global-supervised-tokens: 1.200288E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.411497E-02 | lm-token-buffer-mib: 5.816689E+01 | global_load_balancing_loss: 9.994878E-01 | loss scale: 1.0 | grad norm: 0.084 | params norm: 11323.597 | params rms: 0.061314 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 19:06:46.204076 | finish at 2026-10-10 18:29:59 + [2026-10-08 23:24:35] iteration 5078/ 6855 | consumed samples: 649984 | elapsed time per iteration (ms): 82429.1 | throughput per GPU (TFLOP/s/GPU): 43.3 | MFU 13.88% | FLOPs estimator: workload-aware | logical tokens/s: 203534.7 | valid tokens/s: 203534.7 | physical tokens/s: 203973.3 | loss tokens/s: 156303.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.760345E-01 | lm-global-supervised-tokens: 1.288398E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.153489E-02 | lm-token-buffer-mib: 5.816538E+01 | global_load_balancing_loss: 9.981451E-01 | loss scale: 1.0 | grad norm: 0.054 | params norm: 11323.545 | params rms: 0.061313 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 16:41:16.549319 | finish at 2026-10-10 16:05:52 + [2026-10-08 23:26:04] iteration 5079/ 6855 | consumed samples: 650112 | elapsed time per iteration (ms): 89163.8 | throughput per GPU (TFLOP/s/GPU): 41.3 | MFU 13.23% | FLOPs estimator: workload-aware | logical tokens/s: 188160.4 | valid tokens/s: 188160.4 | physical tokens/s: 188561.9 | loss tokens/s: 141268.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.380503E-01 | lm-global-supervised-tokens: 1.259606E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.250069E-02 | lm-token-buffer-mib: 5.814268E+01 | global_load_balancing_loss: 1.000095E+00 | loss scale: 1.0 | grad norm: 0.076 | params norm: 11323.492 | params rms: 0.061313 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 19:59:14.900757 | finish at 2026-10-10 19:25:19 + [2026-10-08 23:27:55] iteration 5080/ 6855 | consumed samples: 650240 | elapsed time per iteration (ms): 110419.8 | throughput per GPU (TFLOP/s/GPU): 33.8 | MFU 10.85% | FLOPs estimator: workload-aware | logical tokens/s: 151940.0 | valid tokens/s: 151940.0 | physical tokens/s: 152314.7 | loss tokens/s: 110798.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.865942E-01 | lm-global-supervised-tokens: 1.223435E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.740723E-02 | lm-token-buffer-mib: 5.813380E+01 | global_load_balancing_loss: 1.001615E+00 | loss scale: 1.0 | grad norm: 0.043 | params norm: 11323.442 | params rms: 0.061313 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 6:26:35.189089 | finish at 2026-10-11 05:54:30 +times across ranks (ms): + save-checkpoint: + rank 0: 24674.18 + rank 1: 24674.19 + rank 2: 24674.24 + rank 3: 24674.23 + rank 4: 24674.16 + rank 5: 24674.15 + rank 6: 24674.19 + rank 7: 24674.17 + rank 8: 24673.99 + rank 9: 24673.72 + rank 10: 24673.88 + rank 11: 24673.89 + rank 12: 24673.95 + rank 13: 24673.87 + rank 14: 24673.92 + rank 15: 24673.86 + rank 16: 24674.08 + rank 17: 24674.05 + rank 18: 24674.02 + rank 19: 24674.07 + rank 20: 24674.03 + rank 21: 24674.06 + rank 22: 24674.03 + rank 23: 24674.02 + rank 24: 24674.11 + rank 25: 24674.05 + rank 26: 24674.19 + rank 27: 24674.19 + rank 28: 24674.08 + rank 29: 24674.13 + rank 30: 24674.09 + rank 31: 24674.10 + rank 32: 24674.23 + rank 33: 24674.23 + rank 34: 24674.16 + rank 35: 24674.16 + rank 36: 24674.13 + rank 37: 24674.13 + rank 38: 24674.15 + rank 39: 24674.14 + rank 40: 24674.64 + rank 41: 24674.66 + rank 42: 24674.68 + rank 43: 24674.64 + rank 44: 24674.61 + rank 45: 24674.63 + rank 46: 24674.62 + rank 47: 24674.62 + rank 48: 24674.34 + rank 49: 24674.31 + rank 50: 24674.20 + rank 51: 24674.21 + rank 52: 24674.29 + rank 53: 24674.27 + rank 54: 24674.34 + rank 55: 24674.32 + rank 56: 24674.17 + rank 57: 24674.16 + rank 58: 24674.12 + rank 59: 24674.12 + rank 60: 24674.11 + rank 61: 24674.14 + rank 62: 24674.13 + rank 63: 24674.15 + [2026-10-08 23:29:47] iteration 5081/ 6855 | consumed samples: 650368 | elapsed time per iteration (ms): 87143.2 | throughput per GPU (TFLOP/s/GPU): 41.6 | MFU 13.34% | FLOPs estimator: workload-aware | logical tokens/s: 192524.6 | valid tokens/s: 192524.6 | physical tokens/s: 193011.6 | loss tokens/s: 143885.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.477025E-01 | lm-global-supervised-tokens: 1.253861E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 5.352128E-01 | lm-token-buffer-mib: 5.817744E+01 | global_load_balancing_loss: 1.000774E+00 | loss scale: 1.0 | grad norm: 0.105 | params norm: 11323.391 | params rms: 0.061312 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 18:56:32.064997 | finish at 2026-10-10 18:26:19 + [2026-10-08 23:31:18] iteration 5082/ 6855 | consumed samples: 650496 | elapsed time per iteration (ms): 91083.8 | throughput per GPU (TFLOP/s/GPU): 40.8 | MFU 13.08% | FLOPs estimator: workload-aware | logical tokens/s: 184194.7 | valid tokens/s: 184194.7 | physical tokens/s: 184712.6 | loss tokens/s: 136708.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.757129E-01 | lm-global-supervised-tokens: 1.245191E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.049143E-02 | lm-token-buffer-mib: 5.824606E+01 | global_load_balancing_loss: 1.001014E+00 | loss scale: 1.0 | grad norm: 0.052 | params norm: 11323.339 | params rms: 0.061312 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 20:51:31.516666 | finish at 2026-10-10 20:22:49 + [2026-10-08 23:32:41] iteration 5083/ 6855 | consumed samples: 650624 | elapsed time per iteration (ms): 83476.4 | throughput per GPU (TFLOP/s/GPU): 43.6 | MFU 13.96% | FLOPs estimator: workload-aware | logical tokens/s: 200981.4 | valid tokens/s: 200981.4 | physical tokens/s: 201308.5 | loss tokens/s: 148863.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.784322E-01 | lm-global-supervised-tokens: 1.242656E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.193843E-02 | lm-token-buffer-mib: 5.803607E+01 | global_load_balancing_loss: 1.000012E+00 | loss scale: 1.0 | grad norm: 0.079 | params norm: 11323.287 | params rms: 0.061312 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 17:05:20.229628 | finish at 2026-10-10 16:38:01 + [2026-10-08 23:34:06] iteration 5084/ 6855 | consumed samples: 650752 | elapsed time per iteration (ms): 84994.8 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.68% | FLOPs estimator: workload-aware | logical tokens/s: 197390.7 | valid tokens/s: 197390.7 | physical tokens/s: 197912.9 | loss tokens/s: 141886.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.255915E-01 | lm-global-supervised-tokens: 1.205961E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.188213E-02 | lm-token-buffer-mib: 5.804639E+01 | global_load_balancing_loss: 1.000843E+00 | loss scale: 1.0 | grad norm: 0.055 | params norm: 11323.236 | params rms: 0.061312 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 17:48:45.834029 | finish at 2026-10-10 17:22:52 + [2026-10-08 23:35:30] iteration 5085/ 6855 | consumed samples: 650880 | elapsed time per iteration (ms): 83840.1 | throughput per GPU (TFLOP/s/GPU): 43.7 | MFU 14.01% | FLOPs estimator: workload-aware | logical tokens/s: 200109.2 | valid tokens/s: 200109.2 | physical tokens/s: 200548.8 | loss tokens/s: 148011.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.426498E-01 | lm-global-supervised-tokens: 1.240926E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.784906E-02 | lm-token-buffer-mib: 5.807457E+01 | global_load_balancing_loss: 9.987423E-01 | loss scale: 1.0 | grad norm: 0.060 | params norm: 11323.186 | params rms: 0.061311 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 17:13:16.955566 | finish at 2026-10-10 16:48:47 + [2026-10-08 23:36:58] iteration 5086/ 6855 | consumed samples: 651008 | elapsed time per iteration (ms): 88265.3 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.38% | FLOPs estimator: workload-aware | logical tokens/s: 190076.0 | valid tokens/s: 190076.0 | physical tokens/s: 190576.3 | loss tokens/s: 140809.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.226056E-01 | lm-global-supervised-tokens: 1.242862E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.452753E-02 | lm-token-buffer-mib: 5.822717E+01 | global_load_balancing_loss: 9.983484E-01 | loss scale: 1.0 | grad norm: 0.056 | params norm: 11323.134 | params rms: 0.061311 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 19:22:21.345286 | finish at 2026-10-10 18:59:20 + [2026-10-08 23:38:25] iteration 5087/ 6855 | consumed samples: 651136 | elapsed time per iteration (ms): 86382.7 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.79% | FLOPs estimator: workload-aware | logical tokens/s: 194219.2 | valid tokens/s: 194219.2 | physical tokens/s: 194699.2 | loss tokens/s: 143141.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.391646E-01 | lm-global-supervised-tokens: 1.236498E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.664131E-02 | lm-token-buffer-mib: 5.813049E+01 | global_load_balancing_loss: 1.000304E+00 | loss scale: 1.0 | grad norm: 0.044 | params norm: 11323.083 | params rms: 0.061311 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 18:25:24.603003 | finish at 2026-10-10 18:03:49 + [2026-10-08 23:39:50] iteration 5088/ 6855 | consumed samples: 651264 | elapsed time per iteration (ms): 85307.1 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.79% | FLOPs estimator: workload-aware | logical tokens/s: 196668.3 | valid tokens/s: 196668.3 | physical tokens/s: 197216.8 | loss tokens/s: 142752.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.683054E-01 | lm-global-supervised-tokens: 1.217784E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.188179E-02 | lm-token-buffer-mib: 5.817632E+01 | global_load_balancing_loss: 1.001170E+00 | loss scale: 1.0 | grad norm: 0.061 | params norm: 11323.031 | params rms: 0.061311 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 17:52:17.597354 | finish at 2026-10-10 17:32:08 + [2026-10-08 23:41:15] iteration 5089/ 6855 | consumed samples: 651392 | elapsed time per iteration (ms): 84828.0 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.65% | FLOPs estimator: workload-aware | logical tokens/s: 197779.0 | valid tokens/s: 197779.0 | physical tokens/s: 198305.4 | loss tokens/s: 153147.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.781551E-01 | lm-global-supervised-tokens: 1.299118E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.404929E-02 | lm-token-buffer-mib: 5.812590E+01 | global_load_balancing_loss: 1.000457E+00 | loss scale: 1.0 | grad norm: 0.052 | params norm: 11322.978 | params rms: 0.061310 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 17:36:46.304963 | finish at 2026-10-10 17:18:01 + [2026-10-08 23:43:05] iteration 5090/ 6855 | consumed samples: 651520 | elapsed time per iteration (ms): 110260.7 | throughput per GPU (TFLOP/s/GPU): 34.3 | MFU 10.98% | FLOPs estimator: workload-aware | logical tokens/s: 152159.3 | valid tokens/s: 152159.3 | physical tokens/s: 152635.8 | loss tokens/s: 112729.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.622713E-01 | lm-global-supervised-tokens: 1.242965E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.418088E-02 | lm-token-buffer-mib: 5.829897E+01 | global_load_balancing_loss: 1.001159E+00 | loss scale: 1.0 | grad norm: 0.045 | params norm: 11322.924 | params rms: 0.061310 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 6:03:30.109808 | finish at 2026-10-11 05:46:35 + [2026-10-08 23:44:37] iteration 5091/ 6855 | consumed samples: 651648 | elapsed time per iteration (ms): 91927.2 | throughput per GPU (TFLOP/s/GPU): 40.1 | MFU 12.86% | FLOPs estimator: workload-aware | logical tokens/s: 182504.9 | valid tokens/s: 182504.9 | physical tokens/s: 182828.8 | loss tokens/s: 131810.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 5.987909E-01 | lm-global-supervised-tokens: 1.211698E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.553249E-02 | lm-token-buffer-mib: 5.822750E+01 | global_load_balancing_loss: 1.001484E+00 | loss scale: 1.0 | grad norm: 0.096 | params norm: 11322.870 | params rms: 0.061310 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 21:02:39.495143 | finish at 2026-10-10 20:47:17 + [2026-10-08 23:46:05] iteration 5092/ 6855 | consumed samples: 651776 | elapsed time per iteration (ms): 87476.4 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.40% | FLOPs estimator: workload-aware | logical tokens/s: 191790.6 | valid tokens/s: 191790.6 | physical tokens/s: 192120.9 | loss tokens/s: 143835.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.530051E-01 | lm-global-supervised-tokens: 1.258219E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.291090E-02 | lm-token-buffer-mib: 5.806638E+01 | global_load_balancing_loss: 1.000912E+00 | loss scale: 1.0 | grad norm: 0.066 | params norm: 11322.817 | params rms: 0.061309 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 18:50:20.905631 | finish at 2026-10-10 18:36:25 + [2026-10-08 23:47:33] iteration 5093/ 6855 | consumed samples: 651904 | elapsed time per iteration (ms): 88292.5 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.66% | FLOPs estimator: workload-aware | logical tokens/s: 190018.1 | valid tokens/s: 190018.1 | physical tokens/s: 190377.8 | loss tokens/s: 136513.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.404132E-01 | lm-global-supervised-tokens: 1.205313E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.239886E-02 | lm-token-buffer-mib: 5.805721E+01 | global_load_balancing_loss: 9.988415E-01 | loss scale: 1.0 | grad norm: 0.108 | params norm: 11322.764 | params rms: 0.061309 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 19:12:51.311097 | finish at 2026-10-10 19:00:24 + [2026-10-08 23:49:02] iteration 5094/ 6855 | consumed samples: 652032 | elapsed time per iteration (ms): 88840.0 | throughput per GPU (TFLOP/s/GPU): 41.5 | MFU 13.31% | FLOPs estimator: workload-aware | logical tokens/s: 188847.1 | valid tokens/s: 188847.1 | physical tokens/s: 189319.2 | loss tokens/s: 137824.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.857446E-01 | lm-global-supervised-tokens: 1.224435E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.712102E-02 | lm-token-buffer-mib: 5.832340E+01 | global_load_balancing_loss: 9.990386E-01 | loss scale: 1.0 | grad norm: 0.095 | params norm: 11322.711 | params rms: 0.061309 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 19:27:27.252445 | finish at 2026-10-10 19:16:29 + [2026-10-08 23:50:28] iteration 5095/ 6855 | consumed samples: 652160 | elapsed time per iteration (ms): 86817.5 | throughput per GPU (TFLOP/s/GPU): 41.4 | MFU 13.26% | FLOPs estimator: workload-aware | logical tokens/s: 193246.8 | valid tokens/s: 193246.8 | physical tokens/s: 193750.4 | loss tokens/s: 146802.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.175248E-01 | lm-global-supervised-tokens: 1.274503E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.169777E-02 | lm-token-buffer-mib: 5.805599E+01 | global_load_balancing_loss: 9.983233E-01 | loss scale: 1.0 | grad norm: 0.089 | params norm: 11322.658 | params rms: 0.061309 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 18:26:38.788872 | finish at 2026-10-10 18:17:07 + [2026-10-08 23:51:54] iteration 5096/ 6855 | consumed samples: 652288 | elapsed time per iteration (ms): 85207.9 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.66% | FLOPs estimator: workload-aware | logical tokens/s: 196896.8 | valid tokens/s: 196896.8 | physical tokens/s: 197345.0 | loss tokens/s: 142901.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.465111E-01 | lm-global-supervised-tokens: 1.217631E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.162962E-02 | lm-token-buffer-mib: 5.816078E+01 | global_load_balancing_loss: 9.999186E-01 | loss scale: 1.0 | grad norm: 0.085 | params norm: 11322.608 | params rms: 0.061308 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 17:38:00.632018 | finish at 2026-10-10 17:29:54 + [2026-10-08 23:53:19] iteration 5097/ 6855 | consumed samples: 652416 | elapsed time per iteration (ms): 84882.4 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.67% | FLOPs estimator: workload-aware | logical tokens/s: 197652.5 | valid tokens/s: 197652.5 | physical tokens/s: 198064.6 | loss tokens/s: 153866.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.090065E-01 | lm-global-supervised-tokens: 1.306052E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.306281E-02 | lm-token-buffer-mib: 5.813523E+01 | global_load_balancing_loss: 9.997544E-01 | loss scale: 1.0 | grad norm: 0.095 | params norm: 11322.556 | params rms: 0.061308 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 17:27:03.231600 | finish at 2026-10-10 17:20:22 + [2026-10-08 23:54:45] iteration 5098/ 6855 | consumed samples: 652544 | elapsed time per iteration (ms): 86382.5 | throughput per GPU (TFLOP/s/GPU): 43.9 | MFU 14.08% | FLOPs estimator: workload-aware | logical tokens/s: 194219.7 | valid tokens/s: 194219.7 | physical tokens/s: 194711.1 | loss tokens/s: 145891.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.785984E-01 | lm-global-supervised-tokens: 1.260250E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.498954E-02 | lm-token-buffer-mib: 5.820828E+01 | global_load_balancing_loss: 1.000830E+00 | loss scale: 1.0 | grad norm: 0.067 | params norm: 11322.500 | params rms: 0.061308 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 18:09:34.049031 | finish at 2026-10-10 18:04:19 + [2026-10-08 23:56:11] iteration 5099/ 6855 | consumed samples: 652672 | elapsed time per iteration (ms): 86366.8 | throughput per GPU (TFLOP/s/GPU): 41.7 | MFU 13.37% | FLOPs estimator: workload-aware | logical tokens/s: 194255.2 | valid tokens/s: 194255.2 | physical tokens/s: 194847.8 | loss tokens/s: 151164.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.542719E-01 | lm-global-supervised-tokens: 1.305558E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.445225E-02 | lm-token-buffer-mib: 5.809724E+01 | global_load_balancing_loss: 1.002097E+00 | loss scale: 1.0 | grad norm: 0.087 | params norm: 11322.447 | params rms: 0.061307 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 18:07:40.090037 | finish at 2026-10-10 18:03:51 + [2026-10-08 23:57:59] iteration 5100/ 6855 | consumed samples: 652800 | elapsed time per iteration (ms): 107193.8 | throughput per GPU (TFLOP/s/GPU): 34.5 | MFU 11.06% | FLOPs estimator: workload-aware | logical tokens/s: 156512.6 | valid tokens/s: 156512.6 | physical tokens/s: 156928.2 | loss tokens/s: 119069.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.636366E-01 | lm-global-supervised-tokens: 1.276348E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.537310E-02 | lm-token-buffer-mib: 5.808297E+01 | global_load_balancing_loss: 1.001483E+00 | loss scale: 1.0 | grad norm: 0.048 | params norm: 11322.394 | params rms: 0.061307 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 4:15:25.152426 | finish at 2026-10-11 04:13:24 + [2026-10-08 23:59:24] iteration 5101/ 6855 | consumed samples: 652928 | elapsed time per iteration (ms): 85138.6 | throughput per GPU (TFLOP/s/GPU): 43.3 | MFU 13.87% | FLOPs estimator: workload-aware | logical tokens/s: 197057.0 | valid tokens/s: 197057.0 | physical tokens/s: 197566.4 | loss tokens/s: 148269.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.720029E-01 | lm-global-supervised-tokens: 1.262342E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.437149E-02 | lm-token-buffer-mib: 5.806648E+01 | global_load_balancing_loss: 1.000007E+00 | loss scale: 1.0 | grad norm: 0.066 | params norm: 11322.343 | params rms: 0.061307 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 17:28:53.105431 | finish at 2026-10-10 17:28:17 + [2026-10-09 00:00:52] iteration 5102/ 6855 | consumed samples: 653056 | elapsed time per iteration (ms): 88376.1 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.54% | FLOPs estimator: workload-aware | logical tokens/s: 189838.3 | valid tokens/s: 189838.3 | physical tokens/s: 190164.8 | loss tokens/s: 145185.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.237938E-01 | lm-global-supervised-tokens: 1.283096E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.513825E-02 | lm-token-buffer-mib: 5.809945E+01 | global_load_balancing_loss: 9.993239E-01 | loss scale: 1.0 | grad norm: 0.050 | params norm: 11322.292 | params rms: 0.061307 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 19:02:03.351475 | finish at 2026-10-10 19:02:55 + [2026-10-09 00:02:18] iteration 5103/ 6855 | consumed samples: 653184 | elapsed time per iteration (ms): 85992.5 | throughput per GPU (TFLOP/s/GPU): 43.7 | MFU 14.01% | FLOPs estimator: workload-aware | logical tokens/s: 195100.5 | valid tokens/s: 195100.5 | physical tokens/s: 195540.8 | loss tokens/s: 141109.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.707906E-01 | lm-global-supervised-tokens: 1.213434E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.810098E-02 | lm-token-buffer-mib: 5.823541E+01 | global_load_balancing_loss: 1.000487E+00 | loss scale: 1.0 | grad norm: 0.086 | params norm: 11322.244 | params rms: 0.061306 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 17:50:58.802055 | finish at 2026-10-10 17:53:17 + [2026-10-09 00:03:48] iteration 5104/ 6855 | consumed samples: 653312 | elapsed time per iteration (ms): 89794.1 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.52% | FLOPs estimator: workload-aware | logical tokens/s: 186840.0 | valid tokens/s: 186839.9 | physical tokens/s: 187438.5 | loss tokens/s: 128326.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.368990E-01 | lm-global-supervised-tokens: 1.152299E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.149922E-02 | lm-token-buffer-mib: 5.834905E+01 | global_load_balancing_loss: 1.000812E+00 | loss scale: 1.0 | grad norm: 0.086 | params norm: 11322.197 | params rms: 0.061306 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 19:40:29.514685 | finish at 2026-10-10 19:44:17 + [2026-10-09 00:05:12] iteration 5105/ 6855 | consumed samples: 653440 | elapsed time per iteration (ms): 84630.5 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.60% | FLOPs estimator: workload-aware | logical tokens/s: 198240.1 | valid tokens/s: 198240.1 | physical tokens/s: 198841.0 | loss tokens/s: 147084.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.393708E-01 | lm-global-supervised-tokens: 1.244781E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.503578E-02 | lm-token-buffer-mib: 5.816214E+01 | global_load_balancing_loss: 1.000211E+00 | loss scale: 1.0 | grad norm: 0.123 | params norm: 11322.147 | params rms: 0.061306 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 17:08:23.375554 | finish at 2026-10-10 17:13:36 + [2026-10-09 00:06:43] iteration 5106/ 6855 | consumed samples: 653568 | elapsed time per iteration (ms): 90494.6 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.48% | FLOPs estimator: workload-aware | logical tokens/s: 185393.6 | valid tokens/s: 185393.6 | physical tokens/s: 185842.0 | loss tokens/s: 131846.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.856340E-01 | lm-global-supervised-tokens: 1.193139E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.523900E-02 | lm-token-buffer-mib: 5.825038E+01 | global_load_balancing_loss: 9.999295E-01 | loss scale: 1.0 | grad norm: 0.091 | params norm: 11322.098 | params rms: 0.061305 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 19:57:55.063841 | finish at 2026-10-10 20:04:38 + [2026-10-09 00:08:09] iteration 5107/ 6855 | consumed samples: 653696 | elapsed time per iteration (ms): 86509.8 | throughput per GPU (TFLOP/s/GPU): 41.2 | MFU 13.21% | FLOPs estimator: workload-aware | logical tokens/s: 193934.3 | valid tokens/s: 193934.3 | physical tokens/s: 194405.2 | loss tokens/s: 142366.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.280760E-01 | lm-global-supervised-tokens: 1.231611E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.171475E-02 | lm-token-buffer-mib: 5.807141E+01 | global_load_balancing_loss: 9.984709E-01 | loss scale: 1.0 | grad norm: 0.067 | params norm: 11322.049 | params rms: 0.061305 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 18:00:19.102819 | finish at 2026-10-10 18:08:29 + [2026-10-09 00:09:35] iteration 5108/ 6855 | consumed samples: 653824 | elapsed time per iteration (ms): 85324.7 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.69% | FLOPs estimator: workload-aware | logical tokens/s: 196627.3 | valid tokens/s: 196627.2 | physical tokens/s: 197230.7 | loss tokens/s: 145651.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.423790E-01 | lm-global-supervised-tokens: 1.242768E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.133564E-02 | lm-token-buffer-mib: 5.809480E+01 | global_load_balancing_loss: 1.000400E+00 | loss scale: 1.0 | grad norm: 0.142 | params norm: 11321.999 | params rms: 0.061305 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 17:24:22.325245 | finish at 2026-10-10 17:33:57 + [2026-10-09 00:11:01] iteration 5109/ 6855 | consumed samples: 653952 | elapsed time per iteration (ms): 86134.9 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.63% | FLOPs estimator: workload-aware | logical tokens/s: 194778.1 | valid tokens/s: 194778.1 | physical tokens/s: 195187.5 | loss tokens/s: 142764.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.265217E-01 | lm-global-supervised-tokens: 1.229696E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.145765E-02 | lm-token-buffer-mib: 5.814445E+01 | global_load_balancing_loss: 1.000467E+00 | loss scale: 1.0 | grad norm: 0.094 | params norm: 11321.950 | params rms: 0.061305 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 17:46:31.495183 | finish at 2026-10-10 17:57:32 + [2026-10-09 00:12:50] iteration 5110/ 6855 | consumed samples: 654080 | elapsed time per iteration (ms): 108639.8 | throughput per GPU (TFLOP/s/GPU): 33.9 | MFU 10.86% | FLOPs estimator: workload-aware | logical tokens/s: 154429.8 | valid tokens/s: 154429.8 | physical tokens/s: 154770.3 | loss tokens/s: 113535.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.432294E-01 | lm-global-supervised-tokens: 1.233450E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.326066E-02 | lm-token-buffer-mib: 5.818224E+01 | global_load_balancing_loss: 1.000238E+00 | loss scale: 1.0 | grad norm: 0.143 | params norm: 11321.900 | params rms: 0.061304 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 4:39:36.382478 | finish at 2026-10-11 04:52:26 + [2026-10-09 00:14:16] iteration 5111/ 6855 | consumed samples: 654208 | elapsed time per iteration (ms): 86646.3 | throughput per GPU (TFLOP/s/GPU): 42.8 | MFU 13.72% | FLOPs estimator: workload-aware | logical tokens/s: 193628.3 | valid tokens/s: 193628.3 | physical tokens/s: 194087.7 | loss tokens/s: 141174.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.817421E-01 | lm-global-supervised-tokens: 1.223220E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.491783E-02 | lm-token-buffer-mib: 5.815234E+01 | global_load_balancing_loss: 1.000721E+00 | loss scale: 1.0 | grad norm: 0.098 | params norm: 11321.850 | params rms: 0.061304 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 17:58:31.129871 | finish at 2026-10-10 18:12:47 + [2026-10-09 00:15:43] iteration 5112/ 6855 | consumed samples: 654336 | elapsed time per iteration (ms): 87005.9 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.39% | FLOPs estimator: workload-aware | logical tokens/s: 192828.5 | valid tokens/s: 192828.5 | physical tokens/s: 193279.3 | loss tokens/s: 143236.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.491994E-01 | lm-global-supervised-tokens: 1.246244E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.508424E-02 | lm-token-buffer-mib: 5.812922E+01 | global_load_balancing_loss: 1.000540E+00 | loss scale: 1.0 | grad norm: 0.206 | params norm: 11321.799 | params rms: 0.061304 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 18:07:31.226604 | finish at 2026-10-10 18:23:14 + [2026-10-09 00:17:07] iteration 5113/ 6855 | consumed samples: 654464 | elapsed time per iteration (ms): 84253.3 | throughput per GPU (TFLOP/s/GPU): 43.6 | MFU 13.98% | FLOPs estimator: workload-aware | logical tokens/s: 199127.7 | valid tokens/s: 199127.7 | physical tokens/s: 199691.7 | loss tokens/s: 145773.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.961175E-01 | lm-global-supervised-tokens: 1.228192E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.238960E-02 | lm-token-buffer-mib: 5.840631E+01 | global_load_balancing_loss: 1.000832E+00 | loss scale: 1.0 | grad norm: 0.257 | params norm: 11321.748 | params rms: 0.061304 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 16:46:09.269282 | finish at 2026-10-10 17:03:17 + [2026-10-09 00:18:36] iteration 5114/ 6855 | consumed samples: 654592 | elapsed time per iteration (ms): 88962.0 | throughput per GPU (TFLOP/s/GPU): 41.7 | MFU 13.36% | FLOPs estimator: workload-aware | logical tokens/s: 188587.4 | valid tokens/s: 188587.4 | physical tokens/s: 189213.0 | loss tokens/s: 139544.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.805497E-01 | lm-global-supervised-tokens: 1.241416E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.198832E-02 | lm-token-buffer-mib: 5.806769E+01 | global_load_balancing_loss: 1.000545E+00 | loss scale: 1.0 | grad norm: 0.063 | params norm: 11321.695 | params rms: 0.061303 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 19:01:22.815664 | finish at 2026-10-10 19:19:59 + [2026-10-09 00:20:02] iteration 5115/ 6855 | consumed samples: 654720 | elapsed time per iteration (ms): 85123.4 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.56% | FLOPs estimator: workload-aware | logical tokens/s: 197092.6 | valid tokens/s: 197092.6 | physical tokens/s: 197854.1 | loss tokens/s: 147778.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.109019E-01 | lm-global-supervised-tokens: 1.257941E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.359451E-02 | lm-token-buffer-mib: 5.818961E+01 | global_load_balancing_loss: 9.997659E-01 | loss scale: 1.0 | grad norm: 0.172 | params norm: 11321.642 | params rms: 0.061303 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 17:08:34.750385 | finish at 2026-10-10 17:28:36 + [2026-10-09 00:21:26] iteration 5116/ 6855 | consumed samples: 654848 | elapsed time per iteration (ms): 84809.1 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.73% | FLOPs estimator: workload-aware | logical tokens/s: 197823.2 | valid tokens/s: 197823.2 | physical tokens/s: 198521.3 | loss tokens/s: 143288.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.459683E-01 | lm-global-supervised-tokens: 1.215220E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.529499E-02 | lm-token-buffer-mib: 5.851493E+01 | global_load_balancing_loss: 1.001215E+00 | loss scale: 1.0 | grad norm: 0.084 | params norm: 11321.589 | params rms: 0.061303 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 16:58:03.046308 | finish at 2026-10-10 17:19:29 + [2026-10-09 00:22:53] iteration 5117/ 6855 | consumed samples: 654976 | elapsed time per iteration (ms): 86688.8 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.73% | FLOPs estimator: workload-aware | logical tokens/s: 193532.2 | valid tokens/s: 193532.2 | physical tokens/s: 194304.9 | loss tokens/s: 147531.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.868142E-01 | lm-global-supervised-tokens: 1.278931E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.612924E-02 | lm-token-buffer-mib: 5.827182E+01 | global_load_balancing_loss: 9.999764E-01 | loss scale: 1.0 | grad norm: 0.131 | params norm: 11321.538 | params rms: 0.061302 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 17:51:05.212470 | finish at 2026-10-10 18:13:58 + [2026-10-09 00:24:22] iteration 5118/ 6855 | consumed samples: 655104 | elapsed time per iteration (ms): 89318.3 | throughput per GPU (TFLOP/s/GPU): 41.2 | MFU 13.21% | FLOPs estimator: workload-aware | logical tokens/s: 187836.1 | valid tokens/s: 187836.1 | physical tokens/s: 188280.6 | loss tokens/s: 136415.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.391451E-01 | lm-global-supervised-tokens: 1.218442E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.429050E-02 | lm-token-buffer-mib: 5.814317E+01 | global_load_balancing_loss: 1.000446E+00 | loss scale: 1.0 | grad norm: 0.133 | params norm: 11321.486 | params rms: 0.061302 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 19:05:45.810915 | finish at 2026-10-10 19:30:08 + [2026-10-09 00:25:52] iteration 5119/ 6855 | consumed samples: 655232 | elapsed time per iteration (ms): 89783.2 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.59% | FLOPs estimator: workload-aware | logical tokens/s: 186862.9 | valid tokens/s: 186862.9 | physical tokens/s: 187351.4 | loss tokens/s: 133075.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.495248E-01 | lm-global-supervised-tokens: 1.194796E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.179080E-02 | lm-token-buffer-mib: 5.806026E+01 | global_load_balancing_loss: 9.996341E-01 | loss scale: 1.0 | grad norm: 0.055 | params norm: 11321.434 | params rms: 0.061302 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 19:17:43.603788 | finish at 2026-10-10 19:43:36 + [2026-10-09 00:27:43] iteration 5120/ 6855 | consumed samples: 655360 | elapsed time per iteration (ms): 111060.9 | throughput per GPU (TFLOP/s/GPU): 33.4 | MFU 10.70% | FLOPs estimator: workload-aware | logical tokens/s: 151062.5 | valid tokens/s: 151062.5 | physical tokens/s: 151464.6 | loss tokens/s: 111401.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.640236E-01 | lm-global-supervised-tokens: 1.237232E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.149954E-02 | lm-token-buffer-mib: 5.813258E+01 | global_load_balancing_loss: 9.985461E-01 | loss scale: 1.0 | grad norm: 0.102 | params norm: 11321.381 | params rms: 0.061302 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 5:31:30.684204 | finish at 2026-10-11 05:59:14 +times across ranks (ms): + save-checkpoint: + rank 0: 24731.90 + rank 1: 24731.92 + rank 2: 24731.99 + rank 3: 24731.92 + rank 4: 24731.94 + rank 5: 24731.89 + rank 6: 24731.89 + rank 7: 24731.91 + rank 8: 24731.58 + rank 9: 24731.56 + rank 10: 24731.56 + rank 11: 24731.57 + rank 12: 24731.59 + rank 13: 24731.56 + rank 14: 24731.58 + rank 15: 24731.58 + rank 16: 24732.35 + rank 17: 24732.38 + rank 18: 24732.32 + rank 19: 24732.33 + rank 20: 24732.32 + rank 21: 24732.35 + rank 22: 24732.35 + rank 23: 24732.32 + rank 24: 24731.91 + rank 25: 24731.88 + rank 26: 24731.91 + rank 27: 24731.93 + rank 28: 24731.89 + rank 29: 24731.94 + rank 30: 24731.90 + rank 31: 24731.90 + rank 32: 24731.58 + rank 33: 24731.58 + rank 34: 24731.54 + rank 35: 24731.51 + rank 36: 24731.53 + rank 37: 24731.53 + rank 38: 24731.54 + rank 39: 24731.55 + rank 40: 24731.69 + rank 41: 24731.70 + rank 42: 24731.69 + rank 43: 24731.70 + rank 44: 24731.68 + rank 45: 24731.69 + rank 46: 24731.69 + rank 47: 24731.68 + rank 48: 24732.09 + rank 49: 24732.06 + rank 50: 24732.04 + rank 51: 24732.06 + rank 52: 24732.04 + rank 53: 24732.03 + rank 54: 24732.05 + rank 55: 24732.03 + rank 56: 24731.41 + rank 57: 24731.38 + rank 58: 24731.36 + rank 59: 24731.37 + rank 60: 24731.36 + rank 61: 24731.40 + rank 62: 24731.38 + rank 63: 24731.38 + [2026-10-09 00:29:35] iteration 5121/ 6855 | consumed samples: 655488 | elapsed time per iteration (ms): 86732.5 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.68% | FLOPs estimator: workload-aware | logical tokens/s: 193436.2 | valid tokens/s: 193436.2 | physical tokens/s: 193795.7 | loss tokens/s: 141622.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.298504E-01 | lm-global-supervised-tokens: 1.228324E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 9.131474E-01 | lm-token-buffer-mib: 5.804391E+01 | global_load_balancing_loss: 1.000983E+00 | loss scale: 1.0 | grad norm: 0.053 | params norm: 11321.328 | params rms: 0.061301 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 17:46:34.128674 | finish at 2026-10-10 18:16:09 + [2026-10-09 00:31:11] iteration 5122/ 6855 | consumed samples: 655616 | elapsed time per iteration (ms): 95594.4 | throughput per GPU (TFLOP/s/GPU): 39.5 | MFU 12.65% | FLOPs estimator: workload-aware | logical tokens/s: 175503.5 | valid tokens/s: 175503.5 | physical tokens/s: 175929.1 | loss tokens/s: 119995.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.175199E-01 | lm-global-supervised-tokens: 1.147092E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.500768E-02 | lm-token-buffer-mib: 5.814781E+01 | global_load_balancing_loss: 1.000452E+00 | loss scale: 1.0 | grad norm: 0.061 | params norm: 11321.275 | params rms: 0.061301 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 22:01:05.147964 | finish at 2026-10-10 22:32:16 + [2026-10-09 00:32:39] iteration 5123/ 6855 | consumed samples: 655744 | elapsed time per iteration (ms): 88212.6 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.62% | FLOPs estimator: workload-aware | logical tokens/s: 190190.1 | valid tokens/s: 190190.1 | physical tokens/s: 190637.3 | loss tokens/s: 145496.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.840183E-01 | lm-global-supervised-tokens: 1.283463E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.384963E-02 | lm-token-buffer-mib: 5.814143E+01 | global_load_balancing_loss: 1.000519E+00 | loss scale: 1.0 | grad norm: 0.078 | params norm: 11321.222 | params rms: 0.061301 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 18:26:24.202540 | finish at 2026-10-10 18:59:03 + [2026-10-09 00:34:08] iteration 5124/ 6855 | consumed samples: 655872 | elapsed time per iteration (ms): 89137.6 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.41% | FLOPs estimator: workload-aware | logical tokens/s: 188216.4 | valid tokens/s: 188216.4 | physical tokens/s: 188692.2 | loss tokens/s: 138934.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.625106E-01 | lm-global-supervised-tokens: 1.238426E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.407992E-02 | lm-token-buffer-mib: 5.825230E+01 | global_load_balancing_loss: 1.000341E+00 | loss scale: 1.0 | grad norm: 0.061 | params norm: 11321.169 | params rms: 0.061300 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 18:51:37.198299 | finish at 2026-10-10 19:25:45 + [2026-10-09 00:35:32] iteration 5125/ 6855 | consumed samples: 656000 | elapsed time per iteration (ms): 83613.4 | throughput per GPU (TFLOP/s/GPU): 43.8 | MFU 14.05% | FLOPs estimator: workload-aware | logical tokens/s: 200651.7 | valid tokens/s: 200651.7 | physical tokens/s: 201218.6 | loss tokens/s: 151436.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.745226E-01 | lm-global-supervised-tokens: 1.266211E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.210703E-02 | lm-token-buffer-mib: 5.826774E+01 | global_load_balancing_loss: 1.000833E+00 | loss scale: 1.0 | grad norm: 0.089 | params norm: 11321.116 | params rms: 0.061300 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 16:10:51.216617 | finish at 2026-10-10 16:46:23 + [2026-10-09 00:36:55] iteration 5126/ 6855 | consumed samples: 656128 | elapsed time per iteration (ms): 83762.9 | throughput per GPU (TFLOP/s/GPU): 43.4 | MFU 13.91% | FLOPs estimator: workload-aware | logical tokens/s: 200292.8 | valid tokens/s: 200292.8 | physical tokens/s: 200944.0 | loss tokens/s: 143957.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.551560E-01 | lm-global-supervised-tokens: 1.205825E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.263524E-02 | lm-token-buffer-mib: 5.816671E+01 | global_load_balancing_loss: 1.000354E+00 | loss scale: 1.0 | grad norm: 0.055 | params norm: 11321.062 | params rms: 0.061300 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 16:13:46.045228 | finish at 2026-10-10 16:50:41 + [2026-10-09 00:38:23] iteration 5127/ 6855 | consumed samples: 656256 | elapsed time per iteration (ms): 87610.8 | throughput per GPU (TFLOP/s/GPU): 43.8 | MFU 14.04% | FLOPs estimator: workload-aware | logical tokens/s: 191496.8 | valid tokens/s: 191496.7 | physical tokens/s: 191927.0 | loss tokens/s: 145965.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.983606E-01 | lm-global-supervised-tokens: 1.278818E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.156862E-02 | lm-token-buffer-mib: 5.808516E+01 | global_load_balancing_loss: 9.991904E-01 | loss scale: 1.0 | grad norm: 0.057 | params norm: 11321.007 | params rms: 0.061300 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 18:03:11.461624 | finish at 2026-10-10 18:41:34 + [2026-10-09 00:39:51] iteration 5128/ 6855 | consumed samples: 656384 | elapsed time per iteration (ms): 88027.2 | throughput per GPU (TFLOP/s/GPU): 42.8 | MFU 13.71% | FLOPs estimator: workload-aware | logical tokens/s: 190590.9 | valid tokens/s: 190590.9 | physical tokens/s: 190917.5 | loss tokens/s: 142721.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.458074E-01 | lm-global-supervised-tokens: 1.256334E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.477890E-02 | lm-token-buffer-mib: 5.807562E+01 | global_load_balancing_loss: 1.000010E+00 | loss scale: 1.0 | grad norm: 0.060 | params norm: 11320.951 | params rms: 0.061299 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 18:13:42.996194 | finish at 2026-10-10 18:53:34 + [2026-10-09 00:41:19] iteration 5129/ 6855 | consumed samples: 656512 | elapsed time per iteration (ms): 88243.5 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.57% | FLOPs estimator: workload-aware | logical tokens/s: 190124.0 | valid tokens/s: 190124.0 | physical tokens/s: 190538.0 | loss tokens/s: 144633.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.931925E-01 | lm-global-supervised-tokens: 1.276296E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.160010E-02 | lm-token-buffer-mib: 5.820275E+01 | global_load_balancing_loss: 9.997107E-01 | loss scale: 1.0 | grad norm: 0.047 | params norm: 11320.897 | params rms: 0.061299 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 18:18:28.331194 | finish at 2026-10-10 18:59:47 + [2026-10-09 00:43:06] iteration 5130/ 6855 | consumed samples: 656640 | elapsed time per iteration (ms): 106974.4 | throughput per GPU (TFLOP/s/GPU): 33.8 | MFU 10.83% | FLOPs estimator: workload-aware | logical tokens/s: 156834.0 | valid tokens/s: 156834.0 | physical tokens/s: 157225.7 | loss tokens/s: 119081.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.399280E-01 | lm-global-supervised-tokens: 1.273863E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.450182E-02 | lm-token-buffer-mib: 5.805795E+01 | global_load_balancing_loss: 1.000612E+00 | loss scale: 1.0 | grad norm: 0.045 | params norm: 11320.844 | params rms: 0.061299 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 3:15:30.782908 | finish at 2026-10-11 03:58:37 + [2026-10-09 00:44:35] iteration 5131/ 6855 | consumed samples: 656768 | elapsed time per iteration (ms): 88531.0 | throughput per GPU (TFLOP/s/GPU): 41.6 | MFU 13.33% | FLOPs estimator: workload-aware | logical tokens/s: 189506.3 | valid tokens/s: 189506.3 | physical tokens/s: 189815.9 | loss tokens/s: 140494.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.518613E-01 | lm-global-supervised-tokens: 1.243813E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.284300E-02 | lm-token-buffer-mib: 5.810855E+01 | global_load_balancing_loss: 1.001345E+00 | loss scale: 1.0 | grad norm: 0.048 | params norm: 11320.791 | params rms: 0.061298 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 18:23:47.475475 | finish at 2026-10-10 19:08:22 + [2026-10-09 00:46:05] iteration 5132/ 6855 | consumed samples: 656896 | elapsed time per iteration (ms): 90361.0 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.52% | FLOPs estimator: workload-aware | logical tokens/s: 185668.3 | valid tokens/s: 185668.3 | physical tokens/s: 186067.7 | loss tokens/s: 131921.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.731804E-01 | lm-global-supervised-tokens: 1.192053E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.182850E-02 | lm-token-buffer-mib: 5.803462E+01 | global_load_balancing_loss: 1.000426E+00 | loss scale: 1.0 | grad norm: 0.079 | params norm: 11320.739 | params rms: 0.061298 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 19:14:52.040898 | finish at 2026-10-10 20:00:57 + [2026-10-09 00:47:34] iteration 5133/ 6855 | consumed samples: 657024 | elapsed time per iteration (ms): 89395.3 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.53% | FLOPs estimator: workload-aware | logical tokens/s: 187673.2 | valid tokens/s: 187673.1 | physical tokens/s: 188023.0 | loss tokens/s: 142202.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.335724E-01 | lm-global-supervised-tokens: 1.271221E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.509502E-02 | lm-token-buffer-mib: 5.807786E+01 | global_load_balancing_loss: 9.996961E-01 | loss scale: 1.0 | grad norm: 0.058 | params norm: 11320.684 | params rms: 0.061298 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 18:45:38.687562 | finish at 2026-10-10 19:33:13 + [2026-10-09 00:49:02] iteration 5134/ 6855 | consumed samples: 657152 | elapsed time per iteration (ms): 87341.8 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.65% | FLOPs estimator: workload-aware | logical tokens/s: 192085.9 | valid tokens/s: 192085.9 | physical tokens/s: 192489.0 | loss tokens/s: 144740.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.476135E-01 | lm-global-supervised-tokens: 1.264185E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.576812E-02 | lm-token-buffer-mib: 5.817529E+01 | global_load_balancing_loss: 1.000971E+00 | loss scale: 1.0 | grad norm: 0.068 | params norm: 11320.628 | params rms: 0.061298 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 17:45:15.264016 | finish at 2026-10-10 18:34:17 + [2026-10-09 00:50:32] iteration 5135/ 6855 | consumed samples: 657280 | elapsed time per iteration (ms): 89979.6 | throughput per GPU (TFLOP/s/GPU): 41.4 | MFU 13.25% | FLOPs estimator: workload-aware | logical tokens/s: 186455.3 | valid tokens/s: 186455.3 | physical tokens/s: 186823.7 | loss tokens/s: 139577.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.529917E-01 | lm-global-supervised-tokens: 1.255912E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.348692E-02 | lm-token-buffer-mib: 5.813712E+01 | global_load_balancing_loss: 1.000767E+00 | loss scale: 1.0 | grad norm: 0.112 | params norm: 11320.573 | params rms: 0.061297 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 18:59:24.929552 | finish at 2026-10-10 19:49:57 + [2026-10-09 00:52:00] iteration 5136/ 6855 | consumed samples: 657408 | elapsed time per iteration (ms): 88586.5 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.39% | FLOPs estimator: workload-aware | logical tokens/s: 189387.2 | valid tokens/s: 189387.2 | physical tokens/s: 189742.6 | loss tokens/s: 135786.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.101826E-01 | lm-global-supervised-tokens: 1.202883E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.387159E-02 | lm-token-buffer-mib: 5.801283E+01 | global_load_balancing_loss: 1.000473E+00 | loss scale: 1.0 | grad norm: 0.143 | params norm: 11320.521 | params rms: 0.061297 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 18:18:00.143378 | finish at 2026-10-10 19:10:00 + [2026-10-09 00:53:27] iteration 5137/ 6855 | consumed samples: 657536 | elapsed time per iteration (ms): 86923.5 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.60% | FLOPs estimator: workload-aware | logical tokens/s: 193010.4 | valid tokens/s: 193010.4 | physical tokens/s: 193539.1 | loss tokens/s: 143289.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.331121E-01 | lm-global-supervised-tokens: 1.245522E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.393489E-02 | lm-token-buffer-mib: 5.830945E+01 | global_load_balancing_loss: 9.997135E-01 | loss scale: 1.0 | grad norm: 0.117 | params norm: 11320.467 | params rms: 0.061297 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 17:28:54.651749 | finish at 2026-10-10 18:22:22 + [2026-10-09 00:54:55] iteration 5138/ 6855 | consumed samples: 657664 | elapsed time per iteration (ms): 87414.8 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.45% | FLOPs estimator: workload-aware | logical tokens/s: 191926.1 | valid tokens/s: 191926.1 | physical tokens/s: 192483.1 | loss tokens/s: 144734.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.424760E-01 | lm-global-supervised-tokens: 1.265191E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.252802E-02 | lm-token-buffer-mib: 5.809569E+01 | global_load_balancing_loss: 9.996678E-01 | loss scale: 1.0 | grad norm: 0.036 | params norm: 11320.412 | params rms: 0.061296 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 17:41:31.226215 | finish at 2026-10-10 18:36:26 + [2026-10-09 00:56:21] iteration 5139/ 6855 | consumed samples: 657792 | elapsed time per iteration (ms): 86379.7 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.62% | FLOPs estimator: workload-aware | logical tokens/s: 194225.8 | valid tokens/s: 194225.8 | physical tokens/s: 194573.9 | loss tokens/s: 149954.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.524742E-01 | lm-global-supervised-tokens: 1.295298E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.105687E-02 | lm-token-buffer-mib: 5.806052E+01 | global_load_balancing_loss: 9.993194E-01 | loss scale: 1.0 | grad norm: 0.094 | params norm: 11320.356 | params rms: 0.061296 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 17:10:27.539331 | finish at 2026-10-10 18:06:49 + [2026-10-09 00:58:06] iteration 5140/ 6855 | consumed samples: 657920 | elapsed time per iteration (ms): 104821.6 | throughput per GPU (TFLOP/s/GPU): 34.5 | MFU 11.07% | FLOPs estimator: workload-aware | logical tokens/s: 160054.5 | valid tokens/s: 160054.5 | physical tokens/s: 160500.0 | loss tokens/s: 118159.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 5.982787E-01 | lm-global-supervised-tokens: 1.238563E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.102256E-02 | lm-token-buffer-mib: 5.816511E+01 | global_load_balancing_loss: 1.000001E+00 | loss scale: 1.0 | grad norm: 0.058 | params norm: 11320.303 | params rms: 0.061296 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 1:56:09.047612 | finish at 2026-10-11 02:54:15 + [2026-10-09 00:59:32] iteration 5141/ 6855 | consumed samples: 658048 | elapsed time per iteration (ms): 86065.4 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.60% | FLOPs estimator: workload-aware | logical tokens/s: 194934.9 | valid tokens/s: 194934.9 | physical tokens/s: 195678.6 | loss tokens/s: 144242.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.379238E-01 | lm-global-supervised-tokens: 1.241426E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.516724E-02 | lm-token-buffer-mib: 5.810538E+01 | global_load_balancing_loss: 9.997227E-01 | loss scale: 1.0 | grad norm: 0.049 | params norm: 11320.248 | params rms: 0.061295 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 16:58:36.156701 | finish at 2026-10-10 17:58:08 + [2026-10-09 01:00:59] iteration 5142/ 6855 | consumed samples: 658176 | elapsed time per iteration (ms): 86739.5 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.39% | FLOPs estimator: workload-aware | logical tokens/s: 193420.3 | valid tokens/s: 193420.3 | physical tokens/s: 193898.8 | loss tokens/s: 138946.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 5.933084E-01 | lm-global-supervised-tokens: 1.205212E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.498018E-02 | lm-token-buffer-mib: 5.817228E+01 | global_load_balancing_loss: 1.000758E+00 | loss scale: 1.0 | grad norm: 0.033 | params norm: 11320.191 | params rms: 0.061295 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 17:16:24.786858 | finish at 2026-10-10 18:17:23 + [2026-10-09 01:02:28] iteration 5143/ 6855 | consumed samples: 658304 | elapsed time per iteration (ms): 89653.5 | throughput per GPU (TFLOP/s/GPU): 41.5 | MFU 13.31% | FLOPs estimator: workload-aware | logical tokens/s: 187133.3 | valid tokens/s: 187133.3 | physical tokens/s: 187474.5 | loss tokens/s: 145231.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.197562E-01 | lm-global-supervised-tokens: 1.302050E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.544689E-02 | lm-token-buffer-mib: 5.805627E+01 | global_load_balancing_loss: 1.000233E+00 | loss scale: 1.0 | grad norm: 0.041 | params norm: 11320.135 | params rms: 0.061295 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 18:38:06.820709 | finish at 2026-10-10 19:40:35 + [2026-10-09 01:03:54] iteration 5144/ 6855 | consumed samples: 658432 | elapsed time per iteration (ms): 85864.4 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.80% | FLOPs estimator: workload-aware | logical tokens/s: 195390.2 | valid tokens/s: 195390.2 | physical tokens/s: 195897.7 | loss tokens/s: 145344.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.586678E-01 | lm-global-supervised-tokens: 1.247993E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.242068E-02 | lm-token-buffer-mib: 5.823954E+01 | global_load_balancing_loss: 1.000337E+00 | loss scale: 1.0 | grad norm: 0.047 | params norm: 11320.082 | params rms: 0.061295 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 16:48:34.020065 | finish at 2026-10-10 17:52:28 + [2026-10-09 01:05:22] iteration 5145/ 6855 | consumed samples: 658560 | elapsed time per iteration (ms): 87684.6 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.43% | FLOPs estimator: workload-aware | logical tokens/s: 191335.3 | valid tokens/s: 191335.2 | physical tokens/s: 191934.2 | loss tokens/s: 138092.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.638786E-01 | lm-global-supervised-tokens: 1.210861E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.599902E-02 | lm-token-buffer-mib: 5.830662E+01 | global_load_balancing_loss: 1.001411E+00 | loss scale: 1.0 | grad norm: 0.066 | params norm: 11320.026 | params rms: 0.061294 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 17:39:00.606673 | finish at 2026-10-10 18:44:22 + [2026-10-09 01:06:58] iteration 5146/ 6855 | consumed samples: 658688 | elapsed time per iteration (ms): 96585.7 | throughput per GPU (TFLOP/s/GPU): 38.9 | MFU 12.48% | FLOPs estimator: workload-aware | logical tokens/s: 173702.9 | valid tokens/s: 173702.9 | physical tokens/s: 174027.7 | loss tokens/s: 133705.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.971413E-01 | lm-global-supervised-tokens: 1.291399E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.155160E-02 | lm-token-buffer-mib: 5.812573E+01 | global_load_balancing_loss: 9.997120E-01 | loss scale: 1.0 | grad norm: 0.043 | params norm: 11319.970 | params rms: 0.061294 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 21:51:04.885165 | finish at 2026-10-10 22:58:03 + [2026-10-09 01:08:26] iteration 5147/ 6855 | consumed samples: 658816 | elapsed time per iteration (ms): 87536.1 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.47% | FLOPs estimator: workload-aware | logical tokens/s: 191660.6 | valid tokens/s: 191660.6 | physical tokens/s: 192133.9 | loss tokens/s: 143820.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.737722E-01 | lm-global-supervised-tokens: 1.258944E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.481191E-02 | lm-token-buffer-mib: 5.822053E+01 | global_load_balancing_loss: 9.996989E-01 | loss scale: 1.0 | grad norm: 0.080 | params norm: 11319.916 | params rms: 0.061294 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 17:31:51.586570 | finish at 2026-10-10 18:40:18 + [2026-10-09 01:09:54] iteration 5148/ 6855 | consumed samples: 658944 | elapsed time per iteration (ms): 88310.1 | throughput per GPU (TFLOP/s/GPU): 41.5 | MFU 13.31% | FLOPs estimator: workload-aware | logical tokens/s: 189980.2 | valid tokens/s: 189980.2 | physical tokens/s: 190447.7 | loss tokens/s: 139430.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.774498E-01 | lm-global-supervised-tokens: 1.231312E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.283804E-02 | lm-token-buffer-mib: 5.818128E+01 | global_load_balancing_loss: 9.998615E-01 | loss scale: 1.0 | grad norm: 0.072 | params norm: 11319.864 | params rms: 0.061293 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 17:52:25.289148 | finish at 2026-10-10 19:02:20 + [2026-10-09 01:11:20] iteration 5149/ 6855 | consumed samples: 659072 | elapsed time per iteration (ms): 85511.7 | throughput per GPU (TFLOP/s/GPU): 43.1 | MFU 13.83% | FLOPs estimator: workload-aware | logical tokens/s: 196197.9 | valid tokens/s: 196197.9 | physical tokens/s: 196522.2 | loss tokens/s: 149068.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.704156E-01 | lm-global-supervised-tokens: 1.274709E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.845859E-02 | lm-token-buffer-mib: 5.821158E+01 | global_load_balancing_loss: 1.000419E+00 | loss scale: 1.0 | grad norm: 0.043 | params norm: 11319.811 | params rms: 0.061293 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 16:31:22.956394 | finish at 2026-10-10 17:42:43 + [2026-10-09 01:13:06] iteration 5150/ 6855 | consumed samples: 659200 | elapsed time per iteration (ms): 106495.0 | throughput per GPU (TFLOP/s/GPU): 34.2 | MFU 10.97% | FLOPs estimator: workload-aware | logical tokens/s: 157540.0 | valid tokens/s: 157540.0 | physical tokens/s: 158009.7 | loss tokens/s: 115317.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.608936E-01 | lm-global-supervised-tokens: 1.228067E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.155555E-02 | lm-token-buffer-mib: 5.811992E+01 | global_load_balancing_loss: 1.000311E+00 | loss scale: 1.0 | grad norm: 0.054 | params norm: 11319.757 | params rms: 0.061293 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 2:26:13.900415 | finish at 2026-10-11 03:39:20 + [2026-10-09 01:14:32] iteration 5151/ 6855 | consumed samples: 659328 | elapsed time per iteration (ms): 85330.9 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.67% | FLOPs estimator: workload-aware | logical tokens/s: 196613.5 | valid tokens/s: 196613.5 | physical tokens/s: 197041.2 | loss tokens/s: 141906.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.363732E-01 | lm-global-supervised-tokens: 1.210898E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.159051E-02 | lm-token-buffer-mib: 5.817745E+01 | global_load_balancing_loss: 1.001217E+00 | loss scale: 1.0 | grad norm: 0.072 | params norm: 11319.703 | params rms: 0.061293 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 16:23:23.810457 | finish at 2026-10-10 17:37:55 + [2026-10-09 01:15:59] iteration 5152/ 6855 | consumed samples: 659456 | elapsed time per iteration (ms): 86958.4 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.56% | FLOPs estimator: workload-aware | logical tokens/s: 192933.2 | valid tokens/s: 192933.2 | physical tokens/s: 193399.3 | loss tokens/s: 144517.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.270269E-01 | lm-global-supervised-tokens: 1.256699E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.915999E-02 | lm-token-buffer-mib: 5.820010E+01 | global_load_balancing_loss: 9.998271E-01 | loss scale: 1.0 | grad norm: 0.064 | params norm: 11319.654 | params rms: 0.061292 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 17:08:10.195009 | finish at 2026-10-10 18:24:09 + [2026-10-09 01:17:26] iteration 5153/ 6855 | consumed samples: 659584 | elapsed time per iteration (ms): 87105.8 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.41% | FLOPs estimator: workload-aware | logical tokens/s: 192606.4 | valid tokens/s: 192606.4 | physical tokens/s: 193092.3 | loss tokens/s: 141886.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.172446E-01 | lm-global-supervised-tokens: 1.235914E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.168562E-02 | lm-token-buffer-mib: 5.816203E+01 | global_load_balancing_loss: 9.997756E-01 | loss scale: 1.0 | grad norm: 0.064 | params norm: 11319.606 | params rms: 0.061292 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 17:10:54.055627 | finish at 2026-10-10 18:28:20 + [2026-10-09 01:18:53] iteration 5154/ 6855 | consumed samples: 659712 | elapsed time per iteration (ms): 87681.6 | throughput per GPU (TFLOP/s/GPU): 41.6 | MFU 13.34% | FLOPs estimator: workload-aware | logical tokens/s: 191342.1 | valid tokens/s: 191342.1 | physical tokens/s: 191639.0 | loss tokens/s: 136178.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.549588E-01 | lm-global-supervised-tokens: 1.194033E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.907269E-02 | lm-token-buffer-mib: 5.806583E+01 | global_load_balancing_loss: 1.001143E+00 | loss scale: 1.0 | grad norm: 0.033 | params norm: 11319.555 | params rms: 0.061292 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 17:25:46.391215 | finish at 2026-10-10 18:44:40 + [2026-10-09 01:20:20] iteration 5155/ 6855 | consumed samples: 659840 | elapsed time per iteration (ms): 87056.5 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.44% | FLOPs estimator: workload-aware | logical tokens/s: 192715.8 | valid tokens/s: 192715.7 | physical tokens/s: 193190.3 | loss tokens/s: 144143.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.799912E-01 | lm-global-supervised-tokens: 1.254865E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.633444E-02 | lm-token-buffer-mib: 5.804876E+01 | global_load_balancing_loss: 9.991775E-01 | loss scale: 1.0 | grad norm: 0.039 | params norm: 11319.502 | params rms: 0.061291 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 17:06:35.974135 | finish at 2026-10-10 18:26:56 + [2026-10-09 01:21:47] iteration 5156/ 6855 | consumed samples: 659968 | elapsed time per iteration (ms): 86997.7 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.66% | FLOPs estimator: workload-aware | logical tokens/s: 192846.5 | valid tokens/s: 192846.5 | physical tokens/s: 193316.0 | loss tokens/s: 142472.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.358505E-01 | lm-global-supervised-tokens: 1.239478E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.176073E-02 | lm-token-buffer-mib: 5.817181E+01 | global_load_balancing_loss: 9.993432E-01 | loss scale: 1.0 | grad norm: 0.037 | params norm: 11319.450 | params rms: 0.061291 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 17:03:29.078082 | finish at 2026-10-10 18:25:17 + [2026-10-09 01:23:15] iteration 5157/ 6855 | consumed samples: 660096 | elapsed time per iteration (ms): 87998.8 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.61% | FLOPs estimator: workload-aware | logical tokens/s: 190652.3 | valid tokens/s: 190652.3 | physical tokens/s: 191069.0 | loss tokens/s: 135227.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.375625E-01 | lm-global-supervised-tokens: 1.189986E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.097796E-02 | lm-token-buffer-mib: 5.825105E+01 | global_load_balancing_loss: 1.000714E+00 | loss scale: 1.0 | grad norm: 0.039 | params norm: 11319.398 | params rms: 0.061291 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 17:30:22.041409 | finish at 2026-10-10 18:53:37 + [2026-10-09 01:24:41] iteration 5158/ 6855 | consumed samples: 660224 | elapsed time per iteration (ms): 85971.8 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.74% | FLOPs estimator: workload-aware | logical tokens/s: 195147.3 | valid tokens/s: 195147.3 | physical tokens/s: 195675.4 | loss tokens/s: 144207.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.894486E-01 | lm-global-supervised-tokens: 1.239778E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.164155E-02 | lm-token-buffer-mib: 5.836215E+01 | global_load_balancing_loss: 9.997287E-01 | loss scale: 1.0 | grad norm: 0.054 | params norm: 11319.345 | params rms: 0.061291 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 16:31:34.205845 | finish at 2026-10-10 17:56:16 + [2026-10-09 01:26:08] iteration 5159/ 6855 | consumed samples: 660352 | elapsed time per iteration (ms): 86121.0 | throughput per GPU (TFLOP/s/GPU): 41.4 | MFU 13.27% | FLOPs estimator: workload-aware | logical tokens/s: 194809.8 | valid tokens/s: 194809.8 | physical tokens/s: 195360.3 | loss tokens/s: 143852.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.368852E-01 | lm-global-supervised-tokens: 1.238871E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.509394E-02 | lm-token-buffer-mib: 5.812428E+01 | global_load_balancing_loss: 9.997429E-01 | loss scale: 1.0 | grad norm: 0.068 | params norm: 11319.292 | params rms: 0.061290 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 16:34:21.203957 | finish at 2026-10-10 18:00:29 + [2026-10-09 01:27:58] iteration 5160/ 6855 | consumed samples: 660480 | elapsed time per iteration (ms): 110227.1 | throughput per GPU (TFLOP/s/GPU): 33.6 | MFU 10.78% | FLOPs estimator: workload-aware | logical tokens/s: 152205.5 | valid tokens/s: 152205.5 | physical tokens/s: 152636.3 | loss tokens/s: 115746.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.185017E-01 | lm-global-supervised-tokens: 1.275842E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.284991E-02 | lm-token-buffer-mib: 5.820139E+01 | global_load_balancing_loss: 1.000132E+00 | loss scale: 1.0 | grad norm: 0.175 | params norm: 11319.240 | params rms: 0.061290 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 3:53:54.890274 | finish at 2026-10-11 05:21:53 +times across ranks (ms): + save-checkpoint: + rank 0: 24836.96 + rank 1: 24837.00 + rank 2: 24836.98 + rank 3: 24836.97 + rank 4: 24836.99 + rank 5: 24836.94 + rank 6: 24836.94 + rank 7: 24836.98 + rank 8: 24837.62 + rank 9: 24837.57 + rank 10: 24837.56 + rank 11: 24837.57 + rank 12: 24837.59 + rank 13: 24837.55 + rank 14: 24837.57 + rank 15: 24837.56 + rank 16: 24837.88 + rank 17: 24837.90 + rank 18: 24837.87 + rank 19: 24837.91 + rank 20: 24837.86 + rank 21: 24837.89 + rank 22: 24837.85 + rank 23: 24837.86 + rank 24: 24838.31 + rank 25: 24838.26 + rank 26: 24838.29 + rank 27: 24838.29 + rank 28: 24838.29 + rank 29: 24838.26 + rank 30: 24838.27 + rank 31: 24838.26 + rank 32: 24837.81 + rank 33: 24837.76 + rank 34: 24837.69 + rank 35: 24837.68 + rank 36: 24837.66 + rank 37: 24837.65 + rank 38: 24837.65 + rank 39: 24837.65 + rank 40: 24837.45 + rank 41: 24837.44 + rank 42: 24837.44 + rank 43: 24837.45 + rank 44: 24837.42 + rank 45: 24837.46 + rank 46: 24837.43 + rank 47: 24837.42 + rank 48: 24836.86 + rank 49: 24836.95 + rank 50: 24836.86 + rank 51: 24836.85 + rank 52: 24836.96 + rank 53: 24836.83 + rank 54: 24836.86 + rank 55: 24836.88 + rank 56: 24837.46 + rank 57: 24837.46 + rank 58: 24837.47 + rank 59: 24837.47 + rank 60: 24837.45 + rank 61: 24837.44 + rank 62: 24837.49 + rank 63: 24837.43 + [2026-10-09 01:30:07] iteration 5161/ 6855 | consumed samples: 660608 | elapsed time per iteration (ms): 104367.9 | throughput per GPU (TFLOP/s/GPU): 35.7 | MFU 11.44% | FLOPs estimator: workload-aware | logical tokens/s: 160750.1 | valid tokens/s: 160750.1 | physical tokens/s: 161029.9 | loss tokens/s: 116251.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.920484E-01 | lm-global-supervised-tokens: 1.213294E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 9.237215E-01 | lm-token-buffer-mib: 5.823135E+01 | global_load_balancing_loss: 9.993241E-01 | loss scale: 1.0 | grad norm: 0.219 | params norm: 11319.190 | params rms: 0.061290 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 1:06:39.260386 | finish at 2026-10-11 02:36:47 + [2026-10-09 01:31:36] iteration 5162/ 6855 | consumed samples: 660736 | elapsed time per iteration (ms): 88675.4 | throughput per GPU (TFLOP/s/GPU): 41.3 | MFU 13.23% | FLOPs estimator: workload-aware | logical tokens/s: 189198.0 | valid tokens/s: 189198.0 | physical tokens/s: 189734.4 | loss tokens/s: 142970.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.320759E-01 | lm-global-supervised-tokens: 1.267793E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.642308E-02 | lm-token-buffer-mib: 5.813331E+01 | global_load_balancing_loss: 1.000181E+00 | loss scale: 1.0 | grad norm: 0.054 | params norm: 11319.138 | params rms: 0.061289 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 17:42:07.394107 | finish at 2026-10-10 19:13:43 + [2026-10-09 01:33:00] iteration 5163/ 6855 | consumed samples: 660864 | elapsed time per iteration (ms): 84395.1 | throughput per GPU (TFLOP/s/GPU): 42.8 | MFU 13.71% | FLOPs estimator: workload-aware | logical tokens/s: 198793.4 | valid tokens/s: 198793.4 | physical tokens/s: 199607.2 | loss tokens/s: 143647.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.266117E-01 | lm-global-supervised-tokens: 1.212318E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.354012E-02 | lm-token-buffer-mib: 5.812260E+01 | global_load_balancing_loss: 1.001731E+00 | loss scale: 1.0 | grad norm: 0.367 | params norm: 11319.086 | params rms: 0.061289 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 15:39:56.501329 | finish at 2026-10-10 17:12:57 + [2026-10-09 01:34:29] iteration 5164/ 6855 | consumed samples: 660992 | elapsed time per iteration (ms): 88227.4 | throughput per GPU (TFLOP/s/GPU): 41.5 | MFU 13.30% | FLOPs estimator: workload-aware | logical tokens/s: 190158.2 | valid tokens/s: 190158.2 | physical tokens/s: 190622.1 | loss tokens/s: 144784.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.571339E-01 | lm-global-supervised-tokens: 1.277398E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.160147E-02 | lm-token-buffer-mib: 5.818127E+01 | global_load_balancing_loss: 1.001022E+00 | loss scale: 1.0 | grad norm: 0.388 | params norm: 11319.033 | params rms: 0.061289 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 17:26:32.596000 | finish at 2026-10-10 19:01:01 + [2026-10-09 01:35:54] iteration 5165/ 6855 | consumed samples: 661120 | elapsed time per iteration (ms): 85733.9 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.63% | FLOPs estimator: workload-aware | logical tokens/s: 195689.0 | valid tokens/s: 195689.0 | physical tokens/s: 196123.8 | loss tokens/s: 145117.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.474215E-01 | lm-global-supervised-tokens: 1.244152E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.293984E-02 | lm-token-buffer-mib: 5.809115E+01 | global_load_balancing_loss: 1.000475E+00 | loss scale: 1.0 | grad norm: 0.151 | params norm: 11318.980 | params rms: 0.061289 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 16:14:50.342693 | finish at 2026-10-10 17:50:45 + [2026-10-09 01:37:22] iteration 5166/ 6855 | consumed samples: 661248 | elapsed time per iteration (ms): 87275.6 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.59% | FLOPs estimator: workload-aware | logical tokens/s: 192232.1 | valid tokens/s: 192232.1 | physical tokens/s: 192658.1 | loss tokens/s: 141120.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.502254E-01 | lm-global-supervised-tokens: 1.231637E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.334964E-02 | lm-token-buffer-mib: 5.820413E+01 | global_load_balancing_loss: 1.001118E+00 | loss scale: 1.0 | grad norm: 0.123 | params norm: 11318.925 | params rms: 0.061288 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 16:56:48.479467 | finish at 2026-10-10 18:34:10 + [2026-10-09 01:38:50] iteration 5167/ 6855 | consumed samples: 661376 | elapsed time per iteration (ms): 88817.6 | throughput per GPU (TFLOP/s/GPU): 41.7 | MFU 13.38% | FLOPs estimator: workload-aware | logical tokens/s: 188894.2 | valid tokens/s: 188894.2 | physical tokens/s: 189221.6 | loss tokens/s: 138032.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.733671E-01 | lm-global-supervised-tokens: 1.225969E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.452532E-02 | lm-token-buffer-mib: 5.804557E+01 | global_load_balancing_loss: 1.000364E+00 | loss scale: 1.0 | grad norm: 0.134 | params norm: 11318.867 | params rms: 0.061288 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 17:38:44.107210 | finish at 2026-10-10 19:17:35 + [2026-10-09 01:40:19] iteration 5168/ 6855 | consumed samples: 661504 | elapsed time per iteration (ms): 88526.8 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.68% | FLOPs estimator: workload-aware | logical tokens/s: 189515.5 | valid tokens/s: 189515.5 | physical tokens/s: 189775.2 | loss tokens/s: 139813.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.850978E-01 | lm-global-supervised-tokens: 1.237727E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.545971E-02 | lm-token-buffer-mib: 5.807000E+01 | global_load_balancing_loss: 9.993808E-01 | loss scale: 1.0 | grad norm: 0.083 | params norm: 11318.811 | params rms: 0.061288 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 17:29:04.747257 | finish at 2026-10-10 19:09:24 + [2026-10-09 01:41:46] iteration 5169/ 6855 | consumed samples: 661632 | elapsed time per iteration (ms): 86962.6 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.58% | FLOPs estimator: workload-aware | logical tokens/s: 192924.4 | valid tokens/s: 192924.4 | physical tokens/s: 193338.7 | loss tokens/s: 136178.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.168040E-01 | lm-global-supervised-tokens: 1.184239E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.506988E-02 | lm-token-buffer-mib: 5.817949E+01 | global_load_balancing_loss: 9.996302E-01 | loss scale: 1.0 | grad norm: 0.099 | params norm: 11318.757 | params rms: 0.061287 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 16:43:38.914646 | finish at 2026-10-10 18:25:25 + [2026-10-09 01:43:34] iteration 5170/ 6855 | consumed samples: 661760 | elapsed time per iteration (ms): 108193.0 | throughput per GPU (TFLOP/s/GPU): 34.0 | MFU 10.91% | FLOPs estimator: workload-aware | logical tokens/s: 155067.1 | valid tokens/s: 155067.1 | physical tokens/s: 155393.6 | loss tokens/s: 119091.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.913591E-01 | lm-global-supervised-tokens: 1.288488E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.472464E-02 | lm-token-buffer-mib: 5.815493E+01 | global_load_balancing_loss: 9.978827E-01 | loss scale: 1.0 | grad norm: 0.134 | params norm: 11318.701 | params rms: 0.061287 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 2:38:25.168172 | finish at 2026-10-11 04:21:59 + [2026-10-09 01:45:00] iteration 5171/ 6855 | consumed samples: 661888 | elapsed time per iteration (ms): 86129.2 | throughput per GPU (TFLOP/s/GPU): 44.6 | MFU 14.28% | FLOPs estimator: workload-aware | logical tokens/s: 194790.8 | valid tokens/s: 194790.8 | physical tokens/s: 195103.4 | loss tokens/s: 140677.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.822878E-01 | lm-global-supervised-tokens: 1.211644E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.189182E-02 | lm-token-buffer-mib: 5.805641E+01 | global_load_balancing_loss: 9.986889E-01 | loss scale: 1.0 | grad norm: 0.063 | params norm: 11318.646 | params rms: 0.061287 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 16:17:21.548278 | finish at 2026-10-10 18:02:22 + [2026-10-09 01:46:29] iteration 5172/ 6855 | consumed samples: 662016 | elapsed time per iteration (ms): 88376.1 | throughput per GPU (TFLOP/s/GPU): 41.0 | MFU 13.15% | FLOPs estimator: workload-aware | logical tokens/s: 189838.7 | valid tokens/s: 189838.7 | physical tokens/s: 190306.6 | loss tokens/s: 148152.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.769888E-01 | lm-global-supervised-tokens: 1.309318E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.231373E-02 | lm-token-buffer-mib: 5.824055E+01 | global_load_balancing_loss: 1.000291E+00 | loss scale: 1.0 | grad norm: 0.062 | params norm: 11318.591 | params rms: 0.061286 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 17:18:57.008507 | finish at 2026-10-10 19:05:26 + [2026-10-09 01:48:00] iteration 5173/ 6855 | consumed samples: 662144 | elapsed time per iteration (ms): 91045.6 | throughput per GPU (TFLOP/s/GPU): 41.3 | MFU 13.23% | FLOPs estimator: workload-aware | logical tokens/s: 184272.1 | valid tokens/s: 184272.1 | physical tokens/s: 184553.0 | loss tokens/s: 132603.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.588194E-01 | lm-global-supervised-tokens: 1.207297E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.313759E-02 | lm-token-buffer-mib: 5.804129E+01 | global_load_balancing_loss: 1.002314E+00 | loss scale: 1.0 | grad norm: 0.071 | params norm: 11318.537 | params rms: 0.061286 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 18:32:18.708719 | finish at 2026-10-10 20:20:18 + [2026-10-09 01:49:32] iteration 5174/ 6855 | consumed samples: 662272 | elapsed time per iteration (ms): 91812.9 | throughput per GPU (TFLOP/s/GPU): 41.0 | MFU 13.13% | FLOPs estimator: workload-aware | logical tokens/s: 182732.4 | valid tokens/s: 182732.4 | physical tokens/s: 183076.9 | loss tokens/s: 131961.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.853770E-01 | lm-global-supervised-tokens: 1.211578E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.242614E-02 | lm-token-buffer-mib: 5.802336E+01 | global_load_balancing_loss: 1.002394E+00 | loss scale: 1.0 | grad norm: 0.036 | params norm: 11318.482 | params rms: 0.061286 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 18:52:17.426898 | finish at 2026-10-10 20:41:49 + [2026-10-09 01:50:58] iteration 5175/ 6855 | consumed samples: 662400 | elapsed time per iteration (ms): 86779.8 | throughput per GPU (TFLOP/s/GPU): 41.6 | MFU 13.33% | FLOPs estimator: workload-aware | logical tokens/s: 193330.9 | valid tokens/s: 193330.8 | physical tokens/s: 193840.0 | loss tokens/s: 139568.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 5.741221E-01 | lm-global-supervised-tokens: 1.211173E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.108196E-02 | lm-token-buffer-mib: 5.812355E+01 | global_load_balancing_loss: 1.002491E+00 | loss scale: 1.0 | grad norm: 0.064 | params norm: 11318.428 | params rms: 0.061286 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 16:29:50.007019 | finish at 2026-10-10 18:20:48 + [2026-10-09 01:52:26] iteration 5176/ 6855 | consumed samples: 662528 | elapsed time per iteration (ms): 87619.6 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.40% | FLOPs estimator: workload-aware | logical tokens/s: 191477.2 | valid tokens/s: 191477.2 | physical tokens/s: 191946.3 | loss tokens/s: 147260.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.921057E-01 | lm-global-supervised-tokens: 1.290287E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.554461E-02 | lm-token-buffer-mib: 5.813961E+01 | global_load_balancing_loss: 1.000386E+00 | loss scale: 1.0 | grad norm: 0.041 | params norm: 11318.375 | params rms: 0.061285 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 16:51:53.322507 | finish at 2026-10-10 18:44:19 + [2026-10-09 01:53:54] iteration 5177/ 6855 | consumed samples: 662656 | elapsed time per iteration (ms): 87984.7 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.40% | FLOPs estimator: workload-aware | logical tokens/s: 190683.2 | valid tokens/s: 190683.2 | physical tokens/s: 191339.5 | loss tokens/s: 141180.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.670675E-01 | lm-global-supervised-tokens: 1.242172E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.162641E-02 | lm-token-buffer-mib: 5.836219E+01 | global_load_balancing_loss: 9.989432E-01 | loss scale: 1.0 | grad norm: 0.075 | params norm: 11318.324 | params rms: 0.061285 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 17:00:38.296936 | finish at 2026-10-10 18:54:32 + [2026-10-09 01:55:20] iteration 5178/ 6855 | consumed samples: 662784 | elapsed time per iteration (ms): 85650.9 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.65% | FLOPs estimator: workload-aware | logical tokens/s: 195878.9 | valid tokens/s: 195878.9 | physical tokens/s: 196410.3 | loss tokens/s: 148738.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.539503E-01 | lm-global-supervised-tokens: 1.273960E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.732503E-02 | lm-token-buffer-mib: 5.810827E+01 | global_load_balancing_loss: 9.977757E-01 | loss scale: 1.0 | grad norm: 0.044 | params norm: 11318.271 | params rms: 0.061285 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 15:53:56.557911 | finish at 2026-10-10 17:49:16 + [2026-10-09 01:56:44] iteration 5179/ 6855 | consumed samples: 662912 | elapsed time per iteration (ms): 84789.3 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.64% | FLOPs estimator: workload-aware | logical tokens/s: 197869.4 | valid tokens/s: 197869.4 | physical tokens/s: 198377.1 | loss tokens/s: 147077.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.350988E-01 | lm-global-supervised-tokens: 1.247060E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.156451E-02 | lm-token-buffer-mib: 5.813567E+01 | global_load_balancing_loss: 9.988328E-01 | loss scale: 1.0 | grad norm: 0.054 | params norm: 11318.219 | params rms: 0.061284 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 15:28:26.871137 | finish at 2026-10-10 17:25:11 + [2026-10-09 01:58:37] iteration 5180/ 6855 | consumed samples: 663040 | elapsed time per iteration (ms): 112263.0 | throughput per GPU (TFLOP/s/GPU): 33.9 | MFU 10.86% | FLOPs estimator: workload-aware | logical tokens/s: 149445.6 | valid tokens/s: 149445.6 | physical tokens/s: 149714.3 | loss tokens/s: 107129.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.761795E-01 | lm-global-supervised-tokens: 1.202664E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.579220E-02 | lm-token-buffer-mib: 5.806358E+01 | global_load_balancing_loss: 1.000240E+00 | loss scale: 1.0 | grad norm: 0.041 | params norm: 11318.169 | params rms: 0.061284 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 4:14:00.459526 | finish at 2026-10-11 06:12:37 + [2026-10-09 02:00:05] iteration 5181/ 6855 | consumed samples: 663168 | elapsed time per iteration (ms): 88838.9 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.40% | FLOPs estimator: workload-aware | logical tokens/s: 188849.2 | valid tokens/s: 188849.2 | physical tokens/s: 189476.7 | loss tokens/s: 145816.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.275032E-01 | lm-global-supervised-tokens: 1.295419E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.386334E-02 | lm-token-buffer-mib: 5.818362E+01 | global_load_balancing_loss: 1.000487E+00 | loss scale: 1.0 | grad norm: 0.100 | params norm: 11318.117 | params rms: 0.061284 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 17:18:36.379814 | finish at 2026-10-10 19:18:42 + [2026-10-09 02:01:37] iteration 5182/ 6855 | consumed samples: 663296 | elapsed time per iteration (ms): 91741.0 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.51% | FLOPs estimator: workload-aware | logical tokens/s: 182875.6 | valid tokens/s: 182875.6 | physical tokens/s: 183232.5 | loss tokens/s: 132332.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.120627E-01 | lm-global-supervised-tokens: 1.214034E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.436352E-02 | lm-token-buffer-mib: 5.818552E+01 | global_load_balancing_loss: 1.001140E+00 | loss scale: 1.0 | grad norm: 0.063 | params norm: 11318.065 | params rms: 0.061284 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 18:38:02.693692 | finish at 2026-10-10 20:39:40 + [2026-10-09 02:03:04] iteration 5183/ 6855 | consumed samples: 663424 | elapsed time per iteration (ms): 87063.0 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.53% | FLOPs estimator: workload-aware | logical tokens/s: 192701.7 | valid tokens/s: 192701.7 | physical tokens/s: 193168.2 | loss tokens/s: 140333.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.224902E-01 | lm-global-supervised-tokens: 1.221781E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.582045E-02 | lm-token-buffer-mib: 5.811640E+01 | global_load_balancing_loss: 1.001804E+00 | loss scale: 1.0 | grad norm: 0.062 | params norm: 11318.013 | params rms: 0.061283 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 16:26:09.272955 | finish at 2026-10-10 18:29:14 + [2026-10-09 02:04:31] iteration 5184/ 6855 | consumed samples: 663552 | elapsed time per iteration (ms): 86467.7 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.47% | FLOPs estimator: workload-aware | logical tokens/s: 194028.3 | valid tokens/s: 194028.3 | physical tokens/s: 194472.5 | loss tokens/s: 147862.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.545478E-01 | lm-global-supervised-tokens: 1.278536E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.148460E-02 | lm-token-buffer-mib: 5.808202E+01 | global_load_balancing_loss: 1.000442E+00 | loss scale: 1.0 | grad norm: 0.080 | params norm: 11317.962 | params rms: 0.061283 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 16:08:07.579695 | finish at 2026-10-10 18:12:38 + [2026-10-09 02:05:59] iteration 5185/ 6855 | consumed samples: 663680 | elapsed time per iteration (ms): 88024.7 | throughput per GPU (TFLOP/s/GPU): 41.3 | MFU 13.23% | FLOPs estimator: workload-aware | logical tokens/s: 190595.4 | valid tokens/s: 190595.4 | physical tokens/s: 191092.6 | loss tokens/s: 137901.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.011474E-01 | lm-global-supervised-tokens: 1.213877E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.159731E-02 | lm-token-buffer-mib: 5.811011E+01 | global_load_balancing_loss: 9.998307E-01 | loss scale: 1.0 | grad norm: 0.059 | params norm: 11317.912 | params rms: 0.061283 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 16:50:01.318157 | finish at 2026-10-10 18:56:00 + [2026-10-09 02:07:27] iteration 5186/ 6855 | consumed samples: 663808 | elapsed time per iteration (ms): 88134.4 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.43% | FLOPs estimator: workload-aware | logical tokens/s: 190359.2 | valid tokens/s: 190359.2 | physical tokens/s: 191107.4 | loss tokens/s: 144003.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.530455E-01 | lm-global-supervised-tokens: 1.269165E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.184555E-02 | lm-token-buffer-mib: 5.811005E+01 | global_load_balancing_loss: 9.996726E-01 | loss scale: 1.0 | grad norm: 0.050 | params norm: 11317.862 | params rms: 0.061283 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 16:51:36.288349 | finish at 2026-10-10 18:59:03 + [2026-10-09 02:08:54] iteration 5187/ 6855 | consumed samples: 663936 | elapsed time per iteration (ms): 87090.1 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.63% | FLOPs estimator: workload-aware | logical tokens/s: 192641.6 | valid tokens/s: 192641.6 | physical tokens/s: 193026.1 | loss tokens/s: 144967.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.354045E-01 | lm-global-supervised-tokens: 1.262521E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.377258E-02 | lm-token-buffer-mib: 5.810328E+01 | global_load_balancing_loss: 9.998753E-01 | loss scale: 1.0 | grad norm: 0.087 | params norm: 11317.812 | params rms: 0.061282 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 16:21:06.358466 | finish at 2026-10-10 18:30:00 + [2026-10-09 02:10:24] iteration 5188/ 6855 | consumed samples: 664064 | elapsed time per iteration (ms): 90217.0 | throughput per GPU (TFLOP/s/GPU): 41.5 | MFU 13.30% | FLOPs estimator: workload-aware | logical tokens/s: 185964.4 | valid tokens/s: 185964.4 | physical tokens/s: 186409.7 | loss tokens/s: 134893.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.354021E-01 | lm-global-supervised-tokens: 1.216967E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.266764E-02 | lm-token-buffer-mib: 5.816750E+01 | global_load_balancing_loss: 9.995370E-01 | loss scale: 1.0 | grad norm: 0.096 | params norm: 11317.762 | params rms: 0.061282 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 17:46:31.779552 | finish at 2026-10-10 19:56:56 + [2026-10-09 02:11:52] iteration 5189/ 6855 | consumed samples: 664192 | elapsed time per iteration (ms): 88170.0 | throughput per GPU (TFLOP/s/GPU): 41.7 | MFU 13.38% | FLOPs estimator: workload-aware | logical tokens/s: 190282.7 | valid tokens/s: 190282.7 | physical tokens/s: 190726.4 | loss tokens/s: 138340.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.701292E-01 | lm-global-supervised-tokens: 1.219746E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.169889E-02 | lm-token-buffer-mib: 5.829568E+01 | global_load_balancing_loss: 1.000172E+00 | loss scale: 1.0 | grad norm: 0.081 | params norm: 11317.712 | params rms: 0.061282 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 16:48:11.137111 | finish at 2026-10-10 19:00:03 + [2026-10-09 02:13:42] iteration 5190/ 6855 | consumed samples: 664320 | elapsed time per iteration (ms): 109396.0 | throughput per GPU (TFLOP/s/GPU): 33.6 | MFU 10.78% | FLOPs estimator: workload-aware | logical tokens/s: 153362.0 | valid tokens/s: 153362.0 | physical tokens/s: 153599.2 | loss tokens/s: 110310.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.612210E-01 | lm-global-supervised-tokens: 1.206749E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.905234E-02 | lm-token-buffer-mib: 5.805878E+01 | global_load_balancing_loss: 1.000332E+00 | loss scale: 1.0 | grad norm: 0.187 | params norm: 11317.662 | params rms: 0.061281 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 2:35:44.334687 | finish at 2026-10-11 04:49:26 + [2026-10-09 02:15:09] iteration 5191/ 6855 | consumed samples: 664448 | elapsed time per iteration (ms): 87209.2 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.58% | FLOPs estimator: workload-aware | logical tokens/s: 192378.5 | valid tokens/s: 192378.5 | physical tokens/s: 192818.7 | loss tokens/s: 141985.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.447004E-01 | lm-global-supervised-tokens: 1.238243E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.146467E-02 | lm-token-buffer-mib: 5.810831E+01 | global_load_balancing_loss: 1.000212E+00 | loss scale: 1.0 | grad norm: 0.031 | params norm: 11317.612 | params rms: 0.061281 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 16:18:36.162598 | finish at 2026-10-10 18:33:45 + [2026-10-09 02:16:34] iteration 5192/ 6855 | consumed samples: 664576 | elapsed time per iteration (ms): 85389.4 | throughput per GPU (TFLOP/s/GPU): 43.8 | MFU 14.03% | FLOPs estimator: workload-aware | logical tokens/s: 196477.8 | valid tokens/s: 196477.8 | physical tokens/s: 196991.4 | loss tokens/s: 139210.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.811076E-01 | lm-global-supervised-tokens: 1.188709E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.141862E-02 | lm-token-buffer-mib: 5.811118E+01 | global_load_balancing_loss: 9.997580E-01 | loss scale: 1.0 | grad norm: 0.217 | params norm: 11317.561 | params rms: 0.061281 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 15:26:42.531767 | finish at 2026-10-10 17:43:17 + [2026-10-09 02:17:59] iteration 5193/ 6855 | consumed samples: 664704 | elapsed time per iteration (ms): 85052.2 | throughput per GPU (TFLOP/s/GPU): 43.8 | MFU 14.05% | FLOPs estimator: workload-aware | logical tokens/s: 197257.9 | valid tokens/s: 197257.9 | physical tokens/s: 197613.5 | loss tokens/s: 140817.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.579448E-01 | lm-global-supervised-tokens: 1.197681E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.492956E-02 | lm-token-buffer-mib: 5.809003E+01 | global_load_balancing_loss: 1.000709E+00 | loss scale: 1.0 | grad norm: 0.159 | params norm: 11317.511 | params rms: 0.061281 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 15:15:56.707000 | finish at 2026-10-10 17:33:56 + [2026-10-09 02:19:28] iteration 5194/ 6855 | consumed samples: 664832 | elapsed time per iteration (ms): 88223.0 | throughput per GPU (TFLOP/s/GPU): 41.7 | MFU 13.38% | FLOPs estimator: workload-aware | logical tokens/s: 190168.0 | valid tokens/s: 190168.0 | physical tokens/s: 190727.8 | loss tokens/s: 142005.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.633359E-01 | lm-global-supervised-tokens: 1.252818E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.479844E-02 | lm-token-buffer-mib: 5.808088E+01 | global_load_balancing_loss: 1.001043E+00 | loss scale: 1.0 | grad norm: 0.160 | params norm: 11317.460 | params rms: 0.061280 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 16:42:18.485057 | finish at 2026-10-10 19:01:46 + [2026-10-09 02:20:55] iteration 5195/ 6855 | consumed samples: 664960 | elapsed time per iteration (ms): 86891.3 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.80% | FLOPs estimator: workload-aware | logical tokens/s: 193081.9 | valid tokens/s: 193081.9 | physical tokens/s: 193449.8 | loss tokens/s: 149712.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.016588E-01 | lm-global-supervised-tokens: 1.300874E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.147909E-02 | lm-token-buffer-mib: 5.805757E+01 | global_load_balancing_loss: 9.993846E-01 | loss scale: 1.0 | grad norm: 0.082 | params norm: 11317.409 | params rms: 0.061280 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 16:03:59.573770 | finish at 2026-10-10 18:24:54 + [2026-10-09 02:22:21] iteration 5196/ 6855 | consumed samples: 665088 | elapsed time per iteration (ms): 86655.2 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.69% | FLOPs estimator: workload-aware | logical tokens/s: 193608.1 | valid tokens/s: 193608.1 | physical tokens/s: 194110.8 | loss tokens/s: 142205.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.281414E-01 | lm-global-supervised-tokens: 1.232281E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.328747E-02 | lm-token-buffer-mib: 5.821603E+01 | global_load_balancing_loss: 1.000679E+00 | loss scale: 1.0 | grad norm: 0.102 | params norm: 11317.356 | params rms: 0.061280 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 15:56:01.024668 | finish at 2026-10-10 18:18:22 + [2026-10-09 02:23:49] iteration 5197/ 6855 | consumed samples: 665216 | elapsed time per iteration (ms): 88075.0 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.68% | FLOPs estimator: workload-aware | logical tokens/s: 190486.7 | valid tokens/s: 190486.7 | physical tokens/s: 190791.7 | loss tokens/s: 140311.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.196253E-01 | lm-global-supervised-tokens: 1.235792E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.464572E-02 | lm-token-buffer-mib: 5.805606E+01 | global_load_balancing_loss: 1.000583E+00 | loss scale: 1.0 | grad norm: 0.035 | params norm: 11317.304 | params rms: 0.061280 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 16:33:48.395934 | finish at 2026-10-10 18:57:38 + [2026-10-09 02:25:19] iteration 5198/ 6855 | consumed samples: 665344 | elapsed time per iteration (ms): 89285.2 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.55% | FLOPs estimator: workload-aware | logical tokens/s: 187905.9 | valid tokens/s: 187905.9 | physical tokens/s: 188282.8 | loss tokens/s: 132639.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.596392E-01 | lm-global-supervised-tokens: 1.184274E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.222157E-02 | lm-token-buffer-mib: 5.807723E+01 | global_load_balancing_loss: 1.001240E+00 | loss scale: 1.0 | grad norm: 0.100 | params norm: 11317.252 | params rms: 0.061279 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 17:05:45.517733 | finish at 2026-10-10 19:31:04 + [2026-10-09 02:26:44] iteration 5199/ 6855 | consumed samples: 665472 | elapsed time per iteration (ms): 85648.9 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.59% | FLOPs estimator: workload-aware | logical tokens/s: 195883.1 | valid tokens/s: 195883.1 | physical tokens/s: 196381.7 | loss tokens/s: 136255.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 5.986807E-01 | lm-global-supervised-tokens: 1.167012E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.177755E-02 | lm-token-buffer-mib: 5.815467E+01 | global_load_balancing_loss: 1.000318E+00 | loss scale: 1.0 | grad norm: 0.034 | params norm: 11317.199 | params rms: 0.061279 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 15:23:54.644388 | finish at 2026-10-10 17:50:39 + [2026-10-09 02:28:32] iteration 5200/ 6855 | consumed samples: 665600 | elapsed time per iteration (ms): 108266.2 | throughput per GPU (TFLOP/s/GPU): 33.4 | MFU 10.70% | FLOPs estimator: workload-aware | logical tokens/s: 154962.6 | valid tokens/s: 154962.6 | physical tokens/s: 155444.9 | loss tokens/s: 113799.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 5.849680E-01 | lm-global-supervised-tokens: 1.232062E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.751233E-02 | lm-token-buffer-mib: 5.808441E+01 | global_load_balancing_loss: 9.996791E-01 | loss scale: 1.0 | grad norm: 0.087 | params norm: 11317.148 | params rms: 0.061279 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 1:46:20.488505 | finish at 2026-10-11 04:14:53 +times across ranks (ms): + save-checkpoint: + rank 0: 25013.48 + rank 1: 25013.48 + rank 2: 25013.49 + rank 3: 25013.48 + rank 4: 25013.46 + rank 5: 25013.46 + rank 6: 25013.42 + rank 7: 25013.46 + rank 8: 25013.77 + rank 9: 25013.74 + rank 10: 25013.73 + rank 11: 25013.74 + rank 12: 25013.71 + rank 13: 25013.73 + rank 14: 25013.73 + rank 15: 25013.73 + rank 16: 25013.84 + rank 17: 25013.85 + rank 18: 25013.81 + rank 19: 25013.83 + rank 20: 25013.80 + rank 21: 25013.85 + rank 22: 25013.84 + rank 23: 25013.80 + rank 24: 25013.95 + rank 25: 25013.83 + rank 26: 25013.92 + rank 27: 25013.93 + rank 28: 25014.22 + rank 29: 25013.86 + rank 30: 25013.84 + rank 31: 25013.92 + rank 32: 25014.95 + rank 33: 25014.89 + rank 34: 25014.88 + rank 35: 25014.86 + rank 36: 25014.79 + rank 37: 25014.76 + rank 38: 25014.73 + rank 39: 25014.83 + rank 40: 25014.11 + rank 41: 25014.06 + rank 42: 25014.13 + rank 43: 25014.11 + rank 44: 25014.07 + rank 45: 25014.08 + rank 46: 25014.09 + rank 47: 25014.10 + rank 48: 25013.86 + rank 49: 25013.87 + rank 50: 25013.87 + rank 51: 25013.71 + rank 52: 25013.86 + rank 53: 25013.84 + rank 54: 25013.87 + rank 55: 25013.86 + rank 56: 25013.75 + rank 57: 25013.71 + rank 58: 25013.70 + rank 59: 25013.69 + rank 60: 25013.70 + rank 61: 25013.69 + rank 62: 25013.70 + rank 63: 25013.52 + [2026-10-09 02:30:34] iteration 5201/ 6855 | consumed samples: 665728 | elapsed time per iteration (ms): 96052.6 | throughput per GPU (TFLOP/s/GPU): 39.6 | MFU 12.71% | FLOPs estimator: workload-aware | logical tokens/s: 174666.4 | valid tokens/s: 174666.3 | physical tokens/s: 175094.3 | loss tokens/s: 127659.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.477493E-01 | lm-global-supervised-tokens: 1.226207E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.276499E-02 | lm-token-buffer-mib: 5.832694E+01 | global_load_balancing_loss: 9.989215E-01 | loss scale: 1.0 | grad norm: 0.032 | params norm: 11317.097 | params rms: 0.061278 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 20:07:50.981712 | finish at 2026-10-10 22:38:25 + [2026-10-09 02:32:10] iteration 5202/ 6855 | consumed samples: 665856 | elapsed time per iteration (ms): 95934.1 | throughput per GPU (TFLOP/s/GPU): 38.7 | MFU 12.40% | FLOPs estimator: workload-aware | logical tokens/s: 174882.3 | valid tokens/s: 174882.3 | physical tokens/s: 175239.9 | loss tokens/s: 127006.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.446059E-01 | lm-global-supervised-tokens: 1.218429E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.457511E-02 | lm-token-buffer-mib: 5.810548E+01 | global_load_balancing_loss: 9.997054E-01 | loss scale: 1.0 | grad norm: 0.073 | params norm: 11317.046 | params rms: 0.061278 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 20:02:59.103019 | finish at 2026-10-10 22:35:09 + [2026-10-09 02:33:36] iteration 5203/ 6855 | consumed samples: 665984 | elapsed time per iteration (ms): 86838.4 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.69% | FLOPs estimator: workload-aware | logical tokens/s: 193200.2 | valid tokens/s: 193200.2 | physical tokens/s: 193618.7 | loss tokens/s: 140729.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.580842E-01 | lm-global-supervised-tokens: 1.222073E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.789611E-02 | lm-token-buffer-mib: 5.815117E+01 | global_load_balancing_loss: 9.997016E-01 | loss scale: 1.0 | grad norm: 0.034 | params norm: 11316.996 | params rms: 0.061278 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 15:50:57.041340 | finish at 2026-10-10 18:24:33 + [2026-10-09 02:35:02] iteration 5204/ 6855 | consumed samples: 666112 | elapsed time per iteration (ms): 85402.1 | throughput per GPU (TFLOP/s/GPU): 43.2 | MFU 13.84% | FLOPs estimator: workload-aware | logical tokens/s: 196449.5 | valid tokens/s: 196449.5 | physical tokens/s: 197001.4 | loss tokens/s: 152899.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.889894E-01 | lm-global-supervised-tokens: 1.305790E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.174390E-02 | lm-token-buffer-mib: 5.809601E+01 | global_load_balancing_loss: 9.997805E-01 | loss scale: 1.0 | grad norm: 0.072 | params norm: 11316.945 | params rms: 0.061278 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 15:09:58.784974 | finish at 2026-10-10 17:45:01 + [2026-10-09 02:36:32] iteration 5205/ 6855 | consumed samples: 666240 | elapsed time per iteration (ms): 90253.8 | throughput per GPU (TFLOP/s/GPU): 41.6 | MFU 13.33% | FLOPs estimator: workload-aware | logical tokens/s: 185888.8 | valid tokens/s: 185888.8 | physical tokens/s: 186323.3 | loss tokens/s: 133716.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.778221E-01 | lm-global-supervised-tokens: 1.206840E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.201988E-02 | lm-token-buffer-mib: 5.819530E+01 | global_load_balancing_loss: 1.000805E+00 | loss scale: 1.0 | grad norm: 0.053 | params norm: 11316.895 | params rms: 0.061277 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 17:21:58.731308 | finish at 2026-10-10 19:58:31 + [2026-10-09 02:37:56] iteration 5206/ 6855 | consumed samples: 666368 | elapsed time per iteration (ms): 84257.2 | throughput per GPU (TFLOP/s/GPU): 43.6 | MFU 13.97% | FLOPs estimator: workload-aware | logical tokens/s: 199118.6 | valid tokens/s: 199118.6 | physical tokens/s: 199516.5 | loss tokens/s: 140326.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.375636E-01 | lm-global-supervised-tokens: 1.182350E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.182468E-02 | lm-token-buffer-mib: 5.806555E+01 | global_load_balancing_loss: 1.001209E+00 | loss scale: 1.0 | grad norm: 0.055 | params norm: 11316.844 | params rms: 0.061277 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 14:35:40.142462 | finish at 2026-10-10 17:13:36 + [2026-10-09 02:39:20] iteration 5207/ 6855 | consumed samples: 666496 | elapsed time per iteration (ms): 84032.9 | throughput per GPU (TFLOP/s/GPU): 43.5 | MFU 13.95% | FLOPs estimator: workload-aware | logical tokens/s: 199650.1 | valid tokens/s: 199650.1 | physical tokens/s: 200039.2 | loss tokens/s: 149913.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.672718E-01 | lm-global-supervised-tokens: 1.259765E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.168369E-02 | lm-token-buffer-mib: 5.813131E+01 | global_load_balancing_loss: 1.001550E+00 | loss scale: 1.0 | grad norm: 0.044 | params norm: 11316.792 | params rms: 0.061277 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 14:28:06.200890 | finish at 2026-10-10 17:07:27 + [2026-10-09 02:40:49] iteration 5208/ 6855 | consumed samples: 666624 | elapsed time per iteration (ms): 88585.6 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.47% | FLOPs estimator: workload-aware | logical tokens/s: 189388.9 | valid tokens/s: 189388.9 | physical tokens/s: 189975.6 | loss tokens/s: 138683.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.159191E-01 | lm-global-supervised-tokens: 1.228538E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.310732E-02 | lm-token-buffer-mib: 5.806161E+01 | global_load_balancing_loss: 1.000447E+00 | loss scale: 1.0 | grad norm: 0.059 | params norm: 11316.740 | params rms: 0.061276 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 16:31:40.530155 | finish at 2026-10-10 19:12:30 + [2026-10-09 02:42:18] iteration 5209/ 6855 | consumed samples: 666752 | elapsed time per iteration (ms): 89257.8 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.58% | FLOPs estimator: workload-aware | logical tokens/s: 187963.7 | valid tokens/s: 187963.7 | physical tokens/s: 188406.6 | loss tokens/s: 138040.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.764454E-01 | lm-global-supervised-tokens: 1.232119E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.181499E-02 | lm-token-buffer-mib: 5.808585E+01 | global_load_balancing_loss: 9.997261E-01 | loss scale: 1.0 | grad norm: 0.045 | params norm: 11316.689 | params rms: 0.061276 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 16:48:38.274216 | finish at 2026-10-10 19:30:57 + [2026-10-09 02:44:08] iteration 5210/ 6855 | consumed samples: 666880 | elapsed time per iteration (ms): 110114.3 | throughput per GPU (TFLOP/s/GPU): 34.1 | MFU 10.94% | FLOPs estimator: workload-aware | logical tokens/s: 152361.7 | valid tokens/s: 152361.7 | physical tokens/s: 152688.3 | loss tokens/s: 112520.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.869248E-01 | lm-global-supervised-tokens: 1.239009E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.565457E-02 | lm-token-buffer-mib: 5.817728E+01 | global_load_balancing_loss: 1.000133E+00 | loss scale: 1.0 | grad norm: 0.071 | params norm: 11316.638 | params rms: 0.061276 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 2:18:58.055289 | finish at 2026-10-11 05:03:06 + [2026-10-09 02:45:41] iteration 5211/ 6855 | consumed samples: 667008 | elapsed time per iteration (ms): 92679.9 | throughput per GPU (TFLOP/s/GPU): 40.3 | MFU 12.92% | FLOPs estimator: workload-aware | logical tokens/s: 181023.1 | valid tokens/s: 181023.1 | physical tokens/s: 181745.6 | loss tokens/s: 135437.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.612485E-01 | lm-global-supervised-tokens: 1.255235E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.188430E-02 | lm-token-buffer-mib: 5.871960E+01 | global_load_balancing_loss: 9.990411E-01 | loss scale: 1.0 | grad norm: 0.064 | params norm: 11316.586 | params rms: 0.061276 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 18:19:25.693165 | finish at 2026-10-10 21:05:07 + [2026-10-09 02:47:09] iteration 5212/ 6855 | consumed samples: 667136 | elapsed time per iteration (ms): 87665.1 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.57% | FLOPs estimator: workload-aware | logical tokens/s: 191378.1 | valid tokens/s: 191378.1 | physical tokens/s: 191884.5 | loss tokens/s: 140648.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.389707E-01 | lm-global-supervised-tokens: 1.232993E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.141408E-02 | lm-token-buffer-mib: 5.815087E+01 | global_load_balancing_loss: 9.996576E-01 | loss scale: 1.0 | grad norm: 0.121 | params norm: 11316.534 | params rms: 0.061275 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 16:00:33.723422 | finish at 2026-10-10 18:47:42 + [2026-10-09 02:48:36] iteration 5213/ 6855 | consumed samples: 667264 | elapsed time per iteration (ms): 87262.2 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.59% | FLOPs estimator: workload-aware | logical tokens/s: 192261.6 | valid tokens/s: 192261.5 | physical tokens/s: 192829.2 | loss tokens/s: 140840.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.948521E-01 | lm-global-supervised-tokens: 1.229003E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.630435E-02 | lm-token-buffer-mib: 5.819794E+01 | global_load_balancing_loss: 1.000306E+00 | loss scale: 1.0 | grad norm: 0.110 | params norm: 11316.485 | params rms: 0.061275 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 15:48:04.461734 | finish at 2026-10-10 18:36:40 + [2026-10-09 02:50:02] iteration 5214/ 6855 | consumed samples: 667392 | elapsed time per iteration (ms): 86006.0 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.79% | FLOPs estimator: workload-aware | logical tokens/s: 195070.3 | valid tokens/s: 195070.3 | physical tokens/s: 195463.3 | loss tokens/s: 149802.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.194248E-01 | lm-global-supervised-tokens: 1.288393E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.243465E-02 | lm-token-buffer-mib: 5.811469E+01 | global_load_balancing_loss: 9.998745E-01 | loss scale: 1.0 | grad norm: 0.182 | params norm: 11316.437 | params rms: 0.061275 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 15:12:15.794425 | finish at 2026-10-10 18:02:18 + [2026-10-09 02:51:30] iteration 5215/ 6855 | consumed samples: 667520 | elapsed time per iteration (ms): 88434.2 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.64% | FLOPs estimator: workload-aware | logical tokens/s: 189713.5 | valid tokens/s: 189713.5 | physical tokens/s: 190046.8 | loss tokens/s: 135765.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.188169E-01 | lm-global-supervised-tokens: 1.200632E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.416052E-02 | lm-token-buffer-mib: 5.806446E+01 | global_load_balancing_loss: 9.999889E-01 | loss scale: 1.0 | grad norm: 0.313 | params norm: 11316.389 | params rms: 0.061275 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 16:17:12.079477 | finish at 2026-10-10 19:08:42 + [2026-10-09 02:53:00] iteration 5216/ 6855 | consumed samples: 667648 | elapsed time per iteration (ms): 89387.9 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.61% | FLOPs estimator: workload-aware | logical tokens/s: 187689.9 | valid tokens/s: 187689.9 | physical tokens/s: 188144.8 | loss tokens/s: 139921.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.199450E-01 | lm-global-supervised-tokens: 1.250731E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.452713E-02 | lm-token-buffer-mib: 5.818597E+01 | global_load_balancing_loss: 1.001440E+00 | loss scale: 1.0 | grad norm: 0.250 | params norm: 11316.340 | params rms: 0.061274 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 16:41:46.814398 | finish at 2026-10-10 19:34:47 + [2026-10-09 02:54:26] iteration 5217/ 6855 | consumed samples: 667776 | elapsed time per iteration (ms): 85809.9 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.64% | FLOPs estimator: workload-aware | logical tokens/s: 195515.5 | valid tokens/s: 195515.5 | physical tokens/s: 196086.4 | loss tokens/s: 144425.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.751275E-01 | lm-global-supervised-tokens: 1.239313E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.325243E-02 | lm-token-buffer-mib: 5.820062E+01 | global_load_balancing_loss: 1.001347E+00 | loss scale: 1.0 | grad norm: 0.135 | params norm: 11316.292 | params rms: 0.061274 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 15:02:36.665091 | finish at 2026-10-10 17:57:02 + [2026-10-09 02:55:53] iteration 5218/ 6855 | consumed samples: 667904 | elapsed time per iteration (ms): 87231.6 | throughput per GPU (TFLOP/s/GPU): 43.1 | MFU 13.81% | FLOPs estimator: workload-aware | logical tokens/s: 192329.3 | valid tokens/s: 192329.3 | physical tokens/s: 192817.7 | loss tokens/s: 139982.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.739794E-01 | lm-global-supervised-tokens: 1.221086E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.043475E-02 | lm-token-buffer-mib: 5.806210E+01 | global_load_balancing_loss: 1.000727E+00 | loss scale: 1.0 | grad norm: 0.218 | params norm: 11316.242 | params rms: 0.061274 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 15:39:58.075025 | finish at 2026-10-10 18:35:51 + [2026-10-09 02:57:22] iteration 5219/ 6855 | consumed samples: 668032 | elapsed time per iteration (ms): 89657.4 | throughput per GPU (TFLOP/s/GPU): 41.7 | MFU 13.38% | FLOPs estimator: workload-aware | logical tokens/s: 187125.4 | valid tokens/s: 187125.4 | physical tokens/s: 187505.4 | loss tokens/s: 133734.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.422510E-01 | lm-global-supervised-tokens: 1.199026E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.152808E-02 | lm-token-buffer-mib: 5.818871E+01 | global_load_balancing_loss: 9.999542E-01 | loss scale: 1.0 | grad norm: 0.074 | params norm: 11316.191 | params rms: 0.061273 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 16:44:39.514416 | finish at 2026-10-10 19:42:02 + [2026-10-09 02:59:11] iteration 5220/ 6855 | consumed samples: 668160 | elapsed time per iteration (ms): 108981.3 | throughput per GPU (TFLOP/s/GPU): 33.1 | MFU 10.60% | FLOPs estimator: workload-aware | logical tokens/s: 153945.1 | valid tokens/s: 153945.1 | physical tokens/s: 154435.1 | loss tokens/s: 115168.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 5.910481E-01 | lm-global-supervised-tokens: 1.255125E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.630193E-02 | lm-token-buffer-mib: 5.819769E+01 | global_load_balancing_loss: 1.000015E+00 | loss scale: 1.0 | grad norm: 0.139 | params norm: 11316.142 | params rms: 0.061273 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 1:29:44.455705 | finish at 2026-10-11 04:28:56 + [2026-10-09 03:00:39] iteration 5221/ 6855 | consumed samples: 668288 | elapsed time per iteration (ms): 87639.3 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.55% | FLOPs estimator: workload-aware | logical tokens/s: 191433.8 | valid tokens/s: 191433.8 | physical tokens/s: 191834.9 | loss tokens/s: 144982.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.138260E-01 | lm-global-supervised-tokens: 1.270616E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.304428E-02 | lm-token-buffer-mib: 5.815815E+01 | global_load_balancing_loss: 9.993819E-01 | loss scale: 1.0 | grad norm: 0.092 | params norm: 11316.095 | params rms: 0.061273 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 15:46:42.608974 | finish at 2026-10-10 18:47:22 + [2026-10-09 03:02:02] iteration 5222/ 6855 | consumed samples: 668416 | elapsed time per iteration (ms): 82816.3 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.79% | FLOPs estimator: workload-aware | logical tokens/s: 202582.5 | valid tokens/s: 202582.5 | physical tokens/s: 203156.7 | loss tokens/s: 150118.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 5.968603E-01 | lm-global-supervised-tokens: 1.243227E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.299564E-02 | lm-token-buffer-mib: 5.821278E+01 | global_load_balancing_loss: 9.999137E-01 | loss scale: 1.0 | grad norm: 0.157 | params norm: 11316.045 | params rms: 0.061273 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 13:33:59.008418 | finish at 2026-10-10 16:36:01 + [2026-10-09 03:03:29] iteration 5223/ 6855 | consumed samples: 668544 | elapsed time per iteration (ms): 87091.5 | throughput per GPU (TFLOP/s/GPU): 42.8 | MFU 13.72% | FLOPs estimator: workload-aware | logical tokens/s: 192638.4 | valid tokens/s: 192638.4 | physical tokens/s: 193160.5 | loss tokens/s: 140174.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.381299E-01 | lm-global-supervised-tokens: 1.220801E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.515537E-02 | lm-token-buffer-mib: 5.808615E+01 | global_load_balancing_loss: 9.998000E-01 | loss scale: 1.0 | grad norm: 0.113 | params norm: 11315.995 | params rms: 0.061272 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 15:28:53.333519 | finish at 2026-10-10 18:32:22 + [2026-10-09 03:04:55] iteration 5224/ 6855 | consumed samples: 668672 | elapsed time per iteration (ms): 86478.9 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.74% | FLOPs estimator: workload-aware | logical tokens/s: 194003.0 | valid tokens/s: 194003.0 | physical tokens/s: 194406.6 | loss tokens/s: 142494.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.410937E-01 | lm-global-supervised-tokens: 1.232278E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.530359E-02 | lm-token-buffer-mib: 5.811562E+01 | global_load_balancing_loss: 9.995689E-01 | loss scale: 1.0 | grad norm: 0.162 | params norm: 11315.947 | params rms: 0.061272 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 15:10:47.160100 | finish at 2026-10-10 18:15:43 + [2026-10-09 03:06:21] iteration 5225/ 6855 | consumed samples: 668800 | elapsed time per iteration (ms): 85484.5 | throughput per GPU (TFLOP/s/GPU): 44.1 | MFU 14.14% | FLOPs estimator: workload-aware | logical tokens/s: 196258.8 | valid tokens/s: 196258.8 | physical tokens/s: 196635.2 | loss tokens/s: 140239.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.062292E-01 | lm-global-supervised-tokens: 1.198829E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.614768E-02 | lm-token-buffer-mib: 5.809756E+01 | global_load_balancing_loss: 1.001338E+00 | loss scale: 1.0 | grad norm: 0.077 | params norm: 11315.900 | params rms: 0.061272 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 14:42:19.784243 | finish at 2026-10-10 17:48:41 + [2026-10-09 03:07:48] iteration 5226/ 6855 | consumed samples: 668928 | elapsed time per iteration (ms): 86863.4 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.40% | FLOPs estimator: workload-aware | logical tokens/s: 193143.6 | valid tokens/s: 193143.6 | physical tokens/s: 193630.3 | loss tokens/s: 145550.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.304525E-01 | lm-global-supervised-tokens: 1.264297E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.152379E-02 | lm-token-buffer-mib: 5.823537E+01 | global_load_balancing_loss: 1.000406E+00 | loss scale: 1.0 | grad norm: 0.255 | params norm: 11315.851 | params rms: 0.061272 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 15:18:20.425363 | finish at 2026-10-10 18:26:08 + [2026-10-09 03:09:20] iteration 5227/ 6855 | consumed samples: 669056 | elapsed time per iteration (ms): 92136.8 | throughput per GPU (TFLOP/s/GPU): 41.2 | MFU 13.22% | FLOPs estimator: workload-aware | logical tokens/s: 182089.9 | valid tokens/s: 182089.9 | physical tokens/s: 182533.2 | loss tokens/s: 133830.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.806997E-01 | lm-global-supervised-tokens: 1.233068E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.830614E-02 | lm-token-buffer-mib: 5.815082E+01 | global_load_balancing_loss: 9.991570E-01 | loss scale: 1.0 | grad norm: 0.437 | params norm: 11315.801 | params rms: 0.061271 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 17:39:58.738429 | finish at 2026-10-10 20:49:19 + [2026-10-09 03:10:41] iteration 5228/ 6855 | consumed samples: 669184 | elapsed time per iteration (ms): 80736.4 | throughput per GPU (TFLOP/s/GPU): 43.3 | MFU 13.87% | FLOPs estimator: workload-aware | logical tokens/s: 207801.4 | valid tokens/s: 207801.4 | physical tokens/s: 208316.5 | loss tokens/s: 158342.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.197671E-01 | lm-global-supervised-tokens: 1.278397E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.773265E-02 | lm-token-buffer-mib: 5.814303E+01 | global_load_balancing_loss: 1.000918E+00 | loss scale: 1.0 | grad norm: 0.113 | params norm: 11315.748 | params rms: 0.061271 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 12:29:18.075683 | finish at 2026-10-10 15:39:59 + [2026-10-09 03:12:10] iteration 5229/ 6855 | consumed samples: 669312 | elapsed time per iteration (ms): 89210.9 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.53% | FLOPs estimator: workload-aware | logical tokens/s: 188062.0 | valid tokens/s: 188062.0 | physical tokens/s: 188451.4 | loss tokens/s: 132409.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.557221E-01 | lm-global-supervised-tokens: 1.181236E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.245924E-02 | lm-token-buffer-mib: 5.807519E+01 | global_load_balancing_loss: 9.996119E-01 | loss scale: 1.0 | grad norm: 0.406 | params norm: 11315.696 | params rms: 0.061271 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 16:17:36.945996 | finish at 2026-10-10 19:29:47 + [2026-10-09 03:13:58] iteration 5230/ 6855 | consumed samples: 669440 | elapsed time per iteration (ms): 108420.7 | throughput per GPU (TFLOP/s/GPU): 34.2 | MFU 10.96% | FLOPs estimator: workload-aware | logical tokens/s: 154741.5 | valid tokens/s: 154741.4 | physical tokens/s: 155124.9 | loss tokens/s: 111057.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.607084E-01 | lm-global-supervised-tokens: 1.204091E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.323894E-02 | lm-token-buffer-mib: 5.807817E+01 | global_load_balancing_loss: 1.000588E+00 | loss scale: 1.0 | grad norm: 0.060 | params norm: 11315.645 | params rms: 0.061271 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 0:56:23.669001 | finish at 2026-10-11 04:10:22 + [2026-10-09 03:15:26] iteration 5231/ 6855 | consumed samples: 669568 | elapsed time per iteration (ms): 87285.8 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.74% | FLOPs estimator: workload-aware | logical tokens/s: 192210.2 | valid tokens/s: 192210.2 | physical tokens/s: 192601.2 | loss tokens/s: 140310.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.860572E-01 | lm-global-supervised-tokens: 1.224712E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.411695E-02 | lm-token-buffer-mib: 5.806034E+01 | global_load_balancing_loss: 1.000148E+00 | loss scale: 1.0 | grad norm: 0.200 | params norm: 11315.594 | params rms: 0.061270 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 15:22:32.114000 | finish at 2026-10-10 18:37:58 + [2026-10-09 03:16:53] iteration 5232/ 6855 | consumed samples: 669696 | elapsed time per iteration (ms): 87486.9 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.45% | FLOPs estimator: workload-aware | logical tokens/s: 191767.3 | valid tokens/s: 191767.3 | physical tokens/s: 192323.6 | loss tokens/s: 146257.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.601544E-01 | lm-global-supervised-tokens: 1.279561E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.236568E-02 | lm-token-buffer-mib: 5.827402E+01 | global_load_balancing_loss: 1.000382E+00 | loss scale: 1.0 | grad norm: 0.222 | params norm: 11315.542 | params rms: 0.061270 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 15:26:31.203635 | finish at 2026-10-10 18:43:24 + [2026-10-09 03:18:20] iteration 5233/ 6855 | consumed samples: 669824 | elapsed time per iteration (ms): 87348.9 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.58% | FLOPs estimator: workload-aware | logical tokens/s: 192070.8 | valid tokens/s: 192070.8 | physical tokens/s: 192517.3 | loss tokens/s: 137872.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.501343E-01 | lm-global-supervised-tokens: 1.204302E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.137711E-02 | lm-token-buffer-mib: 5.820105E+01 | global_load_balancing_loss: 1.001114E+00 | loss scale: 1.0 | grad norm: 0.114 | params norm: 11315.493 | params rms: 0.061270 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 15:21:19.956534 | finish at 2026-10-10 18:39:40 + [2026-10-09 03:19:47] iteration 5234/ 6855 | consumed samples: 669952 | elapsed time per iteration (ms): 86571.6 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.50% | FLOPs estimator: workload-aware | logical tokens/s: 193795.1 | valid tokens/s: 193795.0 | physical tokens/s: 194330.8 | loss tokens/s: 139162.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 5.972335E-01 | lm-global-supervised-tokens: 1.204754E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.182284E-02 | lm-token-buffer-mib: 5.818639E+01 | global_load_balancing_loss: 9.998525E-01 | loss scale: 1.0 | grad norm: 0.285 | params norm: 11315.444 | params rms: 0.061269 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 14:58:52.622667 | finish at 2026-10-10 18:18:40 + [2026-10-09 03:21:14] iteration 5235/ 6855 | consumed samples: 670080 | elapsed time per iteration (ms): 86974.2 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.52% | FLOPs estimator: workload-aware | logical tokens/s: 192898.2 | valid tokens/s: 192898.2 | physical tokens/s: 193360.1 | loss tokens/s: 141789.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.641199E-01 | lm-global-supervised-tokens: 1.233199E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.357031E-02 | lm-token-buffer-mib: 5.804570E+01 | global_load_balancing_loss: 1.000682E+00 | loss scale: 1.0 | grad norm: 0.054 | params norm: 11315.396 | params rms: 0.061269 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 15:08:18.270450 | finish at 2026-10-10 18:29:32 + [2026-10-09 03:22:43] iteration 5236/ 6855 | consumed samples: 670208 | elapsed time per iteration (ms): 88543.1 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.39% | FLOPs estimator: workload-aware | logical tokens/s: 189480.2 | valid tokens/s: 189480.2 | physical tokens/s: 189851.8 | loss tokens/s: 143534.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.648083E-01 | lm-global-supervised-tokens: 1.270898E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.305374E-02 | lm-token-buffer-mib: 5.809313E+01 | global_load_balancing_loss: 9.991858E-01 | loss scale: 1.0 | grad norm: 0.102 | params norm: 11315.345 | params rms: 0.061269 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 15:49:11.225438 | finish at 2026-10-10 19:11:54 + [2026-10-09 03:24:11] iteration 5237/ 6855 | consumed samples: 670336 | elapsed time per iteration (ms): 88299.8 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.47% | FLOPs estimator: workload-aware | logical tokens/s: 190002.2 | valid tokens/s: 190002.2 | physical tokens/s: 190500.0 | loss tokens/s: 139437.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.431289E-01 | lm-global-supervised-tokens: 1.231234E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.166951E-02 | lm-token-buffer-mib: 5.841957E+01 | global_load_balancing_loss: 9.999487E-01 | loss scale: 1.0 | grad norm: 0.188 | params norm: 11315.294 | params rms: 0.061269 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 15:41:09.147249 | finish at 2026-10-10 19:05:20 + [2026-10-09 03:25:38] iteration 5238/ 6855 | consumed samples: 670464 | elapsed time per iteration (ms): 86770.0 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.52% | FLOPs estimator: workload-aware | logical tokens/s: 193351.8 | valid tokens/s: 193351.8 | physical tokens/s: 194041.2 | loss tokens/s: 147014.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.758892E-01 | lm-global-supervised-tokens: 1.275649E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.946728E-02 | lm-token-buffer-mib: 5.840421E+01 | global_load_balancing_loss: 1.000978E+00 | loss scale: 1.0 | grad norm: 0.104 | params norm: 11315.244 | params rms: 0.061268 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 14:58:27.150111 | finish at 2026-10-10 18:24:05 + [2026-10-09 03:27:06] iteration 5239/ 6855 | consumed samples: 670592 | elapsed time per iteration (ms): 88064.5 | throughput per GPU (TFLOP/s/GPU): 41.5 | MFU 13.31% | FLOPs estimator: workload-aware | logical tokens/s: 190509.9 | valid tokens/s: 190509.9 | physical tokens/s: 190934.9 | loss tokens/s: 135279.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.216659E-01 | lm-global-supervised-tokens: 1.191335E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.153405E-02 | lm-token-buffer-mib: 5.808434E+01 | global_load_balancing_loss: 1.001198E+00 | loss scale: 1.0 | grad norm: 0.167 | params norm: 11315.195 | params rms: 0.061268 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 15:31:52.292641 | finish at 2026-10-10 18:58:58 + [2026-10-09 03:28:54] iteration 5240/ 6855 | consumed samples: 670720 | elapsed time per iteration (ms): 108393.7 | throughput per GPU (TFLOP/s/GPU): 33.6 | MFU 10.78% | FLOPs estimator: workload-aware | logical tokens/s: 154779.9 | valid tokens/s: 154779.9 | physical tokens/s: 155117.4 | loss tokens/s: 112204.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.237261E-01 | lm-global-supervised-tokens: 1.216225E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.648445E-02 | lm-token-buffer-mib: 5.810820E+01 | global_load_balancing_loss: 9.998818E-01 | loss scale: 1.0 | grad norm: 0.106 | params norm: 11315.147 | params rms: 0.061268 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 0:37:35.894622 | finish at 2026-10-11 04:06:30 +times across ranks (ms): + save-checkpoint: + rank 0: 25035.94 + rank 1: 25035.99 + rank 2: 25035.97 + rank 3: 25035.94 + rank 4: 25035.99 + rank 5: 25035.96 + rank 6: 25035.96 + rank 7: 25035.97 + rank 8: 25036.53 + rank 9: 25036.52 + rank 10: 25036.49 + rank 11: 25036.49 + rank 12: 25036.48 + rank 13: 25036.48 + rank 14: 25036.48 + rank 15: 25036.49 + rank 16: 25035.72 + rank 17: 25035.69 + rank 18: 25035.67 + rank 19: 25035.70 + rank 20: 25035.67 + rank 21: 25035.69 + rank 22: 25035.68 + rank 23: 25035.67 + rank 24: 25035.84 + rank 25: 25035.82 + rank 26: 25035.84 + rank 27: 25035.83 + rank 28: 25035.81 + rank 29: 25035.85 + rank 30: 25035.85 + rank 31: 25035.84 + rank 32: 25035.99 + rank 33: 25035.99 + rank 34: 25035.98 + rank 35: 25035.98 + rank 36: 25035.89 + rank 37: 25035.92 + rank 38: 25035.92 + rank 39: 25035.89 + rank 40: 25035.43 + rank 41: 25035.44 + rank 42: 25035.43 + rank 43: 25035.43 + rank 44: 25035.43 + rank 45: 25035.45 + rank 46: 25035.44 + rank 47: 25035.43 + rank 48: 25035.90 + rank 49: 25035.91 + rank 50: 25035.89 + rank 51: 25035.91 + rank 52: 25035.89 + rank 53: 25035.89 + rank 54: 25035.88 + rank 55: 25035.93 + rank 56: 25036.16 + rank 57: 25036.15 + rank 58: 25036.21 + rank 59: 25036.13 + rank 60: 25036.13 + rank 61: 25036.14 + rank 62: 25036.13 + rank 63: 25036.14 + [2026-10-09 03:30:56] iteration 5241/ 6855 | consumed samples: 670848 | elapsed time per iteration (ms): 96657.6 | throughput per GPU (TFLOP/s/GPU): 38.1 | MFU 12.22% | FLOPs estimator: workload-aware | logical tokens/s: 173573.4 | valid tokens/s: 173573.4 | physical tokens/s: 173967.1 | loss tokens/s: 126297.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.410855E-01 | lm-global-supervised-tokens: 1.220762E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 8.528334E-02 | lm-token-buffer-mib: 5.829385E+01 | global_load_balancing_loss: 9.996883E-01 | loss scale: 1.0 | grad norm: 0.099 | params norm: 11315.096 | params rms: 0.061268 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 19:20:05.398604 | finish at 2026-10-10 22:51:02 + [2026-10-09 03:32:34] iteration 5242/ 6855 | consumed samples: 670976 | elapsed time per iteration (ms): 98188.1 | throughput per GPU (TFLOP/s/GPU): 37.9 | MFU 12.15% | FLOPs estimator: workload-aware | logical tokens/s: 170868.2 | valid tokens/s: 170868.1 | physical tokens/s: 171340.9 | loss tokens/s: 125310.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.812601E-01 | lm-global-supervised-tokens: 1.230397E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.325538E-02 | lm-token-buffer-mib: 5.817357E+01 | global_load_balancing_loss: 9.993229E-01 | loss scale: 1.0 | grad norm: 0.082 | params norm: 11315.046 | params rms: 0.061267 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 19:59:37.362773 | finish at 2026-10-10 23:32:12 + [2026-10-09 03:34:03] iteration 5243/ 6855 | consumed samples: 671104 | elapsed time per iteration (ms): 88678.6 | throughput per GPU (TFLOP/s/GPU): 41.6 | MFU 13.34% | FLOPs estimator: workload-aware | logical tokens/s: 189191.0 | valid tokens/s: 189191.0 | physical tokens/s: 189644.0 | loss tokens/s: 139864.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.855832E-01 | lm-global-supervised-tokens: 1.240296E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.531065E-02 | lm-token-buffer-mib: 5.836556E+01 | global_load_balancing_loss: 9.997510E-01 | loss scale: 1.0 | grad norm: 0.076 | params norm: 11314.996 | params rms: 0.061267 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 15:42:29.828373 | finish at 2026-10-10 19:16:33 + [2026-10-09 03:35:28] iteration 5244/ 6855 | consumed samples: 671232 | elapsed time per iteration (ms): 84779.6 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.78% | FLOPs estimator: workload-aware | logical tokens/s: 197891.9 | valid tokens/s: 197891.9 | physical tokens/s: 198426.5 | loss tokens/s: 146572.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.766184E-01 | lm-global-supervised-tokens: 1.242635E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.045747E-02 | lm-token-buffer-mib: 5.808350E+01 | global_load_balancing_loss: 1.000543E+00 | loss scale: 1.0 | grad norm: 0.112 | params norm: 11314.948 | params rms: 0.061267 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 13:56:19.875144 | finish at 2026-10-10 17:31:48 + [2026-10-09 03:36:58] iteration 5245/ 6855 | consumed samples: 671360 | elapsed time per iteration (ms): 90010.2 | throughput per GPU (TFLOP/s/GPU): 41.5 | MFU 13.31% | FLOPs estimator: workload-aware | logical tokens/s: 186392.3 | valid tokens/s: 186392.3 | physical tokens/s: 186870.7 | loss tokens/s: 129279.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 5.988311E-01 | lm-global-supervised-tokens: 1.163646E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.689231E-02 | lm-token-buffer-mib: 5.807801E+01 | global_load_balancing_loss: 1.001522E+00 | loss scale: 1.0 | grad norm: 0.056 | params norm: 11314.898 | params rms: 0.061266 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 16:15:16.413977 | finish at 2026-10-10 19:52:14 + [2026-10-09 03:38:23] iteration 5246/ 6855 | consumed samples: 671488 | elapsed time per iteration (ms): 84919.8 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.53% | FLOPs estimator: workload-aware | logical tokens/s: 197565.1 | valid tokens/s: 197565.1 | physical tokens/s: 197897.9 | loss tokens/s: 153968.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.254097E-01 | lm-global-supervised-tokens: 1.307493E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.164869E-02 | lm-token-buffer-mib: 5.811635E+01 | global_load_balancing_loss: 1.000176E+00 | loss scale: 1.0 | grad norm: 0.077 | params norm: 11314.849 | params rms: 0.061266 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 13:57:15.886917 | finish at 2026-10-10 17:35:39 + [2026-10-09 03:39:46] iteration 5247/ 6855 | consumed samples: 671616 | elapsed time per iteration (ms): 83345.2 | throughput per GPU (TFLOP/s/GPU): 43.3 | MFU 13.88% | FLOPs estimator: workload-aware | logical tokens/s: 201297.4 | valid tokens/s: 201297.4 | physical tokens/s: 201994.1 | loss tokens/s: 154036.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.845385E-01 | lm-global-supervised-tokens: 1.283817E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.615361E-02 | lm-token-buffer-mib: 5.812872E+01 | global_load_balancing_loss: 1.000564E+00 | loss scale: 1.0 | grad norm: 0.060 | params norm: 11314.800 | params rms: 0.061266 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 13:13:39.157991 | finish at 2026-10-10 16:53:25 + [2026-10-09 03:41:12] iteration 5248/ 6855 | consumed samples: 671744 | elapsed time per iteration (ms): 86387.5 | throughput per GPU (TFLOP/s/GPU): 43.8 | MFU 14.02% | FLOPs estimator: workload-aware | logical tokens/s: 194208.8 | valid tokens/s: 194208.8 | physical tokens/s: 194646.2 | loss tokens/s: 142089.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.329732E-01 | lm-global-supervised-tokens: 1.227473E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.456948E-02 | lm-token-buffer-mib: 5.825582E+01 | global_load_balancing_loss: 9.999021E-01 | loss scale: 1.0 | grad norm: 0.102 | params norm: 11314.750 | params rms: 0.061266 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 14:33:44.658937 | finish at 2026-10-10 18:14:57 + [2026-10-09 03:42:40] iteration 5249/ 6855 | consumed samples: 671872 | elapsed time per iteration (ms): 87931.0 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.53% | FLOPs estimator: workload-aware | logical tokens/s: 190799.1 | valid tokens/s: 190799.1 | physical tokens/s: 191358.1 | loss tokens/s: 142766.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.639476E-01 | lm-global-supervised-tokens: 1.255362E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.396791E-02 | lm-token-buffer-mib: 5.818607E+01 | global_load_balancing_loss: 9.988847E-01 | loss scale: 1.0 | grad norm: 0.074 | params norm: 11314.701 | params rms: 0.061265 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 15:13:37.262188 | finish at 2026-10-10 18:56:18 + [2026-10-09 03:44:29] iteration 5250/ 6855 | consumed samples: 672000 | elapsed time per iteration (ms): 109023.6 | throughput per GPU (TFLOP/s/GPU): 33.8 | MFU 10.84% | FLOPs estimator: workload-aware | logical tokens/s: 153885.8 | valid tokens/s: 153885.8 | physical tokens/s: 154180.6 | loss tokens/s: 113474.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.338108E-01 | lm-global-supervised-tokens: 1.237145E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.109276E-02 | lm-token-buffer-mib: 5.815392E+01 | global_load_balancing_loss: 9.994997E-01 | loss scale: 1.0 | grad norm: 0.035 | params norm: 11314.649 | params rms: 0.061265 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 0:36:22.921404 | finish at 2026-10-11 04:20:52 + [2026-10-09 03:45:54] iteration 5251/ 6855 | consumed samples: 672128 | elapsed time per iteration (ms): 84771.4 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.63% | FLOPs estimator: workload-aware | logical tokens/s: 197910.1 | valid tokens/s: 197910.0 | physical tokens/s: 198434.9 | loss tokens/s: 156997.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.950834E-01 | lm-global-supervised-tokens: 1.330892E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.292457E-02 | lm-token-buffer-mib: 5.821320E+01 | global_load_balancing_loss: 9.997714E-01 | loss scale: 1.0 | grad norm: 0.049 | params norm: 11314.597 | params rms: 0.061265 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 13:46:13.405486 | finish at 2026-10-10 17:32:08 + [2026-10-09 03:47:23] iteration 5252/ 6855 | consumed samples: 672256 | elapsed time per iteration (ms): 88421.1 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.63% | FLOPs estimator: workload-aware | logical tokens/s: 189742.0 | valid tokens/s: 189741.9 | physical tokens/s: 190290.2 | loss tokens/s: 138328.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.179519E-01 | lm-global-supervised-tokens: 1.223117E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.382249E-02 | lm-token-buffer-mib: 5.813496E+01 | global_load_balancing_loss: 1.001331E+00 | loss scale: 1.0 | grad norm: 0.060 | params norm: 11314.544 | params rms: 0.061265 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 15:22:19.016262 | finish at 2026-10-10 19:09:42 + [2026-10-09 03:48:51] iteration 5253/ 6855 | consumed samples: 672384 | elapsed time per iteration (ms): 88604.5 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.56% | FLOPs estimator: workload-aware | logical tokens/s: 189349.5 | valid tokens/s: 189349.5 | physical tokens/s: 189712.9 | loss tokens/s: 140954.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.785178E-01 | lm-global-supervised-tokens: 1.248923E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.201780E-02 | lm-token-buffer-mib: 5.809924E+01 | global_load_balancing_loss: 1.000174E+00 | loss scale: 1.0 | grad norm: 0.039 | params norm: 11314.492 | params rms: 0.061264 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 15:25:44.332317 | finish at 2026-10-10 19:14:36 + [2026-10-09 03:50:14] iteration 5254/ 6855 | consumed samples: 672512 | elapsed time per iteration (ms): 82763.3 | throughput per GPU (TFLOP/s/GPU): 43.8 | MFU 14.04% | FLOPs estimator: workload-aware | logical tokens/s: 202712.9 | valid tokens/s: 202712.9 | physical tokens/s: 203049.7 | loss tokens/s: 153707.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.437553E-01 | lm-global-supervised-tokens: 1.272132E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.315006E-02 | lm-token-buffer-mib: 5.807119E+01 | global_load_balancing_loss: 9.999344E-01 | loss scale: 1.0 | grad norm: 0.036 | params norm: 11314.439 | params rms: 0.061264 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 12:48:24.004655 | finish at 2026-10-10 16:38:38 + [2026-10-09 03:51:41] iteration 5255/ 6855 | consumed samples: 672640 | elapsed time per iteration (ms): 86716.4 | throughput per GPU (TFLOP/s/GPU): 43.3 | MFU 13.87% | FLOPs estimator: workload-aware | logical tokens/s: 193471.8 | valid tokens/s: 193471.8 | physical tokens/s: 193823.6 | loss tokens/s: 145818.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.576439E-01 | lm-global-supervised-tokens: 1.264483E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.531698E-02 | lm-token-buffer-mib: 5.810219E+01 | global_load_balancing_loss: 1.000327E+00 | loss scale: 1.0 | grad norm: 0.039 | params norm: 11314.385 | params rms: 0.061264 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 14:32:26.200180 | finish at 2026-10-10 18:24:07 + [2026-10-09 03:53:09] iteration 5256/ 6855 | consumed samples: 672768 | elapsed time per iteration (ms): 88391.0 | throughput per GPU (TFLOP/s/GPU): 41.5 | MFU 13.30% | FLOPs estimator: workload-aware | logical tokens/s: 189806.2 | valid tokens/s: 189806.2 | physical tokens/s: 190278.1 | loss tokens/s: 137450.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.233622E-01 | lm-global-supervised-tokens: 1.214939E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.342779E-02 | lm-token-buffer-mib: 5.834490E+01 | global_load_balancing_loss: 1.000500E+00 | loss scale: 1.0 | grad norm: 0.078 | params norm: 11314.333 | params rms: 0.061263 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 15:15:37.247556 | finish at 2026-10-10 19:08:46 + [2026-10-09 03:54:39] iteration 5257/ 6855 | consumed samples: 672896 | elapsed time per iteration (ms): 89686.0 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.54% | FLOPs estimator: workload-aware | logical tokens/s: 187065.6 | valid tokens/s: 187065.5 | physical tokens/s: 187557.6 | loss tokens/s: 138162.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.338282E-01 | lm-global-supervised-tokens: 1.239121E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.222456E-02 | lm-token-buffer-mib: 5.815195E+01 | global_load_balancing_loss: 1.000311E+00 | loss scale: 1.0 | grad norm: 0.047 | params norm: 11314.280 | params rms: 0.061263 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 15:48:38.237318 | finish at 2026-10-10 19:43:17 + [2026-10-09 03:56:06] iteration 5258/ 6855 | consumed samples: 673024 | elapsed time per iteration (ms): 87655.9 | throughput per GPU (TFLOP/s/GPU): 42.8 | MFU 13.73% | FLOPs estimator: workload-aware | logical tokens/s: 191398.6 | valid tokens/s: 191398.6 | physical tokens/s: 191829.8 | loss tokens/s: 130872.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.608397E-01 | lm-global-supervised-tokens: 1.147178E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.493368E-02 | lm-token-buffer-mib: 5.804317E+01 | global_load_balancing_loss: 1.000631E+00 | loss scale: 1.0 | grad norm: 0.061 | params norm: 11314.225 | params rms: 0.061263 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 14:53:06.437654 | finish at 2026-10-10 18:49:13 + [2026-10-09 03:57:32] iteration 5259/ 6855 | consumed samples: 673152 | elapsed time per iteration (ms): 85884.1 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.66% | FLOPs estimator: workload-aware | logical tokens/s: 195347.1 | valid tokens/s: 195347.1 | physical tokens/s: 195920.6 | loss tokens/s: 143422.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.104736E-01 | lm-global-supervised-tokens: 1.231772E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.237566E-02 | lm-token-buffer-mib: 5.815177E+01 | global_load_balancing_loss: 1.000218E+00 | loss scale: 1.0 | grad norm: 0.060 | params norm: 11314.173 | params rms: 0.061263 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 14:04:31.041801 | finish at 2026-10-10 18:02:03 + [2026-10-09 03:59:26] iteration 5260/ 6855 | consumed samples: 673280 | elapsed time per iteration (ms): 113554.3 | throughput per GPU (TFLOP/s/GPU): 32.5 | MFU 10.41% | FLOPs estimator: workload-aware | logical tokens/s: 147745.3 | valid tokens/s: 147745.3 | physical tokens/s: 148013.7 | loss tokens/s: 109003.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.354654E-01 | lm-global-supervised-tokens: 1.237776E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.342183E-02 | lm-token-buffer-mib: 5.812025E+01 | global_load_balancing_loss: 1.000479E+00 | loss scale: 1.0 | grad norm: 0.042 | params norm: 11314.122 | params rms: 0.061262 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 2:18:39.054612 | finish at 2026-10-11 06:18:05 + [2026-10-09 04:00:54] iteration 5261/ 6855 | consumed samples: 673408 | elapsed time per iteration (ms): 87676.6 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.62% | FLOPs estimator: workload-aware | logical tokens/s: 191352.8 | valid tokens/s: 191352.8 | physical tokens/s: 191856.2 | loss tokens/s: 142210.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.914734E-01 | lm-global-supervised-tokens: 1.246849E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.122433E-02 | lm-token-buffer-mib: 5.804041E+01 | global_load_balancing_loss: 9.997655E-01 | loss scale: 1.0 | grad norm: 0.089 | params norm: 11314.072 | params rms: 0.061262 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 14:49:16.502267 | finish at 2026-10-10 18:50:10 + [2026-10-09 04:02:25] iteration 5262/ 6855 | consumed samples: 673536 | elapsed time per iteration (ms): 91086.0 | throughput per GPU (TFLOP/s/GPU): 41.4 | MFU 13.28% | FLOPs estimator: workload-aware | logical tokens/s: 184189.9 | valid tokens/s: 184189.9 | physical tokens/s: 184557.5 | loss tokens/s: 129167.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.370378E-01 | lm-global-supervised-tokens: 1.176534E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.438562E-02 | lm-token-buffer-mib: 5.806132E+01 | global_load_balancing_loss: 1.001083E+00 | loss scale: 1.0 | grad norm: 0.074 | params norm: 11314.022 | params rms: 0.061262 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 16:18:19.989969 | finish at 2026-10-10 20:20:45 + [2026-10-09 04:03:51] iteration 5263/ 6855 | consumed samples: 673664 | elapsed time per iteration (ms): 86377.8 | throughput per GPU (TFLOP/s/GPU): 43.4 | MFU 13.92% | FLOPs estimator: workload-aware | logical tokens/s: 194230.5 | valid tokens/s: 194230.5 | physical tokens/s: 194638.9 | loss tokens/s: 144478.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.260529E-01 | lm-global-supervised-tokens: 1.247971E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.143518E-02 | lm-token-buffer-mib: 5.807619E+01 | global_load_balancing_loss: 9.992847E-01 | loss scale: 1.0 | grad norm: 0.046 | params norm: 11313.974 | params rms: 0.061261 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 14:11:53.483770 | finish at 2026-10-10 18:15:44 + [2026-10-09 04:05:15] iteration 5264/ 6855 | consumed samples: 673792 | elapsed time per iteration (ms): 84040.6 | throughput per GPU (TFLOP/s/GPU): 42.8 | MFU 13.71% | FLOPs estimator: workload-aware | logical tokens/s: 199630.7 | valid tokens/s: 199630.7 | physical tokens/s: 200196.2 | loss tokens/s: 149514.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.686890E-01 | lm-global-supervised-tokens: 1.256526E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.330022E-02 | lm-token-buffer-mib: 5.813296E+01 | global_load_balancing_loss: 9.987805E-01 | loss scale: 1.0 | grad norm: 0.068 | params norm: 11313.924 | params rms: 0.061261 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 13:08:28.555627 | finish at 2026-10-10 17:13:44 + [2026-10-09 04:06:40] iteration 5265/ 6855 | consumed samples: 673920 | elapsed time per iteration (ms): 85068.4 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.48% | FLOPs estimator: workload-aware | logical tokens/s: 197220.0 | valid tokens/s: 197220.0 | physical tokens/s: 197633.2 | loss tokens/s: 150067.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.207136E-01 | lm-global-supervised-tokens: 1.276604E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.334138E-02 | lm-token-buffer-mib: 5.808829E+01 | global_load_balancing_loss: 9.988403E-01 | loss scale: 1.0 | grad norm: 0.070 | params norm: 11313.876 | params rms: 0.061261 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 13:34:18.711877 | finish at 2026-10-10 17:40:59 + [2026-10-09 04:08:10] iteration 5266/ 6855 | consumed samples: 674048 | elapsed time per iteration (ms): 89430.4 | throughput per GPU (TFLOP/s/GPU): 41.6 | MFU 13.35% | FLOPs estimator: workload-aware | logical tokens/s: 187600.4 | valid tokens/s: 187600.4 | physical tokens/s: 188023.2 | loss tokens/s: 138454.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.731480E-01 | lm-global-supervised-tokens: 1.238206E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.960626E-02 | lm-token-buffer-mib: 5.807504E+01 | global_load_balancing_loss: 1.000494E+00 | loss scale: 1.0 | grad norm: 0.059 | params norm: 11313.829 | params rms: 0.061261 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 15:28:24.946727 | finish at 2026-10-10 19:36:34 + [2026-10-09 04:09:39] iteration 5267/ 6855 | consumed samples: 674176 | elapsed time per iteration (ms): 89370.0 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.39% | FLOPs estimator: workload-aware | logical tokens/s: 187727.1 | valid tokens/s: 187727.1 | physical tokens/s: 188175.9 | loss tokens/s: 137763.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.273614E-01 | lm-global-supervised-tokens: 1.231188E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.121065E-02 | lm-token-buffer-mib: 5.814739E+01 | global_load_balancing_loss: 1.001702E+00 | loss scale: 1.0 | grad norm: 0.106 | params norm: 11313.780 | params rms: 0.061260 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 15:25:19.573070 | finish at 2026-10-10 19:34:58 + [2026-10-09 04:11:08] iteration 5268/ 6855 | consumed samples: 674304 | elapsed time per iteration (ms): 89345.8 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.57% | FLOPs estimator: workload-aware | logical tokens/s: 187778.5 | valid tokens/s: 187778.4 | physical tokens/s: 188275.8 | loss tokens/s: 139899.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.974871E-01 | lm-global-supervised-tokens: 1.249946E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.262877E-02 | lm-token-buffer-mib: 5.805605E+01 | global_load_balancing_loss: 1.000945E+00 | loss scale: 1.0 | grad norm: 0.080 | params norm: 11313.730 | params rms: 0.061260 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 15:23:11.720912 | finish at 2026-10-10 19:34:20 + [2026-10-09 04:12:32] iteration 5269/ 6855 | consumed samples: 674432 | elapsed time per iteration (ms): 84095.0 | throughput per GPU (TFLOP/s/GPU): 43.6 | MFU 13.99% | FLOPs estimator: workload-aware | logical tokens/s: 199502.8 | valid tokens/s: 199502.8 | physical tokens/s: 200172.8 | loss tokens/s: 145889.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.276589E-01 | lm-global-supervised-tokens: 1.226856E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.863755E-02 | lm-token-buffer-mib: 5.808299E+01 | global_load_balancing_loss: 9.995774E-01 | loss scale: 1.0 | grad norm: 0.084 | params norm: 11313.681 | params rms: 0.061260 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 13:02:54.736597 | finish at 2026-10-10 17:15:27 + [2026-10-09 04:14:19] iteration 5270/ 6855 | consumed samples: 674560 | elapsed time per iteration (ms): 106589.9 | throughput per GPU (TFLOP/s/GPU): 35.1 | MFU 11.24% | FLOPs estimator: workload-aware | logical tokens/s: 157399.2 | valid tokens/s: 157399.2 | physical tokens/s: 157705.5 | loss tokens/s: 110669.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.504051E-01 | lm-global-supervised-tokens: 1.179622E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.435063E-02 | lm-token-buffer-mib: 5.806282E+01 | global_load_balancing_loss: 1.000418E+00 | loss scale: 1.0 | grad norm: 0.117 | params norm: 11313.631 | params rms: 0.061260 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 22:55:45.048589 | finish at 2026-10-11 03:10:04 + [2026-10-09 04:15:44] iteration 5271/ 6855 | consumed samples: 674688 | elapsed time per iteration (ms): 85231.4 | throughput per GPU (TFLOP/s/GPU): 43.3 | MFU 13.88% | FLOPs estimator: workload-aware | logical tokens/s: 196842.5 | valid tokens/s: 196842.5 | physical tokens/s: 197250.0 | loss tokens/s: 149074.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.314558E-01 | lm-global-supervised-tokens: 1.270583E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.153287E-02 | lm-token-buffer-mib: 5.817825E+01 | global_load_balancing_loss: 9.998190E-01 | loss scale: 1.0 | grad norm: 0.141 | params norm: 11313.579 | params rms: 0.061259 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 13:30:06.520626 | finish at 2026-10-10 17:45:51 + [2026-10-09 04:17:12] iteration 5272/ 6855 | consumed samples: 674816 | elapsed time per iteration (ms): 87736.3 | throughput per GPU (TFLOP/s/GPU): 41.7 | MFU 13.36% | FLOPs estimator: workload-aware | logical tokens/s: 191222.1 | valid tokens/s: 191222.1 | physical tokens/s: 191670.6 | loss tokens/s: 143179.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.529847E-01 | lm-global-supervised-tokens: 1.256206E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.258292E-02 | lm-token-buffer-mib: 5.812097E+01 | global_load_balancing_loss: 1.001040E+00 | loss scale: 1.0 | grad norm: 0.097 | params norm: 11313.527 | params rms: 0.061259 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 14:34:46.618744 | finish at 2026-10-10 18:51:59 + [2026-10-09 04:18:43] iteration 5273/ 6855 | consumed samples: 674944 | elapsed time per iteration (ms): 90904.0 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.56% | FLOPs estimator: workload-aware | logical tokens/s: 184559.4 | valid tokens/s: 184559.4 | physical tokens/s: 185032.0 | loss tokens/s: 129777.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.120047E-01 | lm-global-supervised-tokens: 1.179727E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.616988E-02 | lm-token-buffer-mib: 5.822453E+01 | global_load_balancing_loss: 1.000855E+00 | loss scale: 1.0 | grad norm: 0.510 | params norm: 11313.474 | params rms: 0.061259 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 15:56:50.124675 | finish at 2026-10-10 20:15:33 + [2026-10-09 04:20:07] iteration 5274/ 6855 | consumed samples: 675072 | elapsed time per iteration (ms): 83761.7 | throughput per GPU (TFLOP/s/GPU): 43.5 | MFU 13.94% | FLOPs estimator: workload-aware | logical tokens/s: 200295.9 | valid tokens/s: 200295.9 | physical tokens/s: 200754.2 | loss tokens/s: 154479.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.230874E-01 | lm-global-supervised-tokens: 1.293943E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.428908E-02 | lm-token-buffer-mib: 5.807942E+01 | global_load_balancing_loss: 9.992938E-01 | loss scale: 1.0 | grad norm: 0.064 | params norm: 11313.406 | params rms: 0.061258 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 12:47:07.177455 | finish at 2026-10-10 17:07:14 + [2026-10-09 04:21:33] iteration 5275/ 6855 | consumed samples: 675200 | elapsed time per iteration (ms): 86720.1 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.44% | FLOPs estimator: workload-aware | logical tokens/s: 193463.7 | valid tokens/s: 193463.7 | physical tokens/s: 194231.2 | loss tokens/s: 145996.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.567126E-01 | lm-global-supervised-tokens: 1.266081E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.312708E-02 | lm-token-buffer-mib: 5.816328E+01 | global_load_balancing_loss: 9.999542E-01 | loss scale: 1.0 | grad norm: 0.167 | params norm: 11313.352 | params rms: 0.061258 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 14:03:37.703638 | finish at 2026-10-10 18:25:11 + [2026-10-09 04:23:02] iteration 5276/ 6855 | consumed samples: 675328 | elapsed time per iteration (ms): 88512.1 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.51% | FLOPs estimator: workload-aware | logical tokens/s: 189546.8 | valid tokens/s: 189546.8 | physical tokens/s: 190020.2 | loss tokens/s: 135197.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.756439E-01 | lm-global-supervised-tokens: 1.196660E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.190660E-02 | lm-token-buffer-mib: 5.801901E+01 | global_load_balancing_loss: 9.991105E-01 | loss scale: 1.0 | grad norm: 0.069 | params norm: 11313.311 | params rms: 0.061258 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 14:49:20.595706 | finish at 2026-10-10 19:12:22 + [2026-10-09 04:24:31] iteration 5277/ 6855 | consumed samples: 675456 | elapsed time per iteration (ms): 88848.0 | throughput per GPU (TFLOP/s/GPU): 41.0 | MFU 13.13% | FLOPs estimator: workload-aware | logical tokens/s: 188830.2 | valid tokens/s: 188830.1 | physical tokens/s: 189254.8 | loss tokens/s: 135940.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 5.766345E-01 | lm-global-supervised-tokens: 1.207808E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.180498E-02 | lm-token-buffer-mib: 5.810527E+01 | global_load_balancing_loss: 9.994587E-01 | loss scale: 1.0 | grad norm: 0.168 | params norm: 11313.278 | params rms: 0.061258 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 14:56:42.154236 | finish at 2026-10-10 19:21:13 + [2026-10-09 04:25:58] iteration 5278/ 6855 | consumed samples: 675584 | elapsed time per iteration (ms): 87388.3 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.62% | FLOPs estimator: workload-aware | logical tokens/s: 191984.4 | valid tokens/s: 191984.4 | physical tokens/s: 192391.7 | loss tokens/s: 140009.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.330619E-01 | lm-global-supervised-tokens: 1.223519E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.833490E-02 | lm-token-buffer-mib: 5.809100E+01 | global_load_balancing_loss: 1.000605E+00 | loss scale: 1.0 | grad norm: 0.052 | params norm: 11313.250 | params rms: 0.061258 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 14:16:51.371568 | finish at 2026-10-10 18:42:49 + [2026-10-09 04:27:26] iteration 5279/ 6855 | consumed samples: 675712 | elapsed time per iteration (ms): 87820.4 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.63% | FLOPs estimator: workload-aware | logical tokens/s: 191039.7 | valid tokens/s: 191039.7 | physical tokens/s: 191407.3 | loss tokens/s: 143634.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.312578E-01 | lm-global-supervised-tokens: 1.261406E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.618823E-02 | lm-token-buffer-mib: 5.807189E+01 | global_load_balancing_loss: 1.000709E+00 | loss scale: 1.0 | grad norm: 0.168 | params norm: 11313.225 | params rms: 0.061257 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 14:26:44.970690 | finish at 2026-10-10 18:54:11 + [2026-10-09 04:29:18] iteration 5280/ 6855 | consumed samples: 675840 | elapsed time per iteration (ms): 111913.8 | throughput per GPU (TFLOP/s/GPU): 32.4 | MFU 10.37% | FLOPs estimator: workload-aware | logical tokens/s: 149911.6 | valid tokens/s: 149911.6 | physical tokens/s: 150234.0 | loss tokens/s: 113861.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.297238E-01 | lm-global-supervised-tokens: 1.274265E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.605722E-02 | lm-token-buffer-mib: 5.810799E+01 | global_load_balancing_loss: 1.000783E+00 | loss scale: 1.0 | grad norm: 0.097 | params norm: 11313.201 | params rms: 0.061257 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 0:57:44.183557 | finish at 2026-10-11 05:27:02 +times across ranks (ms): + save-checkpoint: + rank 0: 25409.91 + rank 1: 25409.93 + rank 2: 25409.95 + rank 3: 25409.92 + rank 4: 25409.91 + rank 5: 25409.89 + rank 6: 25409.89 + rank 7: 25409.90 + rank 8: 25410.06 + rank 9: 25410.01 + rank 10: 25410.03 + rank 11: 25410.04 + rank 12: 25410.04 + rank 13: 25409.93 + rank 14: 25410.07 + rank 15: 25410.01 + rank 16: 25409.83 + rank 17: 25409.80 + rank 18: 25409.83 + rank 19: 25409.83 + rank 20: 25409.79 + rank 21: 25409.83 + rank 22: 25409.84 + rank 23: 25409.80 + rank 24: 25410.80 + rank 25: 25410.79 + rank 26: 25410.71 + rank 27: 25410.81 + rank 28: 25410.73 + rank 29: 25410.78 + rank 30: 25410.76 + rank 31: 25410.78 + rank 32: 25409.81 + rank 33: 25409.80 + rank 34: 25409.73 + rank 35: 25409.79 + rank 36: 25409.71 + rank 37: 25409.69 + rank 38: 25409.76 + rank 39: 25409.47 + rank 40: 25409.95 + rank 41: 25409.90 + rank 42: 25409.93 + rank 43: 25409.94 + rank 44: 25409.90 + rank 45: 25409.91 + rank 46: 25409.93 + rank 47: 25409.90 + rank 48: 25410.27 + rank 49: 25410.14 + rank 50: 25410.21 + rank 51: 25410.06 + rank 52: 25410.18 + rank 53: 25410.09 + rank 54: 25410.16 + rank 55: 25410.21 + rank 56: 25410.23 + rank 57: 25410.22 + rank 58: 25410.20 + rank 59: 25410.16 + rank 60: 25410.18 + rank 61: 25410.19 + rank 62: 25410.18 + rank 63: 25410.29 + [2026-10-09 04:31:15] iteration 5281/ 6855 | consumed samples: 675968 | elapsed time per iteration (ms): 91294.8 | throughput per GPU (TFLOP/s/GPU): 41.1 | MFU 13.17% | FLOPs estimator: workload-aware | logical tokens/s: 183769.6 | valid tokens/s: 183769.6 | physical tokens/s: 184139.3 | loss tokens/s: 139414.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.418657E-01 | lm-global-supervised-tokens: 1.272784E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 8.684707E-01 | lm-token-buffer-mib: 5.812861E+01 | global_load_balancing_loss: 1.001028E+00 | loss scale: 1.0 | grad norm: 0.062 | params norm: 11313.179 | params rms: 0.061257 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 15:54:57.972112 | finish at 2026-10-10 20:26:13 + [2026-10-09 04:32:49] iteration 5282/ 6855 | consumed samples: 676096 | elapsed time per iteration (ms): 94091.1 | throughput per GPU (TFLOP/s/GPU): 38.5 | MFU 12.35% | FLOPs estimator: workload-aware | logical tokens/s: 178307.5 | valid tokens/s: 178307.5 | physical tokens/s: 178605.2 | loss tokens/s: 131585.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.435273E-01 | lm-global-supervised-tokens: 1.238101E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.572149E-02 | lm-token-buffer-mib: 5.806138E+01 | global_load_balancing_loss: 1.001219E+00 | loss scale: 1.0 | grad norm: 0.068 | params norm: 11313.156 | params rms: 0.061257 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 17:06:45.240259 | finish at 2026-10-10 21:39:34 + [2026-10-09 04:34:18] iteration 5283/ 6855 | consumed samples: 676224 | elapsed time per iteration (ms): 89031.8 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.64% | FLOPs estimator: workload-aware | logical tokens/s: 188440.1 | valid tokens/s: 188440.1 | physical tokens/s: 188827.6 | loss tokens/s: 133854.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.717165E-01 | lm-global-supervised-tokens: 1.191728E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.163014E-02 | lm-token-buffer-mib: 5.807479E+01 | global_load_balancing_loss: 1.000786E+00 | loss scale: 1.0 | grad norm: 0.058 | params norm: 11313.132 | params rms: 0.061257 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 14:52:38.039497 | finish at 2026-10-10 19:26:56 + [2026-10-09 04:35:45] iteration 5284/ 6855 | consumed samples: 676352 | elapsed time per iteration (ms): 87170.6 | throughput per GPU (TFLOP/s/GPU): 41.2 | MFU 13.20% | FLOPs estimator: workload-aware | logical tokens/s: 192464.0 | valid tokens/s: 192464.0 | physical tokens/s: 192998.2 | loss tokens/s: 149054.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.598029E-01 | lm-global-supervised-tokens: 1.299319E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.147606E-02 | lm-token-buffer-mib: 5.826466E+01 | global_load_balancing_loss: 1.001730E+00 | loss scale: 1.0 | grad norm: 0.066 | params norm: 11313.104 | params rms: 0.061257 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 14:02:24.934220 | finish at 2026-10-10 18:38:10 + [2026-10-09 04:37:12] iteration 5285/ 6855 | consumed samples: 676480 | elapsed time per iteration (ms): 87477.5 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.66% | FLOPs estimator: workload-aware | logical tokens/s: 191788.6 | valid tokens/s: 191788.6 | physical tokens/s: 192234.4 | loss tokens/s: 144967.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.423668E-01 | lm-global-supervised-tokens: 1.268135E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.160350E-02 | lm-token-buffer-mib: 5.809591E+01 | global_load_balancing_loss: 1.000690E+00 | loss scale: 1.0 | grad norm: 0.049 | params norm: 11313.076 | params rms: 0.061257 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 14:08:59.624033 | finish at 2026-10-10 18:46:12 + [2026-10-09 04:38:38] iteration 5286/ 6855 | consumed samples: 676608 | elapsed time per iteration (ms): 85834.3 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.59% | FLOPs estimator: workload-aware | logical tokens/s: 195460.5 | valid tokens/s: 195460.5 | physical tokens/s: 195920.4 | loss tokens/s: 143438.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.469241E-01 | lm-global-supervised-tokens: 1.231198E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.256831E-02 | lm-token-buffer-mib: 5.835138E+01 | global_load_balancing_loss: 9.998143E-01 | loss scale: 1.0 | grad norm: 0.041 | params norm: 11313.046 | params rms: 0.061256 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 13:24:33.993572 | finish at 2026-10-10 18:03:12 + [2026-10-09 04:40:08] iteration 5287/ 6855 | consumed samples: 676736 | elapsed time per iteration (ms): 89727.9 | throughput per GPU (TFLOP/s/GPU): 41.6 | MFU 13.35% | FLOPs estimator: workload-aware | logical tokens/s: 186978.0 | valid tokens/s: 186978.0 | physical tokens/s: 187448.1 | loss tokens/s: 135312.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.272997E-01 | lm-global-supervised-tokens: 1.214131E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.134687E-02 | lm-token-buffer-mib: 5.829838E+01 | global_load_balancing_loss: 1.000359E+00 | loss scale: 1.0 | grad norm: 0.040 | params norm: 11313.012 | params rms: 0.061256 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 15:04:53.397339 | finish at 2026-10-10 19:45:01 + [2026-10-09 04:41:37] iteration 5288/ 6855 | consumed samples: 676864 | elapsed time per iteration (ms): 88790.9 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.51% | FLOPs estimator: workload-aware | logical tokens/s: 188951.3 | valid tokens/s: 188951.3 | physical tokens/s: 189234.0 | loss tokens/s: 137359.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.564514E-01 | lm-global-supervised-tokens: 1.219627E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.525135E-02 | lm-token-buffer-mib: 5.812131E+01 | global_load_balancing_loss: 9.994921E-01 | loss scale: 1.0 | grad norm: 0.043 | params norm: 11312.977 | params rms: 0.061256 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 14:38:55.323849 | finish at 2026-10-10 19:20:32 + [2026-10-09 04:43:05] iteration 5289/ 6855 | consumed samples: 676992 | elapsed time per iteration (ms): 88441.3 | throughput per GPU (TFLOP/s/GPU): 41.5 | MFU 13.31% | FLOPs estimator: workload-aware | logical tokens/s: 189698.5 | valid tokens/s: 189698.5 | physical tokens/s: 190317.0 | loss tokens/s: 143549.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.454019E-01 | lm-global-supervised-tokens: 1.269570E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.547087E-02 | lm-token-buffer-mib: 5.833200E+01 | global_load_balancing_loss: 9.998099E-01 | loss scale: 1.0 | grad norm: 0.067 | params norm: 11312.938 | params rms: 0.061256 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 14:28:19.097322 | finish at 2026-10-10 19:11:24 + [2026-10-09 04:44:54] iteration 5290/ 6855 | consumed samples: 677120 | elapsed time per iteration (ms): 108402.9 | throughput per GPU (TFLOP/s/GPU): 33.6 | MFU 10.78% | FLOPs estimator: workload-aware | logical tokens/s: 154766.6 | valid tokens/s: 154766.6 | physical tokens/s: 155177.6 | loss tokens/s: 115810.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.749201E-01 | lm-global-supervised-tokens: 1.255423E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.135942E-02 | lm-token-buffer-mib: 5.818083E+01 | global_load_balancing_loss: 9.996991E-01 | loss scale: 1.0 | grad norm: 0.057 | params norm: 11312.894 | params rms: 0.061256 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 23:07:30.591080 | finish at 2026-10-11 03:52:24 + [2026-10-09 04:46:20] iteration 5291/ 6855 | consumed samples: 677248 | elapsed time per iteration (ms): 86128.6 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.50% | FLOPs estimator: workload-aware | logical tokens/s: 194791.8 | valid tokens/s: 194791.8 | physical tokens/s: 195343.9 | loss tokens/s: 151147.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.051152E-01 | lm-global-supervised-tokens: 1.301808E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.652264E-02 | lm-token-buffer-mib: 5.807397E+01 | global_load_balancing_loss: 1.000865E+00 | loss scale: 1.0 | grad norm: 0.067 | params norm: 11312.847 | params rms: 0.061255 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 13:25:05.166386 | finish at 2026-10-10 18:11:25 + [2026-10-09 04:47:48] iteration 5292/ 6855 | consumed samples: 677376 | elapsed time per iteration (ms): 88050.3 | throughput per GPU (TFLOP/s/GPU): 41.4 | MFU 13.28% | FLOPs estimator: workload-aware | logical tokens/s: 190540.6 | valid tokens/s: 190540.5 | physical tokens/s: 191009.0 | loss tokens/s: 145233.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.805168E-01 | lm-global-supervised-tokens: 1.278784E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.419137E-02 | lm-token-buffer-mib: 5.807391E+01 | global_load_balancing_loss: 1.000081E+00 | loss scale: 1.0 | grad norm: 0.108 | params norm: 11312.798 | params rms: 0.061255 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 14:13:42.648156 | finish at 2026-10-10 19:01:30 + [2026-10-09 04:49:14] iteration 5293/ 6855 | consumed samples: 677504 | elapsed time per iteration (ms): 86771.6 | throughput per GPU (TFLOP/s/GPU): 42.8 | MFU 13.70% | FLOPs estimator: workload-aware | logical tokens/s: 193349.2 | valid tokens/s: 193349.2 | physical tokens/s: 193777.6 | loss tokens/s: 141185.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.548574E-01 | lm-global-supervised-tokens: 1.225090E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.102930E-02 | lm-token-buffer-mib: 5.808075E+01 | global_load_balancing_loss: 1.001503E+00 | loss scale: 1.0 | grad norm: 0.213 | params norm: 11312.746 | params rms: 0.061255 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 13:38:57.169572 | finish at 2026-10-10 18:28:12 + [2026-10-09 04:50:44] iteration 5294/ 6855 | consumed samples: 677632 | elapsed time per iteration (ms): 89278.0 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.50% | FLOPs estimator: workload-aware | logical tokens/s: 187920.2 | valid tokens/s: 187920.2 | physical tokens/s: 188343.5 | loss tokens/s: 130148.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.093423E-01 | lm-global-supervised-tokens: 1.161943E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.457487E-02 | lm-token-buffer-mib: 5.811690E+01 | global_load_balancing_loss: 1.001449E+00 | loss scale: 1.0 | grad norm: 0.240 | params norm: 11312.692 | params rms: 0.061255 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 14:42:42.910172 | finish at 2026-10-10 19:33:27 + [2026-10-09 04:52:11] iteration 5295/ 6855 | consumed samples: 677760 | elapsed time per iteration (ms): 87303.8 | throughput per GPU (TFLOP/s/GPU): 43.1 | MFU 13.81% | FLOPs estimator: workload-aware | logical tokens/s: 192170.3 | valid tokens/s: 192170.2 | physical tokens/s: 192762.7 | loss tokens/s: 136230.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.191894E-01 | lm-global-supervised-tokens: 1.189342E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.320160E-02 | lm-token-buffer-mib: 5.824232E+01 | global_load_balancing_loss: 1.000267E+00 | loss scale: 1.0 | grad norm: 0.092 | params norm: 11312.637 | params rms: 0.061254 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 13:49:53.909569 | finish at 2026-10-10 18:42:05 + [2026-10-09 04:53:39] iteration 5296/ 6855 | consumed samples: 677888 | elapsed time per iteration (ms): 88307.3 | throughput per GPU (TFLOP/s/GPU): 41.4 | MFU 13.26% | FLOPs estimator: workload-aware | logical tokens/s: 189985.4 | valid tokens/s: 189985.4 | physical tokens/s: 190465.0 | loss tokens/s: 141951.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.367585E-01 | lm-global-supervised-tokens: 1.253539E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.179021E-02 | lm-token-buffer-mib: 5.811950E+01 | global_load_balancing_loss: 9.993570E-01 | loss scale: 1.0 | grad norm: 0.286 | params norm: 11312.579 | params rms: 0.061254 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 14:14:31.148862 | finish at 2026-10-10 19:08:11 + [2026-10-09 04:55:09] iteration 5297/ 6855 | consumed samples: 678016 | elapsed time per iteration (ms): 89371.7 | throughput per GPU (TFLOP/s/GPU): 41.2 | MFU 13.20% | FLOPs estimator: workload-aware | logical tokens/s: 187724.0 | valid tokens/s: 187724.0 | physical tokens/s: 188182.4 | loss tokens/s: 136586.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.444290E-01 | lm-global-supervised-tokens: 1.220700E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.353403E-02 | lm-token-buffer-mib: 5.817328E+01 | global_load_balancing_loss: 9.984764E-01 | loss scale: 1.0 | grad norm: 0.480 | params norm: 11312.520 | params rms: 0.061254 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 14:40:41.071530 | finish at 2026-10-10 19:35:50 + [2026-10-09 04:56:37] iteration 5298/ 6855 | consumed samples: 678144 | elapsed time per iteration (ms): 87849.1 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.74% | FLOPs estimator: workload-aware | logical tokens/s: 190977.0 | valid tokens/s: 190977.0 | physical tokens/s: 191445.0 | loss tokens/s: 144253.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.877060E-01 | lm-global-supervised-tokens: 1.267255E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.175210E-02 | lm-token-buffer-mib: 5.813081E+01 | global_load_balancing_loss: 9.982220E-01 | loss scale: 1.0 | grad norm: 0.072 | params norm: 11312.458 | params rms: 0.061253 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 13:59:41.044050 | finish at 2026-10-10 18:56:18 + [2026-10-09 04:58:05] iteration 5299/ 6855 | consumed samples: 678272 | elapsed time per iteration (ms): 87971.3 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.56% | FLOPs estimator: workload-aware | logical tokens/s: 190712.3 | valid tokens/s: 190712.3 | physical tokens/s: 191162.7 | loss tokens/s: 142911.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.022315E-01 | lm-global-supervised-tokens: 1.257207E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.529769E-02 | lm-token-buffer-mib: 5.809124E+01 | global_load_balancing_loss: 9.994339E-01 | loss scale: 1.0 | grad norm: 0.199 | params norm: 11312.396 | params rms: 0.061253 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 14:01:23.382689 | finish at 2026-10-10 18:59:28 + [2026-10-09 04:59:53] iteration 5300/ 6855 | consumed samples: 678400 | elapsed time per iteration (ms): 108741.8 | throughput per GPU (TFLOP/s/GPU): 33.8 | MFU 10.82% | FLOPs estimator: workload-aware | logical tokens/s: 154284.7 | valid tokens/s: 154284.7 | physical tokens/s: 154628.1 | loss tokens/s: 115484.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.020344E-01 | lm-global-supervised-tokens: 1.255799E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.163866E-02 | lm-token-buffer-mib: 5.835727E+01 | global_load_balancing_loss: 1.001233E+00 | loss scale: 1.0 | grad norm: 0.136 | params norm: 11312.334 | params rms: 0.061253 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 22:58:13.557315 | finish at 2026-10-11 03:58:07 + [2026-10-09 05:01:20] iteration 5301/ 6855 | consumed samples: 678528 | elapsed time per iteration (ms): 86980.8 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.46% | FLOPs estimator: workload-aware | logical tokens/s: 192884.0 | valid tokens/s: 192884.0 | physical tokens/s: 193228.3 | loss tokens/s: 147327.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.806208E-01 | lm-global-supervised-tokens: 1.281464E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.451896E-02 | lm-token-buffer-mib: 5.814130E+01 | global_load_balancing_loss: 1.000866E+00 | loss scale: 1.0 | grad norm: 0.056 | params norm: 11312.273 | params rms: 0.061252 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 13:32:48.152321 | finish at 2026-10-10 18:34:08 + [2026-10-09 05:02:45] iteration 5302/ 6855 | consumed samples: 678656 | elapsed time per iteration (ms): 84453.0 | throughput per GPU (TFLOP/s/GPU): 43.3 | MFU 13.87% | FLOPs estimator: workload-aware | logical tokens/s: 198656.7 | valid tokens/s: 198656.7 | physical tokens/s: 199152.5 | loss tokens/s: 150346.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.568252E-01 | lm-global-supervised-tokens: 1.269718E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.312888E-02 | lm-token-buffer-mib: 5.820292E+01 | global_load_balancing_loss: 1.001018E+00 | loss scale: 1.0 | grad norm: 0.155 | params norm: 11312.209 | params rms: 0.061252 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 12:25:55.470229 | finish at 2026-10-10 17:28:40 + [2026-10-09 05:04:15] iteration 5303/ 6855 | consumed samples: 678784 | elapsed time per iteration (ms): 90326.2 | throughput per GPU (TFLOP/s/GPU): 41.7 | MFU 13.37% | FLOPs estimator: workload-aware | logical tokens/s: 185739.4 | valid tokens/s: 185739.4 | physical tokens/s: 186042.8 | loss tokens/s: 136855.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.178046E-01 | lm-global-supervised-tokens: 1.236165E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.506076E-02 | lm-token-buffer-mib: 5.807516E+01 | global_load_balancing_loss: 1.000392E+00 | loss scale: 1.0 | grad norm: 0.068 | params norm: 11312.144 | params rms: 0.061252 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 14:56:26.277214 | finish at 2026-10-10 20:00:41 + [2026-10-09 05:05:41] iteration 5304/ 6855 | consumed samples: 678912 | elapsed time per iteration (ms): 85636.3 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.60% | FLOPs estimator: workload-aware | logical tokens/s: 195911.9 | valid tokens/s: 195911.9 | physical tokens/s: 196329.6 | loss tokens/s: 143442.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.917123E-01 | lm-global-supervised-tokens: 1.228393E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.576190E-02 | lm-token-buffer-mib: 5.806641E+01 | global_load_balancing_loss: 1.000268E+00 | loss scale: 1.0 | grad norm: 0.100 | params norm: 11312.078 | params rms: 0.061251 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 12:53:41.955794 | finish at 2026-10-10 17:59:23 + [2026-10-09 05:07:05] iteration 5305/ 6855 | consumed samples: 679040 | elapsed time per iteration (ms): 84140.2 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.70% | FLOPs estimator: workload-aware | logical tokens/s: 199394.7 | valid tokens/s: 199394.7 | physical tokens/s: 199891.1 | loss tokens/s: 149691.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.455705E-01 | lm-global-supervised-tokens: 1.259510E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.059350E-02 | lm-token-buffer-mib: 5.816606E+01 | global_load_balancing_loss: 1.000231E+00 | loss scale: 1.0 | grad norm: 0.066 | params norm: 11312.011 | params rms: 0.061251 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 12:13:37.235935 | finish at 2026-10-10 17:20:42 + [2026-10-09 05:08:32] iteration 5306/ 6855 | consumed samples: 679168 | elapsed time per iteration (ms): 87507.7 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.62% | FLOPs estimator: workload-aware | logical tokens/s: 191722.2 | valid tokens/s: 191722.2 | physical tokens/s: 192181.2 | loss tokens/s: 138769.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.677456E-01 | lm-global-supervised-tokens: 1.214343E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.270115E-02 | lm-token-buffer-mib: 5.810529E+01 | global_load_balancing_loss: 1.001320E+00 | loss scale: 1.0 | grad norm: 0.128 | params norm: 11311.942 | params rms: 0.061250 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 13:39:09.368528 | finish at 2026-10-10 18:47:42 + [2026-10-09 05:10:01] iteration 5307/ 6855 | consumed samples: 679296 | elapsed time per iteration (ms): 88217.6 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.41% | FLOPs estimator: workload-aware | logical tokens/s: 190179.9 | valid tokens/s: 190179.9 | physical tokens/s: 190551.6 | loss tokens/s: 142917.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.731615E-01 | lm-global-supervised-tokens: 1.260787E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.165838E-02 | lm-token-buffer-mib: 5.807246E+01 | global_load_balancing_loss: 1.001481E+00 | loss scale: 1.0 | grad norm: 0.081 | params norm: 11311.874 | params rms: 0.061250 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 13:56:00.792558 | finish at 2026-10-10 19:06:01 + [2026-10-09 05:11:29] iteration 5308/ 6855 | consumed samples: 679424 | elapsed time per iteration (ms): 88676.3 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.51% | FLOPs estimator: workload-aware | logical tokens/s: 189195.5 | valid tokens/s: 189195.5 | physical tokens/s: 189633.4 | loss tokens/s: 140265.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.987151E-01 | lm-global-supervised-tokens: 1.243826E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.300544E-02 | lm-token-buffer-mib: 5.810655E+01 | global_load_balancing_loss: 1.000470E+00 | loss scale: 1.0 | grad norm: 0.131 | params norm: 11311.805 | params rms: 0.061250 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 14:06:22.235807 | finish at 2026-10-10 19:17:51 + [2026-10-09 05:12:58] iteration 5309/ 6855 | consumed samples: 679552 | elapsed time per iteration (ms): 88738.9 | throughput per GPU (TFLOP/s/GPU): 41.5 | MFU 13.31% | FLOPs estimator: workload-aware | logical tokens/s: 189062.3 | valid tokens/s: 189062.3 | physical tokens/s: 189496.8 | loss tokens/s: 135824.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.522743E-01 | lm-global-supervised-tokens: 1.205292E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.155643E-02 | lm-token-buffer-mib: 5.823117E+01 | global_load_balancing_loss: 9.997475E-01 | loss scale: 1.0 | grad norm: 0.050 | params norm: 11311.735 | params rms: 0.061249 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 14:06:30.411930 | finish at 2026-10-10 19:19:28 + [2026-10-09 05:14:47] iteration 5310/ 6855 | consumed samples: 679680 | elapsed time per iteration (ms): 109506.1 | throughput per GPU (TFLOP/s/GPU): 33.2 | MFU 10.64% | FLOPs estimator: workload-aware | logical tokens/s: 153207.1 | valid tokens/s: 153207.1 | physical tokens/s: 153627.9 | loss tokens/s: 113418.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.832086E-01 | lm-global-supervised-tokens: 1.241999E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.335718E-02 | lm-token-buffer-mib: 5.820483E+01 | global_load_balancing_loss: 9.981471E-01 | loss scale: 1.0 | grad norm: 0.083 | params norm: 11311.665 | params rms: 0.061249 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 22:59:46.995867 | finish at 2026-10-11 04:14:34 + [2026-10-09 05:16:18] iteration 5311/ 6855 | consumed samples: 679808 | elapsed time per iteration (ms): 90252.5 | throughput per GPU (TFLOP/s/GPU): 41.6 | MFU 13.32% | FLOPs estimator: workload-aware | logical tokens/s: 185890.6 | valid tokens/s: 185890.6 | physical tokens/s: 186432.8 | loss tokens/s: 141473.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.961810E-01 | lm-global-supervised-tokens: 1.276830E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.166008E-02 | lm-token-buffer-mib: 5.819173E+01 | global_load_balancing_loss: 9.985827E-01 | loss scale: 1.0 | grad norm: 0.051 | params norm: 11311.598 | params rms: 0.061249 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 14:42:29.913465 | finish at 2026-10-10 19:58:48 + [2026-10-09 05:17:46] iteration 5312/ 6855 | consumed samples: 679936 | elapsed time per iteration (ms): 87853.1 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.52% | FLOPs estimator: workload-aware | logical tokens/s: 190968.7 | valid tokens/s: 190968.7 | physical tokens/s: 191435.9 | loss tokens/s: 138135.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.695912E-01 | lm-global-supervised-tokens: 1.213564E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.546819E-02 | lm-token-buffer-mib: 5.811928E+01 | global_load_balancing_loss: 9.994910E-01 | loss scale: 1.0 | grad norm: 0.043 | params norm: 11311.532 | params rms: 0.061248 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 13:39:17.347006 | finish at 2026-10-10 18:57:03 + [2026-10-09 05:19:12] iteration 5313/ 6855 | consumed samples: 680064 | elapsed time per iteration (ms): 86164.0 | throughput per GPU (TFLOP/s/GPU): 43.6 | MFU 13.97% | FLOPs estimator: workload-aware | logical tokens/s: 194712.5 | valid tokens/s: 194712.5 | physical tokens/s: 195114.7 | loss tokens/s: 144539.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.727789E-01 | lm-global-supervised-tokens: 1.245411E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.152438E-02 | lm-token-buffer-mib: 5.819163E+01 | global_load_balancing_loss: 9.998968E-01 | loss scale: 1.0 | grad norm: 0.057 | params norm: 11311.466 | params rms: 0.061248 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 12:54:24.841730 | finish at 2026-10-10 18:13:37 + [2026-10-09 05:20:41] iteration 5314/ 6855 | consumed samples: 680192 | elapsed time per iteration (ms): 89273.5 | throughput per GPU (TFLOP/s/GPU): 41.0 | MFU 13.14% | FLOPs estimator: workload-aware | logical tokens/s: 187929.5 | valid tokens/s: 187929.5 | physical tokens/s: 188376.9 | loss tokens/s: 144143.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.459054E-01 | lm-global-supervised-tokens: 1.286819E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.454732E-02 | lm-token-buffer-mib: 5.806458E+01 | global_load_balancing_loss: 1.000992E+00 | loss scale: 1.0 | grad norm: 0.042 | params norm: 11311.399 | params rms: 0.061248 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 14:12:50.388864 | finish at 2026-10-10 19:33:31 + [2026-10-09 05:22:11] iteration 5315/ 6855 | consumed samples: 680320 | elapsed time per iteration (ms): 89707.1 | throughput per GPU (TFLOP/s/GPU): 41.3 | MFU 13.24% | FLOPs estimator: workload-aware | logical tokens/s: 187022.2 | valid tokens/s: 187022.2 | physical tokens/s: 187421.5 | loss tokens/s: 139350.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.328901E-01 | lm-global-supervised-tokens: 1.250070E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.186916E-02 | lm-token-buffer-mib: 5.808312E+01 | global_load_balancing_loss: 1.001935E+00 | loss scale: 1.0 | grad norm: 0.056 | params norm: 11311.332 | params rms: 0.061247 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 14:22:28.915510 | finish at 2026-10-10 19:44:40 + [2026-10-09 05:23:41] iteration 5316/ 6855 | consumed samples: 680448 | elapsed time per iteration (ms): 90354.4 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.50% | FLOPs estimator: workload-aware | logical tokens/s: 185681.1 | valid tokens/s: 185681.1 | physical tokens/s: 186067.5 | loss tokens/s: 134423.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.725873E-01 | lm-global-supervised-tokens: 1.214578E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.145749E-02 | lm-token-buffer-mib: 5.820431E+01 | global_load_balancing_loss: 1.001186E+00 | loss scale: 1.0 | grad norm: 0.042 | params norm: 11311.265 | params rms: 0.061247 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 14:37:35.393957 | finish at 2026-10-10 20:01:16 + [2026-10-09 05:25:07] iteration 5317/ 6855 | consumed samples: 680576 | elapsed time per iteration (ms): 86056.1 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.50% | FLOPs estimator: workload-aware | logical tokens/s: 194956.2 | valid tokens/s: 194956.2 | physical tokens/s: 195318.6 | loss tokens/s: 145470.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.815547E-01 | lm-global-supervised-tokens: 1.251866E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.222707E-02 | lm-token-buffer-mib: 5.804155E+01 | global_load_balancing_loss: 1.001621E+00 | loss scale: 1.0 | grad norm: 0.039 | params norm: 11311.197 | params rms: 0.061246 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 12:45:54.351304 | finish at 2026-10-10 18:11:02 + [2026-10-09 05:26:32] iteration 5318/ 6855 | consumed samples: 680704 | elapsed time per iteration (ms): 84416.3 | throughput per GPU (TFLOP/s/GPU): 42.8 | MFU 13.71% | FLOPs estimator: workload-aware | logical tokens/s: 198743.6 | valid tokens/s: 198743.6 | physical tokens/s: 199201.8 | loss tokens/s: 153095.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.605320E-01 | lm-global-supervised-tokens: 1.292375E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.762931E-02 | lm-token-buffer-mib: 5.809546E+01 | global_load_balancing_loss: 9.990557E-01 | loss scale: 1.0 | grad norm: 0.037 | params norm: 11311.130 | params rms: 0.061246 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 12:02:27.906325 | finish at 2026-10-10 17:28:59 + [2026-10-09 05:27:58] iteration 5319/ 6855 | consumed samples: 680832 | elapsed time per iteration (ms): 86800.5 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.55% | FLOPs estimator: workload-aware | logical tokens/s: 193284.6 | valid tokens/s: 193284.6 | physical tokens/s: 193946.0 | loss tokens/s: 142858.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.479085E-01 | lm-global-supervised-tokens: 1.240016E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.837536E-02 | lm-token-buffer-mib: 5.813892E+01 | global_load_balancing_loss: 1.000003E+00 | loss scale: 1.0 | grad norm: 0.072 | params norm: 11311.063 | params rms: 0.061246 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 13:02:05.558716 | finish at 2026-10-10 18:30:04 + [2026-10-09 05:29:46] iteration 5320/ 6855 | consumed samples: 680960 | elapsed time per iteration (ms): 107538.8 | throughput per GPU (TFLOP/s/GPU): 34.1 | MFU 10.94% | FLOPs estimator: workload-aware | logical tokens/s: 156010.5 | valid tokens/s: 156010.5 | physical tokens/s: 156499.3 | loss tokens/s: 118514.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.435689E-01 | lm-global-supervised-tokens: 1.274490E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.170401E-02 | lm-token-buffer-mib: 5.806794E+01 | global_load_balancing_loss: 9.992389E-01 | loss scale: 1.0 | grad norm: 0.128 | params norm: 11310.998 | params rms: 0.061245 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 21:51:12.016647 | finish at 2026-10-11 03:20:58 +times across ranks (ms): + save-checkpoint: + rank 0: 23767.53 + rank 1: 23767.54 + rank 2: 23767.57 + rank 3: 23767.51 + rank 4: 23767.55 + rank 5: 23767.54 + rank 6: 23767.53 + rank 7: 23767.53 + rank 8: 23767.91 + rank 9: 23767.82 + rank 10: 23767.84 + rank 11: 23767.82 + rank 12: 23767.82 + rank 13: 23767.86 + rank 14: 23767.86 + rank 15: 23767.83 + rank 16: 23767.35 + rank 17: 23767.35 + rank 18: 23767.36 + rank 19: 23767.36 + rank 20: 23767.33 + rank 21: 23767.41 + rank 22: 23767.34 + rank 23: 23767.34 + rank 24: 23767.60 + rank 25: 23767.60 + rank 26: 23767.64 + rank 27: 23767.65 + rank 28: 23767.62 + rank 29: 23767.63 + rank 30: 23767.61 + rank 31: 23767.61 + rank 32: 23768.15 + rank 33: 23768.15 + rank 34: 23768.13 + rank 35: 23768.11 + rank 36: 23768.10 + rank 37: 23768.10 + rank 38: 23768.10 + rank 39: 23768.07 + rank 40: 23768.08 + rank 41: 23768.06 + rank 42: 23768.05 + rank 43: 23768.10 + rank 44: 23768.05 + rank 45: 23768.06 + rank 46: 23768.06 + rank 47: 23768.04 + rank 48: 23768.13 + rank 49: 23768.11 + rank 50: 23768.13 + rank 51: 23768.09 + rank 52: 23768.09 + rank 53: 23768.10 + rank 54: 23768.05 + rank 55: 23768.07 + rank 56: 23768.13 + rank 57: 23768.12 + rank 58: 23768.11 + rank 59: 23768.11 + rank 60: 23768.10 + rank 61: 23768.14 + rank 62: 23768.12 + rank 63: 23768.12 + [2026-10-09 05:31:36] iteration 5321/ 6855 | consumed samples: 681088 | elapsed time per iteration (ms): 86517.2 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.57% | FLOPs estimator: workload-aware | logical tokens/s: 193917.3 | valid tokens/s: 193917.3 | physical tokens/s: 194299.6 | loss tokens/s: 143129.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.909956E-01 | lm-global-supervised-tokens: 1.238314E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.261187E+00 | lm-token-buffer-mib: 5.823314E+01 | global_load_balancing_loss: 9.996890E-01 | loss scale: 1.0 | grad norm: 0.111 | params norm: 11310.932 | params rms: 0.061245 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 12:51:57.359920 | finish at 2026-10-10 18:23:34 + [2026-10-09 05:33:08] iteration 5322/ 6855 | consumed samples: 681216 | elapsed time per iteration (ms): 91277.1 | throughput per GPU (TFLOP/s/GPU): 39.9 | MFU 12.80% | FLOPs estimator: workload-aware | logical tokens/s: 183805.3 | valid tokens/s: 183805.3 | physical tokens/s: 184325.3 | loss tokens/s: 141487.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.344310E-01 | lm-global-supervised-tokens: 1.291460E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.568989E-02 | lm-token-buffer-mib: 5.811720E+01 | global_load_balancing_loss: 9.997614E-01 | loss scale: 1.0 | grad norm: 0.125 | params norm: 11310.865 | params rms: 0.061245 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 14:52:07.771771 | finish at 2026-10-10 20:25:15 + [2026-10-09 05:34:37] iteration 5323/ 6855 | consumed samples: 681344 | elapsed time per iteration (ms): 89502.3 | throughput per GPU (TFLOP/s/GPU): 41.7 | MFU 13.38% | FLOPs estimator: workload-aware | logical tokens/s: 187449.5 | valid tokens/s: 187449.5 | physical tokens/s: 187791.4 | loss tokens/s: 134614.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.479862E-01 | lm-global-supervised-tokens: 1.204830E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.498540E-02 | lm-token-buffer-mib: 5.805022E+01 | global_load_balancing_loss: 1.000992E+00 | loss scale: 1.0 | grad norm: 0.157 | params norm: 11310.800 | params rms: 0.061244 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 14:05:17.472136 | finish at 2026-10-10 19:39:55 + [2026-10-09 05:36:05] iteration 5324/ 6855 | consumed samples: 681472 | elapsed time per iteration (ms): 88100.7 | throughput per GPU (TFLOP/s/GPU): 41.7 | MFU 13.36% | FLOPs estimator: workload-aware | logical tokens/s: 190431.5 | valid tokens/s: 190431.5 | physical tokens/s: 190843.6 | loss tokens/s: 141080.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.610222E-01 | lm-global-supervised-tokens: 1.242926E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.863839E-02 | lm-token-buffer-mib: 5.804291E+01 | global_load_balancing_loss: 1.001083E+00 | loss scale: 1.0 | grad norm: 0.145 | params norm: 11310.733 | params rms: 0.061244 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 13:28:02.174104 | finish at 2026-10-10 19:04:07 + [2026-10-09 05:37:29] iteration 5325/ 6855 | consumed samples: 681600 | elapsed time per iteration (ms): 83877.2 | throughput per GPU (TFLOP/s/GPU): 43.1 | MFU 13.81% | FLOPs estimator: workload-aware | logical tokens/s: 200019.8 | valid tokens/s: 200019.8 | physical tokens/s: 200473.1 | loss tokens/s: 155924.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.570309E-01 | lm-global-supervised-tokens: 1.307848E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.588487E-02 | lm-token-buffer-mib: 5.817406E+01 | global_load_balancing_loss: 9.997547E-01 | loss scale: 1.0 | grad norm: 0.126 | params norm: 11310.666 | params rms: 0.061244 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 11:38:52.106345 | finish at 2026-10-10 17:16:21 + [2026-10-09 05:38:55] iteration 5326/ 6855 | consumed samples: 681728 | elapsed time per iteration (ms): 85644.6 | throughput per GPU (TFLOP/s/GPU): 43.4 | MFU 13.92% | FLOPs estimator: workload-aware | logical tokens/s: 195892.8 | valid tokens/s: 195892.8 | physical tokens/s: 196238.1 | loss tokens/s: 145755.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.275564E-01 | lm-global-supervised-tokens: 1.248318E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.168549E-02 | lm-token-buffer-mib: 5.813468E+01 | global_load_balancing_loss: 1.000073E+00 | loss scale: 1.0 | grad norm: 0.037 | params norm: 11310.600 | params rms: 0.061243 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 12:22:30.629359 | finish at 2026-10-10 18:01:25 + [2026-10-09 05:40:23] iteration 5327/ 6855 | consumed samples: 681856 | elapsed time per iteration (ms): 87852.2 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.59% | FLOPs estimator: workload-aware | logical tokens/s: 190970.8 | valid tokens/s: 190970.7 | physical tokens/s: 191343.7 | loss tokens/s: 140913.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.779786E-01 | lm-global-supervised-tokens: 1.237954E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.778862E-02 | lm-token-buffer-mib: 5.814660E+01 | global_load_balancing_loss: 9.994032E-01 | loss scale: 1.0 | grad norm: 0.106 | params norm: 11310.534 | params rms: 0.061243 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 13:17:18.170027 | finish at 2026-10-10 18:57:41 + [2026-10-09 05:41:51] iteration 5328/ 6855 | consumed samples: 681984 | elapsed time per iteration (ms): 88082.2 | throughput per GPU (TFLOP/s/GPU): 41.6 | MFU 13.34% | FLOPs estimator: workload-aware | logical tokens/s: 190471.9 | valid tokens/s: 190471.9 | physical tokens/s: 191153.9 | loss tokens/s: 142525.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.326684E-01 | lm-global-supervised-tokens: 1.255396E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.151097E-02 | lm-token-buffer-mib: 5.813332E+01 | global_load_balancing_loss: 1.000389E+00 | loss scale: 1.0 | grad norm: 0.070 | params norm: 11310.465 | params rms: 0.061242 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 13:21:41.483434 | finish at 2026-10-10 19:03:32 + [2026-10-09 05:43:18] iteration 5329/ 6855 | consumed samples: 682112 | elapsed time per iteration (ms): 86802.2 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.63% | FLOPs estimator: workload-aware | logical tokens/s: 193279.8 | valid tokens/s: 193279.8 | physical tokens/s: 193835.0 | loss tokens/s: 146193.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.589490E-01 | lm-global-supervised-tokens: 1.268996E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.308713E-02 | lm-token-buffer-mib: 5.813963E+01 | global_load_balancing_loss: 1.000068E+00 | loss scale: 1.0 | grad norm: 0.182 | params norm: 11310.397 | params rms: 0.061242 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 12:47:40.188246 | finish at 2026-10-10 18:30:58 + [2026-10-09 05:45:08] iteration 5330/ 6855 | consumed samples: 682240 | elapsed time per iteration (ms): 110564.2 | throughput per GPU (TFLOP/s/GPU): 32.8 | MFU 10.51% | FLOPs estimator: workload-aware | logical tokens/s: 151741.1 | valid tokens/s: 151741.1 | physical tokens/s: 152251.2 | loss tokens/s: 114716.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 5.799934E-01 | lm-global-supervised-tokens: 1.268356E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.159367E-02 | lm-token-buffer-mib: 5.819277E+01 | global_load_balancing_loss: 9.993190E-01 | loss scale: 1.0 | grad norm: 0.113 | params norm: 11310.333 | params rms: 0.061242 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 22:50:10.424155 | finish at 2026-10-11 04:35:19 + [2026-10-09 05:46:36] iteration 5331/ 6855 | consumed samples: 682368 | elapsed time per iteration (ms): 87474.6 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.62% | FLOPs estimator: workload-aware | logical tokens/s: 191794.2 | valid tokens/s: 191794.2 | physical tokens/s: 192274.8 | loss tokens/s: 138128.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.498011E-01 | lm-global-supervised-tokens: 1.208271E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.137425E-02 | lm-token-buffer-mib: 5.809957E+01 | global_load_balancing_loss: 1.000155E+00 | loss scale: 1.0 | grad norm: 0.127 | params norm: 11310.269 | params rms: 0.061241 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 13:01:51.330849 | finish at 2026-10-10 18:48:27 + [2026-10-09 05:48:02] iteration 5332/ 6855 | consumed samples: 682496 | elapsed time per iteration (ms): 86561.6 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.78% | FLOPs estimator: workload-aware | logical tokens/s: 193817.6 | valid tokens/s: 193817.6 | physical tokens/s: 194203.1 | loss tokens/s: 140317.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.626997E-01 | lm-global-supervised-tokens: 1.214606E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.332696E-02 | lm-token-buffer-mib: 5.814893E+01 | global_load_balancing_loss: 1.001395E+00 | loss scale: 1.0 | grad norm: 0.133 | params norm: 11310.204 | params rms: 0.061241 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 12:37:13.305135 | finish at 2026-10-10 18:25:15 + [2026-10-09 05:49:28] iteration 5333/ 6855 | consumed samples: 682624 | elapsed time per iteration (ms): 85398.9 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.61% | FLOPs estimator: workload-aware | logical tokens/s: 196456.2 | valid tokens/s: 196456.0 | physical tokens/s: 196928.2 | loss tokens/s: 150998.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.745716E-01 | lm-global-supervised-tokens: 1.289512E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.492884E-02 | lm-token-buffer-mib: 5.815647E+01 | global_load_balancing_loss: 1.000340E+00 | loss scale: 1.0 | grad norm: 0.059 | params norm: 11310.137 | params rms: 0.061241 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 12:06:17.144718 | finish at 2026-10-10 17:55:45 + [2026-10-09 05:50:55] iteration 5334/ 6855 | consumed samples: 682752 | elapsed time per iteration (ms): 87045.3 | throughput per GPU (TFLOP/s/GPU): 43.8 | MFU 14.04% | FLOPs estimator: workload-aware | logical tokens/s: 192740.7 | valid tokens/s: 192740.7 | physical tokens/s: 193091.3 | loss tokens/s: 144117.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.708193E-01 | lm-global-supervised-tokens: 1.254471E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.427439E-02 | lm-token-buffer-mib: 5.805539E+01 | global_load_balancing_loss: 1.000496E+00 | loss scale: 1.0 | grad norm: 0.080 | params norm: 11310.074 | params rms: 0.061240 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 12:46:35.841838 | finish at 2026-10-10 18:37:30 + [2026-10-09 05:52:25] iteration 5335/ 6855 | consumed samples: 682880 | elapsed time per iteration (ms): 90708.3 | throughput per GPU (TFLOP/s/GPU): 41.0 | MFU 13.14% | FLOPs estimator: workload-aware | logical tokens/s: 184957.6 | valid tokens/s: 184957.6 | physical tokens/s: 185513.2 | loss tokens/s: 133212.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.434636E-01 | lm-global-supervised-tokens: 1.208346E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.230431E-02 | lm-token-buffer-mib: 5.835955E+01 | global_load_balancing_loss: 1.000149E+00 | loss scale: 1.0 | grad norm: 0.151 | params norm: 11310.011 | params rms: 0.061240 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 14:17:56.559639 | finish at 2026-10-10 20:10:22 + [2026-10-09 05:53:53] iteration 5336/ 6855 | consumed samples: 683008 | elapsed time per iteration (ms): 87744.3 | throughput per GPU (TFLOP/s/GPU): 41.5 | MFU 13.31% | FLOPs estimator: workload-aware | logical tokens/s: 191205.3 | valid tokens/s: 191205.3 | physical tokens/s: 191808.4 | loss tokens/s: 142195.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.465869E-01 | lm-global-supervised-tokens: 1.247683E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.525161E-02 | lm-token-buffer-mib: 5.826871E+01 | global_load_balancing_loss: 9.998226E-01 | loss scale: 1.0 | grad norm: 0.077 | params norm: 11309.947 | params rms: 0.061240 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 13:01:23.582115 | finish at 2026-10-10 18:55:17 + [2026-10-09 05:55:26] iteration 5337/ 6855 | consumed samples: 683136 | elapsed time per iteration (ms): 93104.8 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.41% | FLOPs estimator: workload-aware | logical tokens/s: 180197.1 | valid tokens/s: 180197.1 | physical tokens/s: 180591.4 | loss tokens/s: 130355.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.151068E-01 | lm-global-supervised-tokens: 1.213675E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.919037E-02 | lm-token-buffer-mib: 5.809939E+01 | global_load_balancing_loss: 9.998122E-01 | loss scale: 1.0 | grad norm: 0.052 | params norm: 11309.884 | params rms: 0.061239 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 15:15:33.070816 | finish at 2026-10-10 21:10:59 + [2026-10-09 05:56:53] iteration 5338/ 6855 | consumed samples: 683264 | elapsed time per iteration (ms): 86605.3 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.76% | FLOPs estimator: workload-aware | logical tokens/s: 193720.5 | valid tokens/s: 193720.5 | physical tokens/s: 194161.5 | loss tokens/s: 145031.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.237523E-01 | lm-global-supervised-tokens: 1.256049E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.234119E-02 | lm-token-buffer-mib: 5.812619E+01 | global_load_balancing_loss: 9.992434E-01 | loss scale: 1.0 | grad norm: 0.134 | params norm: 11309.820 | params rms: 0.061239 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 12:29:40.205302 | finish at 2026-10-10 18:26:33 + [2026-10-09 05:58:25] iteration 5339/ 6855 | consumed samples: 683392 | elapsed time per iteration (ms): 91944.2 | throughput per GPU (TFLOP/s/GPU): 41.1 | MFU 13.18% | FLOPs estimator: workload-aware | logical tokens/s: 182471.4 | valid tokens/s: 182471.4 | physical tokens/s: 182845.4 | loss tokens/s: 135642.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.219952E-01 | lm-global-supervised-tokens: 1.247156E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.322506E-02 | lm-token-buffer-mib: 5.804205E+01 | global_load_balancing_loss: 1.000162E+00 | loss scale: 1.0 | grad norm: 0.099 | params norm: 11309.758 | params rms: 0.061239 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 14:43:07.364970 | finish at 2026-10-10 20:41:32 + [2026-10-09 06:00:15] iteration 5340/ 6855 | consumed samples: 683520 | elapsed time per iteration (ms): 110526.6 | throughput per GPU (TFLOP/s/GPU): 34.0 | MFU 10.91% | FLOPs estimator: workload-aware | logical tokens/s: 151793.3 | valid tokens/s: 151793.3 | physical tokens/s: 152261.7 | loss tokens/s: 112494.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.257345E-01 | lm-global-supervised-tokens: 1.243365E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.298339E-02 | lm-token-buffer-mib: 5.805948E+01 | global_load_balancing_loss: 1.001125E+00 | loss scale: 1.0 | grad norm: 0.092 | params norm: 11309.696 | params rms: 0.061238 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 22:30:47.867453 | finish at 2026-10-11 04:31:03 + [2026-10-09 06:01:38] iteration 5341/ 6855 | consumed samples: 683648 | elapsed time per iteration (ms): 83052.4 | throughput per GPU (TFLOP/s/GPU): 43.2 | MFU 13.85% | FLOPs estimator: workload-aware | logical tokens/s: 202006.4 | valid tokens/s: 202006.3 | physical tokens/s: 202593.7 | loss tokens/s: 152846.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.045462E-01 | lm-global-supervised-tokens: 1.269422E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.357538E-02 | lm-token-buffer-mib: 5.817539E+01 | global_load_balancing_loss: 1.000901E+00 | loss scale: 1.0 | grad norm: 0.069 | params norm: 11309.635 | params rms: 0.061238 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 10:55:41.315361 | finish at 2026-10-10 16:57:20 + [2026-10-09 06:03:07] iteration 5342/ 6855 | consumed samples: 683776 | elapsed time per iteration (ms): 88418.9 | throughput per GPU (TFLOP/s/GPU): 41.7 | MFU 13.35% | FLOPs estimator: workload-aware | logical tokens/s: 189746.9 | valid tokens/s: 189746.9 | physical tokens/s: 190188.2 | loss tokens/s: 145390.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.898469E-01 | lm-global-supervised-tokens: 1.285529E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.134983E-02 | lm-token-buffer-mib: 5.806985E+01 | global_load_balancing_loss: 9.999820E-01 | loss scale: 1.0 | grad norm: 0.053 | params norm: 11309.575 | params rms: 0.061238 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 13:09:37.728624 | finish at 2026-10-10 19:12:44 + [2026-10-09 06:04:36] iteration 5343/ 6855 | consumed samples: 683904 | elapsed time per iteration (ms): 88865.7 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.41% | FLOPs estimator: workload-aware | logical tokens/s: 188792.4 | valid tokens/s: 188792.4 | physical tokens/s: 189203.3 | loss tokens/s: 137548.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.607130E-01 | lm-global-supervised-tokens: 1.222332E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.152139E-02 | lm-token-buffer-mib: 5.809077E+01 | global_load_balancing_loss: 1.000127E+00 | loss scale: 1.0 | grad norm: 0.187 | params norm: 11309.514 | params rms: 0.061237 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 13:19:24.940573 | finish at 2026-10-10 19:24:01 + [2026-10-09 06:06:02] iteration 5344/ 6855 | consumed samples: 684032 | elapsed time per iteration (ms): 86305.0 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.44% | FLOPs estimator: workload-aware | logical tokens/s: 194393.3 | valid tokens/s: 194393.3 | physical tokens/s: 194783.8 | loss tokens/s: 144106.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 5.996895E-01 | lm-global-supervised-tokens: 1.243708E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.162462E-02 | lm-token-buffer-mib: 5.807394E+01 | global_load_balancing_loss: 9.999764E-01 | loss scale: 1.0 | grad norm: 0.398 | params norm: 11309.453 | params rms: 0.061237 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 12:13:26.880318 | finish at 2026-10-10 18:19:29 + [2026-10-09 06:07:30] iteration 5345/ 6855 | consumed samples: 684160 | elapsed time per iteration (ms): 88465.0 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.44% | FLOPs estimator: workload-aware | logical tokens/s: 189647.4 | valid tokens/s: 189647.4 | physical tokens/s: 190001.6 | loss tokens/s: 140993.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.425039E-01 | lm-global-supervised-tokens: 1.247301E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.171108E-02 | lm-token-buffer-mib: 5.807662E+01 | global_load_balancing_loss: 9.997120E-01 | loss scale: 1.0 | grad norm: 0.385 | params norm: 11309.394 | params rms: 0.061237 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 13:06:22.092268 | finish at 2026-10-10 19:13:52 + [2026-10-09 06:09:01] iteration 5346/ 6855 | consumed samples: 684288 | elapsed time per iteration (ms): 90274.9 | throughput per GPU (TFLOP/s/GPU): 42.8 | MFU 13.71% | FLOPs estimator: workload-aware | logical tokens/s: 185845.4 | valid tokens/s: 185845.4 | physical tokens/s: 186269.1 | loss tokens/s: 133087.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.319385E-01 | lm-global-supervised-tokens: 1.201442E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.154599E-02 | lm-token-buffer-mib: 5.806707E+01 | global_load_balancing_loss: 9.997926E-01 | loss scale: 1.0 | grad norm: 0.064 | params norm: 11309.334 | params rms: 0.061236 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 13:50:24.856059 | finish at 2026-10-10 19:59:26 + [2026-10-09 06:10:29] iteration 5347/ 6855 | consumed samples: 684416 | elapsed time per iteration (ms): 88820.5 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.43% | FLOPs estimator: workload-aware | logical tokens/s: 188888.5 | valid tokens/s: 188888.4 | physical tokens/s: 189323.5 | loss tokens/s: 141711.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.486280E-01 | lm-global-supervised-tokens: 1.258690E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.194719E-02 | lm-token-buffer-mib: 5.810214E+01 | global_load_balancing_loss: 9.994417E-01 | loss scale: 1.0 | grad norm: 0.406 | params norm: 11309.271 | params rms: 0.061236 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 13:12:21.306294 | finish at 2026-10-10 19:22:51 + [2026-10-09 06:11:54] iteration 5348/ 6855 | consumed samples: 684544 | elapsed time per iteration (ms): 84865.3 | throughput per GPU (TFLOP/s/GPU): 43.3 | MFU 13.88% | FLOPs estimator: workload-aware | logical tokens/s: 197691.5 | valid tokens/s: 197691.5 | physical tokens/s: 198180.4 | loss tokens/s: 145547.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.984859E-01 | lm-global-supervised-tokens: 1.235189E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.313483E-02 | lm-token-buffer-mib: 5.808656E+01 | global_load_balancing_loss: 1.001142E+00 | loss scale: 1.0 | grad norm: 0.054 | params norm: 11309.208 | params rms: 0.061236 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 11:31:32.033598 | finish at 2026-10-10 17:43:26 + [2026-10-09 06:13:22] iteration 5349/ 6855 | consumed samples: 684672 | elapsed time per iteration (ms): 87603.5 | throughput per GPU (TFLOP/s/GPU): 41.6 | MFU 13.33% | FLOPs estimator: workload-aware | logical tokens/s: 191512.1 | valid tokens/s: 191512.1 | physical tokens/s: 192032.0 | loss tokens/s: 146076.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.342113E-01 | lm-global-supervised-tokens: 1.279678E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.463186E-02 | lm-token-buffer-mib: 5.819212E+01 | global_load_balancing_loss: 1.000530E+00 | loss scale: 1.0 | grad norm: 0.288 | params norm: 11309.145 | params rms: 0.061235 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 12:38:50.837441 | finish at 2026-10-10 18:52:13 + [2026-10-09 06:15:14] iteration 5350/ 6855 | consumed samples: 684800 | elapsed time per iteration (ms): 111682.6 | throughput per GPU (TFLOP/s/GPU): 33.0 | MFU 10.57% | FLOPs estimator: workload-aware | logical tokens/s: 150221.9 | valid tokens/s: 150221.9 | physical tokens/s: 150619.7 | loss tokens/s: 111779.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.420533E-01 | lm-global-supervised-tokens: 1.248384E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.432675E-02 | lm-token-buffer-mib: 5.812334E+01 | global_load_balancing_loss: 1.001859E+00 | loss scale: 1.0 | grad norm: 0.100 | params norm: 11309.083 | params rms: 0.061235 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 22:41:22.327744 | finish at 2026-10-11 04:56:36 + [2026-10-09 06:16:46] iteration 5351/ 6855 | consumed samples: 684928 | elapsed time per iteration (ms): 92812.5 | throughput per GPU (TFLOP/s/GPU): 40.9 | MFU 13.12% | FLOPs estimator: workload-aware | logical tokens/s: 180764.3 | valid tokens/s: 180764.3 | physical tokens/s: 181227.3 | loss tokens/s: 132576.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.049595E-01 | lm-global-supervised-tokens: 1.230479E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.748905E-02 | lm-token-buffer-mib: 5.806700E+01 | global_load_balancing_loss: 1.001121E+00 | loss scale: 1.0 | grad norm: 0.217 | params norm: 11309.024 | params rms: 0.061235 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 14:46:29.975258 | finish at 2026-10-10 21:03:16 + [2026-10-09 06:18:13] iteration 5352/ 6855 | consumed samples: 685056 | elapsed time per iteration (ms): 86256.3 | throughput per GPU (TFLOP/s/GPU): 42.8 | MFU 13.73% | FLOPs estimator: workload-aware | logical tokens/s: 194504.1 | valid tokens/s: 194504.1 | physical tokens/s: 194874.8 | loss tokens/s: 143344.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.796335E-01 | lm-global-supervised-tokens: 1.236435E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.185281E-02 | lm-token-buffer-mib: 5.815614E+01 | global_load_balancing_loss: 9.998487E-01 | loss scale: 1.0 | grad norm: 0.091 | params norm: 11308.966 | params rms: 0.061234 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 12:00:43.263652 | finish at 2026-10-10 18:18:56 + [2026-10-09 06:19:41] iteration 5353/ 6855 | consumed samples: 685184 | elapsed time per iteration (ms): 88662.5 | throughput per GPU (TFLOP/s/GPU): 42.8 | MFU 13.70% | FLOPs estimator: workload-aware | logical tokens/s: 189225.5 | valid tokens/s: 189225.5 | physical tokens/s: 189569.3 | loss tokens/s: 137931.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.420883E-01 | lm-global-supervised-tokens: 1.222937E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.568235E-02 | lm-token-buffer-mib: 5.812058E+01 | global_load_balancing_loss: 9.992659E-01 | loss scale: 1.0 | grad norm: 0.044 | params norm: 11308.908 | params rms: 0.061234 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 12:59:31.144329 | finish at 2026-10-10 19:19:13 + [2026-10-09 06:21:09] iteration 5354/ 6855 | consumed samples: 685312 | elapsed time per iteration (ms): 87676.5 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.53% | FLOPs estimator: workload-aware | logical tokens/s: 191353.4 | valid tokens/s: 191353.4 | physical tokens/s: 191811.6 | loss tokens/s: 139074.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.512936E-01 | lm-global-supervised-tokens: 1.219356E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.156020E-02 | lm-token-buffer-mib: 5.819666E+01 | global_load_balancing_loss: 9.985415E-01 | loss scale: 1.0 | grad norm: 0.113 | params norm: 11308.851 | params rms: 0.061234 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 12:33:22.355050 | finish at 2026-10-10 18:54:31 + [2026-10-09 06:22:37] iteration 5355/ 6855 | consumed samples: 685440 | elapsed time per iteration (ms): 87881.5 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.45% | FLOPs estimator: workload-aware | logical tokens/s: 190906.6 | valid tokens/s: 190906.6 | physical tokens/s: 191430.2 | loss tokens/s: 136699.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.403596E-01 | lm-global-supervised-tokens: 1.201334E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.487927E-02 | lm-token-buffer-mib: 5.813389E+01 | global_load_balancing_loss: 1.000226E+00 | loss scale: 1.0 | grad norm: 0.083 | params norm: 11308.794 | params rms: 0.061233 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 12:37:02.324395 | finish at 2026-10-10 18:59:39 + [2026-10-09 06:24:03] iteration 5356/ 6855 | consumed samples: 685568 | elapsed time per iteration (ms): 86251.2 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.77% | FLOPs estimator: workload-aware | logical tokens/s: 194515.6 | valid tokens/s: 194515.6 | physical tokens/s: 194834.3 | loss tokens/s: 138445.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.299638E-01 | lm-global-supervised-tokens: 1.194110E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.389697E-02 | lm-token-buffer-mib: 5.818037E+01 | global_load_balancing_loss: 1.000249E+00 | loss scale: 1.0 | grad norm: 0.049 | params norm: 11308.739 | params rms: 0.061233 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 11:54:50.616288 | finish at 2026-10-10 18:18:54 + [2026-10-09 06:25:34] iteration 5357/ 6855 | consumed samples: 685696 | elapsed time per iteration (ms): 90371.5 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.60% | FLOPs estimator: workload-aware | logical tokens/s: 185647.2 | valid tokens/s: 185647.2 | physical tokens/s: 186080.9 | loss tokens/s: 131860.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.551493E-01 | lm-global-supervised-tokens: 1.191641E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.156032E-02 | lm-token-buffer-mib: 5.816988E+01 | global_load_balancing_loss: 1.000951E+00 | loss scale: 1.0 | grad norm: 0.068 | params norm: 11308.683 | params rms: 0.061233 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 13:36:16.457736 | finish at 2026-10-10 20:01:50 + [2026-10-09 06:27:03] iteration 5358/ 6855 | consumed samples: 685824 | elapsed time per iteration (ms): 89185.8 | throughput per GPU (TFLOP/s/GPU): 41.4 | MFU 13.27% | FLOPs estimator: workload-aware | logical tokens/s: 188115.2 | valid tokens/s: 188115.2 | physical tokens/s: 188473.9 | loss tokens/s: 137255.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.714702E-01 | lm-global-supervised-tokens: 1.224120E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.757583E-02 | lm-token-buffer-mib: 5.802253E+01 | global_load_balancing_loss: 1.000206E+00 | loss scale: 1.0 | grad norm: 0.082 | params norm: 11308.628 | params rms: 0.061233 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 13:05:11.174835 | finish at 2026-10-10 19:32:14 + [2026-10-09 06:28:31] iteration 5359/ 6855 | consumed samples: 685952 | elapsed time per iteration (ms): 88302.5 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.60% | FLOPs estimator: workload-aware | logical tokens/s: 189996.9 | valid tokens/s: 189996.9 | physical tokens/s: 190278.1 | loss tokens/s: 142049.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.521800E-01 | lm-global-supervised-tokens: 1.254330E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.360118E-02 | lm-token-buffer-mib: 5.809297E+01 | global_load_balancing_loss: 9.995856E-01 | loss scale: 1.0 | grad norm: 0.034 | params norm: 11308.572 | params rms: 0.061232 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 12:41:40.530027 | finish at 2026-10-10 19:10:12 + [2026-10-09 06:30:22] iteration 5360/ 6855 | consumed samples: 686080 | elapsed time per iteration (ms): 110521.1 | throughput per GPU (TFLOP/s/GPU): 33.6 | MFU 10.76% | FLOPs estimator: workload-aware | logical tokens/s: 151800.5 | valid tokens/s: 151800.4 | physical tokens/s: 152092.9 | loss tokens/s: 107427.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.025932E-01 | lm-global-supervised-tokens: 1.187300E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.229592E-02 | lm-token-buffer-mib: 5.819618E+01 | global_load_balancing_loss: 9.999714E-01 | loss scale: 1.0 | grad norm: 0.072 | params norm: 11308.515 | params rms: 0.061232 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 21:53:48.975774 | finish at 2026-10-11 04:24:11 +times across ranks (ms): + save-checkpoint: + rank 0: 25586.97 + rank 1: 25586.97 + rank 2: 25586.97 + rank 3: 25586.94 + rank 4: 25586.99 + rank 5: 25586.95 + rank 6: 25586.95 + rank 7: 25586.98 + rank 8: 25587.20 + rank 9: 25587.15 + rank 10: 25587.15 + rank 11: 25587.19 + rank 12: 25587.15 + rank 13: 25587.14 + rank 14: 25587.16 + rank 15: 25587.15 + rank 16: 25587.18 + rank 17: 25587.19 + rank 18: 25587.19 + rank 19: 25587.16 + rank 20: 25587.16 + rank 21: 25587.18 + rank 22: 25587.14 + rank 23: 25587.16 + rank 24: 25587.34 + rank 25: 25587.34 + rank 26: 25587.35 + rank 27: 25587.34 + rank 28: 25587.32 + rank 29: 25587.34 + rank 30: 25587.34 + rank 31: 25587.32 + rank 32: 25587.02 + rank 33: 25587.02 + rank 34: 25587.09 + rank 35: 25587.00 + rank 36: 25586.97 + rank 37: 25586.97 + rank 38: 25587.01 + rank 39: 25586.97 + rank 40: 25587.25 + rank 41: 25587.22 + rank 42: 25587.24 + rank 43: 25587.22 + rank 44: 25587.21 + rank 45: 25587.22 + rank 46: 25587.24 + rank 47: 25587.22 + rank 48: 25587.36 + rank 49: 25587.37 + rank 50: 25587.36 + rank 51: 25587.38 + rank 52: 25587.35 + rank 53: 25587.46 + rank 54: 25587.33 + rank 55: 25587.33 + rank 56: 25587.36 + rank 57: 25587.35 + rank 58: 25587.35 + rank 59: 25587.35 + rank 60: 25587.36 + rank 61: 25587.36 + rank 62: 25587.34 + rank 63: 25587.33 + [2026-10-09 06:32:31] iteration 5361/ 6855 | consumed samples: 686208 | elapsed time per iteration (ms): 99557.3 | throughput per GPU (TFLOP/s/GPU): 36.7 | MFU 11.76% | FLOPs estimator: workload-aware | logical tokens/s: 168517.7 | valid tokens/s: 168517.7 | physical tokens/s: 168940.9 | loss tokens/s: 127593.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.165649E-01 | lm-global-supervised-tokens: 1.270286E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 9.352165E-01 | lm-token-buffer-mib: 5.804255E+01 | global_load_balancing_loss: 9.988478E-01 | loss scale: 1.0 | grad norm: 0.082 | params norm: 11308.459 | params rms: 0.061232 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 17:18:58.545070 | finish at 2026-10-10 23:51:29 + [2026-10-09 06:34:02] iteration 5362/ 6855 | consumed samples: 686336 | elapsed time per iteration (ms): 90835.5 | throughput per GPU (TFLOP/s/GPU): 39.6 | MFU 12.70% | FLOPs estimator: workload-aware | logical tokens/s: 184698.6 | valid tokens/s: 184698.6 | physical tokens/s: 185192.1 | loss tokens/s: 143126.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.854327E-01 | lm-global-supervised-tokens: 1.300095E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.688490E-02 | lm-token-buffer-mib: 5.812807E+01 | global_load_balancing_loss: 9.999493E-01 | loss scale: 1.0 | grad norm: 0.036 | params norm: 11308.402 | params rms: 0.061231 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 13:40:17.365907 | finish at 2026-10-10 20:14:19 + [2026-10-09 06:35:29] iteration 5363/ 6855 | consumed samples: 686464 | elapsed time per iteration (ms): 86951.9 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.54% | FLOPs estimator: workload-aware | logical tokens/s: 192947.8 | valid tokens/s: 192947.8 | physical tokens/s: 193422.4 | loss tokens/s: 142376.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.602162E-01 | lm-global-supervised-tokens: 1.237987E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.194093E-02 | lm-token-buffer-mib: 5.804543E+01 | global_load_balancing_loss: 1.001677E+00 | loss scale: 1.0 | grad norm: 0.063 | params norm: 11308.345 | params rms: 0.061231 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 12:02:12.289589 | finish at 2026-10-10 18:37:41 + [2026-10-09 06:36:54] iteration 5364/ 6855 | consumed samples: 686592 | elapsed time per iteration (ms): 85729.2 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.74% | FLOPs estimator: workload-aware | logical tokens/s: 195699.1 | valid tokens/s: 195699.1 | physical tokens/s: 196240.0 | loss tokens/s: 148427.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.687626E-01 | lm-global-supervised-tokens: 1.272454E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.585894E-02 | lm-token-buffer-mib: 5.818959E+01 | global_load_balancing_loss: 1.000670E+00 | loss scale: 1.0 | grad norm: 0.059 | params norm: 11308.288 | params rms: 0.061231 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 11:30:22.176599 | finish at 2026-10-10 18:07:17 + [2026-10-09 06:38:21] iteration 5365/ 6855 | consumed samples: 686720 | elapsed time per iteration (ms): 86048.6 | throughput per GPU (TFLOP/s/GPU): 43.2 | MFU 13.84% | FLOPs estimator: workload-aware | logical tokens/s: 194972.1 | valid tokens/s: 194972.1 | physical tokens/s: 195465.1 | loss tokens/s: 146176.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.309846E-01 | lm-global-supervised-tokens: 1.257827E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.575944E-02 | lm-token-buffer-mib: 5.816537E+01 | global_load_balancing_loss: 1.000359E+00 | loss scale: 1.0 | grad norm: 0.040 | params norm: 11308.232 | params rms: 0.061230 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 11:36:52.430990 | finish at 2026-10-10 18:15:13 + [2026-10-09 06:39:44] iteration 5366/ 6855 | consumed samples: 686848 | elapsed time per iteration (ms): 83842.5 | throughput per GPU (TFLOP/s/GPU): 43.3 | MFU 13.89% | FLOPs estimator: workload-aware | logical tokens/s: 200102.8 | valid tokens/s: 200102.8 | physical tokens/s: 200665.5 | loss tokens/s: 147153.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.329411E-01 | lm-global-supervised-tokens: 1.233776E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.167408E-02 | lm-token-buffer-mib: 5.815461E+01 | global_load_balancing_loss: 9.996879E-01 | loss scale: 1.0 | grad norm: 0.050 | params norm: 11308.175 | params rms: 0.061230 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 10:40:41.546358 | finish at 2026-10-10 17:20:26 + [2026-10-09 06:41:09] iteration 5367/ 6855 | consumed samples: 686976 | elapsed time per iteration (ms): 84861.4 | throughput per GPU (TFLOP/s/GPU): 43.7 | MFU 14.01% | FLOPs estimator: workload-aware | logical tokens/s: 197700.8 | valid tokens/s: 197700.8 | physical tokens/s: 198048.9 | loss tokens/s: 149142.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.083217E-01 | lm-global-supervised-tokens: 1.265646E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.142203E-02 | lm-token-buffer-mib: 5.810399E+01 | global_load_balancing_loss: 1.000033E+00 | loss scale: 1.0 | grad norm: 0.046 | params norm: 11308.118 | params rms: 0.061230 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 11:04:33.709465 | finish at 2026-10-10 17:45:43 + [2026-10-09 06:42:40] iteration 5368/ 6855 | consumed samples: 687104 | elapsed time per iteration (ms): 90938.1 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.57% | FLOPs estimator: workload-aware | logical tokens/s: 184490.0 | valid tokens/s: 184490.0 | physical tokens/s: 184759.8 | loss tokens/s: 130430.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.158899E-01 | lm-global-supervised-tokens: 1.186106E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.230332E-02 | lm-token-buffer-mib: 5.803540E+01 | global_load_balancing_loss: 1.001110E+00 | loss scale: 1.0 | grad norm: 0.038 | params norm: 11308.059 | params rms: 0.061229 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 13:33:45.003773 | finish at 2026-10-10 20:16:25 + [2026-10-09 06:44:06] iteration 5369/ 6855 | consumed samples: 687232 | elapsed time per iteration (ms): 86260.5 | throughput per GPU (TFLOP/s/GPU): 43.4 | MFU 13.91% | FLOPs estimator: workload-aware | logical tokens/s: 194494.5 | valid tokens/s: 194494.5 | physical tokens/s: 194882.4 | loss tokens/s: 135283.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.131870E-01 | lm-global-supervised-tokens: 1.166965E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.397696E-02 | lm-token-buffer-mib: 5.808289E+01 | global_load_balancing_loss: 1.000952E+00 | loss scale: 1.0 | grad norm: 0.072 | params norm: 11308.001 | params rms: 0.061229 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 11:36:23.139778 | finish at 2026-10-10 18:20:30 + [2026-10-09 06:45:59] iteration 5370/ 6855 | consumed samples: 687360 | elapsed time per iteration (ms): 112185.6 | throughput per GPU (TFLOP/s/GPU): 33.4 | MFU 10.70% | FLOPs estimator: workload-aware | logical tokens/s: 149548.2 | valid tokens/s: 149548.2 | physical tokens/s: 149785.9 | loss tokens/s: 106854.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.915658E-01 | lm-global-supervised-tokens: 1.198753E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.194038E-02 | lm-token-buffer-mib: 5.817971E+01 | global_load_balancing_loss: 1.000764E+00 | loss scale: 1.0 | grad norm: 0.056 | params norm: 11307.944 | params rms: 0.061229 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 22:16:35.622436 | finish at 2026-10-11 05:02:34 + [2026-10-09 06:47:26] iteration 5371/ 6855 | consumed samples: 687488 | elapsed time per iteration (ms): 87514.0 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.54% | FLOPs estimator: workload-aware | logical tokens/s: 191708.8 | valid tokens/s: 191708.8 | physical tokens/s: 192085.5 | loss tokens/s: 141321.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.247492E-01 | lm-global-supervised-tokens: 1.236761E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.815450E-02 | lm-token-buffer-mib: 5.819200E+01 | global_load_balancing_loss: 9.996772E-01 | loss scale: 1.0 | grad norm: 0.049 | params norm: 11307.885 | params rms: 0.061229 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 12:04:30.805276 | finish at 2026-10-10 18:51:57 + [2026-10-09 06:48:54] iteration 5372/ 6855 | consumed samples: 687616 | elapsed time per iteration (ms): 87899.9 | throughput per GPU (TFLOP/s/GPU): 42.8 | MFU 13.71% | FLOPs estimator: workload-aware | logical tokens/s: 190867.1 | valid tokens/s: 190867.1 | physical tokens/s: 191330.7 | loss tokens/s: 143300.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.285364E-01 | lm-global-supervised-tokens: 1.259606E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.424795E-02 | lm-token-buffer-mib: 5.811634E+01 | global_load_balancing_loss: 9.991390E-01 | loss scale: 1.0 | grad norm: 0.057 | params norm: 11307.826 | params rms: 0.061228 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 12:12:35.555529 | finish at 2026-10-10 19:01:30 + [2026-10-09 06:50:21] iteration 5373/ 6855 | consumed samples: 687744 | elapsed time per iteration (ms): 87086.9 | throughput per GPU (TFLOP/s/GPU): 41.5 | MFU 13.29% | FLOPs estimator: workload-aware | logical tokens/s: 192648.5 | valid tokens/s: 192648.5 | physical tokens/s: 193056.2 | loss tokens/s: 148207.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.200901E-01 | lm-global-supervised-tokens: 1.290697E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.118096E-02 | lm-token-buffer-mib: 5.803556E+01 | global_load_balancing_loss: 9.991255E-01 | loss scale: 1.0 | grad norm: 0.076 | params norm: 11307.767 | params rms: 0.061228 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 11:51:02.821834 | finish at 2026-10-10 18:41:24 + [2026-10-09 06:51:49] iteration 5374/ 6855 | consumed samples: 687872 | elapsed time per iteration (ms): 87545.2 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.45% | FLOPs estimator: workload-aware | logical tokens/s: 191640.5 | valid tokens/s: 191640.5 | physical tokens/s: 192177.4 | loss tokens/s: 138213.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.650673E-01 | lm-global-supervised-tokens: 1.209997E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.161280E-02 | lm-token-buffer-mib: 5.829377E+01 | global_load_balancing_loss: 9.990925E-01 | loss scale: 1.0 | grad norm: 0.132 | params norm: 11307.709 | params rms: 0.061228 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 12:00:54.456545 | finish at 2026-10-10 18:52:43 + [2026-10-09 06:53:12] iteration 5375/ 6855 | consumed samples: 688000 | elapsed time per iteration (ms): 83613.2 | throughput per GPU (TFLOP/s/GPU): 42.8 | MFU 13.70% | FLOPs estimator: workload-aware | logical tokens/s: 200650.7 | valid tokens/s: 200650.7 | physical tokens/s: 201464.9 | loss tokens/s: 153671.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.843610E-01 | lm-global-supervised-tokens: 1.284893E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.116153E-02 | lm-token-buffer-mib: 5.833034E+01 | global_load_balancing_loss: 1.000468E+00 | loss scale: 1.0 | grad norm: 0.187 | params norm: 11307.649 | params rms: 0.061227 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 10:22:27.502050 | finish at 2026-10-10 17:15:40 + [2026-10-09 06:54:39] iteration 5376/ 6855 | consumed samples: 688128 | elapsed time per iteration (ms): 86365.5 | throughput per GPU (TFLOP/s/GPU): 41.2 | MFU 13.21% | FLOPs estimator: workload-aware | logical tokens/s: 194257.4 | valid tokens/s: 194257.4 | physical tokens/s: 194808.6 | loss tokens/s: 140037.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.017032E-01 | lm-global-supervised-tokens: 1.209442E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.602791E-02 | lm-token-buffer-mib: 5.815004E+01 | global_load_balancing_loss: 1.001560E+00 | loss scale: 1.0 | grad norm: 0.297 | params norm: 11307.590 | params rms: 0.061227 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 11:28:54.555772 | finish at 2026-10-10 18:23:33 + [2026-10-09 06:56:08] iteration 5377/ 6855 | consumed samples: 688256 | elapsed time per iteration (ms): 89739.4 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.49% | FLOPs estimator: workload-aware | logical tokens/s: 186954.4 | valid tokens/s: 186954.4 | physical tokens/s: 187254.0 | loss tokens/s: 134069.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.091597E-01 | lm-global-supervised-tokens: 1.203131E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.147587E-02 | lm-token-buffer-mib: 5.806443E+01 | global_load_balancing_loss: 1.000853E+00 | loss scale: 1.0 | grad norm: 0.111 | params norm: 11307.534 | params rms: 0.061227 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 12:50:34.859143 | finish at 2026-10-10 19:46:43 + [2026-10-09 06:57:35] iteration 5378/ 6855 | consumed samples: 688384 | elapsed time per iteration (ms): 86842.2 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.77% | FLOPs estimator: workload-aware | logical tokens/s: 193191.5 | valid tokens/s: 193191.5 | physical tokens/s: 193627.1 | loss tokens/s: 136210.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.221364E-01 | lm-global-supervised-tokens: 1.182884E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.604906E-02 | lm-token-buffer-mib: 5.813208E+01 | global_load_balancing_loss: 1.000910E+00 | loss scale: 1.0 | grad norm: 0.294 | params norm: 11307.477 | params rms: 0.061226 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 11:37:45.981930 | finish at 2026-10-10 18:35:21 + [2026-10-09 06:59:04] iteration 5379/ 6855 | consumed samples: 688512 | elapsed time per iteration (ms): 89018.0 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.41% | FLOPs estimator: workload-aware | logical tokens/s: 188469.7 | valid tokens/s: 188469.7 | physical tokens/s: 188937.0 | loss tokens/s: 139809.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.585153E-01 | lm-global-supervised-tokens: 1.244559E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.161570E-02 | lm-token-buffer-mib: 5.813441E+01 | global_load_balancing_loss: 9.996890E-01 | loss scale: 1.0 | grad norm: 0.112 | params norm: 11307.419 | params rms: 0.061226 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 12:29:50.512233 | finish at 2026-10-10 19:28:55 + [2026-10-09 07:00:52] iteration 5380/ 6855 | consumed samples: 688640 | elapsed time per iteration (ms): 107393.8 | throughput per GPU (TFLOP/s/GPU): 34.8 | MFU 11.15% | FLOPs estimator: workload-aware | logical tokens/s: 156221.4 | valid tokens/s: 156221.4 | physical tokens/s: 156540.6 | loss tokens/s: 116016.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.821001E-01 | lm-global-supervised-tokens: 1.245940E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.436354E-02 | lm-token-buffer-mib: 5.816990E+01 | global_load_balancing_loss: 1.000695E+00 | loss scale: 1.0 | grad norm: 0.313 | params norm: 11307.362 | params rms: 0.061226 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 20:00:05.877888 | finish at 2026-10-11 03:00:58 + [2026-10-09 07:02:19] iteration 5381/ 6855 | consumed samples: 688768 | elapsed time per iteration (ms): 87516.6 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.76% | FLOPs estimator: workload-aware | logical tokens/s: 191702.6 | valid tokens/s: 191702.6 | physical tokens/s: 192075.2 | loss tokens/s: 137216.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.508770E-01 | lm-global-supervised-tokens: 1.200872E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.157439E-02 | lm-token-buffer-mib: 5.819548E+01 | global_load_balancing_loss: 9.993256E-01 | loss scale: 1.0 | grad norm: 0.081 | params norm: 11307.307 | params rms: 0.061225 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 11:49:59.498115 | finish at 2026-10-10 18:52:19 + [2026-10-09 07:03:46] iteration 5382/ 6855 | consumed samples: 688896 | elapsed time per iteration (ms): 86734.8 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.66% | FLOPs estimator: workload-aware | logical tokens/s: 193431.0 | valid tokens/s: 193430.9 | physical tokens/s: 193790.2 | loss tokens/s: 142167.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.640643E-01 | lm-global-supervised-tokens: 1.233083E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.527578E-02 | lm-token-buffer-mib: 5.808628E+01 | global_load_balancing_loss: 9.995942E-01 | loss scale: 1.0 | grad norm: 0.155 | params norm: 11307.253 | params rms: 0.061225 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 11:29:20.343691 | finish at 2026-10-10 18:33:06 + [2026-10-09 07:05:13] iteration 5383/ 6855 | consumed samples: 689024 | elapsed time per iteration (ms): 86921.3 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.74% | FLOPs estimator: workload-aware | logical tokens/s: 193015.8 | valid tokens/s: 193015.8 | physical tokens/s: 193385.9 | loss tokens/s: 139274.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.731030E-01 | lm-global-supervised-tokens: 1.210594E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.345261E-02 | lm-token-buffer-mib: 5.813947E+01 | global_load_balancing_loss: 1.000180E+00 | loss scale: 1.0 | grad norm: 0.169 | params norm: 11307.199 | params rms: 0.061225 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 11:32:28.182983 | finish at 2026-10-10 18:37:41 + [2026-10-09 07:06:37] iteration 5384/ 6855 | consumed samples: 689152 | elapsed time per iteration (ms): 84576.3 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.69% | FLOPs estimator: workload-aware | logical tokens/s: 198367.7 | valid tokens/s: 198367.7 | physical tokens/s: 198898.6 | loss tokens/s: 144040.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.432579E-01 | lm-global-supervised-tokens: 1.218244E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.368740E-02 | lm-token-buffer-mib: 5.817634E+01 | global_load_balancing_loss: 1.000242E+00 | loss scale: 1.0 | grad norm: 0.070 | params norm: 11307.143 | params rms: 0.061225 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 10:33:31.792925 | finish at 2026-10-10 17:40:09 + [2026-10-09 07:08:06] iteration 5385/ 6855 | consumed samples: 689280 | elapsed time per iteration (ms): 89065.6 | throughput per GPU (TFLOP/s/GPU): 41.6 | MFU 13.34% | FLOPs estimator: workload-aware | logical tokens/s: 188368.7 | valid tokens/s: 188368.7 | physical tokens/s: 188982.7 | loss tokens/s: 138977.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.684670E-01 | lm-global-supervised-tokens: 1.237815E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.539219E-02 | lm-token-buffer-mib: 5.815466E+01 | global_load_balancing_loss: 9.996480E-01 | loss scale: 1.0 | grad norm: 0.236 | params norm: 11307.089 | params rms: 0.061224 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 12:22:06.438539 | finish at 2026-10-10 19:30:13 + [2026-10-09 07:09:35] iteration 5386/ 6855 | consumed samples: 689408 | elapsed time per iteration (ms): 88554.4 | throughput per GPU (TFLOP/s/GPU): 41.3 | MFU 13.23% | FLOPs estimator: workload-aware | logical tokens/s: 189456.2 | valid tokens/s: 189456.2 | physical tokens/s: 189933.0 | loss tokens/s: 148853.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.174284E-01 | lm-global-supervised-tokens: 1.318162E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.328677E-02 | lm-token-buffer-mib: 5.824998E+01 | global_load_balancing_loss: 1.000411E+00 | loss scale: 1.0 | grad norm: 0.071 | params norm: 11307.035 | params rms: 0.061224 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 12:08:06.478696 | finish at 2026-10-10 19:17:41 + [2026-10-09 07:11:03] iteration 5387/ 6855 | consumed samples: 689536 | elapsed time per iteration (ms): 87534.8 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.58% | FLOPs estimator: workload-aware | logical tokens/s: 191662.9 | valid tokens/s: 191662.9 | physical tokens/s: 192093.8 | loss tokens/s: 139256.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.959974E-01 | lm-global-supervised-tokens: 1.218975E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.150552E-02 | lm-token-buffer-mib: 5.805838E+01 | global_load_balancing_loss: 1.000476E+00 | loss scale: 1.0 | grad norm: 0.125 | params norm: 11306.981 | params rms: 0.061224 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 11:41:41.128827 | finish at 2026-10-10 18:52:44 + [2026-10-09 07:12:33] iteration 5388/ 6855 | consumed samples: 689664 | elapsed time per iteration (ms): 90092.3 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.46% | FLOPs estimator: workload-aware | logical tokens/s: 186222.4 | valid tokens/s: 186222.4 | physical tokens/s: 186702.1 | loss tokens/s: 131078.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.789840E-01 | lm-global-supervised-tokens: 1.180913E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.148451E-02 | lm-token-buffer-mib: 5.811978E+01 | global_load_balancing_loss: 9.994613E-01 | loss scale: 1.0 | grad norm: 0.057 | params norm: 11306.928 | params rms: 0.061223 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 12:42:45.455774 | finish at 2026-10-10 19:55:18 + [2026-10-09 07:13:59] iteration 5389/ 6855 | consumed samples: 689792 | elapsed time per iteration (ms): 86045.4 | throughput per GPU (TFLOP/s/GPU): 43.5 | MFU 13.95% | FLOPs estimator: workload-aware | logical tokens/s: 194980.8 | valid tokens/s: 194980.8 | physical tokens/s: 195277.0 | loss tokens/s: 145437.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.948403E-01 | lm-global-supervised-tokens: 1.251421E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.253551E-02 | lm-token-buffer-mib: 5.808649E+01 | global_load_balancing_loss: 1.001263E+00 | loss scale: 1.0 | grad norm: 0.076 | params norm: 11306.875 | params rms: 0.061223 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 11:02:22.543677 | finish at 2026-10-10 18:16:21 + [2026-10-09 07:15:48] iteration 5390/ 6855 | consumed samples: 689920 | elapsed time per iteration (ms): 109015.4 | throughput per GPU (TFLOP/s/GPU): 33.3 | MFU 10.69% | FLOPs estimator: workload-aware | logical tokens/s: 153897.4 | valid tokens/s: 153897.4 | physical tokens/s: 154270.9 | loss tokens/s: 114278.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.635069E-01 | lm-global-supervised-tokens: 1.245809E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.465958E-02 | lm-token-buffer-mib: 5.815932E+01 | global_load_balancing_loss: 1.001553E+00 | loss scale: 1.0 | grad norm: 0.160 | params norm: 11306.824 | params rms: 0.061223 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 20:21:47.531213 | finish at 2026-10-11 03:37:35 + [2026-10-09 07:17:16] iteration 5391/ 6855 | consumed samples: 690048 | elapsed time per iteration (ms): 88576.9 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.40% | FLOPs estimator: workload-aware | logical tokens/s: 189408.0 | valid tokens/s: 189408.0 | physical tokens/s: 189992.3 | loss tokens/s: 139269.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.499898E-01 | lm-global-supervised-tokens: 1.233602E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.425838E-02 | lm-token-buffer-mib: 5.815977E+01 | global_load_balancing_loss: 1.000385E+00 | loss scale: 1.0 | grad norm: 0.084 | params norm: 11306.773 | params rms: 0.061222 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 12:01:16.547750 | finish at 2026-10-10 19:18:33 + [2026-10-09 07:18:43] iteration 5392/ 6855 | consumed samples: 690176 | elapsed time per iteration (ms): 87137.9 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.55% | FLOPs estimator: workload-aware | logical tokens/s: 192536.2 | valid tokens/s: 192536.2 | physical tokens/s: 193021.7 | loss tokens/s: 141325.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.352182E-01 | lm-global-supervised-tokens: 1.231478E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.318836E-02 | lm-token-buffer-mib: 5.815221E+01 | global_load_balancing_loss: 1.001265E+00 | loss scale: 1.0 | grad norm: 0.099 | params norm: 11306.721 | params rms: 0.061222 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 11:24:42.713684 | finish at 2026-10-10 18:43:26 + [2026-10-09 07:20:15] iteration 5393/ 6855 | consumed samples: 690304 | elapsed time per iteration (ms): 91841.5 | throughput per GPU (TFLOP/s/GPU): 40.8 | MFU 13.08% | FLOPs estimator: workload-aware | logical tokens/s: 182675.5 | valid tokens/s: 182675.5 | physical tokens/s: 183144.7 | loss tokens/s: 133787.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.599815E-01 | lm-global-supervised-tokens: 1.228722E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.490877E-02 | lm-token-buffer-mib: 5.816724E+01 | global_load_balancing_loss: 9.995289E-01 | loss scale: 1.0 | grad norm: 0.058 | params norm: 11306.670 | params rms: 0.061222 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 13:17:52.291887 | finish at 2026-10-10 20:38:08 + [2026-10-09 07:21:44] iteration 5394/ 6855 | consumed samples: 690432 | elapsed time per iteration (ms): 88723.9 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.48% | FLOPs estimator: workload-aware | logical tokens/s: 189094.4 | valid tokens/s: 189094.4 | physical tokens/s: 189483.6 | loss tokens/s: 139411.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.637681E-01 | lm-global-supervised-tokens: 1.236910E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.725779E-02 | lm-token-buffer-mib: 5.823865E+01 | global_load_balancing_loss: 9.993197E-01 | loss scale: 1.0 | grad norm: 0.070 | params norm: 11306.619 | params rms: 0.061222 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 12:00:25.638220 | finish at 2026-10-10 19:22:10 + [2026-10-09 07:23:12] iteration 5395/ 6855 | consumed samples: 690560 | elapsed time per iteration (ms): 87551.0 | throughput per GPU (TFLOP/s/GPU): 43.3 | MFU 13.89% | FLOPs estimator: workload-aware | logical tokens/s: 191627.7 | valid tokens/s: 191627.7 | physical tokens/s: 192178.2 | loss tokens/s: 138528.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.805961E-01 | lm-global-supervised-tokens: 1.212830E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.147264E-02 | lm-token-buffer-mib: 5.815453E+01 | global_load_balancing_loss: 9.988421E-01 | loss scale: 1.0 | grad norm: 0.047 | params norm: 11306.566 | params rms: 0.061221 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 11:30:24.427104 | finish at 2026-10-10 18:53:36 + [2026-10-09 07:24:39] iteration 5396/ 6855 | consumed samples: 690688 | elapsed time per iteration (ms): 87076.3 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.74% | FLOPs estimator: workload-aware | logical tokens/s: 192672.1 | valid tokens/s: 192672.1 | physical tokens/s: 193112.8 | loss tokens/s: 142296.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.303530E-01 | lm-global-supervised-tokens: 1.239069E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.515379E-02 | lm-token-buffer-mib: 5.808659E+01 | global_load_balancing_loss: 9.987336E-01 | loss scale: 1.0 | grad norm: 0.063 | params norm: 11306.515 | params rms: 0.061221 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 11:17:24.335824 | finish at 2026-10-10 18:42:03 + [2026-10-09 07:26:04] iteration 5397/ 6855 | consumed samples: 690816 | elapsed time per iteration (ms): 84944.2 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.78% | FLOPs estimator: workload-aware | logical tokens/s: 197508.7 | valid tokens/s: 197508.7 | physical tokens/s: 198013.4 | loss tokens/s: 144463.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.045747E-01 | lm-global-supervised-tokens: 1.227130E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.144690E-02 | lm-token-buffer-mib: 5.813440E+01 | global_load_balancing_loss: 1.000351E+00 | loss scale: 1.0 | grad norm: 0.099 | params norm: 11306.463 | params rms: 0.061221 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 10:24:08.579348 | finish at 2026-10-10 17:50:12 + [2026-10-09 07:27:31] iteration 5398/ 6855 | consumed samples: 690944 | elapsed time per iteration (ms): 87511.7 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.46% | FLOPs estimator: workload-aware | logical tokens/s: 191714.0 | valid tokens/s: 191714.0 | physical tokens/s: 192173.1 | loss tokens/s: 142565.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.271431E-01 | lm-global-supervised-tokens: 1.247610E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.216405E-02 | lm-token-buffer-mib: 5.822980E+01 | global_load_balancing_loss: 1.001164E+00 | loss scale: 1.0 | grad norm: 0.074 | params norm: 11306.411 | params rms: 0.061221 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 11:25:04.494670 | finish at 2026-10-10 18:52:36 + [2026-10-09 07:29:00] iteration 5399/ 6855 | consumed samples: 691072 | elapsed time per iteration (ms): 88900.7 | throughput per GPU (TFLOP/s/GPU): 41.1 | MFU 13.18% | FLOPs estimator: workload-aware | logical tokens/s: 188718.3 | valid tokens/s: 188718.3 | physical tokens/s: 189132.9 | loss tokens/s: 135375.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.472853E-01 | lm-global-supervised-tokens: 1.203499E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.769190E-02 | lm-token-buffer-mib: 5.829701E+01 | global_load_balancing_loss: 1.001788E+00 | loss scale: 1.0 | grad norm: 0.049 | params norm: 11306.359 | params rms: 0.061220 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 11:57:19.453354 | finish at 2026-10-10 19:26:19 + [2026-10-09 07:30:50] iteration 5400/ 6855 | consumed samples: 691200 | elapsed time per iteration (ms): 110257.2 | throughput per GPU (TFLOP/s/GPU): 33.6 | MFU 10.77% | FLOPs estimator: workload-aware | logical tokens/s: 152164.0 | valid tokens/s: 152164.0 | physical tokens/s: 152508.6 | loss tokens/s: 115540.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.844293E-01 | lm-global-supervised-tokens: 1.273921E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.201030E-02 | lm-token-buffer-mib: 5.815623E+01 | global_load_balancing_loss: 1.000120E+00 | loss scale: 1.0 | grad norm: 0.060 | params norm: 11306.307 | params rms: 0.061220 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 20:33:44.157318 | finish at 2026-10-11 04:04:34 +times across ranks (ms): + save-checkpoint: + rank 0: 26902.06 + rank 1: 26902.12 + rank 2: 26902.10 + rank 3: 26902.06 + rank 4: 26902.08 + rank 5: 26902.03 + rank 6: 26902.04 + rank 7: 26902.05 + rank 8: 26902.61 + rank 9: 26902.56 + rank 10: 26902.57 + rank 11: 26902.60 + rank 12: 26902.59 + rank 13: 26902.56 + rank 14: 26902.56 + rank 15: 26902.58 + rank 16: 26902.19 + rank 17: 26902.17 + rank 18: 26902.05 + rank 19: 26902.20 + rank 20: 26902.12 + rank 21: 26902.19 + rank 22: 26902.20 + rank 23: 26902.14 + rank 24: 26901.84 + rank 25: 26901.85 + rank 26: 26901.84 + rank 27: 26901.87 + rank 28: 26901.77 + rank 29: 26901.85 + rank 30: 26901.74 + rank 31: 26901.84 + rank 32: 26901.86 + rank 33: 26901.87 + rank 34: 26901.83 + rank 35: 26901.71 + rank 36: 26901.77 + rank 37: 26901.75 + rank 38: 26901.80 + rank 39: 26901.79 + rank 40: 26901.31 + rank 41: 26901.33 + rank 42: 26901.30 + rank 43: 26901.31 + rank 44: 26901.28 + rank 45: 26901.28 + rank 46: 26901.31 + rank 47: 26901.29 + rank 48: 26902.36 + rank 49: 26902.31 + rank 50: 26902.15 + rank 51: 26902.25 + rank 52: 26902.30 + rank 53: 26902.28 + rank 54: 26902.12 + rank 55: 26902.29 + rank 56: 26902.11 + rank 57: 26902.08 + rank 58: 26902.03 + rank 59: 26902.03 + rank 60: 26902.03 + rank 61: 26902.07 + rank 62: 26902.04 + rank 63: 26901.90 + [2026-10-09 07:32:46] iteration 5401/ 6855 | consumed samples: 691328 | elapsed time per iteration (ms): 88553.0 | throughput per GPU (TFLOP/s/GPU): 40.9 | MFU 13.11% | FLOPs estimator: workload-aware | logical tokens/s: 189459.0 | valid tokens/s: 189459.0 | physical tokens/s: 190046.7 | loss tokens/s: 138322.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.900868E-01 | lm-global-supervised-tokens: 1.224886E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.190232E-01 | lm-token-buffer-mib: 5.810092E+01 | global_load_balancing_loss: 1.000674E+00 | loss scale: 1.0 | grad norm: 0.065 | params norm: 11306.253 | params rms: 0.061220 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 11:45:55.999909 | finish at 2026-10-10 19:18:42 + [2026-10-09 07:34:16] iteration 5402/ 6855 | consumed samples: 691456 | elapsed time per iteration (ms): 90304.2 | throughput per GPU (TFLOP/s/GPU): 40.3 | MFU 12.92% | FLOPs estimator: workload-aware | logical tokens/s: 185785.1 | valid tokens/s: 185785.1 | physical tokens/s: 186373.4 | loss tokens/s: 140633.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.391955E-01 | lm-global-supervised-tokens: 1.269978E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.295112E-02 | lm-token-buffer-mib: 5.818304E+01 | global_load_balancing_loss: 1.000552E+00 | loss scale: 1.0 | grad norm: 0.065 | params norm: 11306.200 | params rms: 0.061219 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 12:26:52.018440 | finish at 2026-10-10 20:01:08 + [2026-10-09 07:35:44] iteration 5403/ 6855 | consumed samples: 691584 | elapsed time per iteration (ms): 87712.0 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.74% | FLOPs estimator: workload-aware | logical tokens/s: 191275.8 | valid tokens/s: 191275.8 | physical tokens/s: 191670.7 | loss tokens/s: 140363.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.269967E-01 | lm-global-supervised-tokens: 1.231157E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.286853E-02 | lm-token-buffer-mib: 5.804599E+01 | global_load_balancing_loss: 9.996853E-01 | loss scale: 1.0 | grad norm: 0.060 | params norm: 11306.145 | params rms: 0.061219 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 11:22:37.863656 | finish at 2026-10-10 18:58:22 + [2026-10-09 07:37:07] iteration 5404/ 6855 | consumed samples: 691712 | elapsed time per iteration (ms): 83617.8 | throughput per GPU (TFLOP/s/GPU): 43.6 | MFU 13.97% | FLOPs estimator: workload-aware | logical tokens/s: 200641.3 | valid tokens/s: 200641.3 | physical tokens/s: 201100.5 | loss tokens/s: 153621.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.685551E-01 | lm-global-supervised-tokens: 1.284545E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.235277E-02 | lm-token-buffer-mib: 5.820432E+01 | global_load_balancing_loss: 9.989113E-01 | loss scale: 1.0 | grad norm: 0.055 | params norm: 11306.091 | params rms: 0.061219 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 9:42:09.447515 | finish at 2026-10-10 17:19:17 + [2026-10-09 07:38:37] iteration 5405/ 6855 | consumed samples: 691840 | elapsed time per iteration (ms): 89102.8 | throughput per GPU (TFLOP/s/GPU): 41.3 | MFU 13.25% | FLOPs estimator: workload-aware | logical tokens/s: 188289.7 | valid tokens/s: 188289.7 | physical tokens/s: 188830.5 | loss tokens/s: 139953.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.159972E-01 | lm-global-supervised-tokens: 1.247028E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.188140E-02 | lm-token-buffer-mib: 5.820289E+01 | global_load_balancing_loss: 9.998704E-01 | loss scale: 1.0 | grad norm: 0.045 | params norm: 11306.036 | params rms: 0.061219 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 11:53:19.070561 | finish at 2026-10-10 19:31:56 + [2026-10-09 07:40:05] iteration 5406/ 6855 | consumed samples: 691968 | elapsed time per iteration (ms): 88356.4 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.42% | FLOPs estimator: workload-aware | logical tokens/s: 189880.2 | valid tokens/s: 189880.2 | physical tokens/s: 190475.8 | loss tokens/s: 141609.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.719571E-01 | lm-global-supervised-tokens: 1.251208E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.357027E-02 | lm-token-buffer-mib: 5.811694E+01 | global_load_balancing_loss: 9.996166E-01 | loss scale: 1.0 | grad norm: 0.062 | params norm: 11305.980 | params rms: 0.061218 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 11:33:48.352107 | finish at 2026-10-10 19:13:53 + [2026-10-09 07:41:30] iteration 5407/ 6855 | consumed samples: 692096 | elapsed time per iteration (ms): 84764.9 | throughput per GPU (TFLOP/s/GPU): 44.2 | MFU 14.17% | FLOPs estimator: workload-aware | logical tokens/s: 197925.3 | valid tokens/s: 197925.3 | physical tokens/s: 198301.4 | loss tokens/s: 146996.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.289032E-01 | lm-global-supervised-tokens: 1.246016E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.526912E-02 | lm-token-buffer-mib: 5.807010E+01 | global_load_balancing_loss: 1.001546E+00 | loss scale: 1.0 | grad norm: 0.052 | params norm: 11305.923 | params rms: 0.061218 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 10:05:39.628666 | finish at 2026-10-10 17:47:09 + [2026-10-09 07:42:57] iteration 5408/ 6855 | consumed samples: 692224 | elapsed time per iteration (ms): 86985.9 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.60% | FLOPs estimator: workload-aware | logical tokens/s: 192872.0 | valid tokens/s: 192872.0 | physical tokens/s: 193378.6 | loss tokens/s: 134420.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 5.775072E-01 | lm-global-supervised-tokens: 1.169269E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.111072E-02 | lm-token-buffer-mib: 5.808831E+01 | global_load_balancing_loss: 1.001703E+00 | loss scale: 1.0 | grad norm: 0.046 | params norm: 11305.869 | params rms: 0.061218 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 10:57:48.636521 | finish at 2026-10-10 18:40:45 + [2026-10-09 07:44:26] iteration 5409/ 6855 | consumed samples: 692352 | elapsed time per iteration (ms): 88958.8 | throughput per GPU (TFLOP/s/GPU): 41.0 | MFU 13.13% | FLOPs estimator: workload-aware | logical tokens/s: 188595.4 | valid tokens/s: 188595.3 | physical tokens/s: 189032.8 | loss tokens/s: 143243.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.000075E-01 | lm-global-supervised-tokens: 1.274276E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.218658E-02 | lm-token-buffer-mib: 5.811239E+01 | global_load_balancing_loss: 9.995132E-01 | loss scale: 1.0 | grad norm: 0.041 | params norm: 11305.815 | params rms: 0.061217 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 11:43:54.370786 | finish at 2026-10-10 19:28:20 + [2026-10-09 07:46:16] iteration 5410/ 6855 | consumed samples: 692480 | elapsed time per iteration (ms): 110172.1 | throughput per GPU (TFLOP/s/GPU): 33.2 | MFU 10.64% | FLOPs estimator: workload-aware | logical tokens/s: 152281.8 | valid tokens/s: 152281.8 | physical tokens/s: 152663.3 | loss tokens/s: 115611.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.401895E-01 | lm-global-supervised-tokens: 1.273717E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.155898E-02 | lm-token-buffer-mib: 5.809526E+01 | global_load_balancing_loss: 9.997112E-01 | loss scale: 1.0 | grad norm: 0.070 | params norm: 11305.761 | params rms: 0.061217 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 20:13:18.619828 | finish at 2026-10-11 03:59:34 + [2026-10-09 07:47:41] iteration 5411/ 6855 | consumed samples: 692608 | elapsed time per iteration (ms): 85223.9 | throughput per GPU (TFLOP/s/GPU): 43.8 | MFU 14.05% | FLOPs estimator: workload-aware | logical tokens/s: 196860.3 | valid tokens/s: 196860.3 | physical tokens/s: 197362.9 | loss tokens/s: 149079.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.612556E-01 | lm-global-supervised-tokens: 1.270517E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.171600E-02 | lm-token-buffer-mib: 5.814728E+01 | global_load_balancing_loss: 9.992065E-01 | loss scale: 1.0 | grad norm: 0.051 | params norm: 11305.707 | params rms: 0.061217 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 10:11:03.271435 | finish at 2026-10-10 17:58:44 + [2026-10-09 07:49:07] iteration 5412/ 6855 | consumed samples: 692736 | elapsed time per iteration (ms): 86202.6 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.61% | FLOPs estimator: workload-aware | logical tokens/s: 194624.4 | valid tokens/s: 194624.4 | physical tokens/s: 195050.8 | loss tokens/s: 144880.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.857200E-01 | lm-global-supervised-tokens: 1.248908E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.387139E-02 | lm-token-buffer-mib: 5.819916E+01 | global_load_balancing_loss: 9.995714E-01 | loss scale: 1.0 | grad norm: 0.053 | params norm: 11305.654 | params rms: 0.061216 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 10:33:10.370381 | finish at 2026-10-10 18:22:18 + [2026-10-09 07:50:34] iteration 5413/ 6855 | consumed samples: 692864 | elapsed time per iteration (ms): 86887.3 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.50% | FLOPs estimator: workload-aware | logical tokens/s: 193091.1 | valid tokens/s: 193091.1 | physical tokens/s: 193736.1 | loss tokens/s: 142248.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.539587E-01 | lm-global-supervised-tokens: 1.235960E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.564044E-02 | lm-token-buffer-mib: 5.810725E+01 | global_load_balancing_loss: 1.000264E+00 | loss scale: 1.0 | grad norm: 0.177 | params norm: 11305.600 | params rms: 0.061216 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 10:48:11.537159 | finish at 2026-10-10 18:38:46 + [2026-10-09 07:52:00] iteration 5414/ 6855 | consumed samples: 692992 | elapsed time per iteration (ms): 85448.4 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.62% | FLOPs estimator: workload-aware | logical tokens/s: 196342.4 | valid tokens/s: 196342.4 | physical tokens/s: 196862.7 | loss tokens/s: 148637.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.763643E-01 | lm-global-supervised-tokens: 1.270082E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.163424E-02 | lm-token-buffer-mib: 5.809621E+01 | global_load_balancing_loss: 1.000792E+00 | loss scale: 1.0 | grad norm: 0.243 | params norm: 11305.545 | params rms: 0.061216 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 10:12:11.144430 | finish at 2026-10-10 18:04:11 + [2026-10-09 07:53:29] iteration 5415/ 6855 | consumed samples: 693120 | elapsed time per iteration (ms): 89265.1 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.74% | FLOPs estimator: workload-aware | logical tokens/s: 187948.0 | valid tokens/s: 187948.0 | physical tokens/s: 188323.6 | loss tokens/s: 134339.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.789917E-01 | lm-global-supervised-tokens: 1.199180E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 4.297971E-02 | lm-token-buffer-mib: 5.812262E+01 | global_load_balancing_loss: 1.001112E+00 | loss scale: 1.0 | grad norm: 0.075 | params norm: 11305.490 | params rms: 0.061216 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 11:42:21.806488 | finish at 2026-10-10 19:35:51 + [2026-10-09 07:55:00] iteration 5416/ 6855 | consumed samples: 693248 | elapsed time per iteration (ms): 91333.7 | throughput per GPU (TFLOP/s/GPU): 41.3 | MFU 13.24% | FLOPs estimator: workload-aware | logical tokens/s: 183690.9 | valid tokens/s: 183690.9 | physical tokens/s: 184118.2 | loss tokens/s: 130803.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.303350E-01 | lm-global-supervised-tokens: 1.194675E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.162578E-02 | lm-token-buffer-mib: 5.812052E+01 | global_load_balancing_loss: 1.000204E+00 | loss scale: 1.0 | grad norm: 0.153 | params norm: 11305.436 | params rms: 0.061215 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 12:30:29.131088 | finish at 2026-10-10 20:25:29 + [2026-10-09 07:56:23] iteration 5417/ 6855 | consumed samples: 693376 | elapsed time per iteration (ms): 83354.7 | throughput per GPU (TFLOP/s/GPU): 43.7 | MFU 14.01% | FLOPs estimator: workload-aware | logical tokens/s: 201274.5 | valid tokens/s: 201274.5 | physical tokens/s: 201762.2 | loss tokens/s: 146097.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.585192E-01 | lm-global-supervised-tokens: 1.217787E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.170954E-02 | lm-token-buffer-mib: 5.805561E+01 | global_load_balancing_loss: 9.999551E-01 | loss scale: 1.0 | grad norm: 0.159 | params norm: 11305.383 | params rms: 0.061215 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 9:17:44.036099 | finish at 2026-10-10 17:14:08 + [2026-10-09 07:57:46] iteration 5418/ 6855 | consumed samples: 693504 | elapsed time per iteration (ms): 82921.2 | throughput per GPU (TFLOP/s/GPU): 43.5 | MFU 13.93% | FLOPs estimator: workload-aware | logical tokens/s: 202326.7 | valid tokens/s: 202326.7 | physical tokens/s: 202978.5 | loss tokens/s: 151460.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.007134E-01 | lm-global-supervised-tokens: 1.255930E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.152283E-02 | lm-token-buffer-mib: 5.830820E+01 | global_load_balancing_loss: 1.000828E+00 | loss scale: 1.0 | grad norm: 0.068 | params norm: 11305.329 | params rms: 0.061215 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 9:05:57.718544 | finish at 2026-10-10 17:03:44 + [2026-10-09 07:59:12] iteration 5419/ 6855 | consumed samples: 693632 | elapsed time per iteration (ms): 85925.7 | throughput per GPU (TFLOP/s/GPU): 43.7 | MFU 14.00% | FLOPs estimator: workload-aware | logical tokens/s: 195252.4 | valid tokens/s: 195252.4 | physical tokens/s: 195786.4 | loss tokens/s: 144161.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.855595E-01 | lm-global-supervised-tokens: 1.238715E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.558908E-02 | lm-token-buffer-mib: 5.809868E+01 | global_load_balancing_loss: 9.987044E-01 | loss scale: 1.0 | grad norm: 0.122 | params norm: 11305.277 | params rms: 0.061214 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 10:16:29.245213 | finish at 2026-10-10 18:15:42 + [2026-10-09 08:01:01] iteration 5420/ 6855 | consumed samples: 693760 | elapsed time per iteration (ms): 108716.6 | throughput per GPU (TFLOP/s/GPU): 33.7 | MFU 10.81% | FLOPs estimator: workload-aware | logical tokens/s: 154319.6 | valid tokens/s: 154319.6 | physical tokens/s: 154584.7 | loss tokens/s: 114697.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.685616E-01 | lm-global-supervised-tokens: 1.246956E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.533179E-02 | lm-token-buffer-mib: 5.809212E+01 | global_load_balancing_loss: 9.997262E-01 | loss scale: 1.0 | grad norm: 0.066 | params norm: 11305.223 | params rms: 0.061214 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 19:20:08.263438 | finish at 2026-10-11 03:21:09 + [2026-10-09 08:02:27] iteration 5421/ 6855 | consumed samples: 693888 | elapsed time per iteration (ms): 85552.8 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.76% | FLOPs estimator: workload-aware | logical tokens/s: 196103.7 | valid tokens/s: 196103.7 | physical tokens/s: 196529.2 | loss tokens/s: 145846.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.659272E-01 | lm-global-supervised-tokens: 1.247756E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.220416E-02 | lm-token-buffer-mib: 5.825746E+01 | global_load_balancing_loss: 1.000346E+00 | loss scale: 1.0 | grad norm: 0.105 | params norm: 11305.171 | params rms: 0.061214 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 10:04:42.663488 | finish at 2026-10-10 18:07:09 + [2026-10-09 08:03:56] iteration 5422/ 6855 | consumed samples: 694016 | elapsed time per iteration (ms): 89560.5 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.41% | FLOPs estimator: workload-aware | logical tokens/s: 187328.0 | valid tokens/s: 187328.0 | physical tokens/s: 187908.5 | loss tokens/s: 141221.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.586978E-01 | lm-global-supervised-tokens: 1.264782E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.211278E-02 | lm-token-buffer-mib: 5.809025E+01 | global_load_balancing_loss: 1.001959E+00 | loss scale: 1.0 | grad norm: 0.068 | params norm: 11305.119 | params rms: 0.061214 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 11:39:00.213449 | finish at 2026-10-10 19:42:56 + [2026-10-09 08:05:26] iteration 5423/ 6855 | consumed samples: 694144 | elapsed time per iteration (ms): 89654.5 | throughput per GPU (TFLOP/s/GPU): 41.5 | MFU 13.31% | FLOPs estimator: workload-aware | logical tokens/s: 187131.8 | valid tokens/s: 187131.8 | physical tokens/s: 187491.7 | loss tokens/s: 136603.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.870283E-01 | lm-global-supervised-tokens: 1.224709E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.298287E-02 | lm-token-buffer-mib: 5.806197E+01 | global_load_balancing_loss: 1.001318E+00 | loss scale: 1.0 | grad norm: 0.078 | params norm: 11305.068 | params rms: 0.061213 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 11:39:45.199968 | finish at 2026-10-10 19:45:11 + [2026-10-09 08:06:52] iteration 5424/ 6855 | consumed samples: 694272 | elapsed time per iteration (ms): 86465.6 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.62% | FLOPs estimator: workload-aware | logical tokens/s: 194033.5 | valid tokens/s: 194033.5 | physical tokens/s: 194516.7 | loss tokens/s: 153383.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.355189E-01 | lm-global-supervised-tokens: 1.326236E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.155899E-02 | lm-token-buffer-mib: 5.808208E+01 | global_load_balancing_loss: 1.000538E+00 | loss scale: 1.0 | grad norm: 0.054 | params norm: 11305.019 | params rms: 0.061213 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 10:22:12.218349 | finish at 2026-10-10 18:29:05 + [2026-10-09 08:08:21] iteration 5425/ 6855 | consumed samples: 694400 | elapsed time per iteration (ms): 88386.6 | throughput per GPU (TFLOP/s/GPU): 41.7 | MFU 13.37% | FLOPs estimator: workload-aware | logical tokens/s: 189816.0 | valid tokens/s: 189816.0 | physical tokens/s: 190127.0 | loss tokens/s: 140570.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.305767E-01 | lm-global-supervised-tokens: 1.242457E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.165937E-02 | lm-token-buffer-mib: 5.806879E+01 | global_load_balancing_loss: 9.993080E-01 | loss scale: 1.0 | grad norm: 0.059 | params norm: 11304.970 | params rms: 0.061213 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 11:06:32.886438 | finish at 2026-10-10 19:14:54 + [2026-10-09 08:09:49] iteration 5426/ 6855 | consumed samples: 694528 | elapsed time per iteration (ms): 88390.1 | throughput per GPU (TFLOP/s/GPU): 41.5 | MFU 13.32% | FLOPs estimator: workload-aware | logical tokens/s: 189808.6 | valid tokens/s: 189808.6 | physical tokens/s: 190291.8 | loss tokens/s: 141066.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.310866E-01 | lm-global-supervised-tokens: 1.246886E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.594527E-02 | lm-token-buffer-mib: 5.816953E+01 | global_load_balancing_loss: 9.985871E-01 | loss scale: 1.0 | grad norm: 0.043 | params norm: 11304.922 | params rms: 0.061212 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 11:05:09.382378 | finish at 2026-10-10 19:14:58 + [2026-10-09 08:11:19] iteration 5427/ 6855 | consumed samples: 694656 | elapsed time per iteration (ms): 89564.9 | throughput per GPU (TFLOP/s/GPU): 41.6 | MFU 13.33% | FLOPs estimator: workload-aware | logical tokens/s: 187318.1 | valid tokens/s: 187318.1 | physical tokens/s: 187861.8 | loss tokens/s: 137229.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.607348E-01 | lm-global-supervised-tokens: 1.229096E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.353860E-02 | lm-token-buffer-mib: 5.810173E+01 | global_load_balancing_loss: 9.990998E-01 | loss scale: 1.0 | grad norm: 0.078 | params norm: 11304.874 | params rms: 0.061212 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 11:31:38.696835 | finish at 2026-10-10 19:42:57 + [2026-10-09 08:12:46] iteration 5428/ 6855 | consumed samples: 694784 | elapsed time per iteration (ms): 87136.2 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.63% | FLOPs estimator: workload-aware | logical tokens/s: 192540.0 | valid tokens/s: 192540.0 | physical tokens/s: 192999.6 | loss tokens/s: 138197.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.215807E-01 | lm-global-supervised-tokens: 1.204199E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.993320E-02 | lm-token-buffer-mib: 5.823328E+01 | global_load_balancing_loss: 1.000164E+00 | loss scale: 1.0 | grad norm: 0.044 | params norm: 11304.824 | params rms: 0.061212 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 10:32:23.417210 | finish at 2026-10-10 18:45:09 + [2026-10-09 08:14:15] iteration 5429/ 6855 | consumed samples: 694912 | elapsed time per iteration (ms): 88984.1 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.39% | FLOPs estimator: workload-aware | logical tokens/s: 188541.2 | valid tokens/s: 188541.2 | physical tokens/s: 189012.7 | loss tokens/s: 137180.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.954909E-01 | lm-global-supervised-tokens: 1.220690E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.152724E-02 | lm-token-buffer-mib: 5.816841E+01 | global_load_balancing_loss: 1.001372E+00 | loss scale: 1.0 | grad norm: 0.138 | params norm: 11304.774 | params rms: 0.061212 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 11:14:51.303288 | finish at 2026-10-10 19:29:06 + [2026-10-09 08:16:06] iteration 5430/ 6855 | consumed samples: 695040 | elapsed time per iteration (ms): 110998.6 | throughput per GPU (TFLOP/s/GPU): 33.6 | MFU 10.78% | FLOPs estimator: workload-aware | logical tokens/s: 151147.8 | valid tokens/s: 151147.8 | physical tokens/s: 151374.2 | loss tokens/s: 109904.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.328628E-01 | lm-global-supervised-tokens: 1.219919E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.242217E-02 | lm-token-buffer-mib: 5.804503E+01 | global_load_balancing_loss: 1.001199E+00 | loss scale: 1.0 | grad norm: 0.058 | params norm: 11304.723 | params rms: 0.061211 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 19:56:12.982246 | finish at 2026-10-11 04:12:19 + [2026-10-09 08:17:31] iteration 5431/ 6855 | consumed samples: 695168 | elapsed time per iteration (ms): 84938.7 | throughput per GPU (TFLOP/s/GPU): 43.6 | MFU 13.97% | FLOPs estimator: workload-aware | logical tokens/s: 197520.9 | valid tokens/s: 197520.9 | physical tokens/s: 197860.4 | loss tokens/s: 148282.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.570351E-01 | lm-global-supervised-tokens: 1.259493E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.112859E-02 | lm-token-buffer-mib: 5.811864E+01 | global_load_balancing_loss: 1.000952E+00 | loss scale: 1.0 | grad norm: 0.197 | params norm: 11304.672 | params rms: 0.061211 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 9:35:52.777664 | finish at 2026-10-10 17:53:23 + [2026-10-09 08:18:57] iteration 5432/ 6855 | consumed samples: 695296 | elapsed time per iteration (ms): 85949.8 | throughput per GPU (TFLOP/s/GPU): 42.8 | MFU 13.73% | FLOPs estimator: workload-aware | logical tokens/s: 195197.7 | valid tokens/s: 195197.7 | physical tokens/s: 195686.4 | loss tokens/s: 143598.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.038311E-01 | lm-global-supervised-tokens: 1.234230E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.151415E-02 | lm-token-buffer-mib: 5.809843E+01 | global_load_balancing_loss: 9.992779E-01 | loss scale: 1.0 | grad norm: 0.175 | params norm: 11304.621 | params rms: 0.061211 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 9:58:26.621400 | finish at 2026-10-10 18:17:23 + [2026-10-09 08:20:20] iteration 5433/ 6855 | consumed samples: 695424 | elapsed time per iteration (ms): 83190.8 | throughput per GPU (TFLOP/s/GPU): 44.3 | MFU 14.18% | FLOPs estimator: workload-aware | logical tokens/s: 201671.3 | valid tokens/s: 201671.3 | physical tokens/s: 202324.5 | loss tokens/s: 144033.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.632435E-01 | lm-global-supervised-tokens: 1.198228E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.594509E-02 | lm-token-buffer-mib: 5.815195E+01 | global_load_balancing_loss: 9.996047E-01 | loss scale: 1.0 | grad norm: 0.076 | params norm: 11304.571 | params rms: 0.061211 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 8:51:37.325329 | finish at 2026-10-10 17:11:57 + [2026-10-09 08:21:49] iteration 5434/ 6855 | consumed samples: 695552 | elapsed time per iteration (ms): 88843.1 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.51% | FLOPs estimator: workload-aware | logical tokens/s: 188839.8 | valid tokens/s: 188839.8 | physical tokens/s: 189203.3 | loss tokens/s: 139852.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.521363E-01 | lm-global-supervised-tokens: 1.242489E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.203733E-02 | lm-token-buffer-mib: 5.809361E+01 | global_load_balancing_loss: 1.000086E+00 | loss scale: 1.0 | grad norm: 0.127 | params norm: 11304.519 | params rms: 0.061210 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 11:04:06.038425 | finish at 2026-10-10 19:25:55 + [2026-10-09 08:23:16] iteration 5435/ 6855 | consumed samples: 695680 | elapsed time per iteration (ms): 87368.1 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.44% | FLOPs estimator: workload-aware | logical tokens/s: 192028.8 | valid tokens/s: 192028.8 | physical tokens/s: 192557.8 | loss tokens/s: 143186.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.409427E-01 | lm-global-supervised-tokens: 1.250990E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.669679E-02 | lm-token-buffer-mib: 5.812683E+01 | global_load_balancing_loss: 1.000656E+00 | loss scale: 1.0 | grad norm: 0.037 | params norm: 11304.467 | params rms: 0.061210 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 10:27:42.706299 | finish at 2026-10-10 18:50:59 + [2026-10-09 08:24:47] iteration 5436/ 6855 | consumed samples: 695808 | elapsed time per iteration (ms): 90771.1 | throughput per GPU (TFLOP/s/GPU): 40.6 | MFU 13.00% | FLOPs estimator: workload-aware | logical tokens/s: 184829.2 | valid tokens/s: 184829.2 | physical tokens/s: 185263.5 | loss tokens/s: 141296.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.692993E-01 | lm-global-supervised-tokens: 1.282561E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 4.696647E-02 | lm-token-buffer-mib: 5.810143E+01 | global_load_balancing_loss: 1.000456E+00 | loss scale: 1.0 | grad norm: 0.173 | params norm: 11304.417 | params rms: 0.061210 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 11:46:44.250845 | finish at 2026-10-10 20:11:31 + [2026-10-09 08:26:16] iteration 5437/ 6855 | consumed samples: 695936 | elapsed time per iteration (ms): 88999.4 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.74% | FLOPs estimator: workload-aware | logical tokens/s: 188508.5 | valid tokens/s: 188508.5 | physical tokens/s: 188921.7 | loss tokens/s: 139704.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.059742E-01 | lm-global-supervised-tokens: 1.243360E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.154836E-02 | lm-token-buffer-mib: 5.814085E+01 | global_load_balancing_loss: 9.994156E-01 | loss scale: 1.0 | grad norm: 0.057 | params norm: 11304.368 | params rms: 0.061209 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 11:03:21.165287 | finish at 2026-10-10 19:29:37 + [2026-10-09 08:27:44] iteration 5438/ 6855 | consumed samples: 696064 | elapsed time per iteration (ms): 88408.9 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.54% | FLOPs estimator: workload-aware | logical tokens/s: 189768.5 | valid tokens/s: 189768.5 | physical tokens/s: 190191.4 | loss tokens/s: 139162.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.006042E-01 | lm-global-supervised-tokens: 1.230318E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.141350E-02 | lm-token-buffer-mib: 5.838068E+01 | global_load_balancing_loss: 9.996998E-01 | loss scale: 1.0 | grad norm: 0.120 | params norm: 11304.318 | params rms: 0.061209 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 10:47:55.348156 | finish at 2026-10-10 19:15:40 + [2026-10-09 08:29:12] iteration 5439/ 6855 | consumed samples: 696192 | elapsed time per iteration (ms): 87294.7 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.43% | FLOPs estimator: workload-aware | logical tokens/s: 192190.2 | valid tokens/s: 192190.2 | physical tokens/s: 192731.3 | loss tokens/s: 144292.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.584404E-01 | lm-global-supervised-tokens: 1.259598E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.154735E-02 | lm-token-buffer-mib: 5.806455E+01 | global_load_balancing_loss: 9.998085E-01 | loss scale: 1.0 | grad norm: 0.079 | params norm: 11304.269 | params rms: 0.061209 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 10:20:09.259283 | finish at 2026-10-10 18:49:21 + [2026-10-09 08:30:59] iteration 5440/ 6855 | consumed samples: 696320 | elapsed time per iteration (ms): 107816.9 | throughput per GPU (TFLOP/s/GPU): 34.5 | MFU 11.05% | FLOPs estimator: workload-aware | logical tokens/s: 155607.8 | valid tokens/s: 155607.8 | physical tokens/s: 155946.8 | loss tokens/s: 114239.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.718988E-01 | lm-global-supervised-tokens: 1.231693E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.542447E-02 | lm-token-buffer-mib: 5.813498E+01 | global_load_balancing_loss: 1.000838E+00 | loss scale: 1.0 | grad norm: 0.142 | params norm: 11304.220 | params rms: 0.061209 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 18:22:40.918244 | finish at 2026-10-11 02:53:40 +times across ranks (ms): + save-checkpoint: + rank 0: 24907.49 + rank 1: 24907.51 + rank 2: 24907.56 + rank 3: 24907.50 + rank 4: 24907.54 + rank 5: 24907.51 + rank 6: 24907.49 + rank 7: 24907.51 + rank 8: 24907.39 + rank 9: 24907.30 + rank 10: 24907.31 + rank 11: 24907.32 + rank 12: 24907.33 + rank 13: 24907.32 + rank 14: 24907.33 + rank 15: 24907.31 + rank 16: 24907.20 + rank 17: 24907.21 + rank 18: 24907.22 + rank 19: 24907.19 + rank 20: 24907.19 + rank 21: 24907.20 + rank 22: 24907.19 + rank 23: 24907.20 + rank 24: 24907.54 + rank 25: 24907.49 + rank 26: 24907.51 + rank 27: 24907.54 + rank 28: 24907.51 + rank 29: 24907.50 + rank 30: 24907.50 + rank 31: 24907.63 + rank 32: 24907.24 + rank 33: 24907.24 + rank 34: 24907.24 + rank 35: 24907.20 + rank 36: 24907.19 + rank 37: 24907.18 + rank 38: 24907.19 + rank 39: 24907.17 + rank 40: 24907.60 + rank 41: 24907.58 + rank 42: 24907.62 + rank 43: 24907.59 + rank 44: 24907.58 + rank 45: 24907.60 + rank 46: 24907.59 + rank 47: 24907.57 + rank 48: 24907.01 + rank 49: 24906.98 + rank 50: 24907.00 + rank 51: 24906.97 + rank 52: 24907.02 + rank 53: 24906.97 + rank 54: 24906.98 + rank 55: 24907.01 + rank 56: 24907.01 + rank 57: 24906.99 + rank 58: 24906.99 + rank 59: 24906.99 + rank 60: 24906.98 + rank 61: 24907.00 + rank 62: 24906.99 + rank 63: 24906.98 + [2026-10-09 08:33:00] iteration 5441/ 6855 | consumed samples: 696448 | elapsed time per iteration (ms): 96014.1 | throughput per GPU (TFLOP/s/GPU): 38.4 | MFU 12.31% | FLOPs estimator: workload-aware | logical tokens/s: 174737.0 | valid tokens/s: 174737.0 | physical tokens/s: 175027.9 | loss tokens/s: 128578.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.377432E-01 | lm-global-supervised-tokens: 1.234532E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.328484E-01 | lm-token-buffer-mib: 5.824953E+01 | global_load_balancing_loss: 1.000866E+00 | loss scale: 1.0 | grad norm: 0.045 | params norm: 11304.170 | params rms: 0.061208 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 13:42:43.958745 | finish at 2026-10-10 22:15:44 + [2026-10-09 08:34:29] iteration 5442/ 6855 | consumed samples: 696576 | elapsed time per iteration (ms): 88816.0 | throughput per GPU (TFLOP/s/GPU): 40.7 | MFU 13.04% | FLOPs estimator: workload-aware | logical tokens/s: 188897.9 | valid tokens/s: 188897.9 | physical tokens/s: 189272.2 | loss tokens/s: 143837.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 5.908489E-01 | lm-global-supervised-tokens: 1.277507E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.445385E-02 | lm-token-buffer-mib: 5.811698E+01 | global_load_balancing_loss: 1.000845E+00 | loss scale: 1.0 | grad norm: 0.198 | params norm: 11304.120 | params rms: 0.061208 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 10:51:37.017411 | finish at 2026-10-10 19:26:06 + [2026-10-09 08:35:55] iteration 5443/ 6855 | consumed samples: 696704 | elapsed time per iteration (ms): 85745.7 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.59% | FLOPs estimator: workload-aware | logical tokens/s: 195661.7 | valid tokens/s: 195661.7 | physical tokens/s: 196381.0 | loss tokens/s: 140756.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.293676E-01 | lm-global-supervised-tokens: 1.206926E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.578029E-02 | lm-token-buffer-mib: 5.825327E+01 | global_load_balancing_loss: 1.000458E+00 | loss scale: 1.0 | grad norm: 0.063 | params norm: 11304.069 | params rms: 0.061208 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 9:37:52.864608 | finish at 2026-10-10 18:13:48 + [2026-10-09 08:37:21] iteration 5444/ 6855 | consumed samples: 696832 | elapsed time per iteration (ms): 85813.6 | throughput per GPU (TFLOP/s/GPU): 42.8 | MFU 13.72% | FLOPs estimator: workload-aware | logical tokens/s: 195506.5 | valid tokens/s: 195506.5 | physical tokens/s: 195995.8 | loss tokens/s: 144132.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.495117E-01 | lm-global-supervised-tokens: 1.236851E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.104308E-02 | lm-token-buffer-mib: 5.807497E+01 | global_load_balancing_loss: 1.000146E+00 | loss scale: 1.0 | grad norm: 0.188 | params norm: 11304.018 | params rms: 0.061208 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 9:38:02.956048 | finish at 2026-10-10 18:15:24 + [2026-10-09 08:38:48] iteration 5445/ 6855 | consumed samples: 696960 | elapsed time per iteration (ms): 87510.5 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.78% | FLOPs estimator: workload-aware | logical tokens/s: 191716.1 | valid tokens/s: 191716.1 | physical tokens/s: 192042.7 | loss tokens/s: 142420.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.491141E-01 | lm-global-supervised-tokens: 1.246332E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.246536E-02 | lm-token-buffer-mib: 5.808897E+01 | global_load_balancing_loss: 1.000430E+00 | loss scale: 1.0 | grad norm: 0.097 | params norm: 11303.965 | params rms: 0.061207 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 10:16:29.735012 | finish at 2026-10-10 18:55:18 + [2026-10-09 08:40:15] iteration 5446/ 6855 | consumed samples: 697088 | elapsed time per iteration (ms): 86428.6 | throughput per GPU (TFLOP/s/GPU): 42.8 | MFU 13.73% | FLOPs estimator: workload-aware | logical tokens/s: 194115.8 | valid tokens/s: 194115.8 | physical tokens/s: 194643.2 | loss tokens/s: 137780.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.365314E-01 | lm-global-supervised-tokens: 1.190818E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.604765E-02 | lm-token-buffer-mib: 5.820871E+01 | global_load_balancing_loss: 9.991015E-01 | loss scale: 1.0 | grad norm: 0.151 | params norm: 11303.912 | params rms: 0.061207 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 9:49:37.882722 | finish at 2026-10-10 18:29:53 + [2026-10-09 08:41:42] iteration 5447/ 6855 | consumed samples: 697216 | elapsed time per iteration (ms): 87542.6 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.43% | FLOPs estimator: workload-aware | logical tokens/s: 191646.0 | valid tokens/s: 191646.0 | physical tokens/s: 192377.3 | loss tokens/s: 145260.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.442964E-01 | lm-global-supervised-tokens: 1.271645E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.212220E-02 | lm-token-buffer-mib: 5.815365E+01 | global_load_balancing_loss: 9.984550E-01 | loss scale: 1.0 | grad norm: 0.365 | params norm: 11303.859 | params rms: 0.061207 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 10:14:19.920929 | finish at 2026-10-10 18:56:02 + [2026-10-09 08:43:10] iteration 5448/ 6855 | consumed samples: 697344 | elapsed time per iteration (ms): 87771.1 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.53% | FLOPs estimator: workload-aware | logical tokens/s: 191147.2 | valid tokens/s: 191147.2 | physical tokens/s: 191569.1 | loss tokens/s: 141415.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.011955E-01 | lm-global-supervised-tokens: 1.241222E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.839780E-02 | lm-token-buffer-mib: 5.806352E+01 | global_load_balancing_loss: 9.995057E-01 | loss scale: 1.0 | grad norm: 0.164 | params norm: 11303.805 | params rms: 0.061206 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 10:18:13.979694 | finish at 2026-10-10 19:01:24 + [2026-10-09 08:44:39] iteration 5449/ 6855 | consumed samples: 697472 | elapsed time per iteration (ms): 89214.9 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.48% | FLOPs estimator: workload-aware | logical tokens/s: 188052.7 | valid tokens/s: 188052.7 | physical tokens/s: 188399.4 | loss tokens/s: 134779.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.754630E-01 | lm-global-supervised-tokens: 1.202437E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.240187E-02 | lm-token-buffer-mib: 5.808761E+01 | global_load_balancing_loss: 1.000948E+00 | loss scale: 1.0 | grad norm: 0.419 | params norm: 11303.750 | params rms: 0.061206 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 10:50:36.147412 | finish at 2026-10-10 19:35:15 + [2026-10-09 08:46:31] iteration 5450/ 6855 | consumed samples: 697600 | elapsed time per iteration (ms): 112145.6 | throughput per GPU (TFLOP/s/GPU): 32.3 | MFU 10.35% | FLOPs estimator: workload-aware | logical tokens/s: 149601.7 | valid tokens/s: 149601.7 | physical tokens/s: 150052.1 | loss tokens/s: 113737.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.508725E-01 | lm-global-supervised-tokens: 1.275514E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.637758E-02 | lm-token-buffer-mib: 5.813589E+01 | global_load_balancing_loss: 1.000286E+00 | loss scale: 1.0 | grad norm: 0.160 | params norm: 11303.697 | params rms: 0.061206 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 19:46:04.502792 | finish at 2026-10-11 04:32:36 + [2026-10-09 08:47:58] iteration 5451/ 6855 | consumed samples: 697728 | elapsed time per iteration (ms): 86626.4 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.57% | FLOPs estimator: workload-aware | logical tokens/s: 193672.7 | valid tokens/s: 193672.7 | physical tokens/s: 194251.1 | loss tokens/s: 148830.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.241637E-01 | lm-global-supervised-tokens: 1.289264E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.284260E-02 | lm-token-buffer-mib: 5.817901E+01 | global_load_balancing_loss: 1.000975E+00 | loss scale: 1.0 | grad norm: 0.267 | params norm: 11303.643 | params rms: 0.061206 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 9:47:03.418728 | finish at 2026-10-10 18:35:01 + [2026-10-09 08:49:27] iteration 5452/ 6855 | consumed samples: 697856 | elapsed time per iteration (ms): 88747.2 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.50% | FLOPs estimator: workload-aware | logical tokens/s: 189044.3 | valid tokens/s: 189044.3 | physical tokens/s: 189751.5 | loss tokens/s: 137820.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.719172E-01 | lm-global-supervised-tokens: 1.223122E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.163555E-02 | lm-token-buffer-mib: 5.814857E+01 | global_load_balancing_loss: 1.001369E+00 | loss scale: 1.0 | grad norm: 0.186 | params norm: 11303.587 | params rms: 0.061205 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 10:35:12.340349 | finish at 2026-10-10 19:24:39 + [2026-10-09 08:50:57] iteration 5453/ 6855 | consumed samples: 697984 | elapsed time per iteration (ms): 90252.5 | throughput per GPU (TFLOP/s/GPU): 41.3 | MFU 13.23% | FLOPs estimator: workload-aware | logical tokens/s: 185892.0 | valid tokens/s: 185892.0 | physical tokens/s: 186224.6 | loss tokens/s: 133378.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.310254E-01 | lm-global-supervised-tokens: 1.203778E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.460946E-02 | lm-token-buffer-mib: 5.805202E+01 | global_load_balancing_loss: 1.001533E+00 | loss scale: 1.0 | grad norm: 0.155 | params norm: 11303.532 | params rms: 0.061205 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 11:08:53.958618 | finish at 2026-10-10 19:59:51 + [2026-10-09 08:52:22] iteration 5454/ 6855 | consumed samples: 698112 | elapsed time per iteration (ms): 85407.9 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.75% | FLOPs estimator: workload-aware | logical tokens/s: 196436.2 | valid tokens/s: 196436.2 | physical tokens/s: 196832.6 | loss tokens/s: 148333.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.848483E-01 | lm-global-supervised-tokens: 1.266884E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.529458E-02 | lm-token-buffer-mib: 5.809190E+01 | global_load_balancing_loss: 1.000874E+00 | loss scale: 1.0 | grad norm: 0.111 | params norm: 11303.475 | params rms: 0.061205 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 9:14:16.464359 | finish at 2026-10-10 18:06:39 + [2026-10-09 08:53:53] iteration 5455/ 6855 | consumed samples: 698240 | elapsed time per iteration (ms): 90391.5 | throughput per GPU (TFLOP/s/GPU): 41.4 | MFU 13.29% | FLOPs estimator: workload-aware | logical tokens/s: 185605.8 | valid tokens/s: 185605.8 | physical tokens/s: 185958.6 | loss tokens/s: 140301.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.561877E-01 | lm-global-supervised-tokens: 1.268204E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.167583E-02 | lm-token-buffer-mib: 5.815297E+01 | global_load_balancing_loss: 9.993254E-01 | loss scale: 1.0 | grad norm: 0.171 | params norm: 11303.419 | params rms: 0.061204 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 11:09:08.045921 | finish at 2026-10-10 20:03:01 + [2026-10-09 08:55:19] iteration 5456/ 6855 | consumed samples: 698368 | elapsed time per iteration (ms): 85972.5 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.65% | FLOPs estimator: workload-aware | logical tokens/s: 195145.8 | valid tokens/s: 195145.8 | physical tokens/s: 195755.2 | loss tokens/s: 147721.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.736876E-01 | lm-global-supervised-tokens: 1.270001E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 4.914679E-02 | lm-token-buffer-mib: 5.818399E+01 | global_load_balancing_loss: 9.990696E-01 | loss scale: 1.0 | grad norm: 0.076 | params norm: 11303.367 | params rms: 0.061204 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 9:24:35.546632 | finish at 2026-10-10 18:19:54 + [2026-10-09 08:56:45] iteration 5457/ 6855 | consumed samples: 698496 | elapsed time per iteration (ms): 86123.6 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.64% | FLOPs estimator: workload-aware | logical tokens/s: 194803.7 | valid tokens/s: 194803.7 | physical tokens/s: 195225.4 | loss tokens/s: 144109.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.060745E-01 | lm-global-supervised-tokens: 1.241121E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.254429E-02 | lm-token-buffer-mib: 5.815298E+01 | global_load_balancing_loss: 9.986394E-01 | loss scale: 1.0 | grad norm: 0.176 | params norm: 11303.315 | params rms: 0.061204 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 9:26:40.733146 | finish at 2026-10-10 18:23:26 + [2026-10-09 08:58:12] iteration 5458/ 6855 | consumed samples: 698624 | elapsed time per iteration (ms): 87471.1 | throughput per GPU (TFLOP/s/GPU): 43.4 | MFU 13.90% | FLOPs estimator: workload-aware | logical tokens/s: 191802.9 | valid tokens/s: 191802.9 | physical tokens/s: 192166.2 | loss tokens/s: 148451.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.226544E-01 | lm-global-supervised-tokens: 1.298524E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.558370E-02 | lm-token-buffer-mib: 5.804679E+01 | global_load_balancing_loss: 9.992089E-01 | loss scale: 1.0 | grad norm: 0.127 | params norm: 11303.261 | params rms: 0.061203 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 9:56:37.153474 | finish at 2026-10-10 18:54:50 + [2026-10-09 08:59:36] iteration 5459/ 6855 | consumed samples: 698752 | elapsed time per iteration (ms): 83712.5 | throughput per GPU (TFLOP/s/GPU): 43.6 | MFU 13.99% | FLOPs estimator: workload-aware | logical tokens/s: 200414.2 | valid tokens/s: 200414.2 | physical tokens/s: 200926.1 | loss tokens/s: 145152.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.325475E-01 | lm-global-supervised-tokens: 1.215107E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.182656E-02 | lm-token-buffer-mib: 5.813532E+01 | global_load_balancing_loss: 1.001227E+00 | loss scale: 1.0 | grad norm: 0.069 | params norm: 11303.205 | params rms: 0.061203 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 8:27:42.663114 | finish at 2026-10-10 17:27:19 + [2026-10-09 09:01:23] iteration 5460/ 6855 | consumed samples: 698880 | elapsed time per iteration (ms): 107174.8 | throughput per GPU (TFLOP/s/GPU): 33.9 | MFU 10.87% | FLOPs estimator: workload-aware | logical tokens/s: 156540.6 | valid tokens/s: 156540.6 | physical tokens/s: 156806.6 | loss tokens/s: 117128.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.232181E-01 | lm-global-supervised-tokens: 1.255323E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.345641E-02 | lm-token-buffer-mib: 5.807524E+01 | global_load_balancing_loss: 1.000353E+00 | loss scale: 1.0 | grad norm: 0.054 | params norm: 11303.152 | params rms: 0.061203 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 17:31:48.830588 | finish at 2026-10-11 02:33:12 + [2026-10-09 09:02:52] iteration 5461/ 6855 | consumed samples: 699008 | elapsed time per iteration (ms): 88303.7 | throughput per GPU (TFLOP/s/GPU): 41.6 | MFU 13.34% | FLOPs estimator: workload-aware | logical tokens/s: 189993.8 | valid tokens/s: 189993.8 | physical tokens/s: 190318.5 | loss tokens/s: 135273.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.606817E-01 | lm-global-supervised-tokens: 1.194518E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.158527E-02 | lm-token-buffer-mib: 5.807324E+01 | global_load_balancing_loss: 1.000629E+00 | loss scale: 1.0 | grad norm: 0.091 | params norm: 11303.100 | params rms: 0.061203 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 10:11:35.424230 | finish at 2026-10-10 19:14:27 + [2026-10-09 09:04:19] iteration 5462/ 6855 | consumed samples: 699136 | elapsed time per iteration (ms): 87205.8 | throughput per GPU (TFLOP/s/GPU): 43.3 | MFU 13.87% | FLOPs estimator: workload-aware | logical tokens/s: 192386.0 | valid tokens/s: 192386.0 | physical tokens/s: 192754.1 | loss tokens/s: 139945.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.916191E-01 | lm-global-supervised-tokens: 1.220402E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.464471E-02 | lm-token-buffer-mib: 5.807175E+01 | global_load_balancing_loss: 1.001681E+00 | loss scale: 1.0 | grad norm: 0.096 | params norm: 11303.048 | params rms: 0.061202 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 9:44:37.721990 | finish at 2026-10-10 18:48:57 + [2026-10-09 09:05:50] iteration 5463/ 6855 | consumed samples: 699264 | elapsed time per iteration (ms): 91420.1 | throughput per GPU (TFLOP/s/GPU): 41.4 | MFU 13.27% | FLOPs estimator: workload-aware | logical tokens/s: 183517.7 | valid tokens/s: 183517.7 | physical tokens/s: 183860.0 | loss tokens/s: 131054.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.449522E-01 | lm-global-supervised-tokens: 1.198098E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.733000E-02 | lm-token-buffer-mib: 5.821936E+01 | global_load_balancing_loss: 1.000396E+00 | loss scale: 1.0 | grad norm: 0.036 | params norm: 11302.995 | params rms: 0.061202 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 11:20:56.747303 | finish at 2026-10-10 20:26:47 + [2026-10-09 09:07:18] iteration 5464/ 6855 | consumed samples: 699392 | elapsed time per iteration (ms): 88143.5 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.52% | FLOPs estimator: workload-aware | logical tokens/s: 190338.9 | valid tokens/s: 190338.9 | physical tokens/s: 190771.2 | loss tokens/s: 142197.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.481752E-01 | lm-global-supervised-tokens: 1.253376E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.396989E-02 | lm-token-buffer-mib: 5.817159E+01 | global_load_balancing_loss: 9.996646E-01 | loss scale: 1.0 | grad norm: 0.179 | params norm: 11302.940 | params rms: 0.061202 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 10:03:27.627197 | finish at 2026-10-10 19:10:46 + [2026-10-09 09:08:46] iteration 5465/ 6855 | consumed samples: 699520 | elapsed time per iteration (ms): 87278.6 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.66% | FLOPs estimator: workload-aware | logical tokens/s: 192225.8 | valid tokens/s: 192225.8 | physical tokens/s: 192685.6 | loss tokens/s: 142336.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.059625E-01 | lm-global-supervised-tokens: 1.242291E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.430103E-02 | lm-token-buffer-mib: 5.820911E+01 | global_load_balancing_loss: 9.988125E-01 | loss scale: 1.0 | grad norm: 0.051 | params norm: 11302.887 | params rms: 0.061201 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 9:41:57.268219 | finish at 2026-10-10 18:50:43 + [2026-10-09 09:10:11] iteration 5466/ 6855 | consumed samples: 699648 | elapsed time per iteration (ms): 84927.5 | throughput per GPU (TFLOP/s/GPU): 43.1 | MFU 13.83% | FLOPs estimator: workload-aware | logical tokens/s: 197547.0 | valid tokens/s: 197547.0 | physical tokens/s: 198076.1 | loss tokens/s: 141478.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.612210E-01 | lm-global-supervised-tokens: 1.201540E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.363428E-02 | lm-token-buffer-mib: 5.822506E+01 | global_load_balancing_loss: 9.999445E-01 | loss scale: 1.0 | grad norm: 0.175 | params norm: 11302.835 | params rms: 0.061201 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 8:46:04.348181 | finish at 2026-10-10 17:56:15 + [2026-10-09 09:11:40] iteration 5467/ 6855 | consumed samples: 699776 | elapsed time per iteration (ms): 89641.3 | throughput per GPU (TFLOP/s/GPU): 41.7 | MFU 13.37% | FLOPs estimator: workload-aware | logical tokens/s: 187158.8 | valid tokens/s: 187158.8 | physical tokens/s: 187499.2 | loss tokens/s: 132915.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.032495E-01 | lm-global-supervised-tokens: 1.191473E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.134620E-02 | lm-token-buffer-mib: 5.813660E+01 | global_load_balancing_loss: 1.001096E+00 | loss scale: 1.0 | grad norm: 0.082 | params norm: 11302.782 | params rms: 0.061201 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 10:33:42.191195 | finish at 2026-10-10 19:45:22 + [2026-10-09 09:13:07] iteration 5468/ 6855 | consumed samples: 699904 | elapsed time per iteration (ms): 86787.1 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.58% | FLOPs estimator: workload-aware | logical tokens/s: 193313.4 | valid tokens/s: 193313.3 | physical tokens/s: 193816.7 | loss tokens/s: 148278.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.154912E-01 | lm-global-supervised-tokens: 1.286867E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.130888E-02 | lm-token-buffer-mib: 5.815966E+01 | global_load_balancing_loss: 9.997336E-01 | loss scale: 1.0 | grad norm: 0.093 | params norm: 11302.731 | params rms: 0.061201 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 9:26:13.696894 | finish at 2026-10-10 18:39:21 + [2026-10-09 09:14:36] iteration 5469/ 6855 | consumed samples: 700032 | elapsed time per iteration (ms): 88727.3 | throughput per GPU (TFLOP/s/GPU): 42.8 | MFU 13.72% | FLOPs estimator: workload-aware | logical tokens/s: 189087.3 | valid tokens/s: 189087.3 | physical tokens/s: 189574.7 | loss tokens/s: 129111.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.408558E-01 | lm-global-supervised-tokens: 1.145575E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.159629E-02 | lm-token-buffer-mib: 5.810097E+01 | global_load_balancing_loss: 1.000561E+00 | loss scale: 1.0 | grad norm: 0.104 | params norm: 11302.682 | params rms: 0.061200 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 10:09:36.043980 | finish at 2026-10-10 19:24:12 + [2026-10-09 09:16:27] iteration 5470/ 6855 | consumed samples: 700160 | elapsed time per iteration (ms): 111519.5 | throughput per GPU (TFLOP/s/GPU): 33.4 | MFU 10.70% | FLOPs estimator: workload-aware | logical tokens/s: 150441.3 | valid tokens/s: 150441.3 | physical tokens/s: 150840.8 | loss tokens/s: 113594.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.719538E-01 | lm-global-supervised-tokens: 1.266800E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.350688E-02 | lm-token-buffer-mib: 5.816682E+01 | global_load_balancing_loss: 9.994938E-01 | loss scale: 1.0 | grad norm: 0.056 | params norm: 11302.630 | params rms: 0.061200 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 18:54:14.542856 | finish at 2026-10-11 04:10:42 + [2026-10-09 09:17:56] iteration 5471/ 6855 | consumed samples: 700288 | elapsed time per iteration (ms): 88573.4 | throughput per GPU (TFLOP/s/GPU): 41.4 | MFU 13.26% | FLOPs estimator: workload-aware | logical tokens/s: 189415.3 | valid tokens/s: 189415.3 | physical tokens/s: 189918.7 | loss tokens/s: 140843.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.485209E-01 | lm-global-supervised-tokens: 1.247501E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.880419E-02 | lm-token-buffer-mib: 5.817459E+01 | global_load_balancing_loss: 1.000366E+00 | loss scale: 1.0 | grad norm: 0.042 | params norm: 11302.577 | params rms: 0.061200 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 10:03:05.578699 | finish at 2026-10-10 19:21:01 + [2026-10-09 09:19:28] iteration 5472/ 6855 | consumed samples: 700416 | elapsed time per iteration (ms): 92254.4 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.39% | FLOPs estimator: workload-aware | logical tokens/s: 181857.7 | valid tokens/s: 181857.7 | physical tokens/s: 182113.4 | loss tokens/s: 129613.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.544957E-01 | lm-global-supervised-tokens: 1.195742E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.113835E-02 | lm-token-buffer-mib: 5.813304E+01 | global_load_balancing_loss: 1.000217E+00 | loss scale: 1.0 | grad norm: 0.044 | params norm: 11302.524 | params rms: 0.061199 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 11:26:27.899848 | finish at 2026-10-10 20:45:56 + [2026-10-09 09:20:51] iteration 5473/ 6855 | consumed samples: 700544 | elapsed time per iteration (ms): 82990.0 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.79% | FLOPs estimator: workload-aware | logical tokens/s: 202158.8 | valid tokens/s: 202158.8 | physical tokens/s: 202636.9 | loss tokens/s: 152292.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.331925E-01 | lm-global-supervised-tokens: 1.263877E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.317642E-02 | lm-token-buffer-mib: 5.828373E+01 | global_load_balancing_loss: 1.001302E+00 | loss scale: 1.0 | grad norm: 0.068 | params norm: 11302.471 | params rms: 0.061199 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 7:51:32.140144 | finish at 2026-10-10 17:12:23 + [2026-10-09 09:22:17] iteration 5474/ 6855 | consumed samples: 700672 | elapsed time per iteration (ms): 85874.3 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.70% | FLOPs estimator: workload-aware | logical tokens/s: 195369.5 | valid tokens/s: 195369.5 | physical tokens/s: 195757.4 | loss tokens/s: 146921.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.310483E-01 | lm-global-supervised-tokens: 1.261680E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.157851E-02 | lm-token-buffer-mib: 5.809975E+01 | global_load_balancing_loss: 9.998322E-01 | loss scale: 1.0 | grad norm: 0.046 | params norm: 11302.418 | params rms: 0.061199 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 8:56:32.396451 | finish at 2026-10-10 18:18:49 + [2026-10-09 09:23:43] iteration 5475/ 6855 | consumed samples: 700800 | elapsed time per iteration (ms): 86558.9 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.57% | FLOPs estimator: workload-aware | logical tokens/s: 193823.8 | valid tokens/s: 193823.8 | physical tokens/s: 194118.6 | loss tokens/s: 144120.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.327167E-01 | lm-global-supervised-tokens: 1.247490E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.272800E-02 | lm-token-buffer-mib: 5.817525E+01 | global_load_balancing_loss: 1.000126E+00 | loss scale: 1.0 | grad norm: 0.034 | params norm: 11302.366 | params rms: 0.061199 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 9:10:51.274595 | finish at 2026-10-10 18:34:35 + [2026-10-09 09:25:11] iteration 5476/ 6855 | consumed samples: 700928 | elapsed time per iteration (ms): 87890.4 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.59% | FLOPs estimator: workload-aware | logical tokens/s: 190887.1 | valid tokens/s: 190887.1 | physical tokens/s: 191283.8 | loss tokens/s: 138749.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.512070E-01 | lm-global-supervised-tokens: 1.219473E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.819073E-02 | lm-token-buffer-mib: 5.810345E+01 | global_load_balancing_loss: 1.000118E+00 | loss scale: 1.0 | grad norm: 0.058 | params norm: 11302.314 | params rms: 0.061198 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 9:40:00.927921 | finish at 2026-10-10 19:05:12 + [2026-10-09 09:26:39] iteration 5477/ 6855 | consumed samples: 701056 | elapsed time per iteration (ms): 88106.1 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.47% | FLOPs estimator: workload-aware | logical tokens/s: 190419.8 | valid tokens/s: 190419.8 | physical tokens/s: 190931.5 | loss tokens/s: 142452.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.783163E-01 | lm-global-supervised-tokens: 1.255093E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.617116E-02 | lm-token-buffer-mib: 5.821751E+01 | global_load_balancing_loss: 1.000485E+00 | loss scale: 1.0 | grad norm: 0.053 | params norm: 11302.263 | params rms: 0.061198 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 9:43:30.250595 | finish at 2026-10-10 19:10:10 + [2026-10-09 09:28:08] iteration 5478/ 6855 | consumed samples: 701184 | elapsed time per iteration (ms): 88490.8 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.56% | FLOPs estimator: workload-aware | logical tokens/s: 189592.1 | valid tokens/s: 189592.1 | physical tokens/s: 189907.9 | loss tokens/s: 130662.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.433761E-01 | lm-global-supervised-tokens: 1.156240E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.312613E-02 | lm-token-buffer-mib: 5.814439E+01 | global_load_balancing_loss: 1.001091E+00 | loss scale: 1.0 | grad norm: 0.043 | params norm: 11302.208 | params rms: 0.061198 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 9:50:51.855762 | finish at 2026-10-10 19:19:00 + [2026-10-09 09:29:37] iteration 5479/ 6855 | consumed samples: 701312 | elapsed time per iteration (ms): 89330.9 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.47% | FLOPs estimator: workload-aware | logical tokens/s: 187809.2 | valid tokens/s: 187809.1 | physical tokens/s: 188332.1 | loss tokens/s: 136280.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.760916E-01 | lm-global-supervised-tokens: 1.217403E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.365602E-02 | lm-token-buffer-mib: 5.826214E+01 | global_load_balancing_loss: 9.993874E-01 | loss scale: 1.0 | grad norm: 0.072 | params norm: 11302.154 | params rms: 0.061197 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 10:08:39.311775 | finish at 2026-10-10 19:38:17 + [2026-10-09 09:31:29] iteration 5480/ 6855 | consumed samples: 701440 | elapsed time per iteration (ms): 111627.7 | throughput per GPU (TFLOP/s/GPU): 33.2 | MFU 10.64% | FLOPs estimator: workload-aware | logical tokens/s: 150295.8 | valid tokens/s: 150295.8 | physical tokens/s: 150642.3 | loss tokens/s: 111197.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.679862E-01 | lm-global-supervised-tokens: 1.241273E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.128433E-02 | lm-token-buffer-mib: 5.815971E+01 | global_load_balancing_loss: 9.984914E-01 | loss scale: 1.0 | grad norm: 0.061 | params norm: 11302.101 | params rms: 0.061197 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 18:38:08.142043 | finish at 2026-10-11 04:09:37 +times across ranks (ms): + save-checkpoint: + rank 0: 23855.35 + rank 1: 23855.39 + rank 2: 23855.37 + rank 3: 23855.38 + rank 4: 23855.36 + rank 5: 23855.35 + rank 6: 23855.35 + rank 7: 23855.39 + rank 8: 23855.25 + rank 9: 23855.20 + rank 10: 23855.23 + rank 11: 23855.24 + rank 12: 23855.21 + rank 13: 23855.20 + rank 14: 23855.26 + rank 15: 23856.42 + rank 16: 23854.64 + rank 17: 23854.63 + rank 18: 23854.59 + rank 19: 23854.59 + rank 20: 23854.60 + rank 21: 23854.64 + rank 22: 23854.59 + rank 23: 23854.57 + rank 24: 23855.18 + rank 25: 23855.13 + rank 26: 23855.19 + rank 27: 23855.19 + rank 28: 23855.14 + rank 29: 23855.16 + rank 30: 23855.14 + rank 31: 23855.11 + rank 32: 23855.34 + rank 33: 23855.34 + rank 34: 23855.31 + rank 35: 23855.31 + rank 36: 23855.29 + rank 37: 23855.27 + rank 38: 23855.30 + rank 39: 23855.28 + rank 40: 23855.11 + rank 41: 23855.11 + rank 42: 23855.08 + rank 43: 23855.15 + rank 44: 23855.11 + rank 45: 23855.12 + rank 46: 23855.13 + rank 47: 23855.12 + rank 48: 23855.58 + rank 49: 23855.58 + rank 50: 23855.60 + rank 51: 23855.56 + rank 52: 23855.57 + rank 53: 23855.58 + rank 54: 23855.58 + rank 55: 23855.56 + rank 56: 23855.53 + rank 57: 23855.47 + rank 58: 23855.44 + rank 59: 23855.45 + rank 60: 23855.45 + rank 61: 23855.44 + rank 62: 23855.46 + rank 63: 23855.47 + [2026-10-09 09:33:40] iteration 5481/ 6855 | consumed samples: 701568 | elapsed time per iteration (ms): 106635.6 | throughput per GPU (TFLOP/s/GPU): 34.8 | MFU 11.14% | FLOPs estimator: workload-aware | logical tokens/s: 157331.9 | valid tokens/s: 157331.8 | physical tokens/s: 157619.3 | loss tokens/s: 114083.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.848288E-01 | lm-global-supervised-tokens: 1.216540E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.379182E+00 | lm-token-buffer-mib: 5.808784E+01 | global_load_balancing_loss: 9.994870E-01 | loss scale: 1.0 | grad norm: 0.051 | params norm: 11302.047 | params rms: 0.061197 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 16:41:57.246058 | finish at 2026-10-11 02:15:37 + [2026-10-09 09:35:14] iteration 5482/ 6855 | consumed samples: 701696 | elapsed time per iteration (ms): 94138.3 | throughput per GPU (TFLOP/s/GPU): 40.0 | MFU 12.81% | FLOPs estimator: workload-aware | logical tokens/s: 178218.3 | valid tokens/s: 178218.3 | physical tokens/s: 178679.5 | loss tokens/s: 131657.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.857325E-01 | lm-global-supervised-tokens: 1.239399E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.683586E-02 | lm-token-buffer-mib: 5.809015E+01 | global_load_balancing_loss: 1.000719E+00 | loss scale: 1.0 | grad norm: 0.096 | params norm: 11301.995 | params rms: 0.061197 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 11:54:11.948671 | finish at 2026-10-10 21:29:26 + [2026-10-09 09:36:38] iteration 5483/ 6855 | consumed samples: 701824 | elapsed time per iteration (ms): 84487.9 | throughput per GPU (TFLOP/s/GPU): 43.1 | MFU 13.82% | FLOPs estimator: workload-aware | logical tokens/s: 198575.0 | valid tokens/s: 198575.0 | physical tokens/s: 199053.2 | loss tokens/s: 151305.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.813909E-01 | lm-global-supervised-tokens: 1.278350E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.136911E-02 | lm-token-buffer-mib: 5.813636E+01 | global_load_balancing_loss: 1.001166E+00 | loss scale: 1.0 | grad norm: 0.078 | params norm: 11301.945 | params rms: 0.061196 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 8:11:57.376255 | finish at 2026-10-10 17:48:36 + [2026-10-09 09:38:03] iteration 5484/ 6855 | consumed samples: 701952 | elapsed time per iteration (ms): 85012.3 | throughput per GPU (TFLOP/s/GPU): 43.5 | MFU 13.94% | FLOPs estimator: workload-aware | logical tokens/s: 197349.5 | valid tokens/s: 197349.5 | physical tokens/s: 197986.8 | loss tokens/s: 143955.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.597294E-01 | lm-global-supervised-tokens: 1.223800E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.567072E-02 | lm-token-buffer-mib: 5.826766E+01 | global_load_balancing_loss: 1.001371E+00 | loss scale: 1.0 | grad norm: 0.064 | params norm: 11301.896 | params rms: 0.061196 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 8:22:31.873126 | finish at 2026-10-10 18:00:35 + [2026-10-09 09:39:33] iteration 5485/ 6855 | consumed samples: 702080 | elapsed time per iteration (ms): 89449.8 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.45% | FLOPs estimator: workload-aware | logical tokens/s: 187559.1 | valid tokens/s: 187559.1 | physical tokens/s: 187953.9 | loss tokens/s: 137206.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.457692E-01 | lm-global-supervised-tokens: 1.227306E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.362203E-02 | lm-token-buffer-mib: 5.838728E+01 | global_load_balancing_loss: 1.000313E+00 | loss scale: 1.0 | grad norm: 0.074 | params norm: 11301.846 | params rms: 0.061196 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 10:02:26.201196 | finish at 2026-10-10 19:41:59 + [2026-10-09 09:41:01] iteration 5486/ 6855 | consumed samples: 702208 | elapsed time per iteration (ms): 88808.6 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.64% | FLOPs estimator: workload-aware | logical tokens/s: 188913.8 | valid tokens/s: 188913.8 | physical tokens/s: 189222.8 | loss tokens/s: 130440.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.850017E-01 | lm-global-supervised-tokens: 1.158427E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.171235E-02 | lm-token-buffer-mib: 5.817841E+01 | global_load_balancing_loss: 9.998646E-01 | loss scale: 1.0 | grad norm: 0.061 | params norm: 11301.798 | params rms: 0.061196 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 9:46:19.035998 | finish at 2026-10-10 19:27:20 + [2026-10-09 09:42:29] iteration 5487/ 6855 | consumed samples: 702336 | elapsed time per iteration (ms): 87860.3 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.38% | FLOPs estimator: workload-aware | logical tokens/s: 190952.4 | valid tokens/s: 190952.3 | physical tokens/s: 191537.4 | loss tokens/s: 147696.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.908823E-01 | lm-global-supervised-tokens: 1.297668E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.179812E-02 | lm-token-buffer-mib: 5.806839E+01 | global_load_balancing_loss: 1.000152E+00 | loss scale: 1.0 | grad norm: 0.083 | params norm: 11301.750 | params rms: 0.061195 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 9:23:12.872875 | finish at 2026-10-10 19:05:42 + [2026-10-09 09:43:58] iteration 5488/ 6855 | consumed samples: 702464 | elapsed time per iteration (ms): 88928.9 | throughput per GPU (TFLOP/s/GPU): 41.3 | MFU 13.25% | FLOPs estimator: workload-aware | logical tokens/s: 188658.4 | valid tokens/s: 188658.4 | physical tokens/s: 189025.7 | loss tokens/s: 143037.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.226380E-01 | lm-global-supervised-tokens: 1.272012E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.841314E-02 | lm-token-buffer-mib: 5.813243E+01 | global_load_balancing_loss: 9.989741E-01 | loss scale: 1.0 | grad norm: 0.055 | params norm: 11301.701 | params rms: 0.061195 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 9:46:05.768824 | finish at 2026-10-10 19:30:04 + [2026-10-09 09:45:28] iteration 5489/ 6855 | consumed samples: 702592 | elapsed time per iteration (ms): 89778.0 | throughput per GPU (TFLOP/s/GPU): 41.3 | MFU 13.22% | FLOPs estimator: workload-aware | logical tokens/s: 186873.4 | valid tokens/s: 186873.4 | physical tokens/s: 187353.8 | loss tokens/s: 129349.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.297156E-01 | lm-global-supervised-tokens: 1.161273E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.325167E-02 | lm-token-buffer-mib: 5.813065E+01 | global_load_balancing_loss: 1.000798E+00 | loss scale: 1.0 | grad norm: 0.047 | params norm: 11301.652 | params rms: 0.061195 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 10:03:56.776819 | finish at 2026-10-10 19:49:25 + [2026-10-09 09:47:18] iteration 5490/ 6855 | consumed samples: 702720 | elapsed time per iteration (ms): 109635.5 | throughput per GPU (TFLOP/s/GPU): 33.0 | MFU 10.58% | FLOPs estimator: workload-aware | logical tokens/s: 153027.2 | valid tokens/s: 153027.2 | physical tokens/s: 153396.8 | loss tokens/s: 115155.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.277909E-01 | lm-global-supervised-tokens: 1.262516E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.230473E-02 | lm-token-buffer-mib: 5.829766E+01 | global_load_balancing_loss: 1.000739E+00 | loss scale: 1.0 | grad norm: 0.038 | params norm: 11301.605 | params rms: 0.061195 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 17:34:12.406994 | finish at 2026-10-11 03:21:30 + [2026-10-09 09:48:46] iteration 5491/ 6855 | consumed samples: 702848 | elapsed time per iteration (ms): 88329.7 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.44% | FLOPs estimator: workload-aware | logical tokens/s: 189937.7 | valid tokens/s: 189937.7 | physical tokens/s: 190444.5 | loss tokens/s: 140110.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.341090E-01 | lm-global-supervised-tokens: 1.237589E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.337026E-02 | lm-token-buffer-mib: 5.815658E+01 | global_load_balancing_loss: 1.001035E+00 | loss scale: 1.0 | grad norm: 0.064 | params norm: 11301.558 | params rms: 0.061194 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 9:28:01.726400 | finish at 2026-10-10 19:16:48 + [2026-10-09 09:50:10] iteration 5492/ 6855 | consumed samples: 702976 | elapsed time per iteration (ms): 84278.1 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.67% | FLOPs estimator: workload-aware | logical tokens/s: 199068.8 | valid tokens/s: 199068.8 | physical tokens/s: 199737.9 | loss tokens/s: 152357.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.641938E-01 | lm-global-supervised-tokens: 1.284039E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.606455E-02 | lm-token-buffer-mib: 5.816446E+01 | global_load_balancing_loss: 9.992692E-01 | loss scale: 1.0 | grad norm: 0.182 | params norm: 11301.510 | params rms: 0.061194 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 7:54:31.109137 | finish at 2026-10-10 17:44:41 + [2026-10-09 09:51:38] iteration 5493/ 6855 | consumed samples: 703104 | elapsed time per iteration (ms): 87974.2 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.64% | FLOPs estimator: workload-aware | logical tokens/s: 190704.8 | valid tokens/s: 190704.8 | physical tokens/s: 191356.8 | loss tokens/s: 140226.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.481264E-01 | lm-global-supervised-tokens: 1.233630E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.417209E-02 | lm-token-buffer-mib: 5.818428E+01 | global_load_balancing_loss: 9.993339E-01 | loss scale: 1.0 | grad norm: 0.349 | params norm: 11301.460 | params rms: 0.061194 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 9:17:00.891912 | finish at 2026-10-10 19:08:39 + [2026-10-09 09:53:07] iteration 5494/ 6855 | consumed samples: 703232 | elapsed time per iteration (ms): 89164.8 | throughput per GPU (TFLOP/s/GPU): 41.4 | MFU 13.26% | FLOPs estimator: workload-aware | logical tokens/s: 188159.5 | valid tokens/s: 188159.5 | physical tokens/s: 188517.5 | loss tokens/s: 141922.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.081450E-01 | lm-global-supervised-tokens: 1.265453E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.592250E-02 | lm-token-buffer-mib: 5.810079E+01 | global_load_balancing_loss: 9.995754E-01 | loss scale: 1.0 | grad norm: 0.101 | params norm: 11301.410 | params rms: 0.061193 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 9:42:33.356302 | finish at 2026-10-10 19:35:41 + [2026-10-09 09:54:34] iteration 5495/ 6855 | consumed samples: 703360 | elapsed time per iteration (ms): 86533.0 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.66% | FLOPs estimator: workload-aware | logical tokens/s: 193881.8 | valid tokens/s: 193881.8 | physical tokens/s: 194271.7 | loss tokens/s: 146992.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.282722E-01 | lm-global-supervised-tokens: 1.271970E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.124075E-02 | lm-token-buffer-mib: 5.806643E+01 | global_load_balancing_loss: 1.000670E+00 | loss scale: 1.0 | grad norm: 0.482 | params norm: 11301.361 | params rms: 0.061193 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 8:41:24.880314 | finish at 2026-10-10 18:35:59 + [2026-10-09 09:56:04] iteration 5496/ 6855 | consumed samples: 703488 | elapsed time per iteration (ms): 89706.5 | throughput per GPU (TFLOP/s/GPU): 41.0 | MFU 13.14% | FLOPs estimator: workload-aware | logical tokens/s: 187023.0 | valid tokens/s: 187023.0 | physical tokens/s: 187423.3 | loss tokens/s: 138521.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.386727E-01 | lm-global-supervised-tokens: 1.242623E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.215180E-02 | lm-token-buffer-mib: 5.811885E+01 | global_load_balancing_loss: 9.999209E-01 | loss scale: 1.0 | grad norm: 0.178 | params norm: 11301.313 | params rms: 0.061193 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 9:51:51.106032 | finish at 2026-10-10 19:47:55 + [2026-10-09 09:57:30] iteration 5497/ 6855 | consumed samples: 703616 | elapsed time per iteration (ms): 86461.1 | throughput per GPU (TFLOP/s/GPU): 43.3 | MFU 13.87% | FLOPs estimator: workload-aware | logical tokens/s: 194043.0 | valid tokens/s: 194043.0 | physical tokens/s: 194567.3 | loss tokens/s: 140932.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.748797E-01 | lm-global-supervised-tokens: 1.218519E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.222974E-02 | lm-token-buffer-mib: 5.822807E+01 | global_load_balancing_loss: 1.000775E+00 | loss scale: 1.0 | grad norm: 0.548 | params norm: 11301.264 | params rms: 0.061193 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 8:36:54.148684 | finish at 2026-10-10 18:34:24 + [2026-10-09 09:59:02] iteration 5498/ 6855 | consumed samples: 703744 | elapsed time per iteration (ms): 91929.0 | throughput per GPU (TFLOP/s/GPU): 40.6 | MFU 13.01% | FLOPs estimator: workload-aware | logical tokens/s: 182501.0 | valid tokens/s: 182501.0 | physical tokens/s: 183114.2 | loss tokens/s: 132654.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.630726E-01 | lm-global-supervised-tokens: 1.219478E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.156264E-02 | lm-token-buffer-mib: 5.811488E+01 | global_load_balancing_loss: 1.001093E+00 | loss scale: 1.0 | grad norm: 0.043 | params norm: 11301.213 | params rms: 0.061192 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 10:39:07.632483 | finish at 2026-10-10 20:38:10 + [2026-10-09 10:00:29] iteration 5499/ 6855 | consumed samples: 703872 | elapsed time per iteration (ms): 86912.2 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.77% | FLOPs estimator: workload-aware | logical tokens/s: 193035.6 | valid tokens/s: 193035.6 | physical tokens/s: 193422.6 | loss tokens/s: 146224.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.696641E-01 | lm-global-supervised-tokens: 1.270871E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.370821E-02 | lm-token-buffer-mib: 5.815100E+01 | global_load_balancing_loss: 1.002016E+00 | loss scale: 1.0 | grad norm: 0.245 | params norm: 11301.161 | params rms: 0.061192 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 8:44:12.952864 | finish at 2026-10-10 18:44:42 + [2026-10-09 10:02:17] iteration 5500/ 6855 | consumed samples: 704000 | elapsed time per iteration (ms): 108128.2 | throughput per GPU (TFLOP/s/GPU): 34.2 | MFU 10.97% | FLOPs estimator: workload-aware | logical tokens/s: 155160.4 | valid tokens/s: 155160.4 | physical tokens/s: 155536.0 | loss tokens/s: 112712.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.169091E-01 | lm-global-supervised-tokens: 1.218741E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.241519E-02 | lm-token-buffer-mib: 5.809692E+01 | global_load_balancing_loss: 9.991344E-01 | loss scale: 1.0 | grad norm: 0.273 | params norm: 11301.110 | params rms: 0.061192 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 16:41:53.658738 | finish at 2026-10-11 02:44:11 + [2026-10-09 10:03:42] iteration 5501/ 6855 | consumed samples: 704128 | elapsed time per iteration (ms): 85031.8 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.78% | FLOPs estimator: workload-aware | logical tokens/s: 197304.5 | valid tokens/s: 197304.5 | physical tokens/s: 197711.1 | loss tokens/s: 152005.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.995619E-01 | lm-global-supervised-tokens: 1.292530E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.130788E-02 | lm-token-buffer-mib: 5.804137E+01 | global_load_balancing_loss: 9.996623E-01 | loss scale: 1.0 | grad norm: 0.098 | params norm: 11301.058 | params rms: 0.061192 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 7:58:53.041748 | finish at 2026-10-10 18:02:35 + [2026-10-09 10:05:07] iteration 5502/ 6855 | consumed samples: 704256 | elapsed time per iteration (ms): 84906.2 | throughput per GPU (TFLOP/s/GPU): 43.2 | MFU 13.86% | FLOPs estimator: workload-aware | logical tokens/s: 197596.3 | valid tokens/s: 197596.3 | physical tokens/s: 198149.0 | loss tokens/s: 144147.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.624597E-01 | lm-global-supervised-tokens: 1.223901E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.011025E-02 | lm-token-buffer-mib: 5.830713E+01 | global_load_balancing_loss: 9.997861E-01 | loss scale: 1.0 | grad norm: 0.388 | params norm: 11301.007 | params rms: 0.061191 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 7:54:38.104315 | finish at 2026-10-10 17:59:45 + [2026-10-09 10:06:37] iteration 5503/ 6855 | consumed samples: 704384 | elapsed time per iteration (ms): 89585.9 | throughput per GPU (TFLOP/s/GPU): 41.5 | MFU 13.31% | FLOPs estimator: workload-aware | logical tokens/s: 187274.9 | valid tokens/s: 187274.9 | physical tokens/s: 187537.7 | loss tokens/s: 135946.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.436203E-01 | lm-global-supervised-tokens: 1.217888E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.411253E-02 | lm-token-buffer-mib: 5.817764E+01 | global_load_balancing_loss: 9.996522E-01 | loss scale: 1.0 | grad norm: 0.169 | params norm: 11300.953 | params rms: 0.061191 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 9:38:40.159134 | finish at 2026-10-10 19:45:17 + [2026-10-09 10:08:02] iteration 5504/ 6855 | consumed samples: 704512 | elapsed time per iteration (ms): 85178.7 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.74% | FLOPs estimator: workload-aware | logical tokens/s: 196964.1 | valid tokens/s: 196964.1 | physical tokens/s: 197524.5 | loss tokens/s: 149476.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.615630E-01 | lm-global-supervised-tokens: 1.273220E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.170739E-02 | lm-token-buffer-mib: 5.823860E+01 | global_load_balancing_loss: 9.992989E-01 | loss scale: 1.0 | grad norm: 0.297 | params norm: 11300.902 | params rms: 0.061191 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 7:57:56.399502 | finish at 2026-10-10 18:05:58 + [2026-10-09 10:09:36] iteration 5505/ 6855 | consumed samples: 704640 | elapsed time per iteration (ms): 94350.8 | throughput per GPU (TFLOP/s/GPU): 41.6 | MFU 13.34% | FLOPs estimator: workload-aware | logical tokens/s: 177817.0 | valid tokens/s: 177817.0 | physical tokens/s: 178091.4 | loss tokens/s: 121758.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.949779E-01 | lm-global-supervised-tokens: 1.148804E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.161140E-02 | lm-token-buffer-mib: 5.810237E+01 | global_load_balancing_loss: 1.000603E+00 | loss scale: 1.0 | grad norm: 0.182 | params norm: 11300.851 | params rms: 0.061190 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 11:22:53.591959 | finish at 2026-10-10 21:32:30 + [2026-10-09 10:11:07] iteration 5506/ 6855 | consumed samples: 704768 | elapsed time per iteration (ms): 90413.1 | throughput per GPU (TFLOP/s/GPU): 41.5 | MFU 13.30% | FLOPs estimator: workload-aware | logical tokens/s: 185561.8 | valid tokens/s: 185561.7 | physical tokens/s: 185928.8 | loss tokens/s: 138567.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.420664E-01 | lm-global-supervised-tokens: 1.252829E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.103831E-02 | lm-token-buffer-mib: 5.828732E+01 | global_load_balancing_loss: 1.000999E+00 | loss scale: 1.0 | grad norm: 0.061 | params norm: 11300.798 | params rms: 0.061190 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 9:52:47.259684 | finish at 2026-10-10 20:03:54 + [2026-10-09 10:12:35] iteration 5507/ 6855 | consumed samples: 704896 | elapsed time per iteration (ms): 88674.7 | throughput per GPU (TFLOP/s/GPU): 43.1 | MFU 13.82% | FLOPs estimator: workload-aware | logical tokens/s: 189199.1 | valid tokens/s: 189199.1 | physical tokens/s: 189567.6 | loss tokens/s: 132468.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.777034E-01 | lm-global-supervised-tokens: 1.174659E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.223992E-02 | lm-token-buffer-mib: 5.808938E+01 | global_load_balancing_loss: 9.999993E-01 | loss scale: 1.0 | grad norm: 0.145 | params norm: 11300.745 | params rms: 0.061190 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 9:12:13.430709 | finish at 2026-10-10 19:24:49 + [2026-10-09 10:14:04] iteration 5508/ 6855 | consumed samples: 705024 | elapsed time per iteration (ms): 88692.3 | throughput per GPU (TFLOP/s/GPU): 41.6 | MFU 13.32% | FLOPs estimator: workload-aware | logical tokens/s: 189161.0 | valid tokens/s: 189161.0 | physical tokens/s: 189530.6 | loss tokens/s: 143143.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.549116E-01 | lm-global-supervised-tokens: 1.269574E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.556691E-02 | lm-token-buffer-mib: 5.802122E+01 | global_load_balancing_loss: 1.000239E+00 | loss scale: 1.0 | grad norm: 0.130 | params norm: 11300.691 | params rms: 0.061190 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 9:11:08.536559 | finish at 2026-10-10 19:25:12 + [2026-10-09 10:15:32] iteration 5509/ 6855 | consumed samples: 705152 | elapsed time per iteration (ms): 88098.9 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.63% | FLOPs estimator: workload-aware | logical tokens/s: 190435.6 | valid tokens/s: 190435.6 | physical tokens/s: 190760.4 | loss tokens/s: 133145.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.468430E-01 | lm-global-supervised-tokens: 1.172999E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.474983E-02 | lm-token-buffer-mib: 5.808787E+01 | global_load_balancing_loss: 9.998987E-01 | loss scale: 1.0 | grad norm: 0.126 | params norm: 11300.638 | params rms: 0.061189 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 8:56:21.110522 | finish at 2026-10-10 19:11:53 + [2026-10-09 10:17:22] iteration 5510/ 6855 | consumed samples: 705280 | elapsed time per iteration (ms): 110398.4 | throughput per GPU (TFLOP/s/GPU): 34.0 | MFU 10.88% | FLOPs estimator: workload-aware | logical tokens/s: 151969.5 | valid tokens/s: 151969.4 | physical tokens/s: 152326.6 | loss tokens/s: 110249.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.821753E-01 | lm-global-supervised-tokens: 1.217139E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.214467E-02 | lm-token-buffer-mib: 5.813608E+01 | global_load_balancing_loss: 1.000605E+00 | loss scale: 1.0 | grad norm: 0.085 | params norm: 11300.586 | params rms: 0.061189 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 17:14:45.844885 | finish at 2026-10-11 03:32:08 + [2026-10-09 10:18:48] iteration 5511/ 6855 | consumed samples: 705408 | elapsed time per iteration (ms): 85953.7 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.51% | FLOPs estimator: workload-aware | logical tokens/s: 195188.5 | valid tokens/s: 195188.5 | physical tokens/s: 195654.8 | loss tokens/s: 149631.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.671109E-01 | lm-global-supervised-tokens: 1.286137E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.570203E-02 | lm-token-buffer-mib: 5.826739E+01 | global_load_balancing_loss: 1.000881E+00 | loss scale: 1.0 | grad norm: 0.084 | params norm: 11300.533 | params rms: 0.061189 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 8:05:21.818069 | finish at 2026-10-10 18:24:10 + [2026-10-09 10:20:15] iteration 5512/ 6855 | consumed samples: 705536 | elapsed time per iteration (ms): 86793.5 | throughput per GPU (TFLOP/s/GPU): 43.3 | MFU 13.87% | FLOPs estimator: workload-aware | logical tokens/s: 193300.3 | valid tokens/s: 193300.3 | physical tokens/s: 193722.9 | loss tokens/s: 142816.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.660680E-01 | lm-global-supervised-tokens: 1.239550E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.515196E-02 | lm-token-buffer-mib: 5.819725E+01 | global_load_balancing_loss: 9.999308E-01 | loss scale: 1.0 | grad norm: 0.113 | params norm: 11300.479 | params rms: 0.061188 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 8:22:43.644492 | finish at 2026-10-10 18:42:59 + [2026-10-09 10:21:41] iteration 5513/ 6855 | consumed samples: 705664 | elapsed time per iteration (ms): 85530.9 | throughput per GPU (TFLOP/s/GPU): 43.1 | MFU 13.80% | FLOPs estimator: workload-aware | logical tokens/s: 196153.4 | valid tokens/s: 196153.4 | physical tokens/s: 196726.8 | loss tokens/s: 147314.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.179270E-01 | lm-global-supervised-tokens: 1.259997E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.142349E-02 | lm-token-buffer-mib: 5.814419E+01 | global_load_balancing_loss: 1.000866E+00 | loss scale: 1.0 | grad norm: 0.066 | params norm: 11300.424 | params rms: 0.061188 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 7:53:02.495638 | finish at 2026-10-10 18:14:43 + [2026-10-09 10:23:09] iteration 5514/ 6855 | consumed samples: 705792 | elapsed time per iteration (ms): 88400.8 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.44% | FLOPs estimator: workload-aware | logical tokens/s: 189785.2 | valid tokens/s: 189785.2 | physical tokens/s: 190298.9 | loss tokens/s: 143180.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.799928E-01 | lm-global-supervised-tokens: 1.265724E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.307916E-02 | lm-token-buffer-mib: 5.819949E+01 | global_load_balancing_loss: 1.000742E+00 | loss scale: 1.0 | grad norm: 0.067 | params norm: 11300.370 | params rms: 0.061188 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 8:55:45.450406 | finish at 2026-10-10 19:18:55 + [2026-10-09 10:24:35] iteration 5515/ 6855 | consumed samples: 705920 | elapsed time per iteration (ms): 85406.0 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.76% | FLOPs estimator: workload-aware | logical tokens/s: 196440.1 | valid tokens/s: 196440.1 | physical tokens/s: 196762.8 | loss tokens/s: 152413.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.821172E-01 | lm-global-supervised-tokens: 1.301702E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.263408E-02 | lm-token-buffer-mib: 5.805527E+01 | global_load_balancing_loss: 1.000374E+00 | loss scale: 1.0 | grad norm: 0.096 | params norm: 11300.318 | params rms: 0.061188 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 7:47:24.095454 | finish at 2026-10-10 18:11:59 + [2026-10-09 10:26:04] iteration 5516/ 6855 | consumed samples: 706048 | elapsed time per iteration (ms): 89827.5 | throughput per GPU (TFLOP/s/GPU): 41.7 | MFU 13.38% | FLOPs estimator: workload-aware | logical tokens/s: 186770.9 | valid tokens/s: 186770.9 | physical tokens/s: 187230.4 | loss tokens/s: 135993.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.726139E-01 | lm-global-supervised-tokens: 1.221597E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.343927E-02 | lm-token-buffer-mib: 5.811005E+01 | global_load_balancing_loss: 1.000609E+00 | loss scale: 1.0 | grad norm: 0.076 | params norm: 11300.267 | params rms: 0.061187 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 9:24:38.957834 | finish at 2026-10-10 19:50:43 + [2026-10-09 10:27:31] iteration 5517/ 6855 | consumed samples: 706176 | elapsed time per iteration (ms): 86344.6 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.77% | FLOPs estimator: workload-aware | logical tokens/s: 194305.4 | valid tokens/s: 194305.4 | physical tokens/s: 194683.9 | loss tokens/s: 144303.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.132659E-01 | lm-global-supervised-tokens: 1.245985E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.613090E-02 | lm-token-buffer-mib: 5.820248E+01 | global_load_balancing_loss: 1.000691E+00 | loss scale: 1.0 | grad norm: 0.037 | params norm: 11300.214 | params rms: 0.061187 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 8:05:29.056566 | finish at 2026-10-10 18:33:00 + [2026-10-09 10:28:59] iteration 5518/ 6855 | consumed samples: 706304 | elapsed time per iteration (ms): 88036.1 | throughput per GPU (TFLOP/s/GPU): 41.4 | MFU 13.28% | FLOPs estimator: workload-aware | logical tokens/s: 190571.2 | valid tokens/s: 190571.2 | physical tokens/s: 190970.5 | loss tokens/s: 142720.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.309502E-01 | lm-global-supervised-tokens: 1.256458E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.478719E-02 | lm-token-buffer-mib: 5.820877E+01 | global_load_balancing_loss: 1.000021E+00 | loss scale: 1.0 | grad norm: 0.095 | params norm: 11300.162 | params rms: 0.061187 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 8:41:44.301601 | finish at 2026-10-10 19:10:43 + [2026-10-09 10:30:27] iteration 5519/ 6855 | consumed samples: 706432 | elapsed time per iteration (ms): 88161.8 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.38% | FLOPs estimator: workload-aware | logical tokens/s: 190299.6 | valid tokens/s: 190299.6 | physical tokens/s: 190680.6 | loss tokens/s: 143256.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.627231E-01 | lm-global-supervised-tokens: 1.262979E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.516285E-02 | lm-token-buffer-mib: 5.814779E+01 | global_load_balancing_loss: 1.000366E+00 | loss scale: 1.0 | grad norm: 0.092 | params norm: 11300.112 | params rms: 0.061186 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 8:43:04.226471 | finish at 2026-10-10 19:13:31 + [2026-10-09 10:32:16] iteration 5520/ 6855 | consumed samples: 706560 | elapsed time per iteration (ms): 108794.6 | throughput per GPU (TFLOP/s/GPU): 33.5 | MFU 10.74% | FLOPs estimator: workload-aware | logical tokens/s: 154209.7 | valid tokens/s: 154209.6 | physical tokens/s: 154592.5 | loss tokens/s: 118768.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.342062E-01 | lm-global-supervised-tokens: 1.292141E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.141578E-02 | lm-token-buffer-mib: 5.810717E+01 | global_load_balancing_loss: 9.994346E-01 | loss scale: 1.0 | grad norm: 0.046 | params norm: 11300.062 | params rms: 0.061186 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 16:20:40.849578 | finish at 2026-10-11 02:52:57 +times across ranks (ms): + save-checkpoint: + rank 0: 25113.88 + rank 1: 25113.90 + rank 2: 25113.89 + rank 3: 25113.88 + rank 4: 25113.84 + rank 5: 25113.70 + rank 6: 25113.81 + rank 7: 25113.85 + rank 8: 25114.08 + rank 9: 25114.02 + rank 10: 25114.02 + rank 11: 25114.04 + rank 12: 25114.03 + rank 13: 25114.02 + rank 14: 25114.04 + rank 15: 25114.04 + rank 16: 25115.70 + rank 17: 25115.70 + rank 18: 25115.68 + rank 19: 25115.70 + rank 20: 25115.47 + rank 21: 25115.70 + rank 22: 25115.66 + rank 23: 25115.61 + rank 24: 25113.86 + rank 25: 25114.08 + rank 26: 25113.84 + rank 27: 25113.83 + rank 28: 25113.73 + rank 29: 25113.81 + rank 30: 25113.80 + rank 31: 25113.83 + rank 32: 25113.96 + rank 33: 25113.96 + rank 34: 25113.93 + rank 35: 25113.90 + rank 36: 25113.87 + rank 37: 25113.82 + rank 38: 25113.88 + rank 39: 25113.86 + rank 40: 25115.00 + rank 41: 25114.96 + rank 42: 25114.98 + rank 43: 25114.99 + rank 44: 25115.02 + rank 45: 25114.96 + rank 46: 25114.99 + rank 47: 25114.94 + rank 48: 25114.16 + rank 49: 25114.20 + rank 50: 25114.12 + rank 51: 25114.01 + rank 52: 25114.17 + rank 53: 25114.14 + rank 54: 25113.96 + rank 55: 25114.14 + rank 56: 25113.91 + rank 57: 25113.94 + rank 58: 25113.86 + rank 59: 25113.86 + rank 60: 25113.86 + rank 61: 25113.88 + rank 62: 25113.86 + rank 63: 25113.87 + [2026-10-09 10:34:35] iteration 5521/ 6855 | consumed samples: 706688 | elapsed time per iteration (ms): 112892.5 | throughput per GPU (TFLOP/s/GPU): 33.2 | MFU 10.63% | FLOPs estimator: workload-aware | logical tokens/s: 148611.6 | valid tokens/s: 148611.6 | physical tokens/s: 148916.0 | loss tokens/s: 110394.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.869302E-01 | lm-global-supervised-tokens: 1.246268E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 7.970433E+00 | lm-token-buffer-mib: 5.812029E+01 | global_load_balancing_loss: 1.000166E+00 | loss scale: 1.0 | grad norm: 0.091 | params norm: 11300.011 | params rms: 0.061186 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 17:49:58.604122 | finish at 2026-10-11 04:24:34 + [2026-10-09 10:36:04] iteration 5522/ 6855 | consumed samples: 706816 | elapsed time per iteration (ms): 89204.8 | throughput per GPU (TFLOP/s/GPU): 40.0 | MFU 12.82% | FLOPs estimator: workload-aware | logical tokens/s: 188074.8 | valid tokens/s: 188074.8 | physical tokens/s: 188625.6 | loss tokens/s: 140242.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.660901E-01 | lm-global-supervised-tokens: 1.251033E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.315849E-02 | lm-token-buffer-mib: 5.817531E+01 | global_load_balancing_loss: 9.990542E-01 | loss scale: 1.0 | grad norm: 0.085 | params norm: 11299.958 | params rms: 0.061186 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 9:01:49.973783 | finish at 2026-10-10 19:37:54 + [2026-10-09 10:37:30] iteration 5523/ 6855 | consumed samples: 706944 | elapsed time per iteration (ms): 86093.4 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.69% | FLOPs estimator: workload-aware | logical tokens/s: 194871.7 | valid tokens/s: 194871.7 | physical tokens/s: 195307.2 | loss tokens/s: 144487.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.816274E-01 | lm-global-supervised-tokens: 1.243942E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.419393E-02 | lm-token-buffer-mib: 5.815427E+01 | global_load_balancing_loss: 9.999504E-01 | loss scale: 1.0 | grad norm: 0.090 | params norm: 11299.908 | params rms: 0.061185 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 7:51:16.471999 | finish at 2026-10-10 18:28:47 + [2026-10-09 10:39:00] iteration 5524/ 6855 | consumed samples: 707072 | elapsed time per iteration (ms): 89808.5 | throughput per GPU (TFLOP/s/GPU): 41.7 | MFU 13.36% | FLOPs estimator: workload-aware | logical tokens/s: 186811.1 | valid tokens/s: 186811.1 | physical tokens/s: 187343.2 | loss tokens/s: 136694.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.694152E-01 | lm-global-supervised-tokens: 1.227635E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.491221E-02 | lm-token-buffer-mib: 5.833389E+01 | global_load_balancing_loss: 1.000720E+00 | loss scale: 1.0 | grad norm: 0.062 | params norm: 11299.857 | params rms: 0.061185 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 9:12:15.056448 | finish at 2026-10-10 19:51:15 + [2026-10-09 10:40:28] iteration 5525/ 6855 | consumed samples: 707200 | elapsed time per iteration (ms): 88190.0 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.61% | FLOPs estimator: workload-aware | logical tokens/s: 190239.1 | valid tokens/s: 190239.1 | physical tokens/s: 190584.1 | loss tokens/s: 138909.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 5.828279E-01 | lm-global-supervised-tokens: 1.225045E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.593767E-02 | lm-token-buffer-mib: 5.808204E+01 | global_load_balancing_loss: 1.001997E+00 | loss scale: 1.0 | grad norm: 0.046 | params norm: 11299.807 | params rms: 0.061185 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 8:34:52.714980 | finish at 2026-10-10 19:15:21 + [2026-10-09 10:41:56] iteration 5526/ 6855 | consumed samples: 707328 | elapsed time per iteration (ms): 88079.0 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.54% | FLOPs estimator: workload-aware | logical tokens/s: 190479.0 | valid tokens/s: 190479.0 | physical tokens/s: 190949.4 | loss tokens/s: 144380.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.779629E-01 | lm-global-supervised-tokens: 1.271689E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.582560E-02 | lm-token-buffer-mib: 5.814184E+01 | global_load_balancing_loss: 1.000746E+00 | loss scale: 1.0 | grad norm: 0.061 | params norm: 11299.756 | params rms: 0.061185 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 8:30:57.039157 | finish at 2026-10-10 19:12:53 + [2026-10-09 10:43:28] iteration 5527/ 6855 | consumed samples: 707456 | elapsed time per iteration (ms): 91477.5 | throughput per GPU (TFLOP/s/GPU): 41.5 | MFU 13.29% | FLOPs estimator: workload-aware | logical tokens/s: 183402.5 | valid tokens/s: 183402.5 | physical tokens/s: 183804.4 | loss tokens/s: 134434.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.720632E-01 | lm-global-supervised-tokens: 1.229777E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.133648E-02 | lm-token-buffer-mib: 5.818824E+01 | global_load_balancing_loss: 1.000675E+00 | loss scale: 1.0 | grad norm: 0.071 | params norm: 11299.704 | params rms: 0.061184 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 9:44:42.119316 | finish at 2026-10-10 20:28:10 + [2026-10-09 10:44:55] iteration 5528/ 6855 | consumed samples: 707584 | elapsed time per iteration (ms): 86892.1 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.59% | FLOPs estimator: workload-aware | logical tokens/s: 193080.4 | valid tokens/s: 193080.4 | physical tokens/s: 193591.5 | loss tokens/s: 147560.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.741664E-01 | lm-global-supervised-tokens: 1.282186E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.175722E-02 | lm-token-buffer-mib: 5.804517E+01 | global_load_balancing_loss: 9.992987E-01 | loss scale: 1.0 | grad norm: 0.038 | params norm: 11299.653 | params rms: 0.061184 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 8:01:45.783291 | finish at 2026-10-10 18:46:40 + [2026-10-09 10:46:22] iteration 5529/ 6855 | consumed samples: 707712 | elapsed time per iteration (ms): 87183.6 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.46% | FLOPs estimator: workload-aware | logical tokens/s: 192434.8 | valid tokens/s: 192434.8 | physical tokens/s: 192857.8 | loss tokens/s: 142770.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.507588E-01 | lm-global-supervised-tokens: 1.244724E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.107332E-02 | lm-token-buffer-mib: 5.805857E+01 | global_load_balancing_loss: 9.998859E-01 | loss scale: 1.0 | grad norm: 0.089 | params norm: 11299.604 | params rms: 0.061184 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 8:06:45.444680 | finish at 2026-10-10 18:53:07 + [2026-10-09 10:48:14] iteration 5530/ 6855 | consumed samples: 707840 | elapsed time per iteration (ms): 112099.1 | throughput per GPU (TFLOP/s/GPU): 32.9 | MFU 10.53% | FLOPs estimator: workload-aware | logical tokens/s: 149664.2 | valid tokens/s: 149664.2 | physical tokens/s: 150123.4 | loss tokens/s: 108453.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.604776E-01 | lm-global-supervised-tokens: 1.215756E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.393719E-02 | lm-token-buffer-mib: 5.826237E+01 | global_load_balancing_loss: 9.995186E-01 | loss scale: 1.0 | grad norm: 0.050 | params norm: 11299.556 | params rms: 0.061183 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 17:15:31.262159 | finish at 2026-10-11 04:03:45 + [2026-10-09 10:49:42] iteration 5531/ 6855 | consumed samples: 707968 | elapsed time per iteration (ms): 88416.3 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.57% | FLOPs estimator: workload-aware | logical tokens/s: 189751.9 | valid tokens/s: 189751.9 | physical tokens/s: 190333.2 | loss tokens/s: 138190.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.412690E-01 | lm-global-supervised-tokens: 1.221831E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.364420E-02 | lm-token-buffer-mib: 5.827631E+01 | global_load_balancing_loss: 9.999267E-01 | loss scale: 1.0 | grad norm: 0.080 | params norm: 11299.506 | params rms: 0.061183 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 8:31:03.206531 | finish at 2026-10-10 19:20:46 + [2026-10-09 10:51:08] iteration 5532/ 6855 | consumed samples: 708096 | elapsed time per iteration (ms): 85171.8 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.78% | FLOPs estimator: workload-aware | logical tokens/s: 196980.3 | valid tokens/s: 196980.3 | physical tokens/s: 197483.0 | loss tokens/s: 140463.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.658064E-01 | lm-global-supervised-tokens: 1.196356E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.083657E-02 | lm-token-buffer-mib: 5.802717E+01 | global_load_balancing_loss: 9.995982E-01 | loss scale: 1.0 | grad norm: 0.046 | params norm: 11299.456 | params rms: 0.061183 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 7:18:02.325647 | finish at 2026-10-10 18:09:10 + [2026-10-09 10:52:36] iteration 5533/ 6855 | consumed samples: 708224 | elapsed time per iteration (ms): 88137.2 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.44% | FLOPs estimator: workload-aware | logical tokens/s: 190352.8 | valid tokens/s: 190352.8 | physical tokens/s: 190824.8 | loss tokens/s: 142935.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.244437E-01 | lm-global-supervised-tokens: 1.259791E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.668759E-02 | lm-token-buffer-mib: 5.810407E+01 | global_load_balancing_loss: 1.000231E+00 | loss scale: 1.0 | grad norm: 0.070 | params norm: 11299.408 | params rms: 0.061183 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 8:21:57.319299 | finish at 2026-10-10 19:14:33 + [2026-10-09 10:54:07] iteration 5534/ 6855 | consumed samples: 708352 | elapsed time per iteration (ms): 91321.7 | throughput per GPU (TFLOP/s/GPU): 41.4 | MFU 13.26% | FLOPs estimator: workload-aware | logical tokens/s: 183714.8 | valid tokens/s: 183714.8 | physical tokens/s: 184101.8 | loss tokens/s: 127623.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.496623E-01 | lm-global-supervised-tokens: 1.165479E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.390892E-02 | lm-token-buffer-mib: 5.824415E+01 | global_load_balancing_loss: 1.000405E+00 | loss scale: 1.0 | grad norm: 0.039 | params norm: 11299.359 | params rms: 0.061182 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 9:30:36.024408 | finish at 2026-10-10 20:24:43 + [2026-10-09 10:55:35] iteration 5535/ 6855 | consumed samples: 708480 | elapsed time per iteration (ms): 87505.1 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.56% | FLOPs estimator: workload-aware | logical tokens/s: 191728.0 | valid tokens/s: 191728.0 | physical tokens/s: 192264.7 | loss tokens/s: 145144.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.849971E-01 | lm-global-supervised-tokens: 1.270090E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.795650E-02 | lm-token-buffer-mib: 5.805218E+01 | global_load_balancing_loss: 1.000464E+00 | loss scale: 1.0 | grad norm: 0.075 | params norm: 11299.309 | params rms: 0.061182 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 8:05:06.730127 | finish at 2026-10-10 19:00:41 + [2026-10-09 10:57:05] iteration 5536/ 6855 | consumed samples: 708608 | elapsed time per iteration (ms): 90425.8 | throughput per GPU (TFLOP/s/GPU): 41.4 | MFU 13.27% | FLOPs estimator: workload-aware | logical tokens/s: 185534.7 | valid tokens/s: 185534.7 | physical tokens/s: 186204.2 | loss tokens/s: 134057.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.101575E-01 | lm-global-supervised-tokens: 1.212221E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.404945E-02 | lm-token-buffer-mib: 5.825771E+01 | global_load_balancing_loss: 1.000714E+00 | loss scale: 1.0 | grad norm: 0.048 | params norm: 11299.261 | params rms: 0.061182 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 9:07:51.660816 | finish at 2026-10-10 20:04:57 + [2026-10-09 10:58:31] iteration 5537/ 6855 | consumed samples: 708736 | elapsed time per iteration (ms): 85897.2 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.67% | FLOPs estimator: workload-aware | logical tokens/s: 195315.9 | valid tokens/s: 195315.9 | physical tokens/s: 195825.4 | loss tokens/s: 145016.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.828120E-01 | lm-global-supervised-tokens: 1.245652E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.254126E-02 | lm-token-buffer-mib: 5.813876E+01 | global_load_balancing_loss: 9.996461E-01 | loss scale: 1.0 | grad norm: 0.050 | params norm: 11299.212 | params rms: 0.061182 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 7:26:52.451294 | finish at 2026-10-10 18:25:23 + [2026-10-09 10:59:59] iteration 5538/ 6855 | consumed samples: 708864 | elapsed time per iteration (ms): 88465.4 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.70% | FLOPs estimator: workload-aware | logical tokens/s: 189647.2 | valid tokens/s: 189647.2 | physical tokens/s: 190024.3 | loss tokens/s: 135518.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.818043E-01 | lm-global-supervised-tokens: 1.198870E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.143622E-02 | lm-token-buffer-mib: 5.812222E+01 | global_load_balancing_loss: 1.000012E+00 | loss scale: 1.0 | grad norm: 0.040 | params norm: 11299.161 | params rms: 0.061181 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 8:21:48.954068 | finish at 2026-10-10 19:21:48 + [2026-10-09 11:01:25] iteration 5539/ 6855 | consumed samples: 708992 | elapsed time per iteration (ms): 85855.8 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.45% | FLOPs estimator: workload-aware | logical tokens/s: 195411.1 | valid tokens/s: 195411.1 | physical tokens/s: 196023.2 | loss tokens/s: 154034.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.897528E-01 | lm-global-supervised-tokens: 1.322472E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.585990E-02 | lm-token-buffer-mib: 5.811695E+01 | global_load_balancing_loss: 1.000252E+00 | loss scale: 1.0 | grad norm: 0.031 | params norm: 11299.112 | params rms: 0.061181 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 7:23:06.232059 | finish at 2026-10-10 18:24:31 + [2026-10-09 11:03:17] iteration 5540/ 6855 | consumed samples: 709120 | elapsed time per iteration (ms): 112187.4 | throughput per GPU (TFLOP/s/GPU): 34.2 | MFU 10.95% | FLOPs estimator: workload-aware | logical tokens/s: 149546.2 | valid tokens/s: 149546.2 | physical tokens/s: 149859.9 | loss tokens/s: 104932.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.827411E-01 | lm-global-supervised-tokens: 1.177211E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.173060E-02 | lm-token-buffer-mib: 5.808693E+01 | global_load_balancing_loss: 1.000232E+00 | loss scale: 1.0 | grad norm: 0.056 | params norm: 11299.061 | params rms: 0.061181 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 16:58:46.464682 | finish at 2026-10-11 04:02:04 + [2026-10-09 11:04:43] iteration 5541/ 6855 | consumed samples: 709248 | elapsed time per iteration (ms): 85432.3 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.49% | FLOPs estimator: workload-aware | logical tokens/s: 196380.0 | valid tokens/s: 196380.0 | physical tokens/s: 196832.9 | loss tokens/s: 144550.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.247032E-01 | lm-global-supervised-tokens: 1.234928E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.220920E-02 | lm-token-buffer-mib: 5.807494E+01 | global_load_balancing_loss: 1.001263E+00 | loss scale: 1.0 | grad norm: 0.064 | params norm: 11299.011 | params rms: 0.061180 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 7:10:58.049212 | finish at 2026-10-10 18:15:41 + [2026-10-09 11:06:08] iteration 5542/ 6855 | consumed samples: 709376 | elapsed time per iteration (ms): 85229.3 | throughput per GPU (TFLOP/s/GPU): 43.7 | MFU 14.02% | FLOPs estimator: workload-aware | logical tokens/s: 196847.8 | valid tokens/s: 196847.8 | physical tokens/s: 197208.3 | loss tokens/s: 147406.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.673592E-01 | lm-global-supervised-tokens: 1.256334E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.985857E-02 | lm-token-buffer-mib: 5.803674E+01 | global_load_balancing_loss: 9.997139E-01 | loss scale: 1.0 | grad norm: 0.037 | params norm: 11298.962 | params rms: 0.061180 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 7:05:06.127903 | finish at 2026-10-10 18:11:14 + [2026-10-09 11:07:34] iteration 5543/ 6855 | consumed samples: 709504 | elapsed time per iteration (ms): 85555.8 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.62% | FLOPs estimator: workload-aware | logical tokens/s: 196096.8 | valid tokens/s: 196096.7 | physical tokens/s: 196708.5 | loss tokens/s: 151396.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.584867E-01 | lm-global-supervised-tokens: 1.295286E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.480278E-02 | lm-token-buffer-mib: 5.814622E+01 | global_load_balancing_loss: 9.993610E-01 | loss scale: 1.0 | grad norm: 0.107 | params norm: 11298.913 | params rms: 0.061180 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 7:10:49.180771 | finish at 2026-10-10 18:18:23 + [2026-10-09 11:09:02] iteration 5544/ 6855 | consumed samples: 709632 | elapsed time per iteration (ms): 88568.4 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.62% | FLOPs estimator: workload-aware | logical tokens/s: 189426.4 | valid tokens/s: 189426.4 | physical tokens/s: 189988.7 | loss tokens/s: 136237.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.799411E-01 | lm-global-supervised-tokens: 1.206634E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.783302E-02 | lm-token-buffer-mib: 5.813112E+01 | global_load_balancing_loss: 9.999726E-01 | loss scale: 1.0 | grad norm: 0.073 | params norm: 11298.861 | params rms: 0.061180 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 8:15:13.138832 | finish at 2026-10-10 19:24:15 + [2026-10-09 11:10:29] iteration 5545/ 6855 | consumed samples: 709760 | elapsed time per iteration (ms): 87244.4 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.53% | FLOPs estimator: workload-aware | logical tokens/s: 192301.1 | valid tokens/s: 192301.1 | physical tokens/s: 192819.9 | loss tokens/s: 145357.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.179933E-01 | lm-global-supervised-tokens: 1.268158E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.536354E-02 | lm-token-buffer-mib: 5.802486E+01 | global_load_balancing_loss: 1.000498E+00 | loss scale: 1.0 | grad norm: 0.106 | params norm: 11298.808 | params rms: 0.061179 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 7:44:50.128739 | finish at 2026-10-10 18:55:20 + [2026-10-09 11:11:56] iteration 5546/ 6855 | consumed samples: 709888 | elapsed time per iteration (ms): 86721.7 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.79% | FLOPs estimator: workload-aware | logical tokens/s: 193460.4 | valid tokens/s: 193460.4 | physical tokens/s: 193796.9 | loss tokens/s: 141552.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.140241E-01 | lm-global-supervised-tokens: 1.227566E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.150376E-02 | lm-token-buffer-mib: 5.808707E+01 | global_load_balancing_loss: 1.000168E+00 | loss scale: 1.0 | grad norm: 0.066 | params norm: 11298.755 | params rms: 0.061179 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 7:31:58.736135 | finish at 2026-10-10 18:43:55 + [2026-10-09 11:13:24] iteration 5547/ 6855 | consumed samples: 710016 | elapsed time per iteration (ms): 88325.9 | throughput per GPU (TFLOP/s/GPU): 41.7 | MFU 13.36% | FLOPs estimator: workload-aware | logical tokens/s: 189946.2 | valid tokens/s: 189946.2 | physical tokens/s: 190342.9 | loss tokens/s: 139810.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.340003E-01 | lm-global-supervised-tokens: 1.234888E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.151596E-02 | lm-token-buffer-mib: 5.809141E+01 | global_load_balancing_loss: 1.000778E+00 | loss scale: 1.0 | grad norm: 0.192 | params norm: 11298.703 | params rms: 0.061179 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 8:05:30.267011 | finish at 2026-10-10 19:18:55 + [2026-10-09 11:14:53] iteration 5548/ 6855 | consumed samples: 710144 | elapsed time per iteration (ms): 88759.3 | throughput per GPU (TFLOP/s/GPU): 41.7 | MFU 13.38% | FLOPs estimator: workload-aware | logical tokens/s: 189019.1 | valid tokens/s: 189019.1 | physical tokens/s: 189723.8 | loss tokens/s: 136889.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.878893E-01 | lm-global-supervised-tokens: 1.215018E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.239415E-02 | lm-token-buffer-mib: 5.810351E+01 | global_load_balancing_loss: 1.000270E+00 | loss scale: 1.0 | grad norm: 0.085 | params norm: 11298.651 | params rms: 0.061179 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 8:13:28.448717 | finish at 2026-10-10 19:28:22 + [2026-10-09 11:16:24] iteration 5549/ 6855 | consumed samples: 710272 | elapsed time per iteration (ms): 90773.8 | throughput per GPU (TFLOP/s/GPU): 41.1 | MFU 13.18% | FLOPs estimator: workload-aware | logical tokens/s: 184824.3 | valid tokens/s: 184824.3 | physical tokens/s: 185294.2 | loss tokens/s: 132735.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.826847E-01 | lm-global-supervised-tokens: 1.204894E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.105499E-02 | lm-token-buffer-mib: 5.831711E+01 | global_load_balancing_loss: 1.000764E+00 | loss scale: 1.0 | grad norm: 0.201 | params norm: 11298.597 | params rms: 0.061178 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 8:55:50.644939 | finish at 2026-10-10 20:12:15 + [2026-10-09 11:18:14] iteration 5550/ 6855 | consumed samples: 710400 | elapsed time per iteration (ms): 110033.6 | throughput per GPU (TFLOP/s/GPU): 33.0 | MFU 10.57% | FLOPs estimator: workload-aware | logical tokens/s: 152473.6 | valid tokens/s: 152473.6 | physical tokens/s: 152934.7 | loss tokens/s: 117205.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.551586E-01 | lm-global-supervised-tokens: 1.289648E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.166661E-02 | lm-token-buffer-mib: 5.813895E+01 | global_load_balancing_loss: 9.993984E-01 | loss scale: 1.0 | grad norm: 0.105 | params norm: 11298.544 | params rms: 0.061178 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 15:53:13.820740 | finish at 2026-10-11 03:11:28 + [2026-10-09 11:19:41] iteration 5551/ 6855 | consumed samples: 710528 | elapsed time per iteration (ms): 87289.2 | throughput per GPU (TFLOP/s/GPU): 43.2 | MFU 13.83% | FLOPs estimator: workload-aware | logical tokens/s: 192202.1 | valid tokens/s: 192202.0 | physical tokens/s: 192562.8 | loss tokens/s: 137009.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.640080E-01 | lm-global-supervised-tokens: 1.195943E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.369956E-02 | lm-token-buffer-mib: 5.814439E+01 | global_load_balancing_loss: 9.993852E-01 | loss scale: 1.0 | grad norm: 0.156 | params norm: 11298.493 | params rms: 0.061178 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 7:37:05.162590 | finish at 2026-10-10 18:56:46 + [2026-10-09 11:21:08] iteration 5552/ 6855 | consumed samples: 710656 | elapsed time per iteration (ms): 86773.3 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.38% | FLOPs estimator: workload-aware | logical tokens/s: 193344.5 | valid tokens/s: 193344.5 | physical tokens/s: 193949.0 | loss tokens/s: 148893.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.441180E-01 | lm-global-supervised-tokens: 1.292000E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.540890E-02 | lm-token-buffer-mib: 5.822646E+01 | global_load_balancing_loss: 1.000162E+00 | loss scale: 1.0 | grad norm: 0.066 | params norm: 11298.441 | params rms: 0.061177 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 7:24:25.665420 | finish at 2026-10-10 18:45:34 + [2026-10-09 11:22:35] iteration 5553/ 6855 | consumed samples: 710784 | elapsed time per iteration (ms): 86623.7 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.79% | FLOPs estimator: workload-aware | logical tokens/s: 193679.4 | valid tokens/s: 193679.4 | physical tokens/s: 194115.3 | loss tokens/s: 143399.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.395413E-01 | lm-global-supervised-tokens: 1.242178E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.784303E-02 | lm-token-buffer-mib: 5.803131E+01 | global_load_balancing_loss: 1.000911E+00 | loss scale: 1.0 | grad norm: 0.073 | params norm: 11298.388 | params rms: 0.061177 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 7:19:43.996121 | finish at 2026-10-10 18:42:19 + [2026-10-09 11:24:00] iteration 5554/ 6855 | consumed samples: 710912 | elapsed time per iteration (ms): 85119.3 | throughput per GPU (TFLOP/s/GPU): 43.1 | MFU 13.82% | FLOPs estimator: workload-aware | logical tokens/s: 197100.6 | valid tokens/s: 197100.6 | physical tokens/s: 197501.0 | loss tokens/s: 144200.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.415323E-01 | lm-global-supervised-tokens: 1.227424E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.195069E-02 | lm-token-buffer-mib: 5.810672E+01 | global_load_balancing_loss: 1.000601E+00 | loss scale: 1.0 | grad norm: 0.156 | params norm: 11298.337 | params rms: 0.061177 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 6:45:40.220011 | finish at 2026-10-10 18:09:40 + [2026-10-09 11:25:28] iteration 5555/ 6855 | consumed samples: 711040 | elapsed time per iteration (ms): 88008.6 | throughput per GPU (TFLOP/s/GPU): 41.2 | MFU 13.19% | FLOPs estimator: workload-aware | logical tokens/s: 190631.0 | valid tokens/s: 190631.0 | physical tokens/s: 191078.4 | loss tokens/s: 140873.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.316960E-01 | lm-global-supervised-tokens: 1.239804E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.348294E-02 | lm-token-buffer-mib: 5.809841E+01 | global_load_balancing_loss: 1.000795E+00 | loss scale: 1.0 | grad norm: 0.072 | params norm: 11298.288 | params rms: 0.061177 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 7:46:51.216569 | finish at 2026-10-10 19:12:19 + [2026-10-09 11:26:54] iteration 5556/ 6855 | consumed samples: 711168 | elapsed time per iteration (ms): 86137.6 | throughput per GPU (TFLOP/s/GPU): 43.1 | MFU 13.80% | FLOPs estimator: workload-aware | logical tokens/s: 194772.1 | valid tokens/s: 194772.1 | physical tokens/s: 195118.3 | loss tokens/s: 140441.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.396086E-01 | lm-global-supervised-tokens: 1.209725E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.659902E-02 | lm-token-buffer-mib: 5.809106E+01 | global_load_balancing_loss: 9.994980E-01 | loss scale: 1.0 | grad norm: 0.089 | params norm: 11298.239 | params rms: 0.061176 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 7:04:52.740161 | finish at 2026-10-10 18:31:47 + [2026-10-09 11:28:17] iteration 5557/ 6855 | consumed samples: 711296 | elapsed time per iteration (ms): 83555.4 | throughput per GPU (TFLOP/s/GPU): 42.8 | MFU 13.70% | FLOPs estimator: workload-aware | logical tokens/s: 200791.0 | valid tokens/s: 200791.0 | physical tokens/s: 201205.7 | loss tokens/s: 151634.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.158175E-01 | lm-global-supervised-tokens: 1.266988E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.302499E-02 | lm-token-buffer-mib: 5.807059E+01 | global_load_balancing_loss: 9.994128E-01 | loss scale: 1.0 | grad norm: 0.153 | params norm: 11298.189 | params rms: 0.061176 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 6:07:34.907516 | finish at 2026-10-10 17:35:52 + [2026-10-09 11:29:48] iteration 5558/ 6855 | consumed samples: 711424 | elapsed time per iteration (ms): 90562.6 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.48% | FLOPs estimator: workload-aware | logical tokens/s: 185255.1 | valid tokens/s: 185255.1 | physical tokens/s: 185643.0 | loss tokens/s: 136197.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.586173E-01 | lm-global-supervised-tokens: 1.233444E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.204416E-02 | lm-token-buffer-mib: 5.824677E+01 | global_load_balancing_loss: 9.995300E-01 | loss scale: 1.0 | grad norm: 0.097 | params norm: 11298.139 | params rms: 0.061176 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 8:37:39.724227 | finish at 2026-10-10 20:07:28 + [2026-10-09 11:31:17] iteration 5559/ 6855 | consumed samples: 711552 | elapsed time per iteration (ms): 88973.4 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.40% | FLOPs estimator: workload-aware | logical tokens/s: 188563.9 | valid tokens/s: 188563.9 | physical tokens/s: 189034.0 | loss tokens/s: 137898.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.680203E-01 | lm-global-supervised-tokens: 1.226934E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.324501E-02 | lm-token-buffer-mib: 5.811041E+01 | global_load_balancing_loss: 1.001179E+00 | loss scale: 1.0 | grad norm: 0.320 | params norm: 11298.089 | params rms: 0.061175 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 8:01:49.505230 | finish at 2026-10-10 19:33:07 + [2026-10-09 11:33:03] iteration 5560/ 6855 | consumed samples: 711680 | elapsed time per iteration (ms): 106146.2 | throughput per GPU (TFLOP/s/GPU): 34.5 | MFU 11.06% | FLOPs estimator: workload-aware | logical tokens/s: 158056.9 | valid tokens/s: 158056.9 | physical tokens/s: 158489.5 | loss tokens/s: 115218.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.437855E-01 | lm-global-supervised-tokens: 1.223001E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.139273E-02 | lm-token-buffer-mib: 5.809882E+01 | global_load_balancing_loss: 1.000201E+00 | loss scale: 1.0 | grad norm: 0.127 | params norm: 11298.039 | params rms: 0.061175 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 14:10:59.348993 | finish at 2026-10-11 01:44:03 +times across ranks (ms): + save-checkpoint: + rank 0: 25602.55 + rank 1: 25602.56 + rank 2: 25602.54 + rank 3: 25602.52 + rank 4: 25602.55 + rank 5: 25602.52 + rank 6: 25602.53 + rank 7: 25602.53 + rank 8: 25601.82 + rank 9: 25601.73 + rank 10: 25601.74 + rank 11: 25601.79 + rank 12: 25601.74 + rank 13: 25601.78 + rank 14: 25601.77 + rank 15: 25601.74 + rank 16: 25602.63 + rank 17: 25602.60 + rank 18: 25602.59 + rank 19: 25602.63 + rank 20: 25602.59 + rank 21: 25602.62 + rank 22: 25602.67 + rank 23: 25602.59 + rank 24: 25602.76 + rank 25: 25602.71 + rank 26: 25602.66 + rank 27: 25602.71 + rank 28: 25602.71 + rank 29: 25602.72 + rank 30: 25602.70 + rank 31: 25602.70 + rank 32: 25602.41 + rank 33: 25602.31 + rank 34: 25602.32 + rank 35: 25602.22 + rank 36: 25602.29 + rank 37: 25602.33 + rank 38: 25602.24 + rank 39: 25602.18 + rank 40: 25602.41 + rank 41: 25602.37 + rank 42: 25602.38 + rank 43: 25602.39 + rank 44: 25602.37 + rank 45: 25602.38 + rank 46: 25602.38 + rank 47: 25602.37 + rank 48: 25602.44 + rank 49: 25602.48 + rank 50: 25602.32 + rank 51: 25602.43 + rank 52: 25602.58 + rank 53: 25602.52 + rank 54: 25602.58 + rank 55: 25602.61 + rank 56: 25602.50 + rank 57: 25602.50 + rank 58: 25602.47 + rank 59: 25602.51 + rank 60: 25602.47 + rank 61: 25602.53 + rank 62: 25602.52 + rank 63: 25602.44 + [2026-10-09 11:35:17] iteration 5561/ 6855 | consumed samples: 711808 | elapsed time per iteration (ms): 108436.2 | throughput per GPU (TFLOP/s/GPU): 34.4 | MFU 11.02% | FLOPs estimator: workload-aware | logical tokens/s: 154719.5 | valid tokens/s: 154719.5 | physical tokens/s: 155015.6 | loss tokens/s: 112826.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.911700E-01 | lm-global-supervised-tokens: 1.223444E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 3.914338E-01 | lm-token-buffer-mib: 5.807131E+01 | global_load_balancing_loss: 9.996501E-01 | loss scale: 1.0 | grad norm: 0.314 | params norm: 11297.984 | params rms: 0.061175 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 14:58:36.491420 | finish at 2026-10-11 02:33:54 + [2026-10-09 11:36:51] iteration 5562/ 6855 | consumed samples: 711936 | elapsed time per iteration (ms): 93584.4 | throughput per GPU (TFLOP/s/GPU): 40.4 | MFU 12.95% | FLOPs estimator: workload-aware | logical tokens/s: 179273.6 | valid tokens/s: 179273.6 | physical tokens/s: 179662.7 | loss tokens/s: 125738.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.609970E-01 | lm-global-supervised-tokens: 1.176717E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.385018E-02 | lm-token-buffer-mib: 5.815862E+01 | global_load_balancing_loss: 1.001134E+00 | loss scale: 1.0 | grad norm: 0.063 | params norm: 11297.929 | params rms: 0.061175 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 9:36:44.647001 | finish at 2026-10-10 21:13:36 + [2026-10-09 11:38:20] iteration 5563/ 6855 | consumed samples: 712064 | elapsed time per iteration (ms): 88877.3 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.65% | FLOPs estimator: workload-aware | logical tokens/s: 188767.1 | valid tokens/s: 188767.1 | physical tokens/s: 189250.5 | loss tokens/s: 136103.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.771792E-01 | lm-global-supervised-tokens: 1.209652E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.387993E-02 | lm-token-buffer-mib: 5.805589E+01 | global_load_balancing_loss: 9.998635E-01 | loss scale: 1.0 | grad norm: 0.258 | params norm: 11297.875 | params rms: 0.061174 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 7:53:49.418649 | finish at 2026-10-10 19:32:09 + [2026-10-09 11:39:47] iteration 5564/ 6855 | consumed samples: 712192 | elapsed time per iteration (ms): 87334.8 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.63% | FLOPs estimator: workload-aware | logical tokens/s: 192101.6 | valid tokens/s: 192101.6 | physical tokens/s: 192497.2 | loss tokens/s: 144035.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.109886E-01 | lm-global-supervised-tokens: 1.257930E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.359594E-02 | lm-token-buffer-mib: 5.811693E+01 | global_load_balancing_loss: 1.000317E+00 | loss scale: 1.0 | grad norm: 0.094 | params norm: 11297.824 | params rms: 0.061174 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 7:19:09.261280 | finish at 2026-10-10 18:58:57 + [2026-10-09 11:41:17] iteration 5565/ 6855 | consumed samples: 712320 | elapsed time per iteration (ms): 90142.7 | throughput per GPU (TFLOP/s/GPU): 41.3 | MFU 13.22% | FLOPs estimator: workload-aware | logical tokens/s: 186118.0 | valid tokens/s: 186118.0 | physical tokens/s: 186506.0 | loss tokens/s: 135507.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.764201E-01 | lm-global-supervised-tokens: 1.221499E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.216046E-02 | lm-token-buffer-mib: 5.812996E+01 | global_load_balancing_loss: 1.000514E+00 | loss scale: 1.0 | grad norm: 0.044 | params norm: 11297.774 | params rms: 0.061174 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 8:18:04.140458 | finish at 2026-10-10 19:59:22 + [2026-10-09 11:42:43] iteration 5566/ 6855 | consumed samples: 712448 | elapsed time per iteration (ms): 85915.9 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.62% | FLOPs estimator: workload-aware | logical tokens/s: 195274.0 | valid tokens/s: 195274.0 | physical tokens/s: 195647.9 | loss tokens/s: 150583.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.611427E-01 | lm-global-supervised-tokens: 1.293754E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.191285E-02 | lm-token-buffer-mib: 5.812898E+01 | global_load_balancing_loss: 9.987093E-01 | loss scale: 1.0 | grad norm: 0.169 | params norm: 11297.724 | params rms: 0.061173 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 6:45:45.572655 | finish at 2026-10-10 18:28:29 + [2026-10-09 11:44:09] iteration 5567/ 6855 | consumed samples: 712576 | elapsed time per iteration (ms): 85787.1 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.68% | FLOPs estimator: workload-aware | logical tokens/s: 195567.7 | valid tokens/s: 195567.7 | physical tokens/s: 196115.9 | loss tokens/s: 145953.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.425952E-01 | lm-global-supervised-tokens: 1.252090E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.550431E-02 | lm-token-buffer-mib: 5.810092E+01 | global_load_balancing_loss: 1.000919E+00 | loss scale: 1.0 | grad norm: 0.069 | params norm: 11297.674 | params rms: 0.061173 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 6:41:33.748356 | finish at 2026-10-10 18:25:43 + [2026-10-09 11:45:36] iteration 5568/ 6855 | consumed samples: 712704 | elapsed time per iteration (ms): 86609.4 | throughput per GPU (TFLOP/s/GPU): 41.5 | MFU 13.31% | FLOPs estimator: workload-aware | logical tokens/s: 193710.5 | valid tokens/s: 193710.5 | physical tokens/s: 194164.1 | loss tokens/s: 144532.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.380323E-01 | lm-global-supervised-tokens: 1.251788E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.629430E-02 | lm-token-buffer-mib: 5.815402E+01 | global_load_balancing_loss: 1.000990E+00 | loss scale: 1.0 | grad norm: 0.049 | params norm: 11297.625 | params rms: 0.061173 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 6:57:46.347859 | finish at 2026-10-10 18:43:22 + [2026-10-09 11:47:04] iteration 5569/ 6855 | consumed samples: 712832 | elapsed time per iteration (ms): 88118.3 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.56% | FLOPs estimator: workload-aware | logical tokens/s: 190393.3 | valid tokens/s: 190393.3 | physical tokens/s: 190895.3 | loss tokens/s: 135539.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.272881E-01 | lm-global-supervised-tokens: 1.194350E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.121252E-02 | lm-token-buffer-mib: 5.806065E+01 | global_load_balancing_loss: 1.000546E+00 | loss scale: 1.0 | grad norm: 0.079 | params norm: 11297.574 | params rms: 0.061173 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 7:28:40.151840 | finish at 2026-10-10 19:15:44 + [2026-10-09 11:48:55] iteration 5570/ 6855 | consumed samples: 712960 | elapsed time per iteration (ms): 110984.9 | throughput per GPU (TFLOP/s/GPU): 32.7 | MFU 10.49% | FLOPs estimator: workload-aware | logical tokens/s: 151166.4 | valid tokens/s: 151166.4 | physical tokens/s: 151545.7 | loss tokens/s: 110966.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.355346E-01 | lm-global-supervised-tokens: 1.231563E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.282445E-02 | lm-token-buffer-mib: 5.819613E+01 | global_load_balancing_loss: 1.000401E+00 | loss scale: 1.0 | grad norm: 0.086 | params norm: 11297.522 | params rms: 0.061172 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 15:36:55.603237 | finish at 2026-10-11 03:25:50 + [2026-10-09 11:50:23] iteration 5571/ 6855 | consumed samples: 713088 | elapsed time per iteration (ms): 88058.2 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.48% | FLOPs estimator: workload-aware | logical tokens/s: 190524.1 | valid tokens/s: 190524.1 | physical tokens/s: 191107.4 | loss tokens/s: 141533.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.484256E-01 | lm-global-supervised-tokens: 1.246320E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.163255E-02 | lm-token-buffer-mib: 5.847552E+01 | global_load_balancing_loss: 1.000386E+00 | loss scale: 1.0 | grad norm: 0.075 | params norm: 11297.472 | params rms: 0.061172 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 7:24:26.763854 | finish at 2026-10-10 19:14:50 + [2026-10-09 11:51:49] iteration 5572/ 6855 | consumed samples: 713216 | elapsed time per iteration (ms): 85958.7 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.62% | FLOPs estimator: workload-aware | logical tokens/s: 195176.9 | valid tokens/s: 195176.9 | physical tokens/s: 195671.3 | loss tokens/s: 145053.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.472645E-01 | lm-global-supervised-tokens: 1.246862E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.147776E-02 | lm-token-buffer-mib: 5.831408E+01 | global_load_balancing_loss: 9.989907E-01 | loss scale: 1.0 | grad norm: 0.049 | params norm: 11297.423 | params rms: 0.061172 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 6:38:05.015390 | finish at 2026-10-10 18:29:54 + [2026-10-09 11:53:14] iteration 5573/ 6855 | consumed samples: 713344 | elapsed time per iteration (ms): 85435.9 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.66% | FLOPs estimator: workload-aware | logical tokens/s: 196371.6 | valid tokens/s: 196371.6 | physical tokens/s: 196793.3 | loss tokens/s: 146664.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.790060E-01 | lm-global-supervised-tokens: 1.253042E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.104361E-02 | lm-token-buffer-mib: 5.819811E+01 | global_load_balancing_loss: 9.990693E-01 | loss scale: 1.0 | grad norm: 0.040 | params norm: 11297.374 | params rms: 0.061172 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 6:25:28.847606 | finish at 2026-10-10 18:18:43 + [2026-10-09 11:54:43] iteration 5574/ 6855 | consumed samples: 713472 | elapsed time per iteration (ms): 89093.1 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.57% | FLOPs estimator: workload-aware | logical tokens/s: 188311.0 | valid tokens/s: 188311.0 | physical tokens/s: 188569.9 | loss tokens/s: 134482.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.596032E-01 | lm-global-supervised-tokens: 1.198144E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.141622E-02 | lm-token-buffer-mib: 5.805244E+01 | global_load_balancing_loss: 1.000115E+00 | loss scale: 1.0 | grad norm: 0.048 | params norm: 11297.326 | params rms: 0.061171 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 7:42:08.245004 | finish at 2026-10-10 19:36:52 + [2026-10-09 11:56:10] iteration 5575/ 6855 | consumed samples: 713600 | elapsed time per iteration (ms): 86359.4 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.74% | FLOPs estimator: workload-aware | logical tokens/s: 194270.9 | valid tokens/s: 194270.9 | physical tokens/s: 194814.0 | loss tokens/s: 143495.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.754873E-01 | lm-global-supervised-tokens: 1.239216E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.702526E-02 | lm-token-buffer-mib: 5.817436E+01 | global_load_balancing_loss: 1.000233E+00 | loss scale: 1.0 | grad norm: 0.053 | params norm: 11297.278 | params rms: 0.061171 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 6:42:20.007324 | finish at 2026-10-10 18:38:30 + [2026-10-09 11:57:37] iteration 5576/ 6855 | consumed samples: 713728 | elapsed time per iteration (ms): 87268.7 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.55% | FLOPs estimator: workload-aware | logical tokens/s: 192247.5 | valid tokens/s: 192247.5 | physical tokens/s: 192786.1 | loss tokens/s: 140235.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.592830E-01 | lm-global-supervised-tokens: 1.223814E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.191967E-02 | lm-token-buffer-mib: 5.804156E+01 | global_load_balancing_loss: 9.999986E-01 | loss scale: 1.0 | grad norm: 0.049 | params norm: 11297.230 | params rms: 0.061171 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 7:00:16.689718 | finish at 2026-10-10 18:57:54 + [2026-10-09 11:59:06] iteration 5577/ 6855 | consumed samples: 713856 | elapsed time per iteration (ms): 88868.3 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.63% | FLOPs estimator: workload-aware | logical tokens/s: 188786.9 | valid tokens/s: 188786.9 | physical tokens/s: 189089.3 | loss tokens/s: 135516.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.604732E-01 | lm-global-supervised-tokens: 1.204314E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.579069E-02 | lm-token-buffer-mib: 5.804445E+01 | global_load_balancing_loss: 1.000027E+00 | loss scale: 1.0 | grad norm: 0.048 | params norm: 11297.183 | params rms: 0.061171 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 7:32:53.734274 | finish at 2026-10-10 19:32:00 + [2026-10-09 12:00:34] iteration 5578/ 6855 | consumed samples: 713984 | elapsed time per iteration (ms): 87947.6 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.53% | FLOPs estimator: workload-aware | logical tokens/s: 190762.7 | valid tokens/s: 190762.7 | physical tokens/s: 191254.2 | loss tokens/s: 146675.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.326331E-01 | lm-global-supervised-tokens: 1.289978E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.412948E-02 | lm-token-buffer-mib: 5.814128E+01 | global_load_balancing_loss: 1.000601E+00 | loss scale: 1.0 | grad norm: 0.051 | params norm: 11297.135 | params rms: 0.061170 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 7:11:49.084752 | finish at 2026-10-10 19:12:23 + [2026-10-09 12:01:56] iteration 5579/ 6855 | consumed samples: 714112 | elapsed time per iteration (ms): 81971.4 | throughput per GPU (TFLOP/s/GPU): 43.6 | MFU 13.98% | FLOPs estimator: workload-aware | logical tokens/s: 204671.1 | valid tokens/s: 204671.1 | physical tokens/s: 205114.6 | loss tokens/s: 150474.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.473209E-01 | lm-global-supervised-tokens: 1.233462E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.516691E-02 | lm-token-buffer-mib: 5.811584E+01 | global_load_balancing_loss: 1.002011E+00 | loss scale: 1.0 | grad norm: 0.078 | params norm: 11297.087 | params rms: 0.061170 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 5:03:15.545978 | finish at 2026-10-10 17:05:11 + [2026-10-09 12:03:43] iteration 5580/ 6855 | consumed samples: 714240 | elapsed time per iteration (ms): 107479.3 | throughput per GPU (TFLOP/s/GPU): 33.9 | MFU 10.85% | FLOPs estimator: workload-aware | logical tokens/s: 156096.9 | valid tokens/s: 156096.8 | physical tokens/s: 156493.0 | loss tokens/s: 117272.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.607528E-01 | lm-global-supervised-tokens: 1.260442E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.197594E-02 | lm-token-buffer-mib: 5.804067E+01 | global_load_balancing_loss: 1.000615E+00 | loss scale: 1.0 | grad norm: 0.050 | params norm: 11297.038 | params rms: 0.061170 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 14:03:56.150998 | finish at 2026-10-11 02:07:39 + [2026-10-09 12:05:12] iteration 5581/ 6855 | consumed samples: 714368 | elapsed time per iteration (ms): 88230.4 | throughput per GPU (TFLOP/s/GPU): 41.7 | MFU 13.38% | FLOPs estimator: workload-aware | logical tokens/s: 190151.8 | valid tokens/s: 190151.8 | physical tokens/s: 190860.0 | loss tokens/s: 142435.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.842631E-01 | lm-global-supervised-tokens: 1.256711E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.367770E-02 | lm-token-buffer-mib: 5.808832E+01 | global_load_balancing_loss: 9.998141E-01 | loss scale: 1.0 | grad norm: 0.157 | params norm: 11296.990 | params rms: 0.061170 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 7:13:25.570107 | finish at 2026-10-10 19:18:37 + [2026-10-09 12:06:38] iteration 5582/ 6855 | consumed samples: 714496 | elapsed time per iteration (ms): 86350.0 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.52% | FLOPs estimator: workload-aware | logical tokens/s: 194292.5 | valid tokens/s: 194292.5 | physical tokens/s: 194724.3 | loss tokens/s: 144804.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.513788E-01 | lm-global-supervised-tokens: 1.250385E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.618160E-02 | lm-token-buffer-mib: 5.818922E+01 | global_load_balancing_loss: 9.997258E-01 | loss scale: 1.0 | grad norm: 0.209 | params norm: 11296.942 | params rms: 0.061169 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 6:32:03.557770 | finish at 2026-10-10 18:38:41 + [2026-10-09 12:08:07] iteration 5583/ 6855 | consumed samples: 714624 | elapsed time per iteration (ms): 88925.1 | throughput per GPU (TFLOP/s/GPU): 41.3 | MFU 13.25% | FLOPs estimator: workload-aware | logical tokens/s: 188666.2 | valid tokens/s: 188666.2 | physical tokens/s: 189192.8 | loss tokens/s: 138724.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.761475E-01 | lm-global-supervised-tokens: 1.233610E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.127950E-02 | lm-token-buffer-mib: 5.819067E+01 | global_load_balancing_loss: 1.000035E+00 | loss scale: 1.0 | grad norm: 0.096 | params norm: 11296.894 | params rms: 0.061169 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 7:25:12.711239 | finish at 2026-10-10 19:33:20 + [2026-10-09 12:09:33] iteration 5584/ 6855 | consumed samples: 714752 | elapsed time per iteration (ms): 86625.3 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.74% | FLOPs estimator: workload-aware | logical tokens/s: 193675.6 | valid tokens/s: 193675.6 | physical tokens/s: 194052.4 | loss tokens/s: 139068.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.486307E-01 | lm-global-supervised-tokens: 1.204685E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.559358E-02 | lm-token-buffer-mib: 5.812176E+01 | global_load_balancing_loss: 1.000440E+00 | loss scale: 1.0 | grad norm: 0.111 | params norm: 11296.843 | params rms: 0.061169 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 6:35:00.700757 | finish at 2026-10-10 18:44:34 + [2026-10-09 12:11:02] iteration 5585/ 6855 | consumed samples: 714880 | elapsed time per iteration (ms): 88220.9 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.68% | FLOPs estimator: workload-aware | logical tokens/s: 190172.1 | valid tokens/s: 190172.1 | physical tokens/s: 190473.5 | loss tokens/s: 134275.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.370482E-01 | lm-global-supervised-tokens: 1.184590E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.190916E-02 | lm-token-buffer-mib: 5.809498E+01 | global_load_balancing_loss: 1.000057E+00 | loss scale: 1.0 | grad norm: 0.063 | params norm: 11296.793 | params rms: 0.061168 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 7:07:20.535202 | finish at 2026-10-10 19:18:22 + [2026-10-09 12:12:28] iteration 5586/ 6855 | consumed samples: 715008 | elapsed time per iteration (ms): 86427.0 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.43% | FLOPs estimator: workload-aware | logical tokens/s: 194119.3 | valid tokens/s: 194119.3 | physical tokens/s: 194503.6 | loss tokens/s: 146094.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.407607E-01 | lm-global-supervised-tokens: 1.262655E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.141697E-02 | lm-token-buffer-mib: 5.811489E+01 | global_load_balancing_loss: 9.996680E-01 | loss scale: 1.0 | grad norm: 0.049 | params norm: 11296.744 | params rms: 0.061168 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 6:27:55.886960 | finish at 2026-10-10 18:40:24 + [2026-10-09 12:13:57] iteration 5587/ 6855 | consumed samples: 715136 | elapsed time per iteration (ms): 89379.6 | throughput per GPU (TFLOP/s/GPU): 41.6 | MFU 13.32% | FLOPs estimator: workload-aware | logical tokens/s: 187706.7 | valid tokens/s: 187706.7 | physical tokens/s: 188122.3 | loss tokens/s: 131082.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 5.833951E-01 | lm-global-supervised-tokens: 1.171606E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.974550E-02 | lm-token-buffer-mib: 5.828745E+01 | global_load_balancing_loss: 1.000912E+00 | loss scale: 1.0 | grad norm: 0.046 | params norm: 11296.695 | params rms: 0.061168 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 7:28:53.278747 | finish at 2026-10-10 19:42:51 + [2026-10-09 12:15:24] iteration 5588/ 6855 | consumed samples: 715264 | elapsed time per iteration (ms): 86945.2 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.47% | FLOPs estimator: workload-aware | logical tokens/s: 192962.9 | valid tokens/s: 192962.9 | physical tokens/s: 193353.7 | loss tokens/s: 142544.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.665801E-01 | lm-global-supervised-tokens: 1.239354E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.863346E-02 | lm-token-buffer-mib: 5.818470E+01 | global_load_balancing_loss: 9.993416E-01 | loss scale: 1.0 | grad norm: 0.060 | params norm: 11296.645 | params rms: 0.061168 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 6:35:59.598565 | finish at 2026-10-10 18:51:24 + [2026-10-09 12:16:54] iteration 5589/ 6855 | consumed samples: 715392 | elapsed time per iteration (ms): 89138.8 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.51% | FLOPs estimator: workload-aware | logical tokens/s: 188214.5 | valid tokens/s: 188214.5 | physical tokens/s: 188638.1 | loss tokens/s: 134916.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.178722E-01 | lm-global-supervised-tokens: 1.202626E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.288917E-02 | lm-token-buffer-mib: 5.805293E+01 | global_load_balancing_loss: 1.000414E+00 | loss scale: 1.0 | grad norm: 0.073 | params norm: 11296.596 | params rms: 0.061167 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 7:20:49.758712 | finish at 2026-10-10 19:37:43 + [2026-10-09 12:18:42] iteration 5590/ 6855 | consumed samples: 715520 | elapsed time per iteration (ms): 108765.9 | throughput per GPU (TFLOP/s/GPU): 34.1 | MFU 10.94% | FLOPs estimator: workload-aware | logical tokens/s: 154250.6 | valid tokens/s: 154250.6 | physical tokens/s: 154659.6 | loss tokens/s: 116236.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.584107E-01 | lm-global-supervised-tokens: 1.264260E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.142826E-02 | lm-token-buffer-mib: 5.822327E+01 | global_load_balancing_loss: 9.997075E-01 | loss scale: 1.0 | grad norm: 0.068 | params norm: 11296.547 | params rms: 0.061167 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 14:13:08.892021 | finish at 2026-10-11 02:31:51 + [2026-10-09 12:20:11] iteration 5591/ 6855 | consumed samples: 715648 | elapsed time per iteration (ms): 89004.0 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.52% | FLOPs estimator: workload-aware | logical tokens/s: 188499.1 | valid tokens/s: 188499.1 | physical tokens/s: 188952.7 | loss tokens/s: 136466.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.966467E-01 | lm-global-supervised-tokens: 1.214608E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.521923E-02 | lm-token-buffer-mib: 5.819581E+01 | global_load_balancing_loss: 9.993724E-01 | loss scale: 1.0 | grad norm: 0.127 | params norm: 11296.498 | params rms: 0.061167 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 7:15:01.091797 | finish at 2026-10-10 19:35:12 + [2026-10-09 12:21:42] iteration 5592/ 6855 | consumed samples: 715776 | elapsed time per iteration (ms): 90834.9 | throughput per GPU (TFLOP/s/GPU): 40.2 | MFU 12.90% | FLOPs estimator: workload-aware | logical tokens/s: 184699.6 | valid tokens/s: 184699.6 | physical tokens/s: 185116.8 | loss tokens/s: 136414.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.397184E-01 | lm-global-supervised-tokens: 1.239118E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.104001E-02 | lm-token-buffer-mib: 5.825237E+01 | global_load_balancing_loss: 1.000587E+00 | loss scale: 1.0 | grad norm: 0.122 | params norm: 11296.449 | params rms: 0.061167 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 7:52:04.505377 | finish at 2026-10-10 20:13:47 + [2026-10-09 12:23:06] iteration 5593/ 6855 | consumed samples: 715904 | elapsed time per iteration (ms): 84337.9 | throughput per GPU (TFLOP/s/GPU): 43.5 | MFU 13.94% | FLOPs estimator: workload-aware | logical tokens/s: 198927.8 | valid tokens/s: 198927.8 | physical tokens/s: 199365.6 | loss tokens/s: 146643.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.261674E-01 | lm-global-supervised-tokens: 1.236763E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.136837E-02 | lm-token-buffer-mib: 5.819728E+01 | global_load_balancing_loss: 1.001289E+00 | loss scale: 1.0 | grad norm: 0.161 | params norm: 11296.401 | params rms: 0.061166 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 5:33:54.437225 | finish at 2026-10-10 17:57:01 + [2026-10-09 12:24:32] iteration 5594/ 6855 | consumed samples: 716032 | elapsed time per iteration (ms): 85995.4 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.64% | FLOPs estimator: workload-aware | logical tokens/s: 195094.1 | valid tokens/s: 195094.1 | physical tokens/s: 195506.0 | loss tokens/s: 149017.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.467855E-01 | lm-global-supervised-tokens: 1.281478E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.105566E-02 | lm-token-buffer-mib: 5.816188E+01 | global_load_balancing_loss: 9.998522E-01 | loss scale: 1.0 | grad norm: 0.157 | params norm: 11296.352 | params rms: 0.061166 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 6:07:20.236619 | finish at 2026-10-10 18:31:53 + [2026-10-09 12:25:59] iteration 5595/ 6855 | consumed samples: 716160 | elapsed time per iteration (ms): 86346.1 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.54% | FLOPs estimator: workload-aware | logical tokens/s: 194300.3 | valid tokens/s: 194300.3 | physical tokens/s: 194672.9 | loss tokens/s: 144913.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.273525E-01 | lm-global-supervised-tokens: 1.251270E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.284922E-02 | lm-token-buffer-mib: 5.807877E+01 | global_load_balancing_loss: 1.000603E+00 | loss scale: 1.0 | grad norm: 0.158 | params norm: 11296.304 | params rms: 0.061166 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 6:13:16.107745 | finish at 2026-10-10 18:39:15 + [2026-10-09 12:27:23] iteration 5596/ 6855 | consumed samples: 716288 | elapsed time per iteration (ms): 84654.2 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.75% | FLOPs estimator: workload-aware | logical tokens/s: 198185.3 | valid tokens/s: 198185.3 | physical tokens/s: 198681.1 | loss tokens/s: 150762.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.695249E-01 | lm-global-supervised-tokens: 1.276268E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.294222E-02 | lm-token-buffer-mib: 5.823741E+01 | global_load_balancing_loss: 1.001241E+00 | loss scale: 1.0 | grad norm: 0.141 | params norm: 11296.255 | params rms: 0.061166 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 5:36:19.621688 | finish at 2026-10-10 18:03:43 + [2026-10-09 12:28:49] iteration 5597/ 6855 | consumed samples: 716416 | elapsed time per iteration (ms): 85968.0 | throughput per GPU (TFLOP/s/GPU): 43.1 | MFU 13.83% | FLOPs estimator: workload-aware | logical tokens/s: 195155.9 | valid tokens/s: 195155.9 | physical tokens/s: 195564.2 | loss tokens/s: 145150.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.420295E-01 | lm-global-supervised-tokens: 1.247833E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.148412E-02 | lm-token-buffer-mib: 5.832416E+01 | global_load_balancing_loss: 9.992089E-01 | loss scale: 1.0 | grad norm: 0.044 | params norm: 11296.205 | params rms: 0.061165 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 6:02:27.776611 | finish at 2026-10-10 18:31:17 + [2026-10-09 12:30:13] iteration 5598/ 6855 | consumed samples: 716544 | elapsed time per iteration (ms): 83984.4 | throughput per GPU (TFLOP/s/GPU): 43.5 | MFU 13.95% | FLOPs estimator: workload-aware | logical tokens/s: 199765.5 | valid tokens/s: 199765.5 | physical tokens/s: 200082.6 | loss tokens/s: 148867.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 5.943672E-01 | lm-global-supervised-tokens: 1.250253E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.600560E-02 | lm-token-buffer-mib: 5.805617E+01 | global_load_balancing_loss: 9.991892E-01 | loss scale: 1.0 | grad norm: 0.124 | params norm: 11296.155 | params rms: 0.061165 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 5:19:28.448983 | finish at 2026-10-10 17:49:42 + [2026-10-09 12:31:38] iteration 5599/ 6855 | consumed samples: 716672 | elapsed time per iteration (ms): 84640.1 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.41% | FLOPs estimator: workload-aware | logical tokens/s: 198218.1 | valid tokens/s: 198218.1 | physical tokens/s: 198838.2 | loss tokens/s: 153890.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.220948E-01 | lm-global-supervised-tokens: 1.302529E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.218644E-02 | lm-token-buffer-mib: 5.806384E+01 | global_load_balancing_loss: 9.998659E-01 | loss scale: 1.0 | grad norm: 0.069 | params norm: 11296.105 | params rms: 0.061165 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 5:31:47.942844 | finish at 2026-10-10 18:03:26 + [2026-10-09 12:33:28] iteration 5600/ 6855 | consumed samples: 716800 | elapsed time per iteration (ms): 110243.4 | throughput per GPU (TFLOP/s/GPU): 33.7 | MFU 10.79% | FLOPs estimator: workload-aware | logical tokens/s: 152183.0 | valid tokens/s: 152183.0 | physical tokens/s: 152470.7 | loss tokens/s: 112537.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.531937E-01 | lm-global-supervised-tokens: 1.240654E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.158069E-02 | lm-token-buffer-mib: 5.816100E+01 | global_load_balancing_loss: 9.997395E-01 | loss scale: 1.0 | grad norm: 0.040 | params norm: 11296.055 | params rms: 0.061164 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 14:25:55.496744 | finish at 2026-10-11 02:59:24 +times across ranks (ms): + save-checkpoint: + rank 0: 28368.64 + rank 1: 28368.67 + rank 2: 28368.69 + rank 3: 28368.65 + rank 4: 28368.68 + rank 5: 28368.67 + rank 6: 28368.63 + rank 7: 28368.70 + rank 8: 28368.80 + rank 9: 28368.78 + rank 10: 28368.80 + rank 11: 28368.77 + rank 12: 28368.85 + rank 13: 28368.72 + rank 14: 28368.78 + rank 15: 28368.74 + rank 16: 28368.28 + rank 17: 28368.28 + rank 18: 28368.24 + rank 19: 28368.35 + rank 20: 28368.25 + rank 21: 28368.27 + rank 22: 28368.24 + rank 23: 28368.30 + rank 24: 28368.55 + rank 25: 28368.44 + rank 26: 28368.53 + rank 27: 28368.51 + rank 28: 28368.49 + rank 29: 28368.43 + rank 30: 28368.47 + rank 31: 28368.40 + rank 32: 28368.29 + rank 33: 28368.34 + rank 34: 28368.31 + rank 35: 28368.24 + rank 36: 28368.14 + rank 37: 28368.23 + rank 38: 28368.26 + rank 39: 28368.23 + rank 40: 28368.43 + rank 41: 28368.43 + rank 42: 28368.36 + rank 43: 28368.36 + rank 44: 28368.36 + rank 45: 28368.45 + rank 46: 28368.36 + rank 47: 28368.41 + rank 48: 28368.57 + rank 49: 28368.46 + rank 50: 28368.48 + rank 51: 28368.40 + rank 52: 28368.54 + rank 53: 28368.47 + rank 54: 28368.50 + rank 55: 28368.51 + rank 56: 28368.66 + rank 57: 28368.38 + rank 58: 28368.55 + rank 59: 28368.55 + rank 60: 28368.50 + rank 61: 28368.53 + rank 62: 28368.56 + rank 63: 28368.58 + [2026-10-09 12:35:24] iteration 5601/ 6855 | consumed samples: 716928 | elapsed time per iteration (ms): 86554.7 | throughput per GPU (TFLOP/s/GPU): 41.6 | MFU 13.34% | FLOPs estimator: workload-aware | logical tokens/s: 193833.4 | valid tokens/s: 193833.4 | physical tokens/s: 194387.7 | loss tokens/s: 145992.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.142228E-01 | lm-global-supervised-tokens: 1.263634E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.452099E+00 | lm-token-buffer-mib: 5.810505E+01 | global_load_balancing_loss: 1.000809E+00 | loss scale: 1.0 | grad norm: 0.251 | params norm: 11296.004 | params rms: 0.061164 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 6:08:59.603538 | finish at 2026-10-10 18:44:23 + [2026-10-09 12:36:55] iteration 5602/ 6855 | consumed samples: 717056 | elapsed time per iteration (ms): 91745.7 | throughput per GPU (TFLOP/s/GPU): 40.0 | MFU 12.82% | FLOPs estimator: workload-aware | logical tokens/s: 182866.4 | valid tokens/s: 182866.4 | physical tokens/s: 183260.2 | loss tokens/s: 136930.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.199659E-01 | lm-global-supervised-tokens: 1.256274E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.545312E-02 | lm-token-buffer-mib: 5.813866E+01 | global_load_balancing_loss: 1.001037E+00 | loss scale: 1.0 | grad norm: 0.124 | params norm: 11295.954 | params rms: 0.061164 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 7:55:57.399893 | finish at 2026-10-10 20:32:53 + [2026-10-09 12:38:22] iteration 5603/ 6855 | consumed samples: 717184 | elapsed time per iteration (ms): 87182.0 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.79% | FLOPs estimator: workload-aware | logical tokens/s: 192438.8 | valid tokens/s: 192438.8 | physical tokens/s: 192874.2 | loss tokens/s: 130977.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.182826E-01 | lm-global-supervised-tokens: 1.141892E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.246949E-02 | lm-token-buffer-mib: 5.805324E+01 | global_load_balancing_loss: 1.001863E+00 | loss scale: 1.0 | grad norm: 0.150 | params norm: 11295.904 | params rms: 0.061164 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 6:19:11.901513 | finish at 2026-10-10 18:57:34 + [2026-10-09 12:39:47] iteration 5604/ 6855 | consumed samples: 717312 | elapsed time per iteration (ms): 84161.3 | throughput per GPU (TFLOP/s/GPU): 43.6 | MFU 13.97% | FLOPs estimator: workload-aware | logical tokens/s: 199345.5 | valid tokens/s: 199345.5 | physical tokens/s: 199782.5 | loss tokens/s: 147553.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.650389E-01 | lm-global-supervised-tokens: 1.241832E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.586681E-02 | lm-token-buffer-mib: 5.818568E+01 | global_load_balancing_loss: 1.001318E+00 | loss scale: 1.0 | grad norm: 0.176 | params norm: 11295.853 | params rms: 0.061163 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 5:14:45.759088 | finish at 2026-10-10 17:54:32 + [2026-10-09 12:41:14] iteration 5605/ 6855 | consumed samples: 717440 | elapsed time per iteration (ms): 87127.5 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.50% | FLOPs estimator: workload-aware | logical tokens/s: 192558.1 | valid tokens/s: 192558.1 | physical tokens/s: 193177.3 | loss tokens/s: 143002.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.555403E-01 | lm-global-supervised-tokens: 1.245942E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.196687E-02 | lm-token-buffer-mib: 5.815183E+01 | global_load_balancing_loss: 9.988136E-01 | loss scale: 1.0 | grad norm: 0.158 | params norm: 11295.800 | params rms: 0.061163 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 6:15:09.382224 | finish at 2026-10-10 18:56:23 + [2026-10-09 12:42:42] iteration 5606/ 6855 | consumed samples: 717568 | elapsed time per iteration (ms): 87778.7 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.63% | FLOPs estimator: workload-aware | logical tokens/s: 191130.4 | valid tokens/s: 191130.4 | physical tokens/s: 191540.2 | loss tokens/s: 140881.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.035009E-01 | lm-global-supervised-tokens: 1.236638E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.129594E-02 | lm-token-buffer-mib: 5.810663E+01 | global_load_balancing_loss: 9.999339E-01 | loss scale: 1.0 | grad norm: 0.093 | params norm: 11295.748 | params rms: 0.061163 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 6:27:15.533907 | finish at 2026-10-10 19:09:57 + [2026-10-09 12:44:09] iteration 5607/ 6855 | consumed samples: 717696 | elapsed time per iteration (ms): 87749.9 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.53% | FLOPs estimator: workload-aware | logical tokens/s: 191192.5 | valid tokens/s: 191192.5 | physical tokens/s: 191680.4 | loss tokens/s: 143554.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.717911E-01 | lm-global-supervised-tokens: 1.259685E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.488072E-02 | lm-token-buffer-mib: 5.812772E+01 | global_load_balancing_loss: 9.986529E-01 | loss scale: 1.0 | grad norm: 0.159 | params norm: 11295.695 | params rms: 0.061163 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 6:25:11.868484 | finish at 2026-10-10 19:09:21 + [2026-10-09 12:45:38] iteration 5608/ 6855 | consumed samples: 717824 | elapsed time per iteration (ms): 88475.3 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.68% | FLOPs estimator: workload-aware | logical tokens/s: 189625.6 | valid tokens/s: 189625.6 | physical tokens/s: 190013.8 | loss tokens/s: 141264.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.731697E-01 | lm-global-supervised-tokens: 1.249843E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.641340E-02 | lm-token-buffer-mib: 5.810221E+01 | global_load_balancing_loss: 9.995410E-01 | loss scale: 1.0 | grad norm: 0.038 | params norm: 11295.643 | params rms: 0.061162 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 6:38:48.672137 | finish at 2026-10-10 19:24:26 + [2026-10-09 12:47:08] iteration 5609/ 6855 | consumed samples: 717952 | elapsed time per iteration (ms): 90578.6 | throughput per GPU (TFLOP/s/GPU): 41.1 | MFU 13.18% | FLOPs estimator: workload-aware | logical tokens/s: 185222.8 | valid tokens/s: 185222.8 | physical tokens/s: 185848.1 | loss tokens/s: 137939.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.592786E-01 | lm-global-supervised-tokens: 1.249436E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.224910E-02 | lm-token-buffer-mib: 5.809012E+01 | global_load_balancing_loss: 9.999208E-01 | loss scale: 1.0 | grad norm: 0.165 | params norm: 11295.591 | params rms: 0.061162 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 7:21:00.914421 | finish at 2026-10-10 20:08:09 + [2026-10-09 12:48:59] iteration 5610/ 6855 | consumed samples: 718080 | elapsed time per iteration (ms): 110377.4 | throughput per GPU (TFLOP/s/GPU): 33.6 | MFU 10.76% | FLOPs estimator: workload-aware | logical tokens/s: 151998.0 | valid tokens/s: 151998.0 | physical tokens/s: 152320.1 | loss tokens/s: 108888.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.409016E-01 | lm-global-supervised-tokens: 1.201887E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.235631E-02 | lm-token-buffer-mib: 5.806968E+01 | global_load_balancing_loss: 1.000868E+00 | loss scale: 1.0 | grad norm: 0.066 | params norm: 11295.542 | params rms: 0.061162 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 14:10:19.806207 | finish at 2026-10-11 02:59:19 + [2026-10-09 12:50:29] iteration 5611/ 6855 | consumed samples: 718208 | elapsed time per iteration (ms): 89816.3 | throughput per GPU (TFLOP/s/GPU): 41.4 | MFU 13.28% | FLOPs estimator: workload-aware | logical tokens/s: 186794.6 | valid tokens/s: 186794.6 | physical tokens/s: 187074.0 | loss tokens/s: 138969.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.698011E-01 | lm-global-supervised-tokens: 1.248170E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.585608E-02 | lm-token-buffer-mib: 5.807536E+01 | global_load_balancing_loss: 1.001065E+00 | loss scale: 1.0 | grad norm: 0.098 | params norm: 11295.492 | params rms: 0.061161 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 7:02:11.531964 | finish at 2026-10-10 19:52:40 + [2026-10-09 12:51:58] iteration 5612/ 6855 | consumed samples: 718336 | elapsed time per iteration (ms): 89941.5 | throughput per GPU (TFLOP/s/GPU): 40.9 | MFU 13.10% | FLOPs estimator: workload-aware | logical tokens/s: 186533.6 | valid tokens/s: 186533.6 | physical tokens/s: 186926.7 | loss tokens/s: 139611.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.919912E-01 | lm-global-supervised-tokens: 1.255687E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.332007E-02 | lm-token-buffer-mib: 5.807995E+01 | global_load_balancing_loss: 1.000860E+00 | loss scale: 1.0 | grad norm: 0.037 | params norm: 11295.441 | params rms: 0.061161 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 7:03:17.229610 | finish at 2026-10-10 19:55:16 + [2026-10-09 12:53:27] iteration 5613/ 6855 | consumed samples: 718464 | elapsed time per iteration (ms): 88245.1 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.50% | FLOPs estimator: workload-aware | logical tokens/s: 190120.0 | valid tokens/s: 190120.0 | physical tokens/s: 190543.7 | loss tokens/s: 144654.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.704097E-01 | lm-global-supervised-tokens: 1.276502E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.418931E-02 | lm-token-buffer-mib: 5.817796E+01 | global_load_balancing_loss: 1.000742E+00 | loss scale: 1.0 | grad norm: 0.153 | params norm: 11295.390 | params rms: 0.061161 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 6:26:40.375731 | finish at 2026-10-10 19:20:07 + [2026-10-09 12:54:52] iteration 5614/ 6855 | consumed samples: 718592 | elapsed time per iteration (ms): 85198.1 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.63% | FLOPs estimator: workload-aware | logical tokens/s: 196919.4 | valid tokens/s: 196919.4 | physical tokens/s: 197320.4 | loss tokens/s: 146211.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.221619E-01 | lm-global-supervised-tokens: 1.245690E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.478230E-02 | lm-token-buffer-mib: 5.829741E+01 | global_load_balancing_loss: 1.001021E+00 | loss scale: 1.0 | grad norm: 0.088 | params norm: 11295.341 | params rms: 0.061161 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 5:22:10.781261 | finish at 2026-10-10 18:17:03 + [2026-10-09 12:56:19] iteration 5615/ 6855 | consumed samples: 718720 | elapsed time per iteration (ms): 86807.0 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.49% | FLOPs estimator: workload-aware | logical tokens/s: 193269.6 | valid tokens/s: 193269.6 | physical tokens/s: 193770.6 | loss tokens/s: 149252.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.415122E-01 | lm-global-supervised-tokens: 1.295614E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.339640E-02 | lm-token-buffer-mib: 5.806635E+01 | global_load_balancing_loss: 1.001040E+00 | loss scale: 1.0 | grad norm: 0.195 | params norm: 11295.290 | params rms: 0.061160 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 5:54:00.658913 | finish at 2026-10-10 18:50:19 + [2026-10-09 12:57:45] iteration 5616/ 6855 | consumed samples: 718848 | elapsed time per iteration (ms): 86145.6 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.80% | FLOPs estimator: workload-aware | logical tokens/s: 194753.7 | valid tokens/s: 194753.7 | physical tokens/s: 195130.7 | loss tokens/s: 147778.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.489484E-01 | lm-global-supervised-tokens: 1.273043E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.001403E-02 | lm-token-buffer-mib: 5.817525E+01 | global_load_balancing_loss: 9.989589E-01 | loss scale: 1.0 | grad norm: 0.120 | params norm: 11295.240 | params rms: 0.061160 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 5:38:54.382548 | finish at 2026-10-10 18:36:39 + [2026-10-09 12:59:10] iteration 5617/ 6855 | consumed samples: 718976 | elapsed time per iteration (ms): 85061.5 | throughput per GPU (TFLOP/s/GPU): 43.1 | MFU 13.81% | FLOPs estimator: workload-aware | logical tokens/s: 197236.2 | valid tokens/s: 197236.2 | physical tokens/s: 197939.6 | loss tokens/s: 145657.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.000203E-01 | lm-global-supervised-tokens: 1.238985E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.452190E-02 | lm-token-buffer-mib: 5.808256E+01 | global_load_balancing_loss: 9.990503E-01 | loss scale: 1.0 | grad norm: 0.159 | params norm: 11295.190 | params rms: 0.061160 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 5:15:06.116133 | finish at 2026-10-10 18:14:16 + [2026-10-09 13:00:37] iteration 5618/ 6855 | consumed samples: 719104 | elapsed time per iteration (ms): 86698.7 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.49% | FLOPs estimator: workload-aware | logical tokens/s: 193511.0 | valid tokens/s: 193511.0 | physical tokens/s: 194011.3 | loss tokens/s: 145225.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.800140E-01 | lm-global-supervised-tokens: 1.259083E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.244370E-02 | lm-token-buffer-mib: 5.825491E+01 | global_load_balancing_loss: 1.000125E+00 | loss scale: 1.0 | grad norm: 0.057 | params norm: 11295.142 | params rms: 0.061160 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 5:47:26.283287 | finish at 2026-10-10 18:48:03 + [2026-10-09 13:02:06] iteration 5619/ 6855 | consumed samples: 719232 | elapsed time per iteration (ms): 89132.2 | throughput per GPU (TFLOP/s/GPU): 41.3 | MFU 13.23% | FLOPs estimator: workload-aware | logical tokens/s: 188228.3 | valid tokens/s: 188228.3 | physical tokens/s: 188781.2 | loss tokens/s: 137510.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.574343E-01 | lm-global-supervised-tokens: 1.225657E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.146511E-02 | lm-token-buffer-mib: 5.816174E+01 | global_load_balancing_loss: 1.000482E+00 | loss scale: 1.0 | grad norm: 0.191 | params norm: 11295.094 | params rms: 0.061159 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 6:36:07.382701 | finish at 2026-10-10 19:38:13 + [2026-10-09 13:03:52] iteration 5620/ 6855 | consumed samples: 719360 | elapsed time per iteration (ms): 106351.4 | throughput per GPU (TFLOP/s/GPU): 35.1 | MFU 11.24% | FLOPs estimator: workload-aware | logical tokens/s: 157752.6 | valid tokens/s: 157752.6 | physical tokens/s: 158017.4 | loss tokens/s: 113097.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 5.854108E-01 | lm-global-supervised-tokens: 1.202808E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.130696E-02 | lm-token-buffer-mib: 5.810777E+01 | global_load_balancing_loss: 1.000126E+00 | loss scale: 1.0 | grad norm: 0.104 | params norm: 11295.046 | params rms: 0.061159 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 12:29:04.000958 | finish at 2026-10-11 01:32:56 + [2026-10-09 13:05:14] iteration 5621/ 6855 | consumed samples: 719488 | elapsed time per iteration (ms): 82004.2 | throughput per GPU (TFLOP/s/GPU): 44.2 | MFU 14.17% | FLOPs estimator: workload-aware | logical tokens/s: 204589.3 | valid tokens/s: 204589.3 | physical tokens/s: 204999.4 | loss tokens/s: 149215.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.308452E-01 | lm-global-supervised-tokens: 1.223628E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.174565E-02 | lm-token-buffer-mib: 5.817505E+01 | global_load_balancing_loss: 9.989778E-01 | loss scale: 1.0 | grad norm: 0.126 | params norm: 11294.997 | params rms: 0.061159 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 4:06:33.222790 | finish at 2026-10-10 17:11:47 + [2026-10-09 13:06:39] iteration 5622/ 6855 | consumed samples: 719616 | elapsed time per iteration (ms): 84725.2 | throughput per GPU (TFLOP/s/GPU): 43.1 | MFU 13.80% | FLOPs estimator: workload-aware | logical tokens/s: 198018.7 | valid tokens/s: 198018.7 | physical tokens/s: 198541.3 | loss tokens/s: 151672.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.233502E-01 | lm-global-supervised-tokens: 1.285047E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.434368E-02 | lm-token-buffer-mib: 5.809698E+01 | global_load_balancing_loss: 1.000677E+00 | loss scale: 1.0 | grad norm: 0.152 | params norm: 11294.946 | params rms: 0.061158 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 5:01:06.209739 | finish at 2026-10-10 18:07:45 + [2026-10-09 13:08:05] iteration 5623/ 6855 | consumed samples: 719744 | elapsed time per iteration (ms): 86462.2 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.67% | FLOPs estimator: workload-aware | logical tokens/s: 194040.4 | valid tokens/s: 194040.4 | physical tokens/s: 194501.4 | loss tokens/s: 143147.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.248088E-01 | lm-global-supervised-tokens: 1.237686E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.332608E-02 | lm-token-buffer-mib: 5.811190E+01 | global_load_balancing_loss: 1.000996E+00 | loss scale: 1.0 | grad norm: 0.050 | params norm: 11294.897 | params rms: 0.061158 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 5:35:21.375969 | finish at 2026-10-10 18:43:27 + [2026-10-09 13:09:34] iteration 5624/ 6855 | consumed samples: 719872 | elapsed time per iteration (ms): 88858.0 | throughput per GPU (TFLOP/s/GPU): 41.3 | MFU 13.25% | FLOPs estimator: workload-aware | logical tokens/s: 188808.7 | valid tokens/s: 188808.7 | physical tokens/s: 189136.9 | loss tokens/s: 139318.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.758528E-01 | lm-global-supervised-tokens: 1.237954E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.125776E-02 | lm-token-buffer-mib: 5.809949E+01 | global_load_balancing_loss: 1.001617E+00 | loss scale: 1.0 | grad norm: 0.136 | params norm: 11294.848 | params rms: 0.061158 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 6:23:04.169580 | finish at 2026-10-10 19:32:38 + [2026-10-09 13:11:00] iteration 5625/ 6855 | consumed samples: 720000 | elapsed time per iteration (ms): 85862.8 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.78% | FLOPs estimator: workload-aware | logical tokens/s: 195395.3 | valid tokens/s: 195395.3 | physical tokens/s: 195937.6 | loss tokens/s: 137391.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.394813E-01 | lm-global-supervised-tokens: 1.179684E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.586531E-02 | lm-token-buffer-mib: 5.825700E+01 | global_load_balancing_loss: 1.001599E+00 | loss scale: 1.0 | grad norm: 0.054 | params norm: 11294.798 | params rms: 0.061158 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 5:20:11.287551 | finish at 2026-10-10 18:31:11 + [2026-10-09 13:12:30] iteration 5626/ 6855 | consumed samples: 720128 | elapsed time per iteration (ms): 90371.4 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.49% | FLOPs estimator: workload-aware | logical tokens/s: 185646.9 | valid tokens/s: 185646.9 | physical tokens/s: 185925.9 | loss tokens/s: 133392.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.114315E-01 | lm-global-supervised-tokens: 1.205486E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.521015E-02 | lm-token-buffer-mib: 5.812593E+01 | global_load_balancing_loss: 9.997569E-01 | loss scale: 1.0 | grad norm: 0.094 | params norm: 11294.747 | params rms: 0.061157 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 6:51:06.449573 | finish at 2026-10-10 20:03:37 + [2026-10-09 13:13:58] iteration 5627/ 6855 | consumed samples: 720256 | elapsed time per iteration (ms): 87953.4 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.60% | FLOPs estimator: workload-aware | logical tokens/s: 190750.7 | valid tokens/s: 190750.7 | physical tokens/s: 191211.7 | loss tokens/s: 143594.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.659194E-01 | lm-global-supervised-tokens: 1.262963E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.175154E-02 | lm-token-buffer-mib: 5.803690E+01 | global_load_balancing_loss: 9.979559E-01 | loss scale: 1.0 | grad norm: 0.049 | params norm: 11294.694 | params rms: 0.061157 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 6:00:06.826016 | finish at 2026-10-10 19:14:05 + [2026-10-09 13:15:23] iteration 5628/ 6855 | consumed samples: 720384 | elapsed time per iteration (ms): 85052.6 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.74% | FLOPs estimator: workload-aware | logical tokens/s: 197257.0 | valid tokens/s: 197257.0 | physical tokens/s: 197661.2 | loss tokens/s: 148705.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.200449E-01 | lm-global-supervised-tokens: 1.264782E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.242320E-02 | lm-token-buffer-mib: 5.809873E+01 | global_load_balancing_loss: 9.993293E-01 | loss scale: 1.0 | grad norm: 0.065 | params norm: 11294.642 | params rms: 0.061157 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 4:59:19.535698 | finish at 2026-10-10 18:14:43 + [2026-10-09 13:16:51] iteration 5629/ 6855 | consumed samples: 720512 | elapsed time per iteration (ms): 87725.1 | throughput per GPU (TFLOP/s/GPU): 43.6 | MFU 13.96% | FLOPs estimator: workload-aware | logical tokens/s: 191247.7 | valid tokens/s: 191247.7 | physical tokens/s: 191469.8 | loss tokens/s: 131221.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.735418E-01 | lm-global-supervised-tokens: 1.151137E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.256973E-02 | lm-token-buffer-mib: 5.814483E+01 | global_load_balancing_loss: 9.995325E-01 | loss scale: 1.0 | grad norm: 0.043 | params norm: 11294.590 | params rms: 0.061157 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 5:52:30.929974 | finish at 2026-10-10 19:09:22 + [2026-10-09 13:18:43] iteration 5630/ 6855 | consumed samples: 720640 | elapsed time per iteration (ms): 111428.5 | throughput per GPU (TFLOP/s/GPU): 33.0 | MFU 10.59% | FLOPs estimator: workload-aware | logical tokens/s: 150564.7 | valid tokens/s: 150564.7 | physical tokens/s: 150952.3 | loss tokens/s: 110420.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.199952E-01 | lm-global-supervised-tokens: 1.230393E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.867289E-02 | lm-token-buffer-mib: 5.810009E+01 | global_load_balancing_loss: 1.001751E+00 | loss scale: 1.0 | grad norm: 0.139 | params norm: 11294.538 | params rms: 0.061156 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 13:54:59.858391 | finish at 2026-10-11 03:13:42 + [2026-10-09 13:20:07] iteration 5631/ 6855 | consumed samples: 720768 | elapsed time per iteration (ms): 84732.1 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.77% | FLOPs estimator: workload-aware | logical tokens/s: 198002.4 | valid tokens/s: 198002.4 | physical tokens/s: 198709.5 | loss tokens/s: 145604.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.343387E-01 | lm-global-supervised-tokens: 1.233741E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.537089E-02 | lm-token-buffer-mib: 5.818577E+01 | global_load_balancing_loss: 1.001373E+00 | loss scale: 1.0 | grad norm: 0.081 | params norm: 11294.488 | params rms: 0.061156 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 4:48:32.105879 | finish at 2026-10-10 18:08:39 + [2026-10-09 13:21:38] iteration 5632/ 6855 | consumed samples: 720896 | elapsed time per iteration (ms): 90253.8 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.60% | FLOPs estimator: workload-aware | logical tokens/s: 185889.0 | valid tokens/s: 185889.0 | physical tokens/s: 186168.6 | loss tokens/s: 134053.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.024691E-01 | lm-global-supervised-tokens: 1.209883E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.698677E-02 | lm-token-buffer-mib: 5.806091E+01 | global_load_balancing_loss: 9.997977E-01 | loss scale: 1.0 | grad norm: 0.078 | params norm: 11294.437 | params rms: 0.061156 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 6:39:40.355016 | finish at 2026-10-10 20:01:18 + [2026-10-09 13:23:01] iteration 5633/ 6855 | consumed samples: 721024 | elapsed time per iteration (ms): 83535.0 | throughput per GPU (TFLOP/s/GPU): 43.2 | MFU 13.86% | FLOPs estimator: workload-aware | logical tokens/s: 200840.1 | valid tokens/s: 200840.1 | physical tokens/s: 201342.6 | loss tokens/s: 152588.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.724765E-01 | lm-global-supervised-tokens: 1.274651E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.152058E-02 | lm-token-buffer-mib: 5.821805E+01 | global_load_balancing_loss: 1.000430E+00 | loss scale: 1.0 | grad norm: 0.063 | params norm: 11294.386 | params rms: 0.061155 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 4:21:19.734852 | finish at 2026-10-10 17:44:21 + [2026-10-09 13:24:27] iteration 5634/ 6855 | consumed samples: 721152 | elapsed time per iteration (ms): 85664.9 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.77% | FLOPs estimator: workload-aware | logical tokens/s: 195846.9 | valid tokens/s: 195846.9 | physical tokens/s: 196282.7 | loss tokens/s: 141258.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.357929E-01 | lm-global-supervised-tokens: 1.210089E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.171481E-02 | lm-token-buffer-mib: 5.808183E+01 | global_load_balancing_loss: 1.001305E+00 | loss scale: 1.0 | grad norm: 0.070 | params norm: 11294.334 | params rms: 0.061155 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 5:03:16.797275 | finish at 2026-10-10 18:27:44 + [2026-10-09 13:25:52] iteration 5635/ 6855 | consumed samples: 721280 | elapsed time per iteration (ms): 85429.3 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.64% | FLOPs estimator: workload-aware | logical tokens/s: 196386.7 | valid tokens/s: 196386.7 | physical tokens/s: 196896.0 | loss tokens/s: 149678.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.754562E-01 | lm-global-supervised-tokens: 1.278691E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.231249E-02 | lm-token-buffer-mib: 5.809542E+01 | global_load_balancing_loss: 9.989045E-01 | loss scale: 1.0 | grad norm: 0.115 | params norm: 11294.283 | params rms: 0.061155 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 4:57:03.724561 | finish at 2026-10-10 18:22:56 + [2026-10-09 13:27:16] iteration 5636/ 6855 | consumed samples: 721408 | elapsed time per iteration (ms): 83772.0 | throughput per GPU (TFLOP/s/GPU): 43.7 | MFU 14.00% | FLOPs estimator: workload-aware | logical tokens/s: 200271.9 | valid tokens/s: 200271.9 | physical tokens/s: 200607.5 | loss tokens/s: 145931.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.696010E-01 | lm-global-supervised-tokens: 1.222498E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.476756E-02 | lm-token-buffer-mib: 5.805614E+01 | global_load_balancing_loss: 1.000088E+00 | loss scale: 1.0 | grad norm: 0.056 | params norm: 11294.232 | params rms: 0.061155 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 4:21:58.117789 | finish at 2026-10-10 17:49:14 + [2026-10-09 13:28:41] iteration 5637/ 6855 | consumed samples: 721536 | elapsed time per iteration (ms): 84873.5 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.49% | FLOPs estimator: workload-aware | logical tokens/s: 197673.2 | valid tokens/s: 197673.2 | physical tokens/s: 198378.8 | loss tokens/s: 146890.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.286337E-01 | lm-global-supervised-tokens: 1.246712E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.161949E-02 | lm-token-buffer-mib: 5.819798E+01 | global_load_balancing_loss: 9.996992E-01 | loss scale: 1.0 | grad norm: 0.112 | params norm: 11294.180 | params rms: 0.061154 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 4:42:55.930102 | finish at 2026-10-10 18:11:37 + [2026-10-09 13:30:06] iteration 5638/ 6855 | consumed samples: 721664 | elapsed time per iteration (ms): 84832.9 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.74% | FLOPs estimator: workload-aware | logical tokens/s: 197767.1 | valid tokens/s: 197767.1 | physical tokens/s: 198335.7 | loss tokens/s: 150669.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.139929E-01 | lm-global-supervised-tokens: 1.278171E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.455039E-02 | lm-token-buffer-mib: 5.830068E+01 | global_load_balancing_loss: 9.993553E-01 | loss scale: 1.0 | grad norm: 0.078 | params norm: 11294.129 | params rms: 0.061154 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 4:40:41.639648 | finish at 2026-10-10 18:10:47 + [2026-10-09 13:31:34] iteration 5639/ 6855 | consumed samples: 721792 | elapsed time per iteration (ms): 88246.7 | throughput per GPU (TFLOP/s/GPU): 41.1 | MFU 13.19% | FLOPs estimator: workload-aware | logical tokens/s: 190116.8 | valid tokens/s: 190116.8 | physical tokens/s: 190603.6 | loss tokens/s: 137689.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.347453E-01 | lm-global-supervised-tokens: 1.215063E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.428514E-02 | lm-token-buffer-mib: 5.812817E+01 | global_load_balancing_loss: 1.000549E+00 | loss scale: 1.0 | grad norm: 0.060 | params norm: 11294.079 | params rms: 0.061154 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 5:48:28.011902 | finish at 2026-10-10 19:20:02 + [2026-10-09 13:33:23] iteration 5640/ 6855 | consumed samples: 721920 | elapsed time per iteration (ms): 109575.0 | throughput per GPU (TFLOP/s/GPU): 34.1 | MFU 10.92% | FLOPs estimator: workload-aware | logical tokens/s: 153111.3 | valid tokens/s: 153111.3 | physical tokens/s: 153543.2 | loss tokens/s: 115369.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.294072E-01 | lm-global-supervised-tokens: 1.264158E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.163798E-02 | lm-token-buffer-mib: 5.816606E+01 | global_load_balancing_loss: 1.001988E+00 | loss scale: 1.0 | grad norm: 0.107 | params norm: 11294.029 | params rms: 0.061153 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 12:58:53.620713 | finish at 2026-10-11 02:32:17 +times across ranks (ms): + save-checkpoint: + rank 0: 24718.47 + rank 1: 24718.53 + rank 2: 24718.60 + rank 3: 24718.49 + rank 4: 24718.47 + rank 5: 24718.48 + rank 6: 24718.45 + rank 7: 24718.49 + rank 8: 24718.49 + rank 9: 24718.50 + rank 10: 24718.48 + rank 11: 24718.47 + rank 12: 24718.48 + rank 13: 24718.47 + rank 14: 24718.47 + rank 15: 24718.47 + rank 16: 24718.74 + rank 17: 24718.75 + rank 18: 24718.74 + rank 19: 24718.72 + rank 20: 24718.72 + rank 21: 24718.76 + rank 22: 24718.72 + rank 23: 24718.74 + rank 24: 24717.99 + rank 25: 24717.99 + rank 26: 24718.01 + rank 27: 24718.00 + rank 28: 24717.98 + rank 29: 24718.04 + rank 30: 24717.98 + rank 31: 24717.99 + rank 32: 24718.95 + rank 33: 24719.03 + rank 34: 24718.95 + rank 35: 24718.92 + rank 36: 24718.93 + rank 37: 24718.90 + rank 38: 24718.91 + rank 39: 24718.89 + rank 40: 24718.73 + rank 41: 24718.73 + rank 42: 24718.72 + rank 43: 24718.69 + rank 44: 24718.70 + rank 45: 24718.71 + rank 46: 24718.72 + rank 47: 24718.70 + rank 48: 24717.91 + rank 49: 24717.93 + rank 50: 24717.91 + rank 51: 24717.89 + rank 52: 24717.90 + rank 53: 24718.07 + rank 54: 24717.91 + rank 55: 24717.95 + rank 56: 24719.05 + rank 57: 24719.03 + rank 58: 24719.01 + rank 59: 24719.01 + rank 60: 24719.03 + rank 61: 24719.02 + rank 62: 24719.02 + rank 63: 24719.02 + [2026-10-09 13:35:15] iteration 5641/ 6855 | consumed samples: 722048 | elapsed time per iteration (ms): 86613.2 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.74% | FLOPs estimator: workload-aware | logical tokens/s: 193701.6 | valid tokens/s: 193701.6 | physical tokens/s: 194280.6 | loss tokens/s: 141753.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.575677E-01 | lm-global-supervised-tokens: 1.227774E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 3.224388E-01 | lm-token-buffer-mib: 5.820658E+01 | global_load_balancing_loss: 1.000478E+00 | loss scale: 1.0 | grad norm: 0.060 | params norm: 11293.978 | params rms: 0.061153 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 5:12:28.414897 | finish at 2026-10-10 18:47:44 + [2026-10-09 13:36:49] iteration 5642/ 6855 | consumed samples: 722176 | elapsed time per iteration (ms): 93746.2 | throughput per GPU (TFLOP/s/GPU): 39.8 | MFU 12.75% | FLOPs estimator: workload-aware | logical tokens/s: 178963.5 | valid tokens/s: 178963.5 | physical tokens/s: 179299.4 | loss tokens/s: 130749.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.450915E-01 | lm-global-supervised-tokens: 1.225723E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.325977E-02 | lm-token-buffer-mib: 5.809413E+01 | global_load_balancing_loss: 1.001382E+00 | loss scale: 1.0 | grad norm: 0.099 | params norm: 11293.926 | params rms: 0.061153 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 7:35:14.149668 | finish at 2026-10-10 21:12:03 + [2026-10-09 13:38:16] iteration 5643/ 6855 | consumed samples: 722304 | elapsed time per iteration (ms): 87081.6 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.56% | FLOPs estimator: workload-aware | logical tokens/s: 192659.8 | valid tokens/s: 192659.8 | physical tokens/s: 193182.0 | loss tokens/s: 141631.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.235817E-01 | lm-global-supervised-tokens: 1.233350E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.308060E-02 | lm-token-buffer-mib: 5.810834E+01 | global_load_balancing_loss: 1.000348E+00 | loss scale: 1.0 | grad norm: 0.055 | params norm: 11293.873 | params rms: 0.061153 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 5:19:02.878046 | finish at 2026-10-10 18:57:19 + [2026-10-09 13:39:44] iteration 5644/ 6855 | consumed samples: 722432 | elapsed time per iteration (ms): 88416.9 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.75% | FLOPs estimator: workload-aware | logical tokens/s: 189750.8 | valid tokens/s: 189750.8 | physical tokens/s: 190194.9 | loss tokens/s: 136625.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.102033E-01 | lm-global-supervised-tokens: 1.208000E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.156246E-02 | lm-token-buffer-mib: 5.809071E+01 | global_load_balancing_loss: 9.983181E-01 | loss scale: 1.0 | grad norm: 0.093 | params norm: 11293.823 | params rms: 0.061152 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 5:44:32.921526 | finish at 2026-10-10 19:24:17 + [2026-10-09 13:41:11] iteration 5645/ 6855 | consumed samples: 722560 | elapsed time per iteration (ms): 86362.4 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.66% | FLOPs estimator: workload-aware | logical tokens/s: 194264.2 | valid tokens/s: 194264.1 | physical tokens/s: 194617.2 | loss tokens/s: 144185.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.954989E-01 | lm-global-supervised-tokens: 1.245219E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.998022E-02 | lm-token-buffer-mib: 5.820714E+01 | global_load_balancing_loss: 9.984074E-01 | loss scale: 1.0 | grad norm: 0.057 | params norm: 11293.772 | params rms: 0.061152 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 5:01:38.554263 | finish at 2026-10-10 18:42:49 + [2026-10-09 13:42:39] iteration 5646/ 6855 | consumed samples: 722688 | elapsed time per iteration (ms): 87927.2 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.52% | FLOPs estimator: workload-aware | logical tokens/s: 190807.3 | valid tokens/s: 190807.3 | physical tokens/s: 191215.4 | loss tokens/s: 142759.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.560276E-01 | lm-global-supervised-tokens: 1.255243E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.682148E-02 | lm-token-buffer-mib: 5.814690E+01 | global_load_balancing_loss: 9.989172E-01 | loss scale: 1.0 | grad norm: 0.091 | params norm: 11293.720 | params rms: 0.061152 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 5:31:43.938776 | finish at 2026-10-10 19:14:23 + [2026-10-09 13:44:04] iteration 5647/ 6855 | consumed samples: 722816 | elapsed time per iteration (ms): 85386.7 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.69% | FLOPs estimator: workload-aware | logical tokens/s: 196484.4 | valid tokens/s: 196484.4 | physical tokens/s: 196924.8 | loss tokens/s: 148593.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.348856E-01 | lm-global-supervised-tokens: 1.268794E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.446879E-02 | lm-token-buffer-mib: 5.813418E+01 | global_load_balancing_loss: 1.000128E+00 | loss scale: 1.0 | grad norm: 0.104 | params norm: 11293.669 | params rms: 0.061152 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 4:39:07.180731 | finish at 2026-10-10 18:23:11 + [2026-10-09 13:45:33] iteration 5648/ 6855 | consumed samples: 722944 | elapsed time per iteration (ms): 88916.6 | throughput per GPU (TFLOP/s/GPU): 41.4 | MFU 13.27% | FLOPs estimator: workload-aware | logical tokens/s: 188684.8 | valid tokens/s: 188684.8 | physical tokens/s: 189096.9 | loss tokens/s: 138458.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.173568E-01 | lm-global-supervised-tokens: 1.231127E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.563432E-02 | lm-token-buffer-mib: 5.809116E+01 | global_load_balancing_loss: 1.001531E+00 | loss scale: 1.0 | grad norm: 0.123 | params norm: 11293.618 | params rms: 0.061151 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 5:48:42.337626 | finish at 2026-10-10 19:34:15 + [2026-10-09 13:46:58] iteration 5649/ 6855 | consumed samples: 723072 | elapsed time per iteration (ms): 84639.1 | throughput per GPU (TFLOP/s/GPU): 42.8 | MFU 13.71% | FLOPs estimator: workload-aware | logical tokens/s: 198220.5 | valid tokens/s: 198220.5 | physical tokens/s: 198727.3 | loss tokens/s: 144633.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.528403E-01 | lm-global-supervised-tokens: 1.224164E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.307413E-02 | lm-token-buffer-mib: 5.807378E+01 | global_load_balancing_loss: 1.000785E+00 | loss scale: 1.0 | grad norm: 0.171 | params norm: 11293.569 | params rms: 0.061151 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 4:21:14.740889 | finish at 2026-10-10 18:08:12 + [2026-10-09 13:48:47] iteration 5650/ 6855 | consumed samples: 723200 | elapsed time per iteration (ms): 108977.4 | throughput per GPU (TFLOP/s/GPU): 33.2 | MFU 10.65% | FLOPs estimator: workload-aware | logical tokens/s: 153950.7 | valid tokens/s: 153950.7 | physical tokens/s: 154291.2 | loss tokens/s: 112751.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.293979E-01 | lm-global-supervised-tokens: 1.228738E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.306919E-02 | lm-token-buffer-mib: 5.817858E+01 | global_load_balancing_loss: 1.001140E+00 | loss scale: 1.0 | grad norm: 0.515 | params norm: 11293.519 | params rms: 0.061151 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 12:28:37.787763 | finish at 2026-10-11 02:17:24 + [2026-10-09 13:50:14] iteration 5651/ 6855 | consumed samples: 723328 | elapsed time per iteration (ms): 87169.6 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.60% | FLOPs estimator: workload-aware | logical tokens/s: 192466.0 | valid tokens/s: 192466.0 | physical tokens/s: 192870.4 | loss tokens/s: 146427.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.957416E-01 | lm-global-supervised-tokens: 1.276398E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.143597E-02 | lm-token-buffer-mib: 5.810718E+01 | global_load_balancing_loss: 1.000090E+00 | loss scale: 1.0 | grad norm: 0.209 | params norm: 11293.464 | params rms: 0.061150 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 5:09:12.162817 | finish at 2026-10-10 18:59:26 + [2026-10-09 13:51:44] iteration 5652/ 6855 | consumed samples: 723456 | elapsed time per iteration (ms): 90380.1 | throughput per GPU (TFLOP/s/GPU): 41.1 | MFU 13.16% | FLOPs estimator: workload-aware | logical tokens/s: 185629.1 | valid tokens/s: 185629.1 | physical tokens/s: 185954.4 | loss tokens/s: 133529.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.487916E-01 | lm-global-supervised-tokens: 1.206838E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.454611E-02 | lm-token-buffer-mib: 5.805418E+01 | global_load_balancing_loss: 1.001439E+00 | loss scale: 1.0 | grad norm: 0.199 | params norm: 11293.418 | params rms: 0.061150 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 6:12:07.264616 | finish at 2026-10-10 20:03:52 + [2026-10-09 13:53:10] iteration 5653/ 6855 | consumed samples: 723584 | elapsed time per iteration (ms): 86171.5 | throughput per GPU (TFLOP/s/GPU): 42.8 | MFU 13.71% | FLOPs estimator: workload-aware | logical tokens/s: 194695.3 | valid tokens/s: 194695.3 | physical tokens/s: 195115.5 | loss tokens/s: 139693.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.357936E-01 | lm-global-supervised-tokens: 1.203758E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.794471E-02 | lm-token-buffer-mib: 5.812580E+01 | global_load_balancing_loss: 9.994546E-01 | loss scale: 1.0 | grad norm: 0.185 | params norm: 11293.375 | params rms: 0.061150 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 4:46:18.118602 | finish at 2026-10-10 18:39:29 + [2026-10-09 13:54:40] iteration 5654/ 6855 | consumed samples: 723712 | elapsed time per iteration (ms): 90001.9 | throughput per GPU (TFLOP/s/GPU): 41.6 | MFU 13.33% | FLOPs estimator: workload-aware | logical tokens/s: 186408.6 | valid tokens/s: 186408.6 | physical tokens/s: 186964.4 | loss tokens/s: 136330.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.190288E-01 | lm-global-supervised-tokens: 1.227004E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.658833E-02 | lm-token-buffer-mib: 5.812904E+01 | global_load_balancing_loss: 9.997163E-01 | loss scale: 1.0 | grad norm: 0.268 | params norm: 11293.332 | params rms: 0.061150 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 6:01:32.302179 | finish at 2026-10-10 19:56:13 + [2026-10-09 13:56:09] iteration 5655/ 6855 | consumed samples: 723840 | elapsed time per iteration (ms): 88648.3 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.40% | FLOPs estimator: workload-aware | logical tokens/s: 189255.6 | valid tokens/s: 189255.6 | physical tokens/s: 189755.4 | loss tokens/s: 141255.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.234958E-01 | lm-global-supervised-tokens: 1.252205E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.427861E-02 | lm-token-buffer-mib: 5.822690E+01 | global_load_balancing_loss: 9.992321E-01 | loss scale: 1.0 | grad norm: 0.121 | params norm: 11293.293 | params rms: 0.061150 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 5:32:57.919006 | finish at 2026-10-10 19:29:07 + [2026-10-09 13:57:36] iteration 5656/ 6855 | consumed samples: 723968 | elapsed time per iteration (ms): 86734.1 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.78% | FLOPs estimator: workload-aware | logical tokens/s: 193432.7 | valid tokens/s: 193432.7 | physical tokens/s: 193873.5 | loss tokens/s: 139381.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.526592E-01 | lm-global-supervised-tokens: 1.208908E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.359158E-02 | lm-token-buffer-mib: 5.824304E+01 | global_load_balancing_loss: 1.000069E+00 | loss scale: 1.0 | grad norm: 0.102 | params norm: 11293.255 | params rms: 0.061149 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 4:53:14.187167 | finish at 2026-10-10 18:50:50 + [2026-10-09 13:59:06] iteration 5657/ 6855 | consumed samples: 724096 | elapsed time per iteration (ms): 90087.0 | throughput per GPU (TFLOP/s/GPU): 41.1 | MFU 13.19% | FLOPs estimator: workload-aware | logical tokens/s: 186233.5 | valid tokens/s: 186233.5 | physical tokens/s: 186708.7 | loss tokens/s: 139120.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.696152E-01 | lm-global-supervised-tokens: 1.253296E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.413331E-02 | lm-token-buffer-mib: 5.808362E+01 | global_load_balancing_loss: 9.990684E-01 | loss scale: 1.0 | grad norm: 0.102 | params norm: 11293.212 | params rms: 0.061149 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 5:58:44.179886 | finish at 2026-10-10 19:57:50 + [2026-10-09 14:00:34] iteration 5658/ 6855 | consumed samples: 724224 | elapsed time per iteration (ms): 88267.9 | throughput per GPU (TFLOP/s/GPU): 41.1 | MFU 13.17% | FLOPs estimator: workload-aware | logical tokens/s: 190071.3 | valid tokens/s: 190071.3 | physical tokens/s: 190431.3 | loss tokens/s: 148116.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.478391E-01 | lm-global-supervised-tokens: 1.307391E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.149111E-02 | lm-token-buffer-mib: 5.807855E+01 | global_load_balancing_loss: 1.000215E+00 | loss scale: 1.0 | grad norm: 0.100 | params norm: 11293.173 | params rms: 0.061149 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 5:20:56.632909 | finish at 2026-10-10 19:21:31 + [2026-10-09 14:02:04] iteration 5659/ 6855 | consumed samples: 724352 | elapsed time per iteration (ms): 89665.5 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.66% | FLOPs estimator: workload-aware | logical tokens/s: 187108.6 | valid tokens/s: 187108.6 | physical tokens/s: 187498.6 | loss tokens/s: 133072.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.023432E-01 | lm-global-supervised-tokens: 1.193204E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.155777E-02 | lm-token-buffer-mib: 5.802407E+01 | global_load_balancing_loss: 1.001081E+00 | loss scale: 1.0 | grad norm: 0.068 | params norm: 11293.137 | params rms: 0.061149 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 5:47:19.989523 | finish at 2026-10-10 19:49:24 + [2026-10-09 14:03:55] iteration 5660/ 6855 | consumed samples: 724480 | elapsed time per iteration (ms): 110756.9 | throughput per GPU (TFLOP/s/GPU): 34.0 | MFU 10.90% | FLOPs estimator: workload-aware | logical tokens/s: 151477.4 | valid tokens/s: 151477.4 | physical tokens/s: 151925.7 | loss tokens/s: 109370.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.955606E-01 | lm-global-supervised-tokens: 1.211349E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.602444E-02 | lm-token-buffer-mib: 5.806718E+01 | global_load_balancing_loss: 1.001077E+00 | loss scale: 1.0 | grad norm: 0.047 | params norm: 11293.102 | params rms: 0.061148 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 12:45:54.496156 | finish at 2026-10-11 02:49:49 + [2026-10-09 14:05:24] iteration 5661/ 6855 | consumed samples: 724608 | elapsed time per iteration (ms): 89072.8 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.39% | FLOPs estimator: workload-aware | logical tokens/s: 188353.1 | valid tokens/s: 188353.1 | physical tokens/s: 188777.4 | loss tokens/s: 137071.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.669245E-01 | lm-global-supervised-tokens: 1.220936E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.292857E-02 | lm-token-buffer-mib: 5.809032E+01 | global_load_balancing_loss: 1.000730E+00 | loss scale: 1.0 | grad norm: 0.065 | params norm: 11293.066 | params rms: 0.061148 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 5:32:32.905171 | finish at 2026-10-10 19:37:57 + [2026-10-09 14:06:47] iteration 5662/ 6855 | consumed samples: 724736 | elapsed time per iteration (ms): 83203.5 | throughput per GPU (TFLOP/s/GPU): 43.7 | MFU 14.01% | FLOPs estimator: workload-aware | logical tokens/s: 201640.1 | valid tokens/s: 201640.1 | physical tokens/s: 202068.7 | loss tokens/s: 147372.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.848999E-01 | lm-global-supervised-tokens: 1.226189E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.371326E-02 | lm-token-buffer-mib: 5.803185E+01 | global_load_balancing_loss: 1.001193E+00 | loss scale: 1.0 | grad norm: 0.053 | params norm: 11293.030 | params rms: 0.061148 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 3:34:21.730883 | finish at 2026-10-10 17:41:09 + [2026-10-09 14:08:12] iteration 5663/ 6855 | consumed samples: 724864 | elapsed time per iteration (ms): 85007.5 | throughput per GPU (TFLOP/s/GPU): 43.6 | MFU 13.96% | FLOPs estimator: workload-aware | logical tokens/s: 197361.5 | valid tokens/s: 197361.5 | physical tokens/s: 197718.6 | loss tokens/s: 148646.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.765682E-01 | lm-global-supervised-tokens: 1.263604E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.149870E-02 | lm-token-buffer-mib: 5.811448E+01 | global_load_balancing_loss: 9.995860E-01 | loss scale: 1.0 | grad norm: 0.039 | params norm: 11292.996 | params rms: 0.061148 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 4:08:48.883366 | finish at 2026-10-10 18:17:01 + [2026-10-09 14:09:42] iteration 5664/ 6855 | consumed samples: 724992 | elapsed time per iteration (ms): 90022.4 | throughput per GPU (TFLOP/s/GPU): 41.0 | MFU 13.15% | FLOPs estimator: workload-aware | logical tokens/s: 186367.1 | valid tokens/s: 186367.1 | physical tokens/s: 186836.0 | loss tokens/s: 140603.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.305611E-01 | lm-global-supervised-tokens: 1.265750E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.313410E-02 | lm-token-buffer-mib: 5.819238E+01 | global_load_balancing_loss: 9.998574E-01 | loss scale: 1.0 | grad norm: 0.051 | params norm: 11292.962 | params rms: 0.061148 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 5:46:56.654431 | finish at 2026-10-10 19:56:39 + [2026-10-09 14:11:11] iteration 5665/ 6855 | consumed samples: 725120 | elapsed time per iteration (ms): 89317.8 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.77% | FLOPs estimator: workload-aware | logical tokens/s: 187837.1 | valid tokens/s: 187837.1 | physical tokens/s: 188241.0 | loss tokens/s: 136766.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.615325E-01 | lm-global-supervised-tokens: 1.221569E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.273367E-02 | lm-token-buffer-mib: 5.813960E+01 | global_load_balancing_loss: 9.996043E-01 | loss scale: 1.0 | grad norm: 0.052 | params norm: 11292.927 | params rms: 0.061148 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 5:31:28.220072 | finish at 2026-10-10 19:42:39 + [2026-10-09 14:12:41] iteration 5666/ 6855 | consumed samples: 725248 | elapsed time per iteration (ms): 89673.5 | throughput per GPU (TFLOP/s/GPU): 41.0 | MFU 13.14% | FLOPs estimator: workload-aware | logical tokens/s: 187091.7 | valid tokens/s: 187091.7 | physical tokens/s: 187530.9 | loss tokens/s: 139879.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.444825E-01 | lm-global-supervised-tokens: 1.254347E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.235664E-02 | lm-token-buffer-mib: 5.814003E+01 | global_load_balancing_loss: 9.996927E-01 | loss scale: 1.0 | grad norm: 0.039 | params norm: 11292.891 | params rms: 0.061147 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 5:37:01.843449 | finish at 2026-10-10 19:49:43 + [2026-10-09 14:14:07] iteration 5667/ 6855 | consumed samples: 725376 | elapsed time per iteration (ms): 86147.9 | throughput per GPU (TFLOP/s/GPU): 43.8 | MFU 14.03% | FLOPs estimator: workload-aware | logical tokens/s: 194748.6 | valid tokens/s: 194748.6 | physical tokens/s: 195106.2 | loss tokens/s: 142339.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.136413E-01 | lm-global-supervised-tokens: 1.226222E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.241734E-02 | lm-token-buffer-mib: 5.816703E+01 | global_load_balancing_loss: 1.000020E+00 | loss scale: 1.0 | grad norm: 0.049 | params norm: 11292.853 | params rms: 0.061147 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 4:25:43.671052 | finish at 2026-10-10 18:39:51 + [2026-10-09 14:15:32] iteration 5668/ 6855 | consumed samples: 725504 | elapsed time per iteration (ms): 85302.2 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.39% | FLOPs estimator: workload-aware | logical tokens/s: 196679.4 | valid tokens/s: 196679.4 | physical tokens/s: 197282.2 | loss tokens/s: 149484.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.376843E-01 | lm-global-supervised-tokens: 1.275132E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.444769E-02 | lm-token-buffer-mib: 5.817949E+01 | global_load_balancing_loss: 1.001419E+00 | loss scale: 1.0 | grad norm: 0.055 | params norm: 11292.813 | params rms: 0.061147 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 4:07:33.702438 | finish at 2026-10-10 18:23:06 + [2026-10-09 14:16:57] iteration 5669/ 6855 | consumed samples: 725632 | elapsed time per iteration (ms): 84592.0 | throughput per GPU (TFLOP/s/GPU): 43.6 | MFU 13.97% | FLOPs estimator: workload-aware | logical tokens/s: 198330.7 | valid tokens/s: 198330.7 | physical tokens/s: 198813.4 | loss tokens/s: 146961.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.495442E-01 | lm-global-supervised-tokens: 1.243176E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.806073E-02 | lm-token-buffer-mib: 5.809436E+01 | global_load_balancing_loss: 1.000726E+00 | loss scale: 1.0 | grad norm: 0.039 | params norm: 11292.773 | params rms: 0.061147 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 3:52:06.064505 | finish at 2026-10-10 18:09:03 + [2026-10-09 14:18:45] iteration 5670/ 6855 | consumed samples: 725760 | elapsed time per iteration (ms): 108083.4 | throughput per GPU (TFLOP/s/GPU): 34.2 | MFU 10.95% | FLOPs estimator: workload-aware | logical tokens/s: 155224.5 | valid tokens/s: 155224.5 | physical tokens/s: 155647.0 | loss tokens/s: 112535.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.032806E-01 | lm-global-supervised-tokens: 1.216317E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.674630E-02 | lm-token-buffer-mib: 5.820731E+01 | global_load_balancing_loss: 1.000940E+00 | loss scale: 1.0 | grad norm: 0.042 | params norm: 11292.732 | params rms: 0.061146 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 11:34:38.784374 | finish at 2026-10-11 01:53:24 + [2026-10-09 14:20:11] iteration 5671/ 6855 | consumed samples: 725888 | elapsed time per iteration (ms): 85828.5 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.65% | FLOPs estimator: workload-aware | logical tokens/s: 195473.2 | valid tokens/s: 195473.2 | physical tokens/s: 195928.3 | loss tokens/s: 143278.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.696922E-01 | lm-global-supervised-tokens: 1.229733E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.173970E-02 | lm-token-buffer-mib: 5.811196E+01 | global_load_balancing_loss: 9.999568E-01 | loss scale: 1.0 | grad norm: 0.044 | params norm: 11292.687 | params rms: 0.061146 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 4:13:40.885605 | finish at 2026-10-10 18:33:52 + [2026-10-09 14:21:40] iteration 5672/ 6855 | consumed samples: 726016 | elapsed time per iteration (ms): 89428.5 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.44% | FLOPs estimator: workload-aware | logical tokens/s: 187604.8 | valid tokens/s: 187604.8 | physical tokens/s: 188001.3 | loss tokens/s: 134196.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.433679E-01 | lm-global-supervised-tokens: 1.200095E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.211358E-02 | lm-token-buffer-mib: 5.820916E+01 | global_load_balancing_loss: 9.998471E-01 | loss scale: 1.0 | grad norm: 0.046 | params norm: 11292.642 | params rms: 0.061146 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 5:23:13.916272 | finish at 2026-10-10 19:44:54 + [2026-10-09 14:23:06] iteration 5673/ 6855 | consumed samples: 726144 | elapsed time per iteration (ms): 85471.7 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.66% | FLOPs estimator: workload-aware | logical tokens/s: 196288.5 | valid tokens/s: 196288.5 | physical tokens/s: 196643.0 | loss tokens/s: 150879.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.692093E-01 | lm-global-supervised-tokens: 1.289593E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.418362E-02 | lm-token-buffer-mib: 5.803950E+01 | global_load_balancing_loss: 1.000686E+00 | loss scale: 1.0 | grad norm: 0.060 | params norm: 11292.596 | params rms: 0.061146 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 4:03:47.515527 | finish at 2026-10-10 18:26:53 + [2026-10-09 14:24:32] iteration 5674/ 6855 | consumed samples: 726272 | elapsed time per iteration (ms): 86308.9 | throughput per GPU (TFLOP/s/GPU): 43.3 | MFU 13.88% | FLOPs estimator: workload-aware | logical tokens/s: 194385.2 | valid tokens/s: 194385.2 | physical tokens/s: 194771.0 | loss tokens/s: 142679.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.641795E-01 | lm-global-supervised-tokens: 1.231450E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.261364E-02 | lm-token-buffer-mib: 5.807072E+01 | global_load_balancing_loss: 1.000240E+00 | loss scale: 1.0 | grad norm: 0.083 | params norm: 11292.551 | params rms: 0.061145 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 4:18:50.851586 | finish at 2026-10-10 18:43:23 + [2026-10-09 14:25:59] iteration 5675/ 6855 | consumed samples: 726400 | elapsed time per iteration (ms): 87051.6 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.64% | FLOPs estimator: workload-aware | logical tokens/s: 192726.9 | valid tokens/s: 192726.9 | physical tokens/s: 193161.7 | loss tokens/s: 145759.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.646184E-01 | lm-global-supervised-tokens: 1.268855E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.130802E-02 | lm-token-buffer-mib: 5.822083E+01 | global_load_balancing_loss: 9.993823E-01 | loss scale: 1.0 | grad norm: 0.052 | params norm: 11292.501 | params rms: 0.061145 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 4:32:00.832272 | finish at 2026-10-10 18:58:00 + [2026-10-09 14:27:24] iteration 5676/ 6855 | consumed samples: 726528 | elapsed time per iteration (ms): 85318.9 | throughput per GPU (TFLOP/s/GPU): 43.1 | MFU 13.80% | FLOPs estimator: workload-aware | logical tokens/s: 196640.2 | valid tokens/s: 196640.2 | physical tokens/s: 197066.5 | loss tokens/s: 146200.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.641650E-01 | lm-global-supervised-tokens: 1.247366E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.555497E-02 | lm-token-buffer-mib: 5.804050E+01 | global_load_balancing_loss: 9.998189E-01 | loss scale: 1.0 | grad norm: 0.063 | params norm: 11292.450 | params rms: 0.061145 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 3:56:30.926446 | finish at 2026-10-10 18:23:55 + [2026-10-09 14:28:52] iteration 5677/ 6855 | consumed samples: 726656 | elapsed time per iteration (ms): 87118.3 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.61% | FLOPs estimator: workload-aware | logical tokens/s: 192579.6 | valid tokens/s: 192579.6 | physical tokens/s: 193113.3 | loss tokens/s: 142490.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 5.923550E-01 | lm-global-supervised-tokens: 1.241354E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.210741E-02 | lm-token-buffer-mib: 5.833872E+01 | global_load_balancing_loss: 9.999493E-01 | loss scale: 1.0 | grad norm: 0.084 | params norm: 11292.399 | params rms: 0.061145 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 4:30:25.403134 | finish at 2026-10-10 18:59:17 + [2026-10-09 14:30:20] iteration 5678/ 6855 | consumed samples: 726784 | elapsed time per iteration (ms): 88517.4 | throughput per GPU (TFLOP/s/GPU): 41.5 | MFU 13.31% | FLOPs estimator: workload-aware | logical tokens/s: 189535.8 | valid tokens/s: 189535.8 | physical tokens/s: 190010.5 | loss tokens/s: 140674.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.039596E-01 | lm-global-supervised-tokens: 1.245213E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.292408E-02 | lm-token-buffer-mib: 5.825911E+01 | global_load_balancing_loss: 1.000150E+00 | loss scale: 1.0 | grad norm: 0.070 | params norm: 11292.349 | params rms: 0.061144 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 4:56:24.963555 | finish at 2026-10-10 19:26:45 + [2026-10-09 14:31:44] iteration 5679/ 6855 | consumed samples: 726912 | elapsed time per iteration (ms): 84415.0 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.63% | FLOPs estimator: workload-aware | logical tokens/s: 198746.5 | valid tokens/s: 198746.5 | physical tokens/s: 199215.4 | loss tokens/s: 149082.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.075666E-01 | lm-global-supervised-tokens: 1.258480E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.819811E-02 | lm-token-buffer-mib: 5.823437E+01 | global_load_balancing_loss: 1.000925E+00 | loss scale: 1.0 | grad norm: 0.038 | params norm: 11292.299 | params rms: 0.061144 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 3:34:32.032104 | finish at 2026-10-10 18:06:17 + [2026-10-09 14:33:33] iteration 5680/ 6855 | consumed samples: 727040 | elapsed time per iteration (ms): 108345.6 | throughput per GPU (TFLOP/s/GPU): 33.7 | MFU 10.80% | FLOPs estimator: workload-aware | logical tokens/s: 154848.7 | valid tokens/s: 154848.7 | physical tokens/s: 155191.6 | loss tokens/s: 118374.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.551926E-01 | lm-global-supervised-tokens: 1.282531E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.162069E-02 | lm-token-buffer-mib: 5.825227E+01 | global_load_balancing_loss: 9.996955E-01 | loss scale: 1.0 | grad norm: 0.037 | params norm: 11292.250 | params rms: 0.061144 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 11:21:46.127214 | finish at 2026-10-11 01:55:19 +times across ranks (ms): + save-checkpoint: + rank 0: 26587.88 + rank 1: 26587.91 + rank 2: 26587.87 + rank 3: 26587.90 + rank 4: 26587.88 + rank 5: 26587.85 + rank 6: 26587.85 + rank 7: 26587.86 + rank 8: 26588.74 + rank 9: 26588.69 + rank 10: 26588.69 + rank 11: 26588.69 + rank 12: 26588.70 + rank 13: 26588.71 + rank 14: 26588.71 + rank 15: 26588.67 + rank 16: 26588.82 + rank 17: 26588.78 + rank 18: 26588.79 + rank 19: 26588.78 + rank 20: 26588.76 + rank 21: 26588.84 + rank 22: 26588.76 + rank 23: 26588.81 + rank 24: 26588.44 + rank 25: 26588.41 + rank 26: 26588.46 + rank 27: 26588.46 + rank 28: 26588.38 + rank 29: 26588.44 + rank 30: 26588.44 + rank 31: 26588.43 + rank 32: 26588.03 + rank 33: 26588.02 + rank 34: 26587.95 + rank 35: 26588.00 + rank 36: 26587.92 + rank 37: 26587.92 + rank 38: 26587.90 + rank 39: 26587.92 + rank 40: 26588.09 + rank 41: 26588.06 + rank 42: 26588.08 + rank 43: 26588.08 + rank 44: 26588.06 + rank 45: 26588.06 + rank 46: 26588.04 + rank 47: 26588.06 + rank 48: 26588.24 + rank 49: 26588.36 + rank 50: 26588.34 + rank 51: 26588.23 + rank 52: 26588.51 + rank 53: 26588.35 + rank 54: 26588.38 + rank 55: 26588.37 + rank 56: 26588.50 + rank 57: 26588.49 + rank 58: 26588.45 + rank 59: 26588.45 + rank 60: 26588.44 + rank 61: 26588.50 + rank 62: 26588.45 + rank 63: 26588.49 + [2026-10-09 14:35:38] iteration 5681/ 6855 | consumed samples: 727168 | elapsed time per iteration (ms): 98490.9 | throughput per GPU (TFLOP/s/GPU): 36.2 | MFU 11.60% | FLOPs estimator: workload-aware | logical tokens/s: 170342.2 | valid tokens/s: 170342.2 | physical tokens/s: 170845.6 | loss tokens/s: 128501.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.260936E-01 | lm-global-supervised-tokens: 1.265622E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 9.686153E-01 | lm-token-buffer-mib: 5.821735E+01 | global_load_balancing_loss: 1.000952E+00 | loss scale: 1.0 | grad norm: 0.040 | params norm: 11292.200 | params rms: 0.061144 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 8:07:08.312168 | finish at 2026-10-10 22:42:47 + [2026-10-09 14:37:10] iteration 5682/ 6855 | consumed samples: 727296 | elapsed time per iteration (ms): 91454.5 | throughput per GPU (TFLOP/s/GPU): 41.2 | MFU 13.20% | FLOPs estimator: workload-aware | logical tokens/s: 183448.3 | valid tokens/s: 183448.3 | physical tokens/s: 183861.5 | loss tokens/s: 135983.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.380220E-01 | lm-global-supervised-tokens: 1.243629E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.453599E-02 | lm-token-buffer-mib: 5.812205E+01 | global_load_balancing_loss: 1.000858E+00 | loss scale: 1.0 | grad norm: 0.082 | params norm: 11292.149 | params rms: 0.061143 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 5:47:56.131529 | finish at 2026-10-10 20:25:06 + [2026-10-09 14:38:37] iteration 5683/ 6855 | consumed samples: 727424 | elapsed time per iteration (ms): 86935.9 | throughput per GPU (TFLOP/s/GPU): 43.1 | MFU 13.81% | FLOPs estimator: workload-aware | logical tokens/s: 192983.5 | valid tokens/s: 192983.5 | physical tokens/s: 193311.4 | loss tokens/s: 138496.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.060340E-01 | lm-global-supervised-tokens: 1.204033E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.413782E-02 | lm-token-buffer-mib: 5.808864E+01 | global_load_balancing_loss: 9.999003E-01 | loss scale: 1.0 | grad norm: 0.104 | params norm: 11292.096 | params rms: 0.061143 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 4:18:08.861076 | finish at 2026-10-10 18:56:46 + [2026-10-09 14:40:06] iteration 5684/ 6855 | consumed samples: 727552 | elapsed time per iteration (ms): 89504.5 | throughput per GPU (TFLOP/s/GPU): 41.6 | MFU 13.33% | FLOPs estimator: workload-aware | logical tokens/s: 187445.0 | valid tokens/s: 187445.0 | physical tokens/s: 187914.2 | loss tokens/s: 137411.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.497169E-01 | lm-global-supervised-tokens: 1.229896E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.186300E-02 | lm-token-buffer-mib: 5.808033E+01 | global_load_balancing_loss: 9.994645E-01 | loss scale: 1.0 | grad norm: 0.040 | params norm: 11292.042 | params rms: 0.061143 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 5:06:49.800108 | finish at 2026-10-10 19:46:56 + [2026-10-09 14:41:37] iteration 5685/ 6855 | consumed samples: 727680 | elapsed time per iteration (ms): 90311.6 | throughput per GPU (TFLOP/s/GPU): 41.2 | MFU 13.21% | FLOPs estimator: workload-aware | logical tokens/s: 185770.0 | valid tokens/s: 185770.0 | physical tokens/s: 186157.4 | loss tokens/s: 138374.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.042840E-01 | lm-global-supervised-tokens: 1.249681E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.372054E-02 | lm-token-buffer-mib: 5.824178E+01 | global_load_balancing_loss: 1.000101E+00 | loss scale: 1.0 | grad norm: 0.045 | params norm: 11291.988 | params rms: 0.061142 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 5:21:04.516454 | finish at 2026-10-10 20:02:41 + [2026-10-09 14:42:59] iteration 5686/ 6855 | consumed samples: 727808 | elapsed time per iteration (ms): 82726.3 | throughput per GPU (TFLOP/s/GPU): 44.1 | MFU 14.13% | FLOPs estimator: workload-aware | logical tokens/s: 202803.6 | valid tokens/s: 202803.6 | physical tokens/s: 203288.0 | loss tokens/s: 153849.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.710654E-01 | lm-global-supervised-tokens: 1.272742E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.137030E-02 | lm-token-buffer-mib: 5.805467E+01 | global_load_balancing_loss: 1.000707E+00 | loss scale: 1.0 | grad norm: 0.107 | params norm: 11291.935 | params rms: 0.061142 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 2:51:47.055292 | finish at 2026-10-10 17:34:46 + [2026-10-09 14:44:26] iteration 5687/ 6855 | consumed samples: 727936 | elapsed time per iteration (ms): 86947.1 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.62% | FLOPs estimator: workload-aware | logical tokens/s: 192958.3 | valid tokens/s: 192958.3 | physical tokens/s: 193352.4 | loss tokens/s: 139864.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.534189E-01 | lm-global-supervised-tokens: 1.216080E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.370988E-02 | lm-token-buffer-mib: 5.813303E+01 | global_load_balancing_loss: 1.000788E+00 | loss scale: 1.0 | grad norm: 0.092 | params norm: 11291.883 | params rms: 0.061142 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 4:12:34.182358 | finish at 2026-10-10 18:57:01 + [2026-10-09 14:45:54] iteration 5688/ 6855 | consumed samples: 728064 | elapsed time per iteration (ms): 87959.8 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.62% | FLOPs estimator: workload-aware | logical tokens/s: 190737.4 | valid tokens/s: 190737.4 | physical tokens/s: 191263.3 | loss tokens/s: 135054.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.468080E-01 | lm-global-supervised-tokens: 1.187937E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.428434E-02 | lm-token-buffer-mib: 5.802069E+01 | global_load_balancing_loss: 1.000031E+00 | loss scale: 1.0 | grad norm: 0.068 | params norm: 11291.831 | params rms: 0.061142 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 4:30:49.052661 | finish at 2026-10-10 19:16:43 + [2026-10-09 14:47:23] iteration 5689/ 6855 | consumed samples: 728192 | elapsed time per iteration (ms): 88353.2 | throughput per GPU (TFLOP/s/GPU): 43.4 | MFU 13.90% | FLOPs estimator: workload-aware | logical tokens/s: 189887.3 | valid tokens/s: 189887.3 | physical tokens/s: 190250.9 | loss tokens/s: 134948.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.887158E-01 | lm-global-supervised-tokens: 1.192316E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.758284E-02 | lm-token-buffer-mib: 5.823499E+01 | global_load_balancing_loss: 9.990653E-01 | loss scale: 1.0 | grad norm: 0.052 | params norm: 11291.778 | params rms: 0.061141 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 4:36:59.868403 | finish at 2026-10-10 19:24:23 + [2026-10-09 14:49:10] iteration 5690/ 6855 | consumed samples: 728320 | elapsed time per iteration (ms): 107655.4 | throughput per GPU (TFLOP/s/GPU): 33.4 | MFU 10.69% | FLOPs estimator: workload-aware | logical tokens/s: 155841.8 | valid tokens/s: 155841.8 | physical tokens/s: 156200.1 | loss tokens/s: 114457.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.163061E-01 | lm-global-supervised-tokens: 1.232192E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.192707E-02 | lm-token-buffer-mib: 5.809208E+01 | global_load_balancing_loss: 1.000193E+00 | loss scale: 1.0 | grad norm: 0.096 | params norm: 11291.724 | params rms: 0.061141 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 10:50:18.497158 | finish at 2026-10-11 01:39:29 + [2026-10-09 14:50:41] iteration 5691/ 6855 | consumed samples: 728448 | elapsed time per iteration (ms): 91155.3 | throughput per GPU (TFLOP/s/GPU): 40.8 | MFU 13.08% | FLOPs estimator: workload-aware | logical tokens/s: 184050.8 | valid tokens/s: 184050.8 | physical tokens/s: 184469.1 | loss tokens/s: 136821.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.108518E-01 | lm-global-supervised-tokens: 1.247203E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.596421E-02 | lm-token-buffer-mib: 5.810175E+01 | global_load_balancing_loss: 1.000411E+00 | loss scale: 1.0 | grad norm: 0.076 | params norm: 11291.669 | params rms: 0.061141 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 5:28:24.734396 | finish at 2026-10-10 20:19:06 + [2026-10-09 14:52:11] iteration 5692/ 6855 | consumed samples: 728576 | elapsed time per iteration (ms): 89141.6 | throughput per GPU (TFLOP/s/GPU): 40.9 | MFU 13.10% | FLOPs estimator: workload-aware | logical tokens/s: 188207.6 | valid tokens/s: 188207.6 | physical tokens/s: 188774.3 | loss tokens/s: 142948.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.775641E-01 | lm-global-supervised-tokens: 1.274270E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.468637E-02 | lm-token-buffer-mib: 5.816805E+01 | global_load_balancing_loss: 1.001218E+00 | loss scale: 1.0 | grad norm: 0.125 | params norm: 11291.615 | params rms: 0.061140 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 4:47:51.687886 | finish at 2026-10-10 19:40:02 + [2026-10-09 14:53:39] iteration 5693/ 6855 | consumed samples: 728704 | elapsed time per iteration (ms): 88155.8 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.58% | FLOPs estimator: workload-aware | logical tokens/s: 190312.4 | valid tokens/s: 190312.4 | physical tokens/s: 190893.8 | loss tokens/s: 141401.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.203852E-01 | lm-global-supervised-tokens: 1.246538E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.222631E-02 | lm-token-buffer-mib: 5.810120E+01 | global_load_balancing_loss: 1.000536E+00 | loss scale: 1.0 | grad norm: 0.227 | params norm: 11291.561 | params rms: 0.061140 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 4:27:16.987360 | finish at 2026-10-10 19:20:56 + [2026-10-09 14:55:05] iteration 5694/ 6855 | consumed samples: 728832 | elapsed time per iteration (ms): 86424.8 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.49% | FLOPs estimator: workload-aware | logical tokens/s: 194124.2 | valid tokens/s: 194124.2 | physical tokens/s: 194561.9 | loss tokens/s: 145180.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 5.931149E-01 | lm-global-supervised-tokens: 1.254716E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.283746E-02 | lm-token-buffer-mib: 5.825873E+01 | global_load_balancing_loss: 1.000414E+00 | loss scale: 1.0 | grad norm: 0.068 | params norm: 11291.504 | params rms: 0.061140 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 3:52:19.221771 | finish at 2026-10-10 18:47:24 + [2026-10-09 14:56:33] iteration 5695/ 6855 | consumed samples: 728960 | elapsed time per iteration (ms): 88170.4 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.41% | FLOPs estimator: workload-aware | logical tokens/s: 190280.9 | valid tokens/s: 190280.9 | physical tokens/s: 190688.4 | loss tokens/s: 139438.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.071699E-01 | lm-global-supervised-tokens: 1.229435E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.371815E-02 | lm-token-buffer-mib: 5.815422E+01 | global_load_balancing_loss: 1.000255E+00 | loss scale: 1.0 | grad norm: 0.192 | params norm: 11291.449 | params rms: 0.061140 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 4:24:37.645636 | finish at 2026-10-10 19:21:11 + [2026-10-09 14:57:55] iteration 5696/ 6855 | consumed samples: 729088 | elapsed time per iteration (ms): 82135.7 | throughput per GPU (TFLOP/s/GPU): 43.2 | MFU 13.86% | FLOPs estimator: workload-aware | logical tokens/s: 204261.2 | valid tokens/s: 204261.2 | physical tokens/s: 205088.7 | loss tokens/s: 156952.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.968114E-01 | lm-global-supervised-tokens: 1.289136E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.216657E-02 | lm-token-buffer-mib: 5.818649E+01 | global_load_balancing_loss: 9.993209E-01 | loss scale: 1.0 | grad norm: 0.173 | params norm: 11291.396 | params rms: 0.061139 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 2:26:35.255561 | finish at 2026-10-10 17:24:31 + [2026-10-09 14:59:31] iteration 5697/ 6855 | consumed samples: 729216 | elapsed time per iteration (ms): 95365.5 | throughput per GPU (TFLOP/s/GPU): 40.0 | MFU 12.83% | FLOPs estimator: workload-aware | logical tokens/s: 175924.9 | valid tokens/s: 175924.9 | physical tokens/s: 176272.7 | loss tokens/s: 122284.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.652173E-01 | lm-global-supervised-tokens: 1.166168E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.589078E-02 | lm-token-buffer-mib: 5.813403E+01 | global_load_balancing_loss: 1.000607E+00 | loss scale: 1.0 | grad norm: 0.134 | params norm: 11291.341 | params rms: 0.061139 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 6:40:33.235717 | finish at 2026-10-10 21:40:04 + [2026-10-09 15:00:59] iteration 5698/ 6855 | consumed samples: 729344 | elapsed time per iteration (ms): 88039.6 | throughput per GPU (TFLOP/s/GPU): 41.5 | MFU 13.31% | FLOPs estimator: workload-aware | logical tokens/s: 190562.8 | valid tokens/s: 190562.8 | physical tokens/s: 191224.3 | loss tokens/s: 142252.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.928524E-01 | lm-global-supervised-tokens: 1.252383E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.526993E-02 | lm-token-buffer-mib: 5.820106E+01 | global_load_balancing_loss: 1.000383E+00 | loss scale: 1.0 | grad norm: 0.064 | params norm: 11291.285 | params rms: 0.061139 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 4:17:41.799425 | finish at 2026-10-10 19:18:41 + [2026-10-09 15:02:21] iteration 5699/ 6855 | consumed samples: 729472 | elapsed time per iteration (ms): 82437.0 | throughput per GPU (TFLOP/s/GPU): 44.2 | MFU 14.17% | FLOPs estimator: workload-aware | logical tokens/s: 203515.7 | valid tokens/s: 203515.7 | physical tokens/s: 203943.5 | loss tokens/s: 157350.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.359006E-01 | lm-global-supervised-tokens: 1.297148E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.128347E-02 | lm-token-buffer-mib: 5.810992E+01 | global_load_balancing_loss: 9.998633E-01 | loss scale: 1.0 | grad norm: 0.144 | params norm: 11291.228 | params rms: 0.061138 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 2:28:17.128220 | finish at 2026-10-10 17:30:38 + [2026-10-09 15:04:09] iteration 5700/ 6855 | consumed samples: 729600 | elapsed time per iteration (ms): 107599.9 | throughput per GPU (TFLOP/s/GPU): 34.1 | MFU 10.94% | FLOPs estimator: workload-aware | logical tokens/s: 155921.8 | valid tokens/s: 155921.8 | physical tokens/s: 156539.1 | loss tokens/s: 117812.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.527722E-01 | lm-global-supervised-tokens: 1.267659E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.098381E-02 | lm-token-buffer-mib: 5.823356E+01 | global_load_balancing_loss: 1.000543E+00 | loss scale: 1.0 | grad norm: 0.138 | params norm: 11291.173 | params rms: 0.061138 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 10:31:17.939308 | finish at 2026-10-11 01:35:27 + [2026-10-09 15:05:33] iteration 5701/ 6855 | consumed samples: 729728 | elapsed time per iteration (ms): 84141.5 | throughput per GPU (TFLOP/s/GPU): 43.5 | MFU 13.94% | FLOPs estimator: workload-aware | logical tokens/s: 199392.2 | valid tokens/s: 199392.2 | physical tokens/s: 199947.2 | loss tokens/s: 143880.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.339341E-01 | lm-global-supervised-tokens: 1.210632E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.654363E-02 | lm-token-buffer-mib: 5.819866E+01 | global_load_balancing_loss: 1.000749E+00 | loss scale: 1.0 | grad norm: 0.116 | params norm: 11291.118 | params rms: 0.061138 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 2:58:19.295673 | finish at 2026-10-10 18:03:52 + [2026-10-09 15:07:00] iteration 5702/ 6855 | consumed samples: 729856 | elapsed time per iteration (ms): 87334.4 | throughput per GPU (TFLOP/s/GPU): 41.6 | MFU 13.32% | FLOPs estimator: workload-aware | logical tokens/s: 192102.7 | valid tokens/s: 192102.7 | physical tokens/s: 192621.3 | loss tokens/s: 141420.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.134280E-01 | lm-global-supervised-tokens: 1.235085E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.501569E-02 | lm-token-buffer-mib: 5.808466E+01 | global_load_balancing_loss: 1.000469E+00 | loss scale: 1.0 | grad norm: 0.172 | params norm: 11291.065 | params rms: 0.061137 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 3:58:16.565603 | finish at 2026-10-10 19:05:17 + [2026-10-09 15:08:27] iteration 5703/ 6855 | consumed samples: 729984 | elapsed time per iteration (ms): 86211.4 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.63% | FLOPs estimator: workload-aware | logical tokens/s: 194605.0 | valid tokens/s: 194605.0 | physical tokens/s: 194983.3 | loss tokens/s: 143257.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.778715E-01 | lm-global-supervised-tokens: 1.235038E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.553987E-02 | lm-token-buffer-mib: 5.809698E+01 | global_load_balancing_loss: 1.000498E+00 | loss scale: 1.0 | grad norm: 0.055 | params norm: 11291.010 | params rms: 0.061137 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 3:35:15.483398 | finish at 2026-10-10 18:43:42 + [2026-10-09 15:09:52] iteration 5704/ 6855 | consumed samples: 730112 | elapsed time per iteration (ms): 84958.2 | throughput per GPU (TFLOP/s/GPU): 43.1 | MFU 13.81% | FLOPs estimator: workload-aware | logical tokens/s: 197476.1 | valid tokens/s: 197476.1 | physical tokens/s: 197957.9 | loss tokens/s: 142492.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.443563E-01 | lm-global-supervised-tokens: 1.210592E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.158649E-02 | lm-token-buffer-mib: 5.818396E+01 | global_load_balancing_loss: 9.999154E-01 | loss scale: 1.0 | grad norm: 0.208 | params norm: 11290.954 | params rms: 0.061137 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 3:09:46.856067 | finish at 2026-10-10 18:19:38 + [2026-10-09 15:11:20] iteration 5705/ 6855 | consumed samples: 730240 | elapsed time per iteration (ms): 88689.8 | throughput per GPU (TFLOP/s/GPU): 40.9 | MFU 13.12% | FLOPs estimator: workload-aware | logical tokens/s: 189166.9 | valid tokens/s: 189166.9 | physical tokens/s: 189513.0 | loss tokens/s: 140426.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.654786E-01 | lm-global-supervised-tokens: 1.245442E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.095239E-02 | lm-token-buffer-mib: 5.807004E+01 | global_load_balancing_loss: 9.996595E-01 | loss scale: 1.0 | grad norm: 0.161 | params norm: 11290.899 | params rms: 0.061137 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 4:19:53.282092 | finish at 2026-10-10 19:31:14 + [2026-10-09 15:12:51] iteration 5706/ 6855 | consumed samples: 730368 | elapsed time per iteration (ms): 90306.5 | throughput per GPU (TFLOP/s/GPU): 41.1 | MFU 13.19% | FLOPs estimator: workload-aware | logical tokens/s: 185779.8 | valid tokens/s: 185779.8 | physical tokens/s: 186255.1 | loss tokens/s: 134890.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.360124E-01 | lm-global-supervised-tokens: 1.218148E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.234244E-02 | lm-token-buffer-mib: 5.810278E+01 | global_load_balancing_loss: 9.998977E-01 | loss scale: 1.0 | grad norm: 0.100 | params norm: 11290.844 | params rms: 0.061136 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 4:49:22.170643 | finish at 2026-10-10 20:02:13 + [2026-10-09 15:14:17] iteration 5707/ 6855 | consumed samples: 730496 | elapsed time per iteration (ms): 86539.8 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.53% | FLOPs estimator: workload-aware | logical tokens/s: 193866.6 | valid tokens/s: 193866.6 | physical tokens/s: 194308.3 | loss tokens/s: 142292.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.727284E-01 | lm-global-supervised-tokens: 1.231400E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.480494E-02 | lm-token-buffer-mib: 5.815053E+01 | global_load_balancing_loss: 9.997154E-01 | loss scale: 1.0 | grad norm: 0.255 | params norm: 11290.787 | params rms: 0.061136 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 3:35:47.715225 | finish at 2026-10-10 18:50:05 + [2026-10-09 15:15:47] iteration 5708/ 6855 | consumed samples: 730624 | elapsed time per iteration (ms): 89664.6 | throughput per GPU (TFLOP/s/GPU): 41.7 | MFU 13.38% | FLOPs estimator: workload-aware | logical tokens/s: 187110.8 | valid tokens/s: 187110.8 | physical tokens/s: 187603.8 | loss tokens/s: 134500.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.631523E-01 | lm-global-supervised-tokens: 1.205996E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.044687E-02 | lm-token-buffer-mib: 5.816136E+01 | global_load_balancing_loss: 9.999030E-01 | loss scale: 1.0 | grad norm: 0.049 | params norm: 11290.731 | params rms: 0.061136 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 4:34:05.249317 | finish at 2026-10-10 19:49:52 + [2026-10-09 15:17:13] iteration 5709/ 6855 | consumed samples: 730752 | elapsed time per iteration (ms): 86416.9 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.44% | FLOPs estimator: workload-aware | logical tokens/s: 194142.5 | valid tokens/s: 194142.4 | physical tokens/s: 194460.4 | loss tokens/s: 142075.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.595964E-01 | lm-global-supervised-tokens: 1.227768E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.552953E-02 | lm-token-buffer-mib: 5.811388E+01 | global_load_balancing_loss: 1.000413E+00 | loss scale: 1.0 | grad norm: 0.167 | params norm: 11290.675 | params rms: 0.061135 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 3:30:33.792171 | finish at 2026-10-10 18:47:47 + [2026-10-09 15:19:03] iteration 5710/ 6855 | consumed samples: 730880 | elapsed time per iteration (ms): 109954.2 | throughput per GPU (TFLOP/s/GPU): 33.7 | MFU 10.81% | FLOPs estimator: workload-aware | logical tokens/s: 152583.4 | valid tokens/s: 152583.4 | physical tokens/s: 152889.1 | loss tokens/s: 108874.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.714609E-01 | lm-global-supervised-tokens: 1.197125E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.229964E-02 | lm-token-buffer-mib: 5.811266E+01 | global_load_balancing_loss: 1.000574E+00 | loss scale: 1.0 | grad norm: 0.097 | params norm: 11290.619 | params rms: 0.061135 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 10:58:17.543473 | finish at 2026-10-11 02:17:21 + [2026-10-09 15:20:28] iteration 5711/ 6855 | consumed samples: 731008 | elapsed time per iteration (ms): 84676.3 | throughput per GPU (TFLOP/s/GPU): 43.4 | MFU 13.90% | FLOPs estimator: workload-aware | logical tokens/s: 198133.2 | valid tokens/s: 198133.2 | physical tokens/s: 198520.5 | loss tokens/s: 151507.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.857797E-01 | lm-global-supervised-tokens: 1.282905E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.153627E-02 | lm-token-buffer-mib: 5.814008E+01 | global_load_balancing_loss: 9.998652E-01 | loss scale: 1.0 | grad norm: 0.205 | params norm: 11290.563 | params rms: 0.061135 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 2:54:29.678801 | finish at 2026-10-10 18:14:58 + [2026-10-09 15:21:56] iteration 5712/ 6855 | consumed samples: 731136 | elapsed time per iteration (ms): 88221.9 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.44% | FLOPs estimator: workload-aware | logical tokens/s: 190170.5 | valid tokens/s: 190170.5 | physical tokens/s: 190638.4 | loss tokens/s: 137427.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.516554E-01 | lm-global-supervised-tokens: 1.212411E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.365439E-02 | lm-token-buffer-mib: 5.825789E+01 | global_load_balancing_loss: 1.000832E+00 | loss scale: 1.0 | grad norm: 0.035 | params norm: 11290.505 | params rms: 0.061134 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 4:00:37.615334 | finish at 2026-10-10 19:22:34 + [2026-10-09 15:23:21] iteration 5713/ 6855 | consumed samples: 731264 | elapsed time per iteration (ms): 85030.6 | throughput per GPU (TFLOP/s/GPU): 43.2 | MFU 13.84% | FLOPs estimator: workload-aware | logical tokens/s: 197307.9 | valid tokens/s: 197307.8 | physical tokens/s: 197738.4 | loss tokens/s: 153989.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.118549E-01 | lm-global-supervised-tokens: 1.309381E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.145527E-02 | lm-token-buffer-mib: 5.808592E+01 | global_load_balancing_loss: 1.000310E+00 | loss scale: 1.0 | grad norm: 0.055 | params norm: 11290.449 | params rms: 0.061134 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 2:58:24.988300 | finish at 2026-10-10 18:21:46 + [2026-10-09 15:24:47] iteration 5714/ 6855 | consumed samples: 731392 | elapsed time per iteration (ms): 85631.9 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.53% | FLOPs estimator: workload-aware | logical tokens/s: 195922.5 | valid tokens/s: 195922.5 | physical tokens/s: 196378.5 | loss tokens/s: 148980.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.393068E-01 | lm-global-supervised-tokens: 1.275747E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.567055E-02 | lm-token-buffer-mib: 5.803344E+01 | global_load_balancing_loss: 1.001132E+00 | loss scale: 1.0 | grad norm: 0.160 | params norm: 11290.395 | params rms: 0.061134 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 3:08:25.974315 | finish at 2026-10-10 18:33:13 + [2026-10-09 15:26:14] iteration 5715/ 6855 | consumed samples: 731520 | elapsed time per iteration (ms): 87344.4 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.39% | FLOPs estimator: workload-aware | logical tokens/s: 192080.9 | valid tokens/s: 192080.9 | physical tokens/s: 192542.5 | loss tokens/s: 138843.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.507217E-01 | lm-global-supervised-tokens: 1.212720E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.241828E-02 | lm-token-buffer-mib: 5.812292E+01 | global_load_balancing_loss: 1.001560E+00 | loss scale: 1.0 | grad norm: 0.071 | params norm: 11290.340 | params rms: 0.061134 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 3:39:32.561831 | finish at 2026-10-10 19:05:47 + [2026-10-09 15:27:42] iteration 5716/ 6855 | consumed samples: 731648 | elapsed time per iteration (ms): 87896.6 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.52% | FLOPs estimator: workload-aware | logical tokens/s: 190873.9 | valid tokens/s: 190873.9 | physical tokens/s: 191405.2 | loss tokens/s: 140178.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.399753E-01 | lm-global-supervised-tokens: 1.232125E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.372422E-02 | lm-token-buffer-mib: 5.812249E+01 | global_load_balancing_loss: 9.998950E-01 | loss scale: 1.0 | grad norm: 0.121 | params norm: 11290.285 | params rms: 0.061133 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 3:48:34.249949 | finish at 2026-10-10 19:16:16 + [2026-10-09 15:29:11] iteration 5717/ 6855 | consumed samples: 731776 | elapsed time per iteration (ms): 88577.3 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.62% | FLOPs estimator: workload-aware | logical tokens/s: 189407.3 | valid tokens/s: 189407.3 | physical tokens/s: 189932.4 | loss tokens/s: 142375.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.577005E-01 | lm-global-supervised-tokens: 1.261127E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.150732E-02 | lm-token-buffer-mib: 5.808022E+01 | global_load_balancing_loss: 9.987772E-01 | loss scale: 1.0 | grad norm: 0.069 | params norm: 11290.232 | params rms: 0.061133 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 4:00:00.947404 | finish at 2026-10-10 19:29:11 + [2026-10-09 15:30:40] iteration 5718/ 6855 | consumed samples: 731904 | elapsed time per iteration (ms): 89801.3 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.41% | FLOPs estimator: workload-aware | logical tokens/s: 186825.5 | valid tokens/s: 186825.5 | physical tokens/s: 187243.6 | loss tokens/s: 143243.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.892375E-01 | lm-global-supervised-tokens: 1.286346E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.745635E-02 | lm-token-buffer-mib: 5.815642E+01 | global_load_balancing_loss: 9.999480E-01 | loss scale: 1.0 | grad norm: 0.141 | params norm: 11290.180 | params rms: 0.061133 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 4:21:44.083577 | finish at 2026-10-10 19:52:24 + [2026-10-09 15:32:07] iteration 5719/ 6855 | consumed samples: 732032 | elapsed time per iteration (ms): 86742.6 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.66% | FLOPs estimator: workload-aware | logical tokens/s: 193413.7 | valid tokens/s: 193413.7 | physical tokens/s: 193905.5 | loss tokens/s: 140240.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.694918E-01 | lm-global-supervised-tokens: 1.216484E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.360865E-02 | lm-token-buffer-mib: 5.813256E+01 | global_load_balancing_loss: 9.997489E-01 | loss scale: 1.0 | grad norm: 0.052 | params norm: 11290.127 | params rms: 0.061132 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 3:22:19.598705 | finish at 2026-10-10 18:54:27 + [2026-10-09 15:33:57] iteration 5720/ 6855 | consumed samples: 732160 | elapsed time per iteration (ms): 110185.9 | throughput per GPU (TFLOP/s/GPU): 33.4 | MFU 10.70% | FLOPs estimator: workload-aware | logical tokens/s: 152262.8 | valid tokens/s: 152262.8 | physical tokens/s: 152649.9 | loss tokens/s: 115866.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.035340E-01 | lm-global-supervised-tokens: 1.276685E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.263847E-02 | lm-token-buffer-mib: 5.832660E+01 | global_load_balancing_loss: 9.998103E-01 | loss scale: 1.0 | grad norm: 0.116 | params norm: 11290.074 | params rms: 0.061132 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 10:44:20.956961 | finish at 2026-10-11 02:18:18 +times across ranks (ms): + save-checkpoint: + rank 0: 25541.80 + rank 1: 25541.81 + rank 2: 25541.84 + rank 3: 25541.81 + rank 4: 25541.86 + rank 5: 25541.83 + rank 6: 25541.80 + rank 7: 25541.81 + rank 8: 25542.14 + rank 9: 25542.09 + rank 10: 25542.10 + rank 11: 25542.12 + rank 12: 25542.12 + rank 13: 25542.09 + rank 14: 25542.12 + rank 15: 25542.09 + rank 16: 25542.14 + rank 17: 25542.10 + rank 18: 25542.05 + rank 19: 25542.06 + rank 20: 25542.04 + rank 21: 25542.10 + rank 22: 25542.04 + rank 23: 25542.09 + rank 24: 25542.32 + rank 25: 25542.82 + rank 26: 25542.32 + rank 27: 25542.32 + rank 28: 25542.27 + rank 29: 25542.28 + rank 30: 25542.29 + rank 31: 25542.29 + rank 32: 25542.03 + rank 33: 25542.01 + rank 34: 25542.05 + rank 35: 25541.72 + rank 36: 25541.95 + rank 37: 25541.96 + rank 38: 25541.95 + rank 39: 25541.93 + rank 40: 25542.02 + rank 41: 25542.02 + rank 42: 25542.01 + rank 43: 25542.04 + rank 44: 25542.01 + rank 45: 25542.02 + rank 46: 25542.01 + rank 47: 25542.01 + rank 48: 25542.09 + rank 49: 25542.44 + rank 50: 25542.38 + rank 51: 25542.38 + rank 52: 25542.47 + rank 53: 25542.13 + rank 54: 25542.44 + rank 55: 25542.41 + rank 56: 25541.03 + rank 57: 25541.16 + rank 58: 25540.98 + rank 59: 25540.99 + rank 60: 25540.98 + rank 61: 25541.02 + rank 62: 25540.98 + rank 63: 25541.01 + [2026-10-09 15:36:12] iteration 5721/ 6855 | consumed samples: 732288 | elapsed time per iteration (ms): 108795.9 | throughput per GPU (TFLOP/s/GPU): 33.9 | MFU 10.87% | FLOPs estimator: workload-aware | logical tokens/s: 154207.6 | valid tokens/s: 154207.6 | physical tokens/s: 154498.2 | loss tokens/s: 112786.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.514276E-01 | lm-global-supervised-tokens: 1.227073E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 5.099165E-01 | lm-token-buffer-mib: 5.817357E+01 | global_load_balancing_loss: 1.000214E+00 | loss scale: 1.0 | grad norm: 0.090 | params norm: 11290.021 | params rms: 0.061132 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 10:16:14.556669 | finish at 2026-10-11 01:52:27 + [2026-10-09 15:37:41] iteration 5722/ 6855 | consumed samples: 732416 | elapsed time per iteration (ms): 89280.3 | throughput per GPU (TFLOP/s/GPU): 41.5 | MFU 13.29% | FLOPs estimator: workload-aware | logical tokens/s: 187916.0 | valid tokens/s: 187916.0 | physical tokens/s: 188356.3 | loss tokens/s: 142048.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.402404E-01 | lm-global-supervised-tokens: 1.268214E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.806565E-02 | lm-token-buffer-mib: 5.817414E+01 | global_load_balancing_loss: 1.000805E+00 | loss scale: 1.0 | grad norm: 0.214 | params norm: 11289.967 | params rms: 0.061131 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 4:05:54.603830 | finish at 2026-10-10 19:43:36 + [2026-10-09 15:39:07] iteration 5723/ 6855 | consumed samples: 732544 | elapsed time per iteration (ms): 85965.7 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.55% | FLOPs estimator: workload-aware | logical tokens/s: 195161.1 | valid tokens/s: 195161.0 | physical tokens/s: 195638.3 | loss tokens/s: 143863.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.196334E-01 | lm-global-supervised-tokens: 1.236731E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.223484E-02 | lm-token-buffer-mib: 5.811786E+01 | global_load_balancing_loss: 1.000454E+00 | loss scale: 1.0 | grad norm: 0.069 | params norm: 11289.913 | params rms: 0.061131 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 3:01:53.151788 | finish at 2026-10-10 18:41:00 + [2026-10-09 15:40:35] iteration 5724/ 6855 | consumed samples: 732672 | elapsed time per iteration (ms): 87917.4 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.41% | FLOPs estimator: workload-aware | logical tokens/s: 190828.3 | valid tokens/s: 190828.3 | physical tokens/s: 191261.6 | loss tokens/s: 141199.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.595199E-01 | lm-global-supervised-tokens: 1.241392E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.169321E-02 | lm-token-buffer-mib: 5.808951E+01 | global_load_balancing_loss: 9.999672E-01 | loss scale: 1.0 | grad norm: 0.201 | params norm: 11289.859 | params rms: 0.061131 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 3:37:14.533697 | finish at 2026-10-10 19:17:50 + [2026-10-09 15:42:00] iteration 5725/ 6855 | consumed samples: 732800 | elapsed time per iteration (ms): 84411.4 | throughput per GPU (TFLOP/s/GPU): 43.8 | MFU 14.03% | FLOPs estimator: workload-aware | logical tokens/s: 198754.9 | valid tokens/s: 198754.9 | physical tokens/s: 199081.1 | loss tokens/s: 153731.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.066717E-01 | lm-global-supervised-tokens: 1.297673E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.341992E-02 | lm-token-buffer-mib: 5.810069E+01 | global_load_balancing_loss: 9.999862E-01 | loss scale: 1.0 | grad norm: 0.038 | params norm: 11289.805 | params rms: 0.061131 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 2:29:44.917922 | finish at 2026-10-10 18:11:44 + [2026-10-09 15:43:27] iteration 5726/ 6855 | consumed samples: 732928 | elapsed time per iteration (ms): 87178.4 | throughput per GPU (TFLOP/s/GPU): 43.3 | MFU 13.88% | FLOPs estimator: workload-aware | logical tokens/s: 192446.0 | valid tokens/s: 192446.0 | physical tokens/s: 192768.2 | loss tokens/s: 142136.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.744133E-01 | lm-global-supervised-tokens: 1.239120E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.322258E-02 | lm-token-buffer-mib: 5.806051E+01 | global_load_balancing_loss: 1.000168E+00 | loss scale: 1.0 | grad norm: 0.123 | params norm: 11289.753 | params rms: 0.061130 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 3:20:24.450522 | finish at 2026-10-10 19:03:51 + [2026-10-09 15:44:53] iteration 5727/ 6855 | consumed samples: 733056 | elapsed time per iteration (ms): 86461.0 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.59% | FLOPs estimator: workload-aware | logical tokens/s: 194043.6 | valid tokens/s: 194043.6 | physical tokens/s: 194635.2 | loss tokens/s: 147461.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.642833E-01 | lm-global-supervised-tokens: 1.274966E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.119078E-02 | lm-token-buffer-mib: 5.850159E+01 | global_load_balancing_loss: 1.000892E+00 | loss scale: 1.0 | grad norm: 0.103 | params norm: 11289.701 | params rms: 0.061130 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 3:05:27.984026 | finish at 2026-10-10 18:50:21 + [2026-10-09 15:46:21] iteration 5728/ 6855 | consumed samples: 733184 | elapsed time per iteration (ms): 88051.2 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.46% | FLOPs estimator: workload-aware | logical tokens/s: 190538.8 | valid tokens/s: 190538.8 | physical tokens/s: 190846.8 | loss tokens/s: 143863.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.657137E-01 | lm-global-supervised-tokens: 1.266738E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.515231E-02 | lm-token-buffer-mib: 5.810999E+01 | global_load_balancing_loss: 1.000403E+00 | loss scale: 1.0 | grad norm: 0.066 | params norm: 11289.649 | params rms: 0.061130 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 3:33:53.678119 | finish at 2026-10-10 19:20:15 + [2026-10-09 15:47:51] iteration 5729/ 6855 | consumed samples: 733312 | elapsed time per iteration (ms): 89373.1 | throughput per GPU (TFLOP/s/GPU): 41.3 | MFU 13.25% | FLOPs estimator: workload-aware | logical tokens/s: 187720.7 | valid tokens/s: 187720.7 | physical tokens/s: 188203.3 | loss tokens/s: 138674.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.086034E-01 | lm-global-supervised-tokens: 1.239375E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.193784E-02 | lm-token-buffer-mib: 5.838426E+01 | global_load_balancing_loss: 1.000277E+00 | loss scale: 1.0 | grad norm: 0.150 | params norm: 11289.596 | params rms: 0.061129 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 3:57:14.080043 | finish at 2026-10-10 19:45:05 + [2026-10-09 15:49:40] iteration 5730/ 6855 | consumed samples: 733440 | elapsed time per iteration (ms): 108966.6 | throughput per GPU (TFLOP/s/GPU): 33.5 | MFU 10.75% | FLOPs estimator: workload-aware | logical tokens/s: 153966.1 | valid tokens/s: 153966.1 | physical tokens/s: 154396.3 | loss tokens/s: 114502.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.064895E-01 | lm-global-supervised-tokens: 1.247694E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.462298E-02 | lm-token-buffer-mib: 5.821011E+01 | global_load_balancing_loss: 9.994137E-01 | loss scale: 1.0 | grad norm: 0.054 | params norm: 11289.544 | params rms: 0.061129 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 10:03:07.388992 | finish at 2026-10-11 01:52:47 + [2026-10-09 15:51:09] iteration 5731/ 6855 | consumed samples: 733568 | elapsed time per iteration (ms): 89543.0 | throughput per GPU (TFLOP/s/GPU): 41.4 | MFU 13.27% | FLOPs estimator: workload-aware | logical tokens/s: 187364.5 | valid tokens/s: 187364.5 | physical tokens/s: 187830.0 | loss tokens/s: 138582.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.674946E-01 | lm-global-supervised-tokens: 1.240910E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.462889E-02 | lm-token-buffer-mib: 5.810626E+01 | global_load_balancing_loss: 1.001077E+00 | loss scale: 1.0 | grad norm: 0.278 | params norm: 11289.492 | params rms: 0.061129 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 3:57:26.276776 | finish at 2026-10-10 19:48:35 + [2026-10-09 15:52:39] iteration 5732/ 6855 | consumed samples: 733696 | elapsed time per iteration (ms): 89803.1 | throughput per GPU (TFLOP/s/GPU): 41.7 | MFU 13.36% | FLOPs estimator: workload-aware | logical tokens/s: 186821.0 | valid tokens/s: 186821.0 | physical tokens/s: 187304.0 | loss tokens/s: 134441.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.363640E-01 | lm-global-supervised-tokens: 1.207327E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.333044E-02 | lm-token-buffer-mib: 5.811362E+01 | global_load_balancing_loss: 1.000459E+00 | loss scale: 1.0 | grad norm: 0.121 | params norm: 11289.439 | params rms: 0.061129 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 4:00:48.899629 | finish at 2026-10-10 19:53:28 + [2026-10-09 15:54:11] iteration 5733/ 6855 | consumed samples: 733824 | elapsed time per iteration (ms): 91945.4 | throughput per GPU (TFLOP/s/GPU): 41.0 | MFU 13.15% | FLOPs estimator: workload-aware | logical tokens/s: 182468.9 | valid tokens/s: 182468.9 | physical tokens/s: 183014.8 | loss tokens/s: 137126.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.673796E-01 | lm-global-supervised-tokens: 1.260811E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.600298E-02 | lm-token-buffer-mib: 5.821671E+01 | global_load_balancing_loss: 1.000252E+00 | loss scale: 1.0 | grad norm: 0.246 | params norm: 11289.386 | params rms: 0.061128 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 4:39:22.699209 | finish at 2026-10-10 20:33:34 + [2026-10-09 15:55:40] iteration 5734/ 6855 | consumed samples: 733952 | elapsed time per iteration (ms): 89289.5 | throughput per GPU (TFLOP/s/GPU): 41.7 | MFU 13.36% | FLOPs estimator: workload-aware | logical tokens/s: 187896.4 | valid tokens/s: 187896.4 | physical tokens/s: 188328.2 | loss tokens/s: 136520.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.511756E-01 | lm-global-supervised-tokens: 1.218984E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.484999E-02 | lm-token-buffer-mib: 5.814290E+01 | global_load_balancing_loss: 1.001092E+00 | loss scale: 1.0 | grad norm: 0.119 | params norm: 11289.336 | params rms: 0.061128 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 3:48:13.492615 | finish at 2026-10-10 19:43:54 + [2026-10-09 15:57:14] iteration 5735/ 6855 | consumed samples: 734080 | elapsed time per iteration (ms): 93900.0 | throughput per GPU (TFLOP/s/GPU): 40.6 | MFU 13.01% | FLOPs estimator: workload-aware | logical tokens/s: 178670.9 | valid tokens/s: 178670.9 | physical tokens/s: 178904.0 | loss tokens/s: 121093.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.668472E-01 | lm-global-supervised-tokens: 1.137064E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.170597E-02 | lm-token-buffer-mib: 5.807014E+01 | global_load_balancing_loss: 9.995382E-01 | loss scale: 1.0 | grad norm: 0.079 | params norm: 11289.287 | params rms: 0.061128 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 5:12:47.988358 | finish at 2026-10-10 21:10:02 + [2026-10-09 15:58:44] iteration 5736/ 6855 | consumed samples: 734208 | elapsed time per iteration (ms): 89696.9 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.54% | FLOPs estimator: workload-aware | logical tokens/s: 187043.4 | valid tokens/s: 187043.3 | physical tokens/s: 187289.7 | loss tokens/s: 135434.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.365059E-01 | lm-global-supervised-tokens: 1.214809E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.215459E-02 | lm-token-buffer-mib: 5.808685E+01 | global_load_balancing_loss: 9.984644E-01 | loss scale: 1.0 | grad norm: 0.085 | params norm: 11289.237 | params rms: 0.061128 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 3:52:50.793627 | finish at 2026-10-10 19:51:35 + [2026-10-09 16:00:10] iteration 5737/ 6855 | consumed samples: 734336 | elapsed time per iteration (ms): 86593.1 | throughput per GPU (TFLOP/s/GPU): 42.8 | MFU 13.72% | FLOPs estimator: workload-aware | logical tokens/s: 193747.2 | valid tokens/s: 193747.1 | physical tokens/s: 194106.5 | loss tokens/s: 147827.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.121429E-01 | lm-global-supervised-tokens: 1.280088E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.122640E-02 | lm-token-buffer-mib: 5.812389E+01 | global_load_balancing_loss: 9.993168E-01 | loss scale: 1.0 | grad norm: 0.116 | params norm: 11289.187 | params rms: 0.061127 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 2:53:31.094676 | finish at 2026-10-10 18:53:41 + [2026-10-09 16:01:34] iteration 5738/ 6855 | consumed samples: 734464 | elapsed time per iteration (ms): 83265.5 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.75% | FLOPs estimator: workload-aware | logical tokens/s: 201489.7 | valid tokens/s: 201489.7 | physical tokens/s: 201972.0 | loss tokens/s: 152143.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.468889E-01 | lm-global-supervised-tokens: 1.266832E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.163308E-02 | lm-token-buffer-mib: 5.813246E+01 | global_load_balancing_loss: 1.000286E+00 | loss scale: 1.0 | grad norm: 0.236 | params norm: 11289.138 | params rms: 0.061127 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 1:50:07.557185 | finish at 2026-10-10 17:51:41 + [2026-10-09 16:02:58] iteration 5739/ 6855 | consumed samples: 734592 | elapsed time per iteration (ms): 84467.6 | throughput per GPU (TFLOP/s/GPU): 43.1 | MFU 13.82% | FLOPs estimator: workload-aware | logical tokens/s: 198622.9 | valid tokens/s: 198622.9 | physical tokens/s: 199029.1 | loss tokens/s: 145831.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.411857E-01 | lm-global-supervised-tokens: 1.231806E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.427069E-02 | lm-token-buffer-mib: 5.805584E+01 | global_load_balancing_loss: 1.000697E+00 | loss scale: 1.0 | grad norm: 0.093 | params norm: 11289.090 | params rms: 0.061127 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 2:11:05.883759 | finish at 2026-10-10 18:14:04 + [2026-10-09 16:04:46] iteration 5740/ 6855 | consumed samples: 734720 | elapsed time per iteration (ms): 108189.3 | throughput per GPU (TFLOP/s/GPU): 33.5 | MFU 10.75% | FLOPs estimator: workload-aware | logical tokens/s: 155072.6 | valid tokens/s: 155072.6 | physical tokens/s: 155482.1 | loss tokens/s: 116679.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.792194E-01 | lm-global-supervised-tokens: 1.262344E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.186323E-02 | lm-token-buffer-mib: 5.818749E+01 | global_load_balancing_loss: 1.000046E+00 | loss scale: 1.0 | grad norm: 0.134 | params norm: 11289.042 | params rms: 0.061126 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 9:30:31.067175 | finish at 2026-10-11 01:35:17 + [2026-10-09 16:06:15] iteration 5741/ 6855 | consumed samples: 734848 | elapsed time per iteration (ms): 88841.7 | throughput per GPU (TFLOP/s/GPU): 41.6 | MFU 13.33% | FLOPs estimator: workload-aware | logical tokens/s: 188843.3 | valid tokens/s: 188843.3 | physical tokens/s: 189380.0 | loss tokens/s: 134867.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.604453E-01 | lm-global-supervised-tokens: 1.198184E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.268504E-02 | lm-token-buffer-mib: 5.823303E+01 | global_load_balancing_loss: 1.001708E+00 | loss scale: 1.0 | grad norm: 0.136 | params norm: 11288.994 | params rms: 0.061126 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 3:29:29.676462 | finish at 2026-10-10 19:35:45 + [2026-10-09 16:07:42] iteration 5742/ 6855 | consumed samples: 734976 | elapsed time per iteration (ms): 87069.0 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.74% | FLOPs estimator: workload-aware | logical tokens/s: 192688.7 | valid tokens/s: 192688.7 | physical tokens/s: 193097.8 | loss tokens/s: 145484.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.746392E-01 | lm-global-supervised-tokens: 1.266717E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.170241E-02 | lm-token-buffer-mib: 5.808688E+01 | global_load_balancing_loss: 1.001963E+00 | loss scale: 1.0 | grad norm: 0.051 | params norm: 11288.945 | params rms: 0.061126 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 2:55:07.788515 | finish at 2026-10-10 19:02:50 + [2026-10-09 16:09:11] iteration 5743/ 6855 | consumed samples: 735104 | elapsed time per iteration (ms): 88495.7 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.40% | FLOPs estimator: workload-aware | logical tokens/s: 189581.7 | valid tokens/s: 189581.7 | physical tokens/s: 189955.6 | loss tokens/s: 140084.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.732165E-01 | lm-global-supervised-tokens: 1.239691E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.227563E-02 | lm-token-buffer-mib: 5.802598E+01 | global_load_balancing_loss: 1.000741E+00 | loss scale: 1.0 | grad norm: 0.134 | params norm: 11288.898 | params rms: 0.061126 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 3:20:07.250614 | finish at 2026-10-10 19:29:18 + [2026-10-09 16:10:39] iteration 5744/ 6855 | consumed samples: 735232 | elapsed time per iteration (ms): 88392.8 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.43% | FLOPs estimator: workload-aware | logical tokens/s: 189802.5 | valid tokens/s: 189802.5 | physical tokens/s: 190163.7 | loss tokens/s: 141805.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.172636E-01 | lm-global-supervised-tokens: 1.253459E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.133608E-02 | lm-token-buffer-mib: 5.806026E+01 | global_load_balancing_loss: 9.998303E-01 | loss scale: 1.0 | grad norm: 0.069 | params norm: 11288.850 | params rms: 0.061125 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 3:16:44.431365 | finish at 2026-10-10 19:27:24 + [2026-10-09 16:12:06] iteration 5745/ 6855 | consumed samples: 735360 | elapsed time per iteration (ms): 87071.6 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.55% | FLOPs estimator: workload-aware | logical tokens/s: 192682.7 | valid tokens/s: 192682.7 | physical tokens/s: 193134.0 | loss tokens/s: 140726.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.557276E-01 | lm-global-supervised-tokens: 1.225331E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.149115E-02 | lm-token-buffer-mib: 5.822777E+01 | global_load_balancing_loss: 9.991892E-01 | loss scale: 1.0 | grad norm: 0.064 | params norm: 11288.801 | params rms: 0.061125 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 2:50:49.457695 | finish at 2026-10-10 19:02:56 + [2026-10-09 16:13:37] iteration 5746/ 6855 | consumed samples: 735488 | elapsed time per iteration (ms): 90822.8 | throughput per GPU (TFLOP/s/GPU): 41.5 | MFU 13.31% | FLOPs estimator: workload-aware | logical tokens/s: 184724.2 | valid tokens/s: 184724.2 | physical tokens/s: 185034.4 | loss tokens/s: 136468.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.895422E-01 | lm-global-supervised-tokens: 1.239445E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.184143E-02 | lm-token-buffer-mib: 5.806117E+01 | global_load_balancing_loss: 9.997111E-01 | loss scale: 1.0 | grad norm: 0.093 | params norm: 11288.750 | params rms: 0.061125 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 3:58:42.476123 | finish at 2026-10-10 20:12:19 + [2026-10-09 16:15:07] iteration 5747/ 6855 | consumed samples: 735616 | elapsed time per iteration (ms): 89671.4 | throughput per GPU (TFLOP/s/GPU): 41.2 | MFU 13.20% | FLOPs estimator: workload-aware | logical tokens/s: 187096.5 | valid tokens/s: 187096.5 | physical tokens/s: 187485.2 | loss tokens/s: 137315.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.423010E-01 | lm-global-supervised-tokens: 1.231331E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.581010E-02 | lm-token-buffer-mib: 5.822931E+01 | global_load_balancing_loss: 1.000102E+00 | loss scale: 1.0 | grad norm: 0.071 | params norm: 11288.701 | params rms: 0.061125 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 3:35:55.921316 | finish at 2026-10-10 19:51:03 + [2026-10-09 16:16:32] iteration 5748/ 6855 | consumed samples: 735744 | elapsed time per iteration (ms): 85281.1 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.69% | FLOPs estimator: workload-aware | logical tokens/s: 196728.0 | valid tokens/s: 196728.0 | physical tokens/s: 197408.3 | loss tokens/s: 146546.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.334248E-01 | lm-global-supervised-tokens: 1.249762E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.272676E-02 | lm-token-buffer-mib: 5.808492E+01 | global_load_balancing_loss: 9.999178E-01 | loss scale: 1.0 | grad norm: 0.051 | params norm: 11288.651 | params rms: 0.061124 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 2:13:26.122577 | finish at 2026-10-10 18:29:58 + [2026-10-09 16:18:01] iteration 5749/ 6855 | consumed samples: 735872 | elapsed time per iteration (ms): 88824.8 | throughput per GPU (TFLOP/s/GPU): 41.6 | MFU 13.32% | FLOPs estimator: workload-aware | logical tokens/s: 188879.3 | valid tokens/s: 188879.3 | physical tokens/s: 189360.1 | loss tokens/s: 139332.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.355526E-01 | lm-global-supervised-tokens: 1.237621E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.364649E-02 | lm-token-buffer-mib: 5.825311E+01 | global_load_balancing_loss: 1.001126E+00 | loss scale: 1.0 | grad norm: 0.102 | params norm: 11288.601 | params rms: 0.061124 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 3:17:20.218532 | finish at 2026-10-10 19:35:21 + [2026-10-09 16:19:53] iteration 5750/ 6855 | consumed samples: 736000 | elapsed time per iteration (ms): 112495.3 | throughput per GPU (TFLOP/s/GPU): 32.9 | MFU 10.53% | FLOPs estimator: workload-aware | logical tokens/s: 149136.6 | valid tokens/s: 149136.6 | physical tokens/s: 149469.7 | loss tokens/s: 112136.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.551663E-01 | lm-global-supervised-tokens: 1.261478E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.524539E-02 | lm-token-buffer-mib: 5.807102E+01 | global_load_balancing_loss: 9.994091E-01 | loss scale: 1.0 | grad norm: 0.047 | params norm: 11288.548 | params rms: 0.061124 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 10:31:47.301555 | finish at 2026-10-11 02:51:41 + [2026-10-09 16:21:20] iteration 5751/ 6855 | consumed samples: 736128 | elapsed time per iteration (ms): 86473.4 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.62% | FLOPs estimator: workload-aware | logical tokens/s: 194015.5 | valid tokens/s: 194015.5 | physical tokens/s: 194562.4 | loss tokens/s: 144988.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.545789E-01 | lm-global-supervised-tokens: 1.253763E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.168150E-02 | lm-token-buffer-mib: 5.826575E+01 | global_load_balancing_loss: 1.000545E+00 | loss scale: 1.0 | grad norm: 0.134 | params norm: 11288.494 | params rms: 0.061124 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 2:31:06.599773 | finish at 2026-10-10 18:52:26 + [2026-10-09 16:22:43] iteration 5752/ 6855 | consumed samples: 736256 | elapsed time per iteration (ms): 83257.0 | throughput per GPU (TFLOP/s/GPU): 43.8 | MFU 14.04% | FLOPs estimator: workload-aware | logical tokens/s: 201510.7 | valid tokens/s: 201510.7 | physical tokens/s: 201960.4 | loss tokens/s: 155095.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.688315E-01 | lm-global-supervised-tokens: 1.291281E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.155386E-02 | lm-token-buffer-mib: 5.804204E+01 | global_load_balancing_loss: 1.000198E+00 | loss scale: 1.0 | grad norm: 0.047 | params norm: 11288.443 | params rms: 0.061123 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 1:30:32.418897 | finish at 2026-10-10 17:53:15 + [2026-10-09 16:24:12] iteration 5753/ 6855 | consumed samples: 736384 | elapsed time per iteration (ms): 88726.8 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.55% | FLOPs estimator: workload-aware | logical tokens/s: 189088.4 | valid tokens/s: 189088.4 | physical tokens/s: 189468.6 | loss tokens/s: 133380.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.618968E-01 | lm-global-supervised-tokens: 1.183445E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.131827E-02 | lm-token-buffer-mib: 5.806088E+01 | global_load_balancing_loss: 1.001402E+00 | loss scale: 1.0 | grad norm: 0.099 | params norm: 11288.394 | params rms: 0.061123 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 3:09:36.951690 | finish at 2026-10-10 19:33:49 + [2026-10-09 16:25:41] iteration 5754/ 6855 | consumed samples: 736512 | elapsed time per iteration (ms): 89413.5 | throughput per GPU (TFLOP/s/GPU): 41.3 | MFU 13.24% | FLOPs estimator: workload-aware | logical tokens/s: 187635.6 | valid tokens/s: 187635.6 | physical tokens/s: 188160.9 | loss tokens/s: 135251.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.819789E-01 | lm-global-supervised-tokens: 1.209334E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.411786E-02 | lm-token-buffer-mib: 5.826021E+01 | global_load_balancing_loss: 1.001415E+00 | loss scale: 1.0 | grad norm: 0.084 | params norm: 11288.345 | params rms: 0.061123 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 3:20:44.270403 | finish at 2026-10-10 19:46:25 + [2026-10-09 16:27:09] iteration 5755/ 6855 | consumed samples: 736640 | elapsed time per iteration (ms): 88217.4 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.60% | FLOPs estimator: workload-aware | logical tokens/s: 190179.4 | valid tokens/s: 190179.4 | physical tokens/s: 190569.1 | loss tokens/s: 138568.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.547679E-01 | lm-global-supervised-tokens: 1.222416E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.146827E-02 | lm-token-buffer-mib: 5.813358E+01 | global_load_balancing_loss: 9.999328E-01 | loss scale: 1.0 | grad norm: 0.093 | params norm: 11288.296 | params rms: 0.061122 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 2:57:19.090252 | finish at 2026-10-10 19:24:28 + [2026-10-09 16:28:35] iteration 5756/ 6855 | consumed samples: 736768 | elapsed time per iteration (ms): 86054.4 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.51% | FLOPs estimator: workload-aware | logical tokens/s: 194960.1 | valid tokens/s: 194960.0 | physical tokens/s: 195467.9 | loss tokens/s: 148093.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.616459E-01 | lm-global-supervised-tokens: 1.274412E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.305996E-02 | lm-token-buffer-mib: 5.817540E+01 | global_load_balancing_loss: 9.990681E-01 | loss scale: 1.0 | grad norm: 0.161 | params norm: 11288.250 | params rms: 0.061122 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 2:16:13.832204 | finish at 2026-10-10 18:44:49 + [2026-10-09 16:30:01] iteration 5757/ 6855 | consumed samples: 736896 | elapsed time per iteration (ms): 85840.3 | throughput per GPU (TFLOP/s/GPU): 43.6 | MFU 13.99% | FLOPs estimator: workload-aware | logical tokens/s: 195446.8 | valid tokens/s: 195446.8 | physical tokens/s: 195806.9 | loss tokens/s: 151391.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.814303E-01 | lm-global-supervised-tokens: 1.299552E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.153659E-02 | lm-token-buffer-mib: 5.802368E+01 | global_load_balancing_loss: 9.995663E-01 | loss scale: 1.0 | grad norm: 0.078 | params norm: 11288.206 | params rms: 0.061122 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 2:10:52.602894 | finish at 2026-10-10 18:40:54 + [2026-10-09 16:31:31] iteration 5758/ 6855 | consumed samples: 737024 | elapsed time per iteration (ms): 89620.0 | throughput per GPU (TFLOP/s/GPU): 40.9 | MFU 13.12% | FLOPs estimator: workload-aware | logical tokens/s: 187203.6 | valid tokens/s: 187203.6 | physical tokens/s: 187674.4 | loss tokens/s: 142925.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.491076E-01 | lm-global-supervised-tokens: 1.280900E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.123821E-02 | lm-token-buffer-mib: 5.817178E+01 | global_load_balancing_loss: 9.992939E-01 | loss scale: 1.0 | grad norm: 0.441 | params norm: 11288.163 | params rms: 0.061122 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 3:18:33.139613 | finish at 2026-10-10 19:50:04 + [2026-10-09 16:32:57] iteration 5759/ 6855 | consumed samples: 737152 | elapsed time per iteration (ms): 85895.1 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.66% | FLOPs estimator: workload-aware | logical tokens/s: 195321.4 | valid tokens/s: 195321.4 | physical tokens/s: 195885.4 | loss tokens/s: 147733.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.429172E-01 | lm-global-supervised-tokens: 1.268957E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.381524E-02 | lm-token-buffer-mib: 5.826212E+01 | global_load_balancing_loss: 1.000093E+00 | loss scale: 1.0 | grad norm: 0.291 | params norm: 11288.120 | params rms: 0.061121 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 2:09:01.035477 | finish at 2026-10-10 18:41:58 + [2026-10-09 16:34:49] iteration 5760/ 6855 | consumed samples: 737280 | elapsed time per iteration (ms): 111840.3 | throughput per GPU (TFLOP/s/GPU): 33.0 | MFU 10.58% | FLOPs estimator: workload-aware | logical tokens/s: 150010.1 | valid tokens/s: 150010.1 | physical tokens/s: 150316.9 | loss tokens/s: 116649.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.258028E-01 | lm-global-supervised-tokens: 1.304614E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.550918E-01 | lm-token-buffer-mib: 5.806216E+01 | global_load_balancing_loss: 9.999568E-01 | loss scale: 1.0 | grad norm: 0.344 | params norm: 11288.076 | params rms: 0.061121 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 10:01:05.137728 | finish at 2026-10-11 02:35:54 +times across ranks (ms): + save-checkpoint: + rank 0: 24910.75 + rank 1: 24910.77 + rank 2: 24910.78 + rank 3: 24910.78 + rank 4: 24910.76 + rank 5: 24910.75 + rank 6: 24910.77 + rank 7: 24910.78 + rank 8: 24912.35 + rank 9: 24912.34 + rank 10: 24912.36 + rank 11: 24912.34 + rank 12: 24912.34 + rank 13: 24912.36 + rank 14: 24912.34 + rank 15: 24912.34 + rank 16: 24910.63 + rank 17: 24910.59 + rank 18: 24910.60 + rank 19: 24910.58 + rank 20: 24910.59 + rank 21: 24910.61 + rank 22: 24910.58 + rank 23: 24910.58 + rank 24: 24914.12 + rank 25: 24914.13 + rank 26: 24914.14 + rank 27: 24914.17 + rank 28: 24914.09 + rank 29: 24914.14 + rank 30: 24914.12 + rank 31: 24914.11 + rank 32: 24912.52 + rank 33: 24912.53 + rank 34: 24912.50 + rank 35: 24912.49 + rank 36: 24912.44 + rank 37: 24912.47 + rank 38: 24912.47 + rank 39: 24912.45 + rank 40: 24911.46 + rank 41: 24911.44 + rank 42: 24911.46 + rank 43: 24911.46 + rank 44: 24911.45 + rank 45: 24911.45 + rank 46: 24911.44 + rank 47: 24911.45 + rank 48: 24911.16 + rank 49: 24911.15 + rank 50: 24911.11 + rank 51: 24911.15 + rank 52: 24911.12 + rank 53: 24911.13 + rank 54: 24911.13 + rank 55: 24911.12 + rank 56: 24910.85 + rank 57: 24910.83 + rank 58: 24910.81 + rank 59: 24910.82 + rank 60: 24910.81 + rank 61: 24910.83 + rank 62: 24910.82 + rank 63: 24910.81 + [2026-10-09 16:36:50] iteration 5761/ 6855 | consumed samples: 737408 | elapsed time per iteration (ms): 96208.3 | throughput per GPU (TFLOP/s/GPU): 38.9 | MFU 12.45% | FLOPs estimator: workload-aware | logical tokens/s: 174384.3 | valid tokens/s: 174384.3 | physical tokens/s: 174706.6 | loss tokens/s: 130274.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.341026E-01 | lm-global-supervised-tokens: 1.253345E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 3.025913E-01 | lm-token-buffer-mib: 5.809789E+01 | global_load_balancing_loss: 1.001356E+00 | loss scale: 1.0 | grad norm: 0.075 | params norm: 11288.030 | params rms: 0.061121 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 5:14:11.871456 | finish at 2026-10-10 21:51:02 + [2026-10-09 16:38:27] iteration 5762/ 6855 | consumed samples: 737536 | elapsed time per iteration (ms): 96958.9 | throughput per GPU (TFLOP/s/GPU): 40.4 | MFU 12.94% | FLOPs estimator: workload-aware | logical tokens/s: 173034.2 | valid tokens/s: 173034.2 | physical tokens/s: 173311.8 | loss tokens/s: 122092.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.551486E-01 | lm-global-supervised-tokens: 1.183797E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.830357E-02 | lm-token-buffer-mib: 5.805646E+01 | global_load_balancing_loss: 1.002133E+00 | loss scale: 1.0 | grad norm: 0.277 | params norm: 11287.985 | params rms: 0.061121 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 5:26:16.023209 | finish at 2026-10-10 22:04:43 + [2026-10-09 16:39:54] iteration 5763/ 6855 | consumed samples: 737664 | elapsed time per iteration (ms): 86925.2 | throughput per GPU (TFLOP/s/GPU): 43.4 | MFU 13.91% | FLOPs estimator: workload-aware | logical tokens/s: 193007.4 | valid tokens/s: 193007.4 | physical tokens/s: 193384.1 | loss tokens/s: 135973.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.635298E-01 | lm-global-supervised-tokens: 1.181952E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.246941E-02 | lm-token-buffer-mib: 5.809470E+01 | global_load_balancing_loss: 1.000843E+00 | loss scale: 1.0 | grad norm: 0.200 | params norm: 11287.941 | params rms: 0.061121 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 2:22:02.319165 | finish at 2026-10-10 19:01:56 + [2026-10-09 16:41:20] iteration 5764/ 6855 | consumed samples: 737792 | elapsed time per iteration (ms): 85784.8 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.66% | FLOPs estimator: workload-aware | logical tokens/s: 195573.3 | valid tokens/s: 195573.3 | physical tokens/s: 196079.2 | loss tokens/s: 148943.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.238754E-01 | lm-global-supervised-tokens: 1.277704E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.275525E-02 | lm-token-buffer-mib: 5.802802E+01 | global_load_balancing_loss: 9.994255E-01 | loss scale: 1.0 | grad norm: 0.141 | params norm: 11287.898 | params rms: 0.061120 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 1:59:51.172118 | finish at 2026-10-10 18:41:11 + [2026-10-09 16:42:43] iteration 5765/ 6855 | consumed samples: 737920 | elapsed time per iteration (ms): 83726.0 | throughput per GPU (TFLOP/s/GPU): 43.3 | MFU 13.89% | FLOPs estimator: workload-aware | logical tokens/s: 200381.5 | valid tokens/s: 200381.5 | physical tokens/s: 200859.1 | loss tokens/s: 149068.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.192071E-01 | lm-global-supervised-tokens: 1.248088E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.129032E-02 | lm-token-buffer-mib: 5.805418E+01 | global_load_balancing_loss: 9.989879E-01 | loss scale: 1.0 | grad norm: 0.208 | params norm: 11287.856 | params rms: 0.061120 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 1:21:01.370482 | finish at 2026-10-10 18:03:45 + [2026-10-09 16:44:08] iteration 5766/ 6855 | consumed samples: 738048 | elapsed time per iteration (ms): 85033.1 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.66% | FLOPs estimator: workload-aware | logical tokens/s: 197301.5 | valid tokens/s: 197301.5 | physical tokens/s: 197782.4 | loss tokens/s: 144507.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.624237E-01 | lm-global-supervised-tokens: 1.228795E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.512165E-02 | lm-token-buffer-mib: 5.813448E+01 | global_load_balancing_loss: 9.994441E-01 | loss scale: 1.0 | grad norm: 0.188 | params norm: 11287.813 | params rms: 0.061120 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 1:43:21.036433 | finish at 2026-10-10 18:27:29 + [2026-10-09 16:45:36] iteration 5767/ 6855 | consumed samples: 738176 | elapsed time per iteration (ms): 88006.8 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.67% | FLOPs estimator: workload-aware | logical tokens/s: 190635.4 | valid tokens/s: 190635.4 | physical tokens/s: 191030.6 | loss tokens/s: 130959.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 5.974135E-01 | lm-global-supervised-tokens: 1.152535E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.161177E-02 | lm-token-buffer-mib: 5.828093E+01 | global_load_balancing_loss: 1.000042E+00 | loss scale: 1.0 | grad norm: 0.079 | params norm: 11287.769 | params rms: 0.061120 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 2:35:51.352676 | finish at 2026-10-10 19:21:28 + [2026-10-09 16:47:07] iteration 5768/ 6855 | consumed samples: 738304 | elapsed time per iteration (ms): 91090.4 | throughput per GPU (TFLOP/s/GPU): 41.0 | MFU 13.14% | FLOPs estimator: workload-aware | logical tokens/s: 184181.7 | valid tokens/s: 184181.7 | physical tokens/s: 184704.9 | loss tokens/s: 133767.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.195433E-01 | lm-global-supervised-tokens: 1.218495E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.620047E-02 | lm-token-buffer-mib: 5.814544E+01 | global_load_balancing_loss: 9.995278E-01 | loss scale: 1.0 | grad norm: 0.100 | params norm: 11287.725 | params rms: 0.061119 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 3:30:15.289658 | finish at 2026-10-10 20:17:23 + [2026-10-09 16:48:40] iteration 5769/ 6855 | consumed samples: 738432 | elapsed time per iteration (ms): 92901.1 | throughput per GPU (TFLOP/s/GPU): 40.7 | MFU 13.04% | FLOPs estimator: workload-aware | logical tokens/s: 180591.8 | valid tokens/s: 180591.8 | physical tokens/s: 180896.4 | loss tokens/s: 126171.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.606315E-01 | lm-global-supervised-tokens: 1.172142E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.492351E-02 | lm-token-buffer-mib: 5.809393E+01 | global_load_balancing_loss: 1.000568E+00 | loss scale: 1.0 | grad norm: 0.079 | params norm: 11287.679 | params rms: 0.061119 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 4:01:30.566550 | finish at 2026-10-10 20:50:11 + [2026-10-09 16:50:27] iteration 5770/ 6855 | consumed samples: 738560 | elapsed time per iteration (ms): 106519.6 | throughput per GPU (TFLOP/s/GPU): 34.4 | MFU 11.04% | FLOPs estimator: workload-aware | logical tokens/s: 157503.2 | valid tokens/s: 157503.2 | physical tokens/s: 157847.2 | loss tokens/s: 121114.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.101911E-01 | lm-global-supervised-tokens: 1.290102E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.256194E-02 | lm-token-buffer-mib: 5.805566E+01 | global_load_balancing_loss: 1.000816E+00 | loss scale: 1.0 | grad norm: 0.129 | params norm: 11287.631 | params rms: 0.061119 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 8:06:13.800830 | finish at 2026-10-11 00:56:41 + [2026-10-09 16:51:53] iteration 5771/ 6855 | consumed samples: 738688 | elapsed time per iteration (ms): 86589.3 | throughput per GPU (TFLOP/s/GPU): 42.8 | MFU 13.72% | FLOPs estimator: workload-aware | logical tokens/s: 193756.1 | valid tokens/s: 193756.1 | physical tokens/s: 194276.9 | loss tokens/s: 147272.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.679848E-01 | lm-global-supervised-tokens: 1.275226E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.256328E-02 | lm-token-buffer-mib: 5.803409E+01 | global_load_balancing_loss: 1.001156E+00 | loss scale: 1.0 | grad norm: 0.048 | params norm: 11287.581 | params rms: 0.061119 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 2:04:22.822820 | finish at 2026-10-10 18:56:16 + [2026-10-09 16:53:23] iteration 5772/ 6855 | consumed samples: 738816 | elapsed time per iteration (ms): 90003.6 | throughput per GPU (TFLOP/s/GPU): 41.0 | MFU 13.13% | FLOPs estimator: workload-aware | logical tokens/s: 186406.1 | valid tokens/s: 186406.1 | physical tokens/s: 186809.8 | loss tokens/s: 133622.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.457723E-01 | lm-global-supervised-tokens: 1.202652E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.172209E-02 | lm-token-buffer-mib: 5.811381E+01 | global_load_balancing_loss: 1.000292E+00 | loss scale: 1.0 | grad norm: 0.041 | params norm: 11287.530 | params rms: 0.061118 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 3:04:33.887828 | finish at 2026-10-10 19:57:57 + [2026-10-09 16:54:50] iteration 5773/ 6855 | consumed samples: 738944 | elapsed time per iteration (ms): 86171.2 | throughput per GPU (TFLOP/s/GPU): 43.1 | MFU 13.83% | FLOPs estimator: workload-aware | logical tokens/s: 194695.6 | valid tokens/s: 194695.6 | physical tokens/s: 195183.8 | loss tokens/s: 143286.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.465515E-01 | lm-global-supervised-tokens: 1.234717E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.169768E-02 | lm-token-buffer-mib: 5.807323E+01 | global_load_balancing_loss: 1.000208E+00 | loss scale: 1.0 | grad norm: 0.100 | params norm: 11287.479 | params rms: 0.061118 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 1:53:57.278683 | finish at 2026-10-10 18:48:47 + [2026-10-09 16:56:19] iteration 5774/ 6855 | consumed samples: 739072 | elapsed time per iteration (ms): 89801.6 | throughput per GPU (TFLOP/s/GPU): 41.5 | MFU 13.31% | FLOPs estimator: workload-aware | logical tokens/s: 186825.3 | valid tokens/s: 186825.3 | physical tokens/s: 187239.9 | loss tokens/s: 133700.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.491390E-01 | lm-global-supervised-tokens: 1.200651E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.450602E-02 | lm-token-buffer-mib: 5.825858E+01 | global_load_balancing_loss: 1.000143E+00 | loss scale: 1.0 | grad norm: 0.066 | params norm: 11287.428 | params rms: 0.061118 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 2:57:55.537567 | finish at 2026-10-10 19:54:15 + [2026-10-09 16:57:45] iteration 5775/ 6855 | consumed samples: 739200 | elapsed time per iteration (ms): 86007.6 | throughput per GPU (TFLOP/s/GPU): 44.2 | MFU 14.16% | FLOPs estimator: workload-aware | logical tokens/s: 195066.4 | valid tokens/s: 195066.4 | physical tokens/s: 195552.3 | loss tokens/s: 141794.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.108291E-01 | lm-global-supervised-tokens: 1.219538E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.903600E-02 | lm-token-buffer-mib: 5.807960E+01 | global_load_balancing_loss: 1.000335E+00 | loss scale: 1.0 | grad norm: 0.035 | params norm: 11287.375 | params rms: 0.061117 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 1:48:08.154774 | finish at 2026-10-10 18:45:54 + [2026-10-09 16:59:15] iteration 5776/ 6855 | consumed samples: 739328 | elapsed time per iteration (ms): 89769.6 | throughput per GPU (TFLOP/s/GPU): 40.5 | MFU 12.98% | FLOPs estimator: workload-aware | logical tokens/s: 186891.3 | valid tokens/s: 186891.3 | physical tokens/s: 187419.1 | loss tokens/s: 144624.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.290945E-01 | lm-global-supervised-tokens: 1.298291E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.424042E-02 | lm-token-buffer-mib: 5.820181E+01 | global_load_balancing_loss: 1.000491E+00 | loss scale: 1.0 | grad norm: 0.079 | params norm: 11287.323 | params rms: 0.061117 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 2:54:21.444871 | finish at 2026-10-10 19:53:37 + [2026-10-09 17:00:44] iteration 5777/ 6855 | consumed samples: 739456 | elapsed time per iteration (ms): 88622.8 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.45% | FLOPs estimator: workload-aware | logical tokens/s: 189309.9 | valid tokens/s: 189309.9 | physical tokens/s: 189744.1 | loss tokens/s: 138999.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.629506E-01 | lm-global-supervised-tokens: 1.231849E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.383100E-02 | lm-token-buffer-mib: 5.815777E+01 | global_load_balancing_loss: 1.000442E+00 | loss scale: 1.0 | grad norm: 0.052 | params norm: 11287.272 | params rms: 0.061117 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 2:32:15.338169 | finish at 2026-10-10 19:32:59 + [2026-10-09 17:02:10] iteration 5778/ 6855 | consumed samples: 739584 | elapsed time per iteration (ms): 86375.7 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.52% | FLOPs estimator: workload-aware | logical tokens/s: 194235.4 | valid tokens/s: 194235.3 | physical tokens/s: 194666.4 | loss tokens/s: 145794.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.630273E-01 | lm-global-supervised-tokens: 1.259312E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.342896E-02 | lm-token-buffer-mib: 5.821999E+01 | global_load_balancing_loss: 1.000336E+00 | loss scale: 1.0 | grad norm: 0.079 | params norm: 11287.220 | params rms: 0.061117 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 1:50:26.628897 | finish at 2026-10-10 18:52:37 + [2026-10-09 17:03:40] iteration 5779/ 6855 | consumed samples: 739712 | elapsed time per iteration (ms): 89602.7 | throughput per GPU (TFLOP/s/GPU): 41.4 | MFU 13.26% | FLOPs estimator: workload-aware | logical tokens/s: 187239.5 | valid tokens/s: 187239.5 | physical tokens/s: 187552.6 | loss tokens/s: 134397.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.006472E-01 | lm-global-supervised-tokens: 1.204234E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.335208E-02 | lm-token-buffer-mib: 5.806479E+01 | global_load_balancing_loss: 1.000715E+00 | loss scale: 1.0 | grad norm: 0.056 | params norm: 11287.168 | params rms: 0.061116 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 2:46:52.482876 | finish at 2026-10-10 19:50:32 + [2026-10-09 17:05:28] iteration 5780/ 6855 | consumed samples: 739840 | elapsed time per iteration (ms): 108653.0 | throughput per GPU (TFLOP/s/GPU): 34.7 | MFU 11.14% | FLOPs estimator: workload-aware | logical tokens/s: 154410.7 | valid tokens/s: 154410.7 | physical tokens/s: 154772.8 | loss tokens/s: 112790.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.096000E-01 | lm-global-supervised-tokens: 1.225504E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.240894E-02 | lm-token-buffer-mib: 5.827895E+01 | global_load_balancing_loss: 1.000095E+00 | loss scale: 1.0 | grad norm: 0.161 | params norm: 11287.116 | params rms: 0.061116 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 8:26:41.976663 | finish at 2026-10-11 01:32:10 + [2026-10-09 17:06:55] iteration 5781/ 6855 | consumed samples: 739968 | elapsed time per iteration (ms): 86326.9 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.45% | FLOPs estimator: workload-aware | logical tokens/s: 194344.6 | valid tokens/s: 194344.5 | physical tokens/s: 194776.9 | loss tokens/s: 147423.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.428298E-01 | lm-global-supervised-tokens: 1.272665E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.650729E-02 | lm-token-buffer-mib: 5.830896E+01 | global_load_balancing_loss: 9.996135E-01 | loss scale: 1.0 | grad norm: 0.066 | params norm: 11287.064 | params rms: 0.061116 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 1:45:15.140282 | finish at 2026-10-10 18:52:10 + [2026-10-09 17:08:26] iteration 5782/ 6855 | consumed samples: 740096 | elapsed time per iteration (ms): 90993.1 | throughput per GPU (TFLOP/s/GPU): 41.4 | MFU 13.28% | FLOPs estimator: workload-aware | logical tokens/s: 184378.3 | valid tokens/s: 184378.2 | physical tokens/s: 184795.8 | loss tokens/s: 134079.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.451723E-01 | lm-global-supervised-tokens: 1.220029E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.141281E-02 | lm-token-buffer-mib: 5.806328E+01 | global_load_balancing_loss: 9.992023E-01 | loss scale: 1.0 | grad norm: 0.114 | params norm: 11287.011 | params rms: 0.061115 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 3:07:15.560663 | finish at 2026-10-10 20:15:41 + [2026-10-09 17:09:53] iteration 5783/ 6855 | consumed samples: 740224 | elapsed time per iteration (ms): 87361.6 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.50% | FLOPs estimator: workload-aware | logical tokens/s: 192043.4 | valid tokens/s: 192043.4 | physical tokens/s: 192420.6 | loss tokens/s: 139987.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.217805E-01 | lm-global-supervised-tokens: 1.222951E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.281842E-02 | lm-token-buffer-mib: 5.808233E+01 | global_load_balancing_loss: 9.998797E-01 | loss scale: 1.0 | grad norm: 0.044 | params norm: 11286.959 | params rms: 0.061115 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 2:00:51.590900 | finish at 2026-10-10 19:10:45 + [2026-10-09 17:11:19] iteration 5784/ 6855 | consumed samples: 740352 | elapsed time per iteration (ms): 85875.8 | throughput per GPU (TFLOP/s/GPU): 43.6 | MFU 13.96% | FLOPs estimator: workload-aware | logical tokens/s: 195365.8 | valid tokens/s: 195365.8 | physical tokens/s: 195911.9 | loss tokens/s: 142844.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.651263E-01 | lm-global-supervised-tokens: 1.226688E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.348171E-02 | lm-token-buffer-mib: 5.830314E+01 | global_load_balancing_loss: 1.000356E+00 | loss scale: 1.0 | grad norm: 0.090 | params norm: 11286.905 | params rms: 0.061115 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 1:32:52.930107 | finish at 2026-10-10 18:44:12 + [2026-10-09 17:12:45] iteration 5785/ 6855 | consumed samples: 740480 | elapsed time per iteration (ms): 85591.3 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.65% | FLOPs estimator: workload-aware | logical tokens/s: 196015.2 | valid tokens/s: 196015.2 | physical tokens/s: 196562.8 | loss tokens/s: 145441.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.082757E-01 | lm-global-supervised-tokens: 1.244854E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.156010E-02 | lm-token-buffer-mib: 5.815470E+01 | global_load_balancing_loss: 9.999805E-01 | loss scale: 1.0 | grad norm: 0.054 | params norm: 11286.851 | params rms: 0.061115 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 1:26:22.661929 | finish at 2026-10-10 18:39:07 + [2026-10-09 17:14:12] iteration 5786/ 6855 | consumed samples: 740608 | elapsed time per iteration (ms): 87459.6 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.59% | FLOPs estimator: workload-aware | logical tokens/s: 191828.2 | valid tokens/s: 191828.2 | physical tokens/s: 192211.7 | loss tokens/s: 143412.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.423211E-01 | lm-global-supervised-tokens: 1.254279E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.114362E-02 | lm-token-buffer-mib: 5.808581E+01 | global_load_balancing_loss: 1.001062E+00 | loss scale: 1.0 | grad norm: 0.089 | params norm: 11286.798 | params rms: 0.061114 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 1:58:14.273120 | finish at 2026-10-10 19:12:26 + [2026-10-09 17:15:41] iteration 5787/ 6855 | consumed samples: 740736 | elapsed time per iteration (ms): 88574.5 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.40% | FLOPs estimator: workload-aware | logical tokens/s: 189413.5 | valid tokens/s: 189413.5 | physical tokens/s: 189825.8 | loss tokens/s: 140868.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.659902E-01 | lm-global-supervised-tokens: 1.247732E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.751058E-02 | lm-token-buffer-mib: 5.816924E+01 | global_load_balancing_loss: 1.001479E+00 | loss scale: 1.0 | grad norm: 0.056 | params norm: 11286.746 | params rms: 0.061114 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 2:16:37.596645 | finish at 2026-10-10 19:32:18 + [2026-10-09 17:17:10] iteration 5788/ 6855 | consumed samples: 740864 | elapsed time per iteration (ms): 89603.9 | throughput per GPU (TFLOP/s/GPU): 41.6 | MFU 13.33% | FLOPs estimator: workload-aware | logical tokens/s: 187237.2 | valid tokens/s: 187237.2 | physical tokens/s: 187613.6 | loss tokens/s: 139815.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.797221E-01 | lm-global-supervised-tokens: 1.252805E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.537985E-02 | lm-token-buffer-mib: 5.808151E+01 | global_load_balancing_loss: 1.000871E+00 | loss scale: 1.0 | grad norm: 0.066 | params norm: 11286.692 | params rms: 0.061114 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 2:33:27.345989 | finish at 2026-10-10 19:50:38 + [2026-10-09 17:18:33] iteration 5789/ 6855 | consumed samples: 740992 | elapsed time per iteration (ms): 82386.9 | throughput per GPU (TFLOP/s/GPU): 43.6 | MFU 13.97% | FLOPs estimator: workload-aware | logical tokens/s: 203638.8 | valid tokens/s: 203638.8 | physical tokens/s: 204321.7 | loss tokens/s: 155366.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.451606E-01 | lm-global-supervised-tokens: 1.280014E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.207659E-02 | lm-token-buffer-mib: 5.821867E+01 | global_load_balancing_loss: 1.000321E+00 | loss scale: 1.0 | grad norm: 0.439 | params norm: 11286.638 | params rms: 0.061113 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 0:23:44.389343 | finish at 2026-10-10 17:42:17 + [2026-10-09 17:20:21] iteration 5790/ 6855 | consumed samples: 741120 | elapsed time per iteration (ms): 108863.9 | throughput per GPU (TFLOP/s/GPU): 33.6 | MFU 10.76% | FLOPs estimator: workload-aware | logical tokens/s: 154111.5 | valid tokens/s: 154111.5 | physical tokens/s: 154394.1 | loss tokens/s: 112562.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.539243E-01 | lm-global-supervised-tokens: 1.225402E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.378147E-02 | lm-token-buffer-mib: 5.806150E+01 | global_load_balancing_loss: 9.985636E-01 | loss scale: 1.0 | grad norm: 0.148 | params norm: 11286.584 | params rms: 0.061113 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 8:12:20.007992 | finish at 2026-10-11 01:32:41 + [2026-10-09 17:21:52] iteration 5791/ 6855 | consumed samples: 741248 | elapsed time per iteration (ms): 90577.6 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.47% | FLOPs estimator: workload-aware | logical tokens/s: 185224.2 | valid tokens/s: 185224.2 | physical tokens/s: 185589.7 | loss tokens/s: 140378.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.155955E-01 | lm-global-supervised-tokens: 1.271518E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.410049E-02 | lm-token-buffer-mib: 5.811591E+01 | global_load_balancing_loss: 9.978260E-01 | loss scale: 1.0 | grad norm: 0.052 | params norm: 11286.532 | params rms: 0.061113 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 2:46:14.513891 | finish at 2026-10-10 20:08:07 + [2026-10-09 17:23:24] iteration 5792/ 6855 | consumed samples: 741376 | elapsed time per iteration (ms): 92197.9 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.48% | FLOPs estimator: workload-aware | logical tokens/s: 181969.5 | valid tokens/s: 181969.5 | physical tokens/s: 182439.5 | loss tokens/s: 130484.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.443085E-01 | lm-global-supervised-tokens: 1.203044E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.164863E-02 | lm-token-buffer-mib: 5.813210E+01 | global_load_balancing_loss: 1.000558E+00 | loss scale: 1.0 | grad norm: 0.054 | params norm: 11286.479 | params rms: 0.061113 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 3:13:26.406790 | finish at 2026-10-10 20:36:51 + [2026-10-09 17:24:54] iteration 5793/ 6855 | consumed samples: 741504 | elapsed time per iteration (ms): 90165.1 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.49% | FLOPs estimator: workload-aware | logical tokens/s: 186071.9 | valid tokens/s: 186071.9 | physical tokens/s: 186572.1 | loss tokens/s: 136063.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.999100E-01 | lm-global-supervised-tokens: 1.226819E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.124990E-02 | lm-token-buffer-mib: 5.822226E+01 | global_load_balancing_loss: 1.001529E+00 | loss scale: 1.0 | grad norm: 0.048 | params norm: 11286.426 | params rms: 0.061112 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 2:35:55.326682 | finish at 2026-10-10 20:00:50 + [2026-10-09 17:26:27] iteration 5794/ 6855 | consumed samples: 741632 | elapsed time per iteration (ms): 92776.1 | throughput per GPU (TFLOP/s/GPU): 41.6 | MFU 13.32% | FLOPs estimator: workload-aware | logical tokens/s: 180835.4 | valid tokens/s: 180835.4 | physical tokens/s: 181128.6 | loss tokens/s: 129539.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.709639E-01 | lm-global-supervised-tokens: 1.201820E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.562916E-02 | lm-token-buffer-mib: 5.807388E+01 | global_load_balancing_loss: 1.000958E+00 | loss scale: 1.0 | grad norm: 0.054 | params norm: 11286.375 | params rms: 0.061112 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 3:20:35.493367 | finish at 2026-10-10 20:47:03 + [2026-10-09 17:28:00] iteration 5795/ 6855 | consumed samples: 741760 | elapsed time per iteration (ms): 92353.9 | throughput per GPU (TFLOP/s/GPU): 41.4 | MFU 13.28% | FLOPs estimator: workload-aware | logical tokens/s: 181661.6 | valid tokens/s: 181661.6 | physical tokens/s: 182135.3 | loss tokens/s: 129316.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.473531E-01 | lm-global-supervised-tokens: 1.194288E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.775068E-02 | lm-token-buffer-mib: 5.811809E+01 | global_load_balancing_loss: 1.001195E+00 | loss scale: 1.0 | grad norm: 0.036 | params norm: 11286.326 | params rms: 0.061112 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 3:11:35.096297 | finish at 2026-10-10 20:39:35 + [2026-10-09 17:29:25] iteration 5796/ 6855 | consumed samples: 741888 | elapsed time per iteration (ms): 85291.2 | throughput per GPU (TFLOP/s/GPU): 43.2 | MFU 13.86% | FLOPs estimator: workload-aware | logical tokens/s: 196704.8 | valid tokens/s: 196704.8 | physical tokens/s: 197100.5 | loss tokens/s: 144883.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.358572E-01 | lm-global-supervised-tokens: 1.235728E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.256553E-02 | lm-token-buffer-mib: 5.808206E+01 | global_load_balancing_loss: 1.001036E+00 | loss scale: 1.0 | grad norm: 0.080 | params norm: 11286.277 | params rms: 0.061112 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 1:05:23.400917 | finish at 2026-10-10 18:34:48 + [2026-10-09 17:30:49] iteration 5797/ 6855 | consumed samples: 742016 | elapsed time per iteration (ms): 84043.8 | throughput per GPU (TFLOP/s/GPU): 43.5 | MFU 13.94% | FLOPs estimator: workload-aware | logical tokens/s: 199624.1 | valid tokens/s: 199624.1 | physical tokens/s: 200046.5 | loss tokens/s: 151180.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.598473E-01 | lm-global-supervised-tokens: 1.270582E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.998246E-02 | lm-token-buffer-mib: 5.825803E+01 | global_load_balancing_loss: 9.994742E-01 | loss scale: 1.0 | grad norm: 0.048 | params norm: 11286.226 | params rms: 0.061111 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 0:41:58.364233 | finish at 2026-10-10 18:12:47 + [2026-10-09 17:32:18] iteration 5798/ 6855 | consumed samples: 742144 | elapsed time per iteration (ms): 89532.3 | throughput per GPU (TFLOP/s/GPU): 43.2 | MFU 13.84% | FLOPs estimator: workload-aware | logical tokens/s: 187387.1 | valid tokens/s: 187387.1 | physical tokens/s: 187643.1 | loss tokens/s: 132196.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.148891E-01 | lm-global-supervised-tokens: 1.183583E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.154996E-02 | lm-token-buffer-mib: 5.806400E+01 | global_load_balancing_loss: 9.994555E-01 | loss scale: 1.0 | grad norm: 0.039 | params norm: 11286.177 | params rms: 0.061111 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 2:17:15.597246 | finish at 2026-10-10 19:49:34 + [2026-10-09 17:33:49] iteration 5799/ 6855 | consumed samples: 742272 | elapsed time per iteration (ms): 90932.9 | throughput per GPU (TFLOP/s/GPU): 41.4 | MFU 13.28% | FLOPs estimator: workload-aware | logical tokens/s: 184500.7 | valid tokens/s: 184500.7 | physical tokens/s: 184886.6 | loss tokens/s: 132840.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.336815E-01 | lm-global-supervised-tokens: 1.207960E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.165173E-02 | lm-token-buffer-mib: 5.807326E+01 | global_load_balancing_loss: 9.994794E-01 | loss scale: 1.0 | grad norm: 0.041 | params norm: 11286.130 | params rms: 0.061111 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 2:40:25.137817 | finish at 2026-10-10 20:14:14 + [2026-10-09 17:35:39] iteration 5800/ 6855 | consumed samples: 742400 | elapsed time per iteration (ms): 109379.2 | throughput per GPU (TFLOP/s/GPU): 34.9 | MFU 11.19% | FLOPs estimator: workload-aware | logical tokens/s: 153385.5 | valid tokens/s: 153385.4 | physical tokens/s: 153628.6 | loss tokens/s: 113571.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.829053E-01 | lm-global-supervised-tokens: 1.242240E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.813899E-02 | lm-token-buffer-mib: 5.809741E+01 | global_load_balancing_loss: 1.000588E+00 | loss scale: 1.0 | grad norm: 0.038 | params norm: 11286.082 | params rms: 0.061110 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 8:03:15.068809 | finish at 2026-10-11 01:38:54 +times across ranks (ms): + save-checkpoint: + rank 0: 24890.90 + rank 1: 24890.92 + rank 2: 24890.98 + rank 3: 24890.93 + rank 4: 24890.93 + rank 5: 24890.89 + rank 6: 24890.88 + rank 7: 24890.93 + rank 8: 24890.85 + rank 9: 24890.72 + rank 10: 24890.73 + rank 11: 24890.73 + rank 12: 24890.79 + rank 13: 24890.75 + rank 14: 24890.85 + rank 15: 24890.71 + rank 16: 24891.07 + rank 17: 24891.10 + rank 18: 24891.08 + rank 19: 24891.08 + rank 20: 24891.07 + rank 21: 24891.08 + rank 22: 24891.05 + rank 23: 24891.05 + rank 24: 24890.22 + rank 25: 24890.21 + rank 26: 24890.24 + rank 27: 24890.23 + rank 28: 24890.17 + rank 29: 24890.19 + rank 30: 24890.19 + rank 31: 24890.21 + rank 32: 24891.13 + rank 33: 24891.12 + rank 34: 24891.10 + rank 35: 24891.08 + rank 36: 24891.04 + rank 37: 24891.06 + rank 38: 24891.08 + rank 39: 24891.05 + rank 40: 24891.05 + rank 41: 24891.05 + rank 42: 24891.04 + rank 43: 24891.08 + rank 44: 24891.04 + rank 45: 24891.04 + rank 46: 24891.03 + rank 47: 24891.04 + rank 48: 24891.25 + rank 49: 24891.23 + rank 50: 24891.20 + rank 51: 24891.27 + rank 52: 24891.22 + rank 53: 24891.21 + rank 54: 24891.21 + rank 55: 24891.21 + rank 56: 24891.60 + rank 57: 24891.60 + rank 58: 24891.56 + rank 59: 24891.57 + rank 60: 24891.55 + rank 61: 24891.56 + rank 62: 24891.57 + rank 63: 24891.55 + [2026-10-09 17:37:30] iteration 5801/ 6855 | consumed samples: 742528 | elapsed time per iteration (ms): 86425.7 | throughput per GPU (TFLOP/s/GPU): 43.2 | MFU 13.85% | FLOPs estimator: workload-aware | logical tokens/s: 194121.8 | valid tokens/s: 194121.8 | physical tokens/s: 194639.2 | loss tokens/s: 145348.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.647607E-01 | lm-global-supervised-tokens: 1.256180E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.006774E-01 | lm-token-buffer-mib: 5.810780E+01 | global_load_balancing_loss: 9.991623E-01 | loss scale: 1.0 | grad norm: 0.059 | params norm: 11286.033 | params rms: 0.061110 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 1:18:12.652314 | finish at 2026-10-10 18:55:43 + [2026-10-09 17:39:06] iteration 5802/ 6855 | consumed samples: 742656 | elapsed time per iteration (ms): 95565.2 | throughput per GPU (TFLOP/s/GPU): 38.9 | MFU 12.46% | FLOPs estimator: workload-aware | logical tokens/s: 175557.8 | valid tokens/s: 175557.8 | physical tokens/s: 176056.9 | loss tokens/s: 127669.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.404775E-01 | lm-global-supervised-tokens: 1.220077E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.410435E-02 | lm-token-buffer-mib: 5.831569E+01 | global_load_balancing_loss: 1.000336E+00 | loss scale: 1.0 | grad norm: 0.045 | params norm: 11285.984 | params rms: 0.061110 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 3:57:10.134607 | finish at 2026-10-10 21:36:16 + [2026-10-09 17:40:35] iteration 5803/ 6855 | consumed samples: 742784 | elapsed time per iteration (ms): 89051.0 | throughput per GPU (TFLOP/s/GPU): 40.5 | MFU 12.99% | FLOPs estimator: workload-aware | logical tokens/s: 188399.4 | valid tokens/s: 188399.4 | physical tokens/s: 188768.1 | loss tokens/s: 143751.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.544268E-01 | lm-global-supervised-tokens: 1.280125E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.267322E-02 | lm-token-buffer-mib: 5.808946E+01 | global_load_balancing_loss: 1.000073E+00 | loss scale: 1.0 | grad norm: 0.038 | params norm: 11285.936 | params rms: 0.061110 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 2:01:21.618013 | finish at 2026-10-10 19:41:56 + [2026-10-09 17:42:04] iteration 5804/ 6855 | consumed samples: 742912 | elapsed time per iteration (ms): 89342.5 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.47% | FLOPs estimator: workload-aware | logical tokens/s: 187784.8 | valid tokens/s: 187784.8 | physical tokens/s: 188305.1 | loss tokens/s: 139388.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.162197E-01 | lm-global-supervised-tokens: 1.245333E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.544715E-02 | lm-token-buffer-mib: 5.811677E+01 | global_load_balancing_loss: 1.000720E+00 | loss scale: 1.0 | grad norm: 0.051 | params norm: 11285.887 | params rms: 0.061109 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 2:04:58.978495 | finish at 2026-10-10 19:47:03 + [2026-10-09 17:43:31] iteration 5805/ 6855 | consumed samples: 743040 | elapsed time per iteration (ms): 86521.1 | throughput per GPU (TFLOP/s/GPU): 43.1 | MFU 13.82% | FLOPs estimator: workload-aware | logical tokens/s: 193908.0 | valid tokens/s: 193908.0 | physical tokens/s: 194265.6 | loss tokens/s: 146655.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.634011E-01 | lm-global-supervised-tokens: 1.268879E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.120701E-02 | lm-token-buffer-mib: 5.810941E+01 | global_load_balancing_loss: 1.000574E+00 | loss scale: 1.0 | grad norm: 0.049 | params norm: 11285.837 | params rms: 0.061109 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 1:14:07.108233 | finish at 2026-10-10 18:57:38 + [2026-10-09 17:47:00] iteration 5806/ 6855 | consumed samples: 743168 | elapsed time per iteration (ms): 209669.6 | throughput per GPU (TFLOP/s/GPU): 17.7 | MFU 5.69% | FLOPs estimator: workload-aware | logical tokens/s: 80017.3 | valid tokens/s: 80017.3 | physical tokens/s: 80187.6 | loss tokens/s: 59924.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.580293E-01 | lm-global-supervised-tokens: 1.256443E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.196940E+02 | lm-token-buffer-mib: 5.807389E+01 | global_load_balancing_loss: 1.001090E+00 | loss scale: 1.0 | grad norm: 0.045 | params norm: 11285.789 | params rms: 0.061109 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 13:05:43.396905 | finish at 2026-10-12 06:52:44 + [2026-10-09 17:48:26] iteration 5807/ 6855 | consumed samples: 743296 | elapsed time per iteration (ms): 85788.0 | throughput per GPU (TFLOP/s/GPU): 43.3 | MFU 13.87% | FLOPs estimator: workload-aware | logical tokens/s: 195564.6 | valid tokens/s: 195564.6 | physical tokens/s: 195929.2 | loss tokens/s: 142822.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.612270E-01 | lm-global-supervised-tokens: 1.225244E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.304713E-02 | lm-token-buffer-mib: 5.812744E+01 | global_load_balancing_loss: 1.001959E+00 | loss scale: 1.0 | grad norm: 0.055 | params norm: 11285.741 | params rms: 0.061109 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 0:58:25.874584 | finish at 2026-10-10 18:46:52 + [2026-10-09 17:49:53] iteration 5808/ 6855 | consumed samples: 743424 | elapsed time per iteration (ms): 86937.2 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.67% | FLOPs estimator: workload-aware | logical tokens/s: 192980.0 | valid tokens/s: 192980.0 | physical tokens/s: 193377.6 | loss tokens/s: 140496.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.975137E-01 | lm-global-supervised-tokens: 1.221436E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.180477E-02 | lm-token-buffer-mib: 5.803658E+01 | global_load_balancing_loss: 1.000714E+00 | loss scale: 1.0 | grad norm: 0.065 | params norm: 11285.692 | params rms: 0.061108 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 1:17:03.258957 | finish at 2026-10-10 19:06:56 + [2026-10-09 17:54:09] iteration 5809/ 6855 | consumed samples: 743552 | elapsed time per iteration (ms): 255777.3 | throughput per GPU (TFLOP/s/GPU): 14.5 | MFU 4.63% | FLOPs estimator: workload-aware | logical tokens/s: 65592.9 | valid tokens/s: 65592.9 | physical tokens/s: 65800.0 | loss tokens/s: 47795.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.865494E-01 | lm-global-supervised-tokens: 1.222510E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.684206E+02 | lm-token-buffer-mib: 5.808701E+01 | global_load_balancing_loss: 9.995432E-01 | loss scale: 1.0 | grad norm: 0.114 | params norm: 11285.641 | params rms: 0.061108 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3 days, 2:19:03.105802 | finish at 2026-10-12 20:13:12 + [2026-10-09 17:55:57] iteration 5810/ 6855 | consumed samples: 743680 | elapsed time per iteration (ms): 107969.5 | throughput per GPU (TFLOP/s/GPU): 33.9 | MFU 10.85% | FLOPs estimator: workload-aware | logical tokens/s: 155388.4 | valid tokens/s: 155388.4 | physical tokens/s: 155847.9 | loss tokens/s: 118838.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.628187E-01 | lm-global-supervised-tokens: 1.283097E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.546356E-02 | lm-token-buffer-mib: 5.817938E+01 | global_load_balancing_loss: 9.987442E-01 | loss scale: 1.0 | grad norm: 0.150 | params norm: 11285.592 | params rms: 0.061108 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 7:20:28.088202 | finish at 2026-10-11 01:16:25 + [2026-10-09 17:57:26] iteration 5811/ 6855 | consumed samples: 743808 | elapsed time per iteration (ms): 89264.5 | throughput per GPU (TFLOP/s/GPU): 41.7 | MFU 13.37% | FLOPs estimator: workload-aware | logical tokens/s: 187949.3 | valid tokens/s: 187949.3 | physical tokens/s: 188353.8 | loss tokens/s: 136193.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.350898E-01 | lm-global-supervised-tokens: 1.215720E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.438109E-02 | lm-token-buffer-mib: 5.805009E+01 | global_load_balancing_loss: 9.986437E-01 | loss scale: 1.0 | grad norm: 0.090 | params norm: 11285.543 | params rms: 0.061108 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 1:53:12.136903 | finish at 2026-10-10 19:50:38 + [2026-10-09 17:58:53] iteration 5812/ 6855 | consumed samples: 743936 | elapsed time per iteration (ms): 86754.6 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.58% | FLOPs estimator: workload-aware | logical tokens/s: 193386.7 | valid tokens/s: 193386.7 | physical tokens/s: 193869.2 | loss tokens/s: 144144.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.394575E-01 | lm-global-supervised-tokens: 1.250521E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.265696E-02 | lm-token-buffer-mib: 5.807707E+01 | global_load_balancing_loss: 9.993355E-01 | loss scale: 1.0 | grad norm: 0.101 | params norm: 11285.493 | params rms: 0.061107 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 1:08:05.012788 | finish at 2026-10-10 19:06:58 + [2026-10-09 18:00:22] iteration 5813/ 6855 | consumed samples: 744064 | elapsed time per iteration (ms): 89010.5 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.52% | FLOPs estimator: workload-aware | logical tokens/s: 188485.3 | valid tokens/s: 188485.3 | physical tokens/s: 188850.5 | loss tokens/s: 137870.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.641902E-01 | lm-global-supervised-tokens: 1.227188E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.152410E-02 | lm-token-buffer-mib: 5.810906E+01 | global_load_balancing_loss: 1.000600E+00 | loss scale: 1.0 | grad norm: 0.153 | params norm: 11285.440 | params rms: 0.061107 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 1:45:48.948654 | finish at 2026-10-10 19:46:11 + [2026-10-09 18:01:51] iteration 5814/ 6855 | consumed samples: 744192 | elapsed time per iteration (ms): 88822.9 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.56% | FLOPs estimator: workload-aware | logical tokens/s: 188883.5 | valid tokens/s: 188883.5 | physical tokens/s: 189344.4 | loss tokens/s: 141065.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.281182E-01 | lm-global-supervised-tokens: 1.252987E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.474126E-02 | lm-token-buffer-mib: 5.809895E+01 | global_load_balancing_loss: 1.002160E+00 | loss scale: 1.0 | grad norm: 0.042 | params norm: 11285.389 | params rms: 0.061107 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 1:41:04.621089 | finish at 2026-10-10 19:42:55 + [2026-10-09 18:03:16] iteration 5815/ 6855 | consumed samples: 744320 | elapsed time per iteration (ms): 85719.0 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.76% | FLOPs estimator: workload-aware | logical tokens/s: 195722.5 | valid tokens/s: 195722.5 | physical tokens/s: 196237.1 | loss tokens/s: 140317.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.234131E-01 | lm-global-supervised-tokens: 1.202788E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.122366E-02 | lm-token-buffer-mib: 5.817479E+01 | global_load_balancing_loss: 1.000235E+00 | loss scale: 1.0 | grad norm: 0.117 | params norm: 11285.339 | params rms: 0.061106 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 0:45:47.742004 | finish at 2026-10-10 18:49:04 + [2026-10-09 18:04:40] iteration 5816/ 6855 | consumed samples: 744448 | elapsed time per iteration (ms): 83766.8 | throughput per GPU (TFLOP/s/GPU): 44.1 | MFU 14.13% | FLOPs estimator: workload-aware | logical tokens/s: 200283.7 | valid tokens/s: 200283.7 | physical tokens/s: 200688.0 | loss tokens/s: 146489.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.374090E-01 | lm-global-supervised-tokens: 1.227098E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.188334E-02 | lm-token-buffer-mib: 5.829084E+01 | global_load_balancing_loss: 1.000839E+00 | loss scale: 1.0 | grad norm: 0.043 | params norm: 11285.288 | params rms: 0.061106 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 0:10:33.701656 | finish at 2026-10-10 18:15:14 + [2026-10-09 18:06:08] iteration 5817/ 6855 | consumed samples: 744576 | elapsed time per iteration (ms): 88214.9 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.57% | FLOPs estimator: workload-aware | logical tokens/s: 190185.8 | valid tokens/s: 190185.8 | physical tokens/s: 190494.5 | loss tokens/s: 139032.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.007431E-01 | lm-global-supervised-tokens: 1.226474E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.423311E-02 | lm-token-buffer-mib: 5.809989E+01 | global_load_balancing_loss: 1.000833E+00 | loss scale: 1.0 | grad norm: 0.040 | params norm: 11285.236 | params rms: 0.061106 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 1:26:07.023899 | finish at 2026-10-10 19:32:15 + [2026-10-09 18:07:36] iteration 5818/ 6855 | consumed samples: 744704 | elapsed time per iteration (ms): 88077.7 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.74% | FLOPs estimator: workload-aware | logical tokens/s: 190481.7 | valid tokens/s: 190481.7 | physical tokens/s: 191034.1 | loss tokens/s: 132177.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.921887E-01 | lm-global-supervised-tokens: 1.164188E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.346228E-02 | lm-token-buffer-mib: 5.809488E+01 | global_load_balancing_loss: 9.993320E-01 | loss scale: 1.0 | grad norm: 0.090 | params norm: 11285.184 | params rms: 0.061106 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 1:22:16.571087 | finish at 2026-10-10 19:29:53 + [2026-10-09 18:09:32] iteration 5819/ 6855 | consumed samples: 744832 | elapsed time per iteration (ms): 115448.2 | throughput per GPU (TFLOP/s/GPU): 32.0 | MFU 10.27% | FLOPs estimator: workload-aware | logical tokens/s: 145322.0 | valid tokens/s: 145322.0 | physical tokens/s: 145694.7 | loss tokens/s: 106808.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.378163E-01 | lm-global-supervised-tokens: 1.233090E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.613494E+01 | lm-token-buffer-mib: 5.809732E+01 | global_load_balancing_loss: 9.997010E-01 | loss scale: 1.0 | grad norm: 0.045 | params norm: 11285.132 | params rms: 0.061105 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 9:13:24.371249 | finish at 2026-10-11 03:22:56 + [2026-10-09 18:11:19] iteration 5820/ 6855 | consumed samples: 744960 | elapsed time per iteration (ms): 107294.8 | throughput per GPU (TFLOP/s/GPU): 34.1 | MFU 10.94% | FLOPs estimator: workload-aware | logical tokens/s: 156365.1 | valid tokens/s: 156365.1 | physical tokens/s: 156845.6 | loss tokens/s: 118470.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.652661E-01 | lm-global-supervised-tokens: 1.271130E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.176407E-02 | lm-token-buffer-mib: 5.817266E+01 | global_load_balancing_loss: 9.994600E-01 | loss scale: 1.0 | grad norm: 0.099 | params norm: 11285.079 | params rms: 0.061105 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 6:50:50.161228 | finish at 2026-10-11 01:02:09 + [2026-10-09 18:12:45] iteration 5821/ 6855 | consumed samples: 745088 | elapsed time per iteration (ms): 85674.1 | throughput per GPU (TFLOP/s/GPU): 41.6 | MFU 13.33% | FLOPs estimator: workload-aware | logical tokens/s: 195825.5 | valid tokens/s: 195825.5 | physical tokens/s: 196610.7 | loss tokens/s: 146941.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.425868E-01 | lm-global-supervised-tokens: 1.258906E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.542271E-02 | lm-token-buffer-mib: 5.815259E+01 | global_load_balancing_loss: 9.998217E-01 | loss scale: 1.0 | grad norm: 0.047 | params norm: 11285.027 | params rms: 0.061105 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 0:36:27.027455 | finish at 2026-10-10 18:49:12 + [2026-10-09 18:14:12] iteration 5822/ 6855 | consumed samples: 745216 | elapsed time per iteration (ms): 86626.2 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.41% | FLOPs estimator: workload-aware | logical tokens/s: 193672.8 | valid tokens/s: 193672.8 | physical tokens/s: 194284.4 | loss tokens/s: 143569.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.037869E-01 | lm-global-supervised-tokens: 1.243684E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.176258E-02 | lm-token-buffer-mib: 5.816945E+01 | global_load_balancing_loss: 1.000449E+00 | loss scale: 1.0 | grad norm: 0.141 | params norm: 11284.972 | params rms: 0.061104 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 0:51:24.847566 | finish at 2026-10-10 19:05:36 + [2026-10-09 18:15:41] iteration 5823/ 6855 | consumed samples: 745344 | elapsed time per iteration (ms): 88982.0 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.64% | FLOPs estimator: workload-aware | logical tokens/s: 188546.2 | valid tokens/s: 188546.2 | physical tokens/s: 188862.9 | loss tokens/s: 139842.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.355164E-01 | lm-global-supervised-tokens: 1.244348E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.167514E-02 | lm-token-buffer-mib: 5.811005E+01 | global_load_balancing_loss: 1.000754E+00 | loss scale: 1.0 | grad norm: 0.065 | params norm: 11284.918 | params rms: 0.061104 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 1:30:29.382534 | finish at 2026-10-10 19:46:10 + [2026-10-09 18:17:07] iteration 5824/ 6855 | consumed samples: 745472 | elapsed time per iteration (ms): 86096.7 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.43% | FLOPs estimator: workload-aware | logical tokens/s: 194864.1 | valid tokens/s: 194864.1 | physical tokens/s: 195338.7 | loss tokens/s: 144082.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.011963E-01 | lm-global-supervised-tokens: 1.240503E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.604902E-02 | lm-token-buffer-mib: 5.815133E+01 | global_load_balancing_loss: 1.000782E+00 | loss scale: 1.0 | grad norm: 0.176 | params norm: 11284.863 | params rms: 0.061104 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 0:39:25.706255 | finish at 2026-10-10 18:56:32 + [2026-10-09 18:18:36] iteration 5825/ 6855 | consumed samples: 745600 | elapsed time per iteration (ms): 88950.8 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.38% | FLOPs estimator: workload-aware | logical tokens/s: 188611.7 | valid tokens/s: 188611.7 | physical tokens/s: 188978.6 | loss tokens/s: 134068.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.029021E-01 | lm-global-supervised-tokens: 1.192551E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.458165E-02 | lm-token-buffer-mib: 5.813548E+01 | global_load_balancing_loss: 1.000709E+00 | loss scale: 1.0 | grad norm: 0.458 | params norm: 11284.806 | params rms: 0.061104 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 1:26:59.289315 | finish at 2026-10-10 19:45:35 + [2026-10-09 18:20:05] iteration 5826/ 6855 | consumed samples: 745728 | elapsed time per iteration (ms): 89140.8 | throughput per GPU (TFLOP/s/GPU): 41.0 | MFU 13.15% | FLOPs estimator: workload-aware | logical tokens/s: 188210.1 | valid tokens/s: 188210.1 | physical tokens/s: 188593.5 | loss tokens/s: 138333.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.934548E-01 | lm-global-supervised-tokens: 1.233120E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.107496E-02 | lm-token-buffer-mib: 5.819025E+01 | global_load_balancing_loss: 1.000403E+00 | loss scale: 1.0 | grad norm: 0.152 | params norm: 11284.738 | params rms: 0.061103 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 1:28:45.883935 | finish at 2026-10-10 19:48:51 + [2026-10-09 18:21:31] iteration 5827/ 6855 | consumed samples: 745856 | elapsed time per iteration (ms): 86786.0 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.61% | FLOPs estimator: workload-aware | logical tokens/s: 193316.2 | valid tokens/s: 193316.2 | physical tokens/s: 193764.5 | loss tokens/s: 148703.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.757731E-01 | lm-global-supervised-tokens: 1.290535E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.333398E-02 | lm-token-buffer-mib: 5.820197E+01 | global_load_balancing_loss: 9.987696E-01 | loss scale: 1.0 | grad norm: 0.069 | params norm: 11284.678 | params rms: 0.061103 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 0:46:56.029337 | finish at 2026-10-10 19:08:28 + [2026-10-09 18:22:55] iteration 5828/ 6855 | consumed samples: 745984 | elapsed time per iteration (ms): 83978.5 | throughput per GPU (TFLOP/s/GPU): 43.1 | MFU 13.81% | FLOPs estimator: workload-aware | logical tokens/s: 199779.6 | valid tokens/s: 199779.6 | physical tokens/s: 200396.6 | loss tokens/s: 152897.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.013564E-01 | lm-global-supervised-tokens: 1.284014E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.143995E-02 | lm-token-buffer-mib: 5.819796E+01 | global_load_balancing_loss: 9.992304E-01 | loss scale: 1.0 | grad norm: 0.169 | params norm: 11284.624 | params rms: 0.061103 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 23:57:25.914000 | finish at 2026-10-10 18:20:21 + [2026-10-09 18:24:23] iteration 5829/ 6855 | consumed samples: 746112 | elapsed time per iteration (ms): 87339.3 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.65% | FLOPs estimator: workload-aware | logical tokens/s: 192092.0 | valid tokens/s: 192092.0 | physical tokens/s: 192702.4 | loss tokens/s: 137600.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.329656E-01 | lm-global-supervised-tokens: 1.201796E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.157926E-02 | lm-token-buffer-mib: 5.821069E+01 | global_load_balancing_loss: 1.000741E+00 | loss scale: 1.0 | grad norm: 0.108 | params norm: 11284.575 | params rms: 0.061102 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 0:53:30.154494 | finish at 2026-10-10 19:17:53 + [2026-10-09 18:26:13] iteration 5830/ 6855 | consumed samples: 746240 | elapsed time per iteration (ms): 109944.1 | throughput per GPU (TFLOP/s/GPU): 33.7 | MFU 10.80% | FLOPs estimator: workload-aware | logical tokens/s: 152597.7 | valid tokens/s: 152597.7 | physical tokens/s: 153019.1 | loss tokens/s: 113958.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.724313E-01 | lm-global-supervised-tokens: 1.252908E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.173138E-02 | lm-token-buffer-mib: 5.804446E+01 | global_load_balancing_loss: 9.995139E-01 | loss scale: 1.0 | grad norm: 0.146 | params norm: 11284.529 | params rms: 0.061102 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 7:18:12.716330 | finish at 2026-10-11 01:44:25 + [2026-10-09 18:27:39] iteration 5831/ 6855 | consumed samples: 746368 | elapsed time per iteration (ms): 86243.0 | throughput per GPU (TFLOP/s/GPU): 43.9 | MFU 14.06% | FLOPs estimator: workload-aware | logical tokens/s: 194534.3 | valid tokens/s: 194534.3 | physical tokens/s: 195003.5 | loss tokens/s: 148138.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.569310E-01 | lm-global-supervised-tokens: 1.277593E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.127072E-02 | lm-token-buffer-mib: 5.811106E+01 | global_load_balancing_loss: 9.994387E-01 | loss scale: 1.0 | grad norm: 0.119 | params norm: 11284.484 | params rms: 0.061102 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 0:31:52.809814 | finish at 2026-10-10 18:59:32 + [2026-10-09 18:29:05] iteration 5832/ 6855 | consumed samples: 746496 | elapsed time per iteration (ms): 86334.1 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.66% | FLOPs estimator: workload-aware | logical tokens/s: 194328.6 | valid tokens/s: 194328.5 | physical tokens/s: 195057.6 | loss tokens/s: 147394.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.466486E-01 | lm-global-supervised-tokens: 1.272520E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.962155E-02 | lm-token-buffer-mib: 5.831959E+01 | global_load_balancing_loss: 1.000840E+00 | loss scale: 1.0 | grad norm: 0.062 | params norm: 11284.442 | params rms: 0.061102 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 0:31:59.752113 | finish at 2026-10-10 19:01:05 + [2026-10-09 18:30:32] iteration 5833/ 6855 | consumed samples: 746624 | elapsed time per iteration (ms): 86546.7 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.61% | FLOPs estimator: workload-aware | logical tokens/s: 193851.2 | valid tokens/s: 193851.2 | physical tokens/s: 194391.1 | loss tokens/s: 146160.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.284064E-01 | lm-global-supervised-tokens: 1.264966E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.137336E-02 | lm-token-buffer-mib: 5.829014E+01 | global_load_balancing_loss: 1.000887E+00 | loss scale: 1.0 | grad norm: 0.144 | params norm: 11284.400 | params rms: 0.061101 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 0:34:10.697674 | finish at 2026-10-10 19:04:43 + [2026-10-09 18:32:02] iteration 5834/ 6855 | consumed samples: 746752 | elapsed time per iteration (ms): 89938.3 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.49% | FLOPs estimator: workload-aware | logical tokens/s: 186540.9 | valid tokens/s: 186540.9 | physical tokens/s: 187027.2 | loss tokens/s: 131292.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.620319E-01 | lm-global-supervised-tokens: 1.180818E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.440468E-02 | lm-token-buffer-mib: 5.815104E+01 | global_load_balancing_loss: 1.001624E+00 | loss scale: 1.0 | grad norm: 0.152 | params norm: 11284.356 | params rms: 0.061101 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 1:30:27.010278 | finish at 2026-10-10 20:02:29 + [2026-10-09 18:33:28] iteration 5835/ 6855 | consumed samples: 746880 | elapsed time per iteration (ms): 86596.5 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.58% | FLOPs estimator: workload-aware | logical tokens/s: 193739.6 | valid tokens/s: 193739.6 | physical tokens/s: 194107.7 | loss tokens/s: 146185.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.255571E-01 | lm-global-supervised-tokens: 1.265911E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.139920E-02 | lm-token-buffer-mib: 5.807847E+01 | global_load_balancing_loss: 1.000635E+00 | loss scale: 1.0 | grad norm: 0.111 | params norm: 11284.313 | params rms: 0.061101 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 0:32:08.464208 | finish at 2026-10-10 19:05:37 + [2026-10-09 18:34:56] iteration 5836/ 6855 | consumed samples: 747008 | elapsed time per iteration (ms): 87928.3 | throughput per GPU (TFLOP/s/GPU): 41.7 | MFU 13.37% | FLOPs estimator: workload-aware | logical tokens/s: 190805.5 | valid tokens/s: 190805.5 | physical tokens/s: 191306.3 | loss tokens/s: 142416.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.883252E-01 | lm-global-supervised-tokens: 1.252245E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.160066E-02 | lm-token-buffer-mib: 5.814040E+01 | global_load_balancing_loss: 1.000554E+00 | loss scale: 1.0 | grad norm: 0.155 | params norm: 11284.270 | params rms: 0.061101 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 0:53:18.980847 | finish at 2026-10-10 19:28:15 + [2026-10-09 18:36:22] iteration 5837/ 6855 | consumed samples: 747136 | elapsed time per iteration (ms): 85864.0 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.61% | FLOPs estimator: workload-aware | logical tokens/s: 195392.9 | valid tokens/s: 195392.9 | physical tokens/s: 195747.9 | loss tokens/s: 147578.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.196865E-01 | lm-global-supervised-tokens: 1.267163E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.200643E-02 | lm-token-buffer-mib: 5.812268E+01 | global_load_balancing_loss: 9.996529E-01 | loss scale: 1.0 | grad norm: 0.065 | params norm: 11284.226 | params rms: 0.061100 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 0:16:49.534851 | finish at 2026-10-10 18:53:12 + [2026-10-09 18:37:52] iteration 5838/ 6855 | consumed samples: 747264 | elapsed time per iteration (ms): 89870.0 | throughput per GPU (TFLOP/s/GPU): 41.5 | MFU 13.29% | FLOPs estimator: workload-aware | logical tokens/s: 186682.9 | valid tokens/s: 186682.9 | physical tokens/s: 187228.2 | loss tokens/s: 132484.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.653352E-01 | lm-global-supervised-tokens: 1.190633E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.168481E-02 | lm-token-buffer-mib: 5.814410E+01 | global_load_balancing_loss: 9.997345E-01 | loss scale: 1.0 | grad norm: 0.105 | params norm: 11284.183 | params rms: 0.061100 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 1:23:17.741632 | finish at 2026-10-10 20:01:10 + [2026-10-09 18:39:20] iteration 5839/ 6855 | consumed samples: 747392 | elapsed time per iteration (ms): 87654.3 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.79% | FLOPs estimator: workload-aware | logical tokens/s: 191401.9 | valid tokens/s: 191401.9 | physical tokens/s: 191729.1 | loss tokens/s: 134997.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.587858E-01 | lm-global-supervised-tokens: 1.183312E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.520163E-02 | lm-token-buffer-mib: 5.813628E+01 | global_load_balancing_loss: 9.989226E-01 | loss scale: 1.0 | grad norm: 0.059 | params norm: 11284.140 | params rms: 0.061100 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 0:44:16.811892 | finish at 2026-10-10 19:23:37 + [2026-10-09 18:41:11] iteration 5840/ 6855 | consumed samples: 747520 | elapsed time per iteration (ms): 111734.3 | throughput per GPU (TFLOP/s/GPU): 33.1 | MFU 10.60% | FLOPs estimator: workload-aware | logical tokens/s: 150152.2 | valid tokens/s: 150152.2 | physical tokens/s: 150469.3 | loss tokens/s: 115557.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.469815E-01 | lm-global-supervised-tokens: 1.291172E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.482909E-02 | lm-token-buffer-mib: 5.815553E+01 | global_load_balancing_loss: 9.998765E-01 | loss scale: 1.0 | grad norm: 0.063 | params norm: 11284.096 | params rms: 0.061100 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 7:30:10.346340 | finish at 2026-10-11 02:11:22 +times across ranks (ms): + save-checkpoint: + rank 0: 26748.08 + rank 1: 26748.08 + rank 2: 26748.08 + rank 3: 26748.10 + rank 4: 26748.05 + rank 5: 26748.02 + rank 6: 26748.04 + rank 7: 26748.02 + rank 8: 26748.37 + rank 9: 26748.34 + rank 10: 26748.36 + rank 11: 26748.35 + rank 12: 26748.36 + rank 13: 26748.35 + rank 14: 26748.38 + rank 15: 26748.44 + rank 16: 26748.47 + rank 17: 26748.49 + rank 18: 26748.45 + rank 19: 26748.46 + rank 20: 26748.45 + rank 21: 26748.47 + rank 22: 26748.45 + rank 23: 26748.44 + rank 24: 26748.17 + rank 25: 26748.51 + rank 26: 26748.18 + rank 27: 26748.22 + rank 28: 26748.12 + rank 29: 26748.15 + rank 30: 26748.13 + rank 31: 26748.20 + rank 32: 26748.60 + rank 33: 26748.60 + rank 34: 26748.61 + rank 35: 26748.62 + rank 36: 26748.43 + rank 37: 26748.50 + rank 38: 26748.50 + rank 39: 26748.51 + rank 40: 26748.34 + rank 41: 26748.30 + rank 42: 26748.34 + rank 43: 26748.29 + rank 44: 26748.14 + rank 45: 26748.31 + rank 46: 26748.29 + rank 47: 26748.27 + rank 48: 26748.03 + rank 49: 26748.06 + rank 50: 26748.12 + rank 51: 26748.04 + rank 52: 26748.19 + rank 53: 26748.21 + rank 54: 26748.18 + rank 55: 26748.02 + rank 56: 26747.77 + rank 57: 26747.85 + rank 58: 26747.72 + rank 59: 26747.73 + rank 60: 26747.74 + rank 61: 26747.75 + rank 62: 26747.73 + rank 63: 26747.76 + [2026-10-09 18:43:03] iteration 5841/ 6855 | consumed samples: 747648 | elapsed time per iteration (ms): 85082.1 | throughput per GPU (TFLOP/s/GPU): 43.2 | MFU 13.86% | FLOPs estimator: workload-aware | logical tokens/s: 197188.6 | valid tokens/s: 197188.6 | physical tokens/s: 197613.4 | loss tokens/s: 147117.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.640131E-01 | lm-global-supervised-tokens: 1.251704E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 6.078845E-01 | lm-token-buffer-mib: 5.817315E+01 | global_load_balancing_loss: 9.990055E-01 | loss scale: 1.0 | grad norm: 0.070 | params norm: 11284.052 | params rms: 0.061099 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 23:57:53.217402 | finish at 2026-10-10 18:40:57 + [2026-10-09 18:44:36] iteration 5842/ 6855 | consumed samples: 747776 | elapsed time per iteration (ms): 92424.1 | throughput per GPU (TFLOP/s/GPU): 40.5 | MFU 12.98% | FLOPs estimator: workload-aware | logical tokens/s: 181524.2 | valid tokens/s: 181524.2 | physical tokens/s: 181894.2 | loss tokens/s: 140027.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.155096E-01 | lm-global-supervised-tokens: 1.294190E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.366600E-02 | lm-token-buffer-mib: 5.806971E+01 | global_load_balancing_loss: 1.001305E+00 | loss scale: 1.0 | grad norm: 0.040 | params norm: 11284.007 | params rms: 0.061099 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 2:00:25.592451 | finish at 2026-10-10 20:45:01 + [2026-10-09 18:46:00] iteration 5843/ 6855 | consumed samples: 747904 | elapsed time per iteration (ms): 84398.9 | throughput per GPU (TFLOP/s/GPU): 43.3 | MFU 13.89% | FLOPs estimator: workload-aware | logical tokens/s: 198784.7 | valid tokens/s: 198784.7 | physical tokens/s: 199259.2 | loss tokens/s: 148340.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.548851E-01 | lm-global-supervised-tokens: 1.251974E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.278384E-02 | lm-token-buffer-mib: 5.818889E+01 | global_load_balancing_loss: 1.001677E+00 | loss scale: 1.0 | grad norm: 0.102 | params norm: 11283.960 | params rms: 0.061099 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 23:43:31.700344 | finish at 2026-10-10 18:29:32 + [2026-10-09 18:47:26] iteration 5844/ 6855 | consumed samples: 748032 | elapsed time per iteration (ms): 85700.1 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.61% | FLOPs estimator: workload-aware | logical tokens/s: 195765.9 | valid tokens/s: 195765.9 | physical tokens/s: 196282.2 | loss tokens/s: 147343.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.792645E-01 | lm-global-supervised-tokens: 1.262740E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.384373E-02 | lm-token-buffer-mib: 5.810786E+01 | global_load_balancing_loss: 1.001273E+00 | loss scale: 1.0 | grad norm: 0.048 | params norm: 11283.915 | params rms: 0.061099 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 0:04:02.835513 | finish at 2026-10-10 18:51:29 + [2026-10-09 18:48:55] iteration 5845/ 6855 | consumed samples: 748160 | elapsed time per iteration (ms): 89366.4 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.45% | FLOPs estimator: workload-aware | logical tokens/s: 187735.0 | valid tokens/s: 187735.0 | physical tokens/s: 188364.7 | loss tokens/s: 136145.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.205714E-01 | lm-global-supervised-tokens: 1.216684E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.690096E-02 | lm-token-buffer-mib: 5.804196E+01 | global_load_balancing_loss: 1.001315E+00 | loss scale: 1.0 | grad norm: 0.088 | params norm: 11283.868 | params rms: 0.061098 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 1:04:20.111682 | finish at 2026-10-10 19:53:15 + [2026-10-09 18:50:23] iteration 5846/ 6855 | consumed samples: 748288 | elapsed time per iteration (ms): 87615.5 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.55% | FLOPs estimator: workload-aware | logical tokens/s: 191486.8 | valid tokens/s: 191486.8 | physical tokens/s: 191944.2 | loss tokens/s: 144958.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.283405E-01 | lm-global-supervised-tokens: 1.270060E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.154807E-02 | lm-token-buffer-mib: 5.813024E+01 | global_load_balancing_loss: 1.000548E+00 | loss scale: 1.0 | grad norm: 0.067 | params norm: 11283.819 | params rms: 0.061098 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 0:33:24.018063 | finish at 2026-10-10 19:23:47 + [2026-10-09 18:51:52] iteration 5847/ 6855 | consumed samples: 748416 | elapsed time per iteration (ms): 89155.2 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.68% | FLOPs estimator: workload-aware | logical tokens/s: 188179.5 | valid tokens/s: 188179.5 | physical tokens/s: 188541.9 | loss tokens/s: 134657.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.598233E-01 | lm-global-supervised-tokens: 1.200544E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.170099E-02 | lm-token-buffer-mib: 5.804034E+01 | global_load_balancing_loss: 1.000278E+00 | loss scale: 1.0 | grad norm: 0.035 | params norm: 11283.769 | params rms: 0.061098 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 0:57:48.426224 | finish at 2026-10-10 19:49:41 + [2026-10-09 18:53:18] iteration 5848/ 6855 | consumed samples: 748544 | elapsed time per iteration (ms): 85509.9 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.54% | FLOPs estimator: workload-aware | logical tokens/s: 196200.8 | valid tokens/s: 196200.8 | physical tokens/s: 196821.1 | loss tokens/s: 148025.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 5.966892E-01 | lm-global-supervised-tokens: 1.265762E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.179360E-02 | lm-token-buffer-mib: 5.814964E+01 | global_load_balancing_loss: 9.988016E-01 | loss scale: 1.0 | grad norm: 0.137 | params norm: 11283.718 | params rms: 0.061098 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 23:55:08.516931 | finish at 2026-10-10 18:48:26 + [2026-10-09 18:54:54] iteration 5849/ 6855 | consumed samples: 748672 | elapsed time per iteration (ms): 96547.0 | throughput per GPU (TFLOP/s/GPU): 38.2 | MFU 12.24% | FLOPs estimator: workload-aware | logical tokens/s: 173772.6 | valid tokens/s: 173772.6 | physical tokens/s: 174096.1 | loss tokens/s: 129644.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.616876E-01 | lm-global-supervised-tokens: 1.251676E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.182908E-02 | lm-token-buffer-mib: 5.818656E+01 | global_load_balancing_loss: 9.995338E-01 | loss scale: 1.0 | grad norm: 0.162 | params norm: 11283.667 | params rms: 0.061097 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 2:58:46.246673 | finish at 2026-10-10 21:53:40 + [2026-10-09 18:58:18] iteration 5850/ 6855 | consumed samples: 748800 | elapsed time per iteration (ms): 203632.2 | throughput per GPU (TFLOP/s/GPU): 18.2 | MFU 5.83% | FLOPs estimator: workload-aware | logical tokens/s: 82389.3 | valid tokens/s: 82389.3 | physical tokens/s: 82639.3 | loss tokens/s: 60172.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.293218E-01 | lm-global-supervised-tokens: 1.225298E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.823551E-02 | lm-token-buffer-mib: 5.811563E+01 | global_load_balancing_loss: 1.000134E+00 | loss scale: 1.0 | grad norm: 0.058 | params norm: 11283.615 | params rms: 0.061097 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2 days, 8:50:50.387599 | finish at 2026-10-12 03:49:08 + [2026-10-09 19:00:09] iteration 5851/ 6855 | consumed samples: 748928 | elapsed time per iteration (ms): 111504.8 | throughput per GPU (TFLOP/s/GPU): 33.5 | MFU 10.72% | FLOPs estimator: workload-aware | logical tokens/s: 150461.7 | valid tokens/s: 150461.7 | physical tokens/s: 150968.3 | loss tokens/s: 114183.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.768426E-01 | lm-global-supervised-tokens: 1.273205E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.799543E-02 | lm-token-buffer-mib: 5.806117E+01 | global_load_balancing_loss: 1.000701E+00 | loss scale: 1.0 | grad norm: 0.156 | params norm: 11283.561 | params rms: 0.061097 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 7:05:50.847049 | finish at 2026-10-11 02:06:00 + [2026-10-09 19:01:34] iteration 5852/ 6855 | consumed samples: 749056 | elapsed time per iteration (ms): 84913.8 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.66% | FLOPs estimator: workload-aware | logical tokens/s: 197579.2 | valid tokens/s: 197579.2 | physical tokens/s: 198141.7 | loss tokens/s: 149250.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.601426E-01 | lm-global-supervised-tokens: 1.267347E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.446376E-02 | lm-token-buffer-mib: 5.812356E+01 | global_load_balancing_loss: 9.999008E-01 | loss scale: 1.0 | grad norm: 0.084 | params norm: 11283.507 | params rms: 0.061097 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 23:39:28.583728 | finish at 2026-10-10 18:41:03 + [2026-10-09 19:03:05] iteration 5853/ 6855 | consumed samples: 749184 | elapsed time per iteration (ms): 90572.6 | throughput per GPU (TFLOP/s/GPU): 40.8 | MFU 13.09% | FLOPs estimator: workload-aware | logical tokens/s: 185234.2 | valid tokens/s: 185234.2 | physical tokens/s: 185846.5 | loss tokens/s: 135121.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.701603E-01 | lm-global-supervised-tokens: 1.223831E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.165254E-02 | lm-token-buffer-mib: 5.809092E+01 | global_load_balancing_loss: 9.998542E-01 | loss scale: 1.0 | grad norm: 0.153 | params norm: 11283.451 | params rms: 0.061096 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 1:12:33.794539 | finish at 2026-10-10 20:15:39 + [2026-10-09 19:04:34] iteration 5854/ 6855 | consumed samples: 749312 | elapsed time per iteration (ms): 89533.1 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.56% | FLOPs estimator: workload-aware | logical tokens/s: 187384.6 | valid tokens/s: 187384.6 | physical tokens/s: 187773.0 | loss tokens/s: 141704.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.059797E-01 | lm-global-supervised-tokens: 1.268724E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.155538E-02 | lm-token-buffer-mib: 5.804854E+01 | global_load_balancing_loss: 1.001344E+00 | loss scale: 1.0 | grad norm: 0.150 | params norm: 11283.394 | params rms: 0.061096 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 0:53:42.637285 | finish at 2026-10-10 19:58:17 + [2026-10-09 19:06:00] iteration 5855/ 6855 | consumed samples: 749440 | elapsed time per iteration (ms): 85697.6 | throughput per GPU (TFLOP/s/GPU): 43.3 | MFU 13.87% | FLOPs estimator: workload-aware | logical tokens/s: 195771.1 | valid tokens/s: 195771.1 | physical tokens/s: 196383.9 | loss tokens/s: 147045.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.929699E-01 | lm-global-supervised-tokens: 1.260142E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.394597E-02 | lm-token-buffer-mib: 5.825262E+01 | global_load_balancing_loss: 1.000277E+00 | loss scale: 1.0 | grad norm: 0.062 | params norm: 11283.336 | params rms: 0.061096 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 23:48:17.637081 | finish at 2026-10-10 18:54:18 + [2026-10-09 19:07:32] iteration 5856/ 6855 | consumed samples: 749568 | elapsed time per iteration (ms): 92315.2 | throughput per GPU (TFLOP/s/GPU): 39.8 | MFU 12.76% | FLOPs estimator: workload-aware | logical tokens/s: 181737.6 | valid tokens/s: 181737.6 | physical tokens/s: 182374.9 | loss tokens/s: 143165.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.976802E-01 | lm-global-supervised-tokens: 1.321634E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.458325E-02 | lm-token-buffer-mib: 5.834243E+01 | global_load_balancing_loss: 9.993263E-01 | loss scale: 1.0 | grad norm: 0.111 | params norm: 11283.280 | params rms: 0.061095 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 1:37:02.909899 | finish at 2026-10-10 20:44:35 + [2026-10-09 19:09:01] iteration 5857/ 6855 | consumed samples: 749696 | elapsed time per iteration (ms): 88801.3 | throughput per GPU (TFLOP/s/GPU): 40.8 | MFU 13.08% | FLOPs estimator: workload-aware | logical tokens/s: 188929.2 | valid tokens/s: 188929.2 | physical tokens/s: 189460.6 | loss tokens/s: 139143.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.602904E-01 | lm-global-supervised-tokens: 1.235615E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.544800E-02 | lm-token-buffer-mib: 5.809715E+01 | global_load_balancing_loss: 9.992207E-01 | loss scale: 1.0 | grad norm: 0.150 | params norm: 11283.226 | params rms: 0.061095 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 0:37:03.731712 | finish at 2026-10-10 19:46:05 + [2026-10-09 19:10:28] iteration 5858/ 6855 | consumed samples: 749824 | elapsed time per iteration (ms): 86552.1 | throughput per GPU (TFLOP/s/GPU): 43.1 | MFU 13.80% | FLOPs estimator: workload-aware | logical tokens/s: 193838.5 | valid tokens/s: 193838.5 | physical tokens/s: 194363.3 | loss tokens/s: 146862.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.544892E-01 | lm-global-supervised-tokens: 1.271127E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.584830E-02 | lm-token-buffer-mib: 5.809448E+01 | global_load_balancing_loss: 9.998620E-01 | loss scale: 1.0 | grad norm: 0.166 | params norm: 11283.172 | params rms: 0.061095 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 23:58:12.481438 | finish at 2026-10-10 19:08:40 + [2026-10-09 19:11:55] iteration 5859/ 6855 | consumed samples: 749952 | elapsed time per iteration (ms): 87290.5 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.50% | FLOPs estimator: workload-aware | logical tokens/s: 192199.2 | valid tokens/s: 192199.2 | physical tokens/s: 192601.4 | loss tokens/s: 141854.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.911551E-01 | lm-global-supervised-tokens: 1.238258E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.582180E-02 | lm-token-buffer-mib: 5.811901E+01 | global_load_balancing_loss: 1.000301E+00 | loss scale: 1.0 | grad norm: 0.043 | params norm: 11283.118 | params rms: 0.061094 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 0:09:01.335076 | finish at 2026-10-10 19:20:56 + [2026-10-09 19:13:46] iteration 5860/ 6855 | consumed samples: 750080 | elapsed time per iteration (ms): 111195.3 | throughput per GPU (TFLOP/s/GPU): 34.3 | MFU 10.98% | FLOPs estimator: workload-aware | logical tokens/s: 150880.3 | valid tokens/s: 150880.3 | physical tokens/s: 151131.0 | loss tokens/s: 105506.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.677911E-01 | lm-global-supervised-tokens: 1.173179E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.335303E-02 | lm-token-buffer-mib: 5.808324E+01 | global_load_balancing_loss: 1.000987E+00 | loss scale: 1.0 | grad norm: 0.047 | params norm: 11283.065 | params rms: 0.061094 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 6:43:59.331905 | finish at 2026-10-11 01:57:46 + [2026-10-09 19:15:11] iteration 5861/ 6855 | consumed samples: 750208 | elapsed time per iteration (ms): 85073.7 | throughput per GPU (TFLOP/s/GPU): 42.8 | MFU 13.73% | FLOPs estimator: workload-aware | logical tokens/s: 197207.7 | valid tokens/s: 197207.7 | physical tokens/s: 197542.5 | loss tokens/s: 148069.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.608529E-01 | lm-global-supervised-tokens: 1.259680E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.166391E-02 | lm-token-buffer-mib: 5.816393E+01 | global_load_balancing_loss: 1.000600E+00 | loss scale: 1.0 | grad norm: 0.099 | params norm: 11283.012 | params rms: 0.061094 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 23:29:23.211065 | finish at 2026-10-10 18:44:34 + [2026-10-09 19:16:40] iteration 5862/ 6855 | consumed samples: 750336 | elapsed time per iteration (ms): 89224.1 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.42% | FLOPs estimator: workload-aware | logical tokens/s: 188034.5 | valid tokens/s: 188034.4 | physical tokens/s: 188459.9 | loss tokens/s: 134615.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.752661E-01 | lm-global-supervised-tokens: 1.201098E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.243288E-02 | lm-token-buffer-mib: 5.826919E+01 | global_load_balancing_loss: 1.000782E+00 | loss scale: 1.0 | grad norm: 0.162 | params norm: 11282.959 | params rms: 0.061094 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 0:36:39.543960 | finish at 2026-10-10 19:53:20 + [2026-10-09 19:18:07] iteration 5863/ 6855 | consumed samples: 750464 | elapsed time per iteration (ms): 86349.8 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.45% | FLOPs estimator: workload-aware | logical tokens/s: 194293.3 | valid tokens/s: 194293.3 | physical tokens/s: 194881.3 | loss tokens/s: 147355.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.695122E-01 | lm-global-supervised-tokens: 1.272416E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.379488E-02 | lm-token-buffer-mib: 5.820892E+01 | global_load_balancing_loss: 9.995407E-01 | loss scale: 1.0 | grad norm: 0.065 | params norm: 11282.904 | params rms: 0.061093 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 23:47:39.043571 | finish at 2026-10-10 19:05:46 + [2026-10-09 19:19:30] iteration 5864/ 6855 | consumed samples: 750592 | elapsed time per iteration (ms): 83589.0 | throughput per GPU (TFLOP/s/GPU): 43.3 | MFU 13.89% | FLOPs estimator: workload-aware | logical tokens/s: 200710.7 | valid tokens/s: 200710.7 | physical tokens/s: 201196.9 | loss tokens/s: 158091.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.329744E-01 | lm-global-supervised-tokens: 1.321467E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.349314E-02 | lm-token-buffer-mib: 5.817539E+01 | global_load_balancing_loss: 9.984733E-01 | loss scale: 1.0 | grad norm: 0.095 | params norm: 11282.850 | params rms: 0.061093 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 23:00:36.670161 | finish at 2026-10-10 18:20:07 + [2026-10-09 19:20:58] iteration 5865/ 6855 | consumed samples: 750720 | elapsed time per iteration (ms): 87090.2 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.68% | FLOPs estimator: workload-aware | logical tokens/s: 192641.0 | valid tokens/s: 192641.0 | physical tokens/s: 193371.0 | loss tokens/s: 138483.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.388967E-01 | lm-global-supervised-tokens: 1.206054E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.152684E-02 | lm-token-buffer-mib: 5.815550E+01 | global_load_balancing_loss: 1.001243E+00 | loss scale: 1.0 | grad norm: 0.064 | params norm: 11282.795 | params rms: 0.061093 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 23:56:59.313054 | finish at 2026-10-10 19:17:57 + [2026-10-09 19:22:26] iteration 5866/ 6855 | consumed samples: 750848 | elapsed time per iteration (ms): 88856.2 | throughput per GPU (TFLOP/s/GPU): 40.7 | MFU 13.05% | FLOPs estimator: workload-aware | logical tokens/s: 188813.0 | valid tokens/s: 188813.0 | physical tokens/s: 189249.6 | loss tokens/s: 144660.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.219496E-01 | lm-global-supervised-tokens: 1.285402E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.223604E-02 | lm-token-buffer-mib: 5.821394E+01 | global_load_balancing_loss: 1.001271E+00 | loss scale: 1.0 | grad norm: 0.163 | params norm: 11282.741 | params rms: 0.061092 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 0:24:38.775178 | finish at 2026-10-10 19:47:05 + [2026-10-09 19:23:52] iteration 5867/ 6855 | consumed samples: 750976 | elapsed time per iteration (ms): 85631.4 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.50% | FLOPs estimator: workload-aware | logical tokens/s: 195923.5 | valid tokens/s: 195923.5 | physical tokens/s: 196461.7 | loss tokens/s: 148958.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.497298E-01 | lm-global-supervised-tokens: 1.275550E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.730615E-02 | lm-token-buffer-mib: 5.806014E+01 | global_load_balancing_loss: 1.000009E+00 | loss scale: 1.0 | grad norm: 0.375 | params norm: 11282.687 | params rms: 0.061092 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 23:30:03.782320 | finish at 2026-10-10 18:53:56 + [2026-10-09 19:25:19] iteration 5868/ 6855 | consumed samples: 751104 | elapsed time per iteration (ms): 86758.8 | throughput per GPU (TFLOP/s/GPU): 43.2 | MFU 13.86% | FLOPs estimator: workload-aware | logical tokens/s: 193377.6 | valid tokens/s: 193377.6 | physical tokens/s: 193898.2 | loss tokens/s: 146653.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.791194E-01 | lm-global-supervised-tokens: 1.272351E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.235214E-02 | lm-token-buffer-mib: 5.822172E+01 | global_load_balancing_loss: 1.000781E+00 | loss scale: 1.0 | grad norm: 0.518 | params norm: 11282.631 | params rms: 0.061092 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 23:47:10.902920 | finish at 2026-10-10 19:12:30 + [2026-10-09 19:26:45] iteration 5869/ 6855 | consumed samples: 751232 | elapsed time per iteration (ms): 86591.3 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.54% | FLOPs estimator: workload-aware | logical tokens/s: 193751.4 | valid tokens/s: 193751.4 | physical tokens/s: 194221.9 | loss tokens/s: 148387.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.216048E-01 | lm-global-supervised-tokens: 1.284903E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.551450E-02 | lm-token-buffer-mib: 5.812615E+01 | global_load_balancing_loss: 1.000594E+00 | loss scale: 1.0 | grad norm: 0.098 | params norm: 11282.574 | params rms: 0.061091 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 23:42:59.008411 | finish at 2026-10-10 19:09:44 + [2026-10-09 19:28:36] iteration 5870/ 6855 | consumed samples: 751360 | elapsed time per iteration (ms): 110199.2 | throughput per GPU (TFLOP/s/GPU): 34.4 | MFU 11.01% | FLOPs estimator: workload-aware | logical tokens/s: 152244.0 | valid tokens/s: 152244.0 | physical tokens/s: 152563.2 | loss tokens/s: 111302.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.739023E-01 | lm-global-supervised-tokens: 1.226544E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.133144E-02 | lm-token-buffer-mib: 5.820912E+01 | global_load_balancing_loss: 1.000448E+00 | loss scale: 1.0 | grad norm: 0.235 | params norm: 11282.519 | params rms: 0.061091 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 6:09:06.246203 | finish at 2026-10-11 01:37:42 + [2026-10-09 19:30:02] iteration 5871/ 6855 | consumed samples: 751488 | elapsed time per iteration (ms): 86566.5 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.61% | FLOPs estimator: workload-aware | logical tokens/s: 193806.8 | valid tokens/s: 193806.8 | physical tokens/s: 194228.5 | loss tokens/s: 143249.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.689523E-01 | lm-global-supervised-tokens: 1.240063E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.556717E-02 | lm-token-buffer-mib: 5.811396E+01 | global_load_balancing_loss: 9.988460E-01 | loss scale: 1.0 | grad norm: 0.056 | params norm: 11282.463 | params rms: 0.061091 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 23:39:41.445099 | finish at 2026-10-10 19:09:44 + [2026-10-09 19:31:27] iteration 5872/ 6855 | consumed samples: 751616 | elapsed time per iteration (ms): 84526.7 | throughput per GPU (TFLOP/s/GPU): 43.4 | MFU 13.90% | FLOPs estimator: workload-aware | logical tokens/s: 198483.8 | valid tokens/s: 198483.8 | physical tokens/s: 198856.9 | loss tokens/s: 146143.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.488290E-01 | lm-global-supervised-tokens: 1.235302E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.139134E-02 | lm-token-buffer-mib: 5.811916E+01 | global_load_balancing_loss: 9.999466E-01 | loss scale: 1.0 | grad norm: 0.066 | params norm: 11282.407 | params rms: 0.061091 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 23:04:49.770728 | finish at 2026-10-10 18:36:16 + [2026-10-09 19:32:54] iteration 5873/ 6855 | consumed samples: 751744 | elapsed time per iteration (ms): 87042.9 | throughput per GPU (TFLOP/s/GPU): 43.8 | MFU 14.03% | FLOPs estimator: workload-aware | logical tokens/s: 192746.1 | valid tokens/s: 192746.1 | physical tokens/s: 193108.5 | loss tokens/s: 141788.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.061636E-01 | lm-global-supervised-tokens: 1.234168E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.292815E-02 | lm-token-buffer-mib: 5.814562E+01 | global_load_balancing_loss: 9.993623E-01 | loss scale: 1.0 | grad norm: 0.104 | params norm: 11282.352 | params rms: 0.061090 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 23:44:36.116646 | finish at 2026-10-10 19:17:30 + [2026-10-09 19:34:22] iteration 5874/ 6855 | consumed samples: 751872 | elapsed time per iteration (ms): 88398.7 | throughput per GPU (TFLOP/s/GPU): 41.6 | MFU 13.35% | FLOPs estimator: workload-aware | logical tokens/s: 189789.5 | valid tokens/s: 189789.5 | physical tokens/s: 190097.5 | loss tokens/s: 143336.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.868785E-01 | lm-global-supervised-tokens: 1.267074E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.456100E-02 | lm-token-buffer-mib: 5.808339E+01 | global_load_balancing_loss: 1.000524E+00 | loss scale: 1.0 | grad norm: 0.055 | params norm: 11282.295 | params rms: 0.061090 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 0:05:19.169839 | finish at 2026-10-10 19:39:41 + [2026-10-09 19:35:47] iteration 5875/ 6855 | consumed samples: 752000 | elapsed time per iteration (ms): 85349.1 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.67% | FLOPs estimator: workload-aware | logical tokens/s: 196571.6 | valid tokens/s: 196571.6 | physical tokens/s: 197027.6 | loss tokens/s: 143670.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 5.743011E-01 | lm-global-supervised-tokens: 1.226215E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.850943E-02 | lm-token-buffer-mib: 5.822192E+01 | global_load_balancing_loss: 1.001642E+00 | loss scale: 1.0 | grad norm: 0.043 | params norm: 11282.241 | params rms: 0.061090 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 23:14:02.122083 | finish at 2026-10-10 18:49:50 + [2026-10-09 19:37:13] iteration 5876/ 6855 | consumed samples: 752128 | elapsed time per iteration (ms): 85547.1 | throughput per GPU (TFLOP/s/GPU): 42.8 | MFU 13.71% | FLOPs estimator: workload-aware | logical tokens/s: 196116.4 | valid tokens/s: 196116.4 | physical tokens/s: 196518.8 | loss tokens/s: 142522.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.607382E-01 | lm-global-supervised-tokens: 1.219234E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.279821E-02 | lm-token-buffer-mib: 5.820459E+01 | global_load_balancing_loss: 1.000861E+00 | loss scale: 1.0 | grad norm: 0.098 | params norm: 11282.188 | params rms: 0.061089 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 23:15:50.614000 | finish at 2026-10-10 18:53:04 + [2026-10-09 19:38:38] iteration 5877/ 6855 | consumed samples: 752256 | elapsed time per iteration (ms): 84587.7 | throughput per GPU (TFLOP/s/GPU): 43.2 | MFU 13.85% | FLOPs estimator: workload-aware | logical tokens/s: 198340.6 | valid tokens/s: 198340.6 | physical tokens/s: 198882.1 | loss tokens/s: 150873.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.244633E-01 | lm-global-supervised-tokens: 1.276204E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.162978E-02 | lm-token-buffer-mib: 5.812205E+01 | global_load_balancing_loss: 1.000095E+00 | loss scale: 1.0 | grad norm: 0.083 | params norm: 11282.135 | params rms: 0.061089 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 22:58:46.811997 | finish at 2026-10-10 18:37:24 + [2026-10-09 19:40:03] iteration 5878/ 6855 | consumed samples: 752384 | elapsed time per iteration (ms): 85901.0 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.42% | FLOPs estimator: workload-aware | logical tokens/s: 195308.3 | valid tokens/s: 195308.3 | physical tokens/s: 195747.9 | loss tokens/s: 144844.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.636176E-01 | lm-global-supervised-tokens: 1.244232E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.170780E-02 | lm-token-buffer-mib: 5.806273E+01 | global_load_balancing_loss: 1.000381E+00 | loss scale: 1.0 | grad norm: 0.069 | params norm: 11282.080 | params rms: 0.061089 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 23:18:45.291464 | finish at 2026-10-10 18:58:49 + [2026-10-09 19:41:30] iteration 5879/ 6855 | consumed samples: 752512 | elapsed time per iteration (ms): 86585.5 | throughput per GPU (TFLOP/s/GPU): 41.4 | MFU 13.29% | FLOPs estimator: workload-aware | logical tokens/s: 193764.5 | valid tokens/s: 193764.5 | physical tokens/s: 194255.8 | loss tokens/s: 152309.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.432830E-01 | lm-global-supervised-tokens: 1.318777E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.477175E-02 | lm-token-buffer-mib: 5.815781E+01 | global_load_balancing_loss: 9.986387E-01 | loss scale: 1.0 | grad norm: 0.091 | params norm: 11282.027 | params rms: 0.061089 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 23:28:27.404488 | finish at 2026-10-10 19:09:57 + [2026-10-09 19:43:18] iteration 5880/ 6855 | consumed samples: 752640 | elapsed time per iteration (ms): 108233.9 | throughput per GPU (TFLOP/s/GPU): 34.0 | MFU 10.90% | FLOPs estimator: workload-aware | logical tokens/s: 155008.6 | valid tokens/s: 155008.6 | physical tokens/s: 155366.6 | loss tokens/s: 115338.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.821707E-01 | lm-global-supervised-tokens: 1.248349E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.303784E-02 | lm-token-buffer-mib: 5.816727E+01 | global_load_balancing_loss: 9.991016E-01 | loss scale: 1.0 | grad norm: 0.074 | params norm: 11281.975 | params rms: 0.061088 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 5:18:48.046292 | finish at 2026-10-11 01:02:06 +times across ranks (ms): + save-checkpoint: + rank 0: 23959.18 + rank 1: 23959.20 + rank 2: 23959.20 + rank 3: 23959.21 + rank 4: 23959.20 + rank 5: 23959.19 + rank 6: 23959.17 + rank 7: 23959.19 + rank 8: 23959.89 + rank 9: 23959.87 + rank 10: 23959.87 + rank 11: 23959.87 + rank 12: 23959.88 + rank 13: 23959.90 + rank 14: 23959.88 + rank 15: 23959.88 + rank 16: 23959.12 + rank 17: 23959.12 + rank 18: 23959.12 + rank 19: 23959.12 + rank 20: 23959.10 + rank 21: 23959.16 + rank 22: 23959.11 + rank 23: 23959.10 + rank 24: 23960.58 + rank 25: 23960.51 + rank 26: 23960.56 + rank 27: 23960.58 + rank 28: 23960.51 + rank 29: 23960.58 + rank 30: 23960.51 + rank 31: 23960.64 + rank 32: 23960.00 + rank 33: 23960.03 + rank 34: 23960.02 + rank 35: 23960.00 + rank 36: 23959.98 + rank 37: 23959.98 + rank 38: 23959.99 + rank 39: 23959.98 + rank 40: 23959.61 + rank 41: 23959.61 + rank 42: 23959.59 + rank 43: 23959.63 + rank 44: 23959.60 + rank 45: 23959.61 + rank 46: 23959.60 + rank 47: 23959.59 + rank 48: 23959.21 + rank 49: 23959.23 + rank 50: 23959.20 + rank 51: 23959.17 + rank 52: 23959.17 + rank 53: 23959.19 + rank 54: 23959.18 + rank 55: 23959.23 + rank 56: 23959.69 + rank 57: 23959.67 + rank 58: 23959.64 + rank 59: 23959.64 + rank 60: 23959.76 + rank 61: 23959.66 + rank 62: 23959.64 + rank 63: 23959.64 + [2026-10-09 19:45:17] iteration 5881/ 6855 | consumed samples: 752768 | elapsed time per iteration (ms): 94338.4 | throughput per GPU (TFLOP/s/GPU): 39.8 | MFU 12.75% | FLOPs estimator: workload-aware | logical tokens/s: 177840.0 | valid tokens/s: 177840.0 | physical tokens/s: 178257.4 | loss tokens/s: 133289.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.813312E-01 | lm-global-supervised-tokens: 1.257428E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.019325E+00 | lm-token-buffer-mib: 5.809999E+01 | global_load_balancing_loss: 1.000450E+00 | loss scale: 1.0 | grad norm: 0.045 | params norm: 11281.920 | params rms: 0.061088 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 1:31:25.560619 | finish at 2026-10-10 21:16:42 + [2026-10-09 19:46:50] iteration 5882/ 6855 | consumed samples: 752896 | elapsed time per iteration (ms): 92891.5 | throughput per GPU (TFLOP/s/GPU): 39.3 | MFU 12.59% | FLOPs estimator: workload-aware | logical tokens/s: 180610.8 | valid tokens/s: 180610.8 | physical tokens/s: 181077.4 | loss tokens/s: 129662.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 5.798298E-01 | lm-global-supervised-tokens: 1.204457E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.372618E-02 | lm-token-buffer-mib: 5.828661E+01 | global_load_balancing_loss: 1.001808E+00 | loss scale: 1.0 | grad norm: 0.138 | params norm: 11281.864 | params rms: 0.061088 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 1:06:23.454318 | finish at 2026-10-10 20:53:13 + [2026-10-09 19:48:14] iteration 5883/ 6855 | consumed samples: 753024 | elapsed time per iteration (ms): 83856.6 | throughput per GPU (TFLOP/s/GPU): 43.6 | MFU 13.96% | FLOPs estimator: workload-aware | logical tokens/s: 200070.3 | valid tokens/s: 200070.3 | physical tokens/s: 200489.5 | loss tokens/s: 151777.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.774544E-01 | lm-global-supervised-tokens: 1.272755E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.453008E-02 | lm-token-buffer-mib: 5.814294E+01 | global_load_balancing_loss: 1.000909E+00 | loss scale: 1.0 | grad norm: 0.049 | params norm: 11281.808 | params rms: 0.061087 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 22:38:28.620807 | finish at 2026-10-10 18:26:42 + [2026-10-09 19:49:39] iteration 5884/ 6855 | consumed samples: 753152 | elapsed time per iteration (ms): 85044.6 | throughput per GPU (TFLOP/s/GPU): 43.4 | MFU 13.92% | FLOPs estimator: workload-aware | logical tokens/s: 197274.6 | valid tokens/s: 197274.6 | physical tokens/s: 197911.9 | loss tokens/s: 158500.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.264558E-01 | lm-global-supervised-tokens: 1.347958E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.297951E-02 | lm-token-buffer-mib: 5.811424E+01 | global_load_balancing_loss: 9.995191E-01 | loss scale: 1.0 | grad norm: 0.094 | params norm: 11281.751 | params rms: 0.061087 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 22:56:18.339483 | finish at 2026-10-10 18:45:57 + [2026-10-09 19:51:06] iteration 5885/ 6855 | consumed samples: 753280 | elapsed time per iteration (ms): 87654.0 | throughput per GPU (TFLOP/s/GPU): 43.3 | MFU 13.87% | FLOPs estimator: workload-aware | logical tokens/s: 191402.7 | valid tokens/s: 191402.7 | physical tokens/s: 191792.9 | loss tokens/s: 139339.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.761111E-01 | lm-global-supervised-tokens: 1.221366E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.138071E-02 | lm-token-buffer-mib: 5.816085E+01 | global_load_balancing_loss: 1.001029E+00 | loss scale: 1.0 | grad norm: 0.045 | params norm: 11281.693 | params rms: 0.061087 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 23:37:04.347203 | finish at 2026-10-10 19:28:11 + [2026-10-09 19:52:32] iteration 5886/ 6855 | consumed samples: 753408 | elapsed time per iteration (ms): 85712.7 | throughput per GPU (TFLOP/s/GPU): 41.7 | MFU 13.37% | FLOPs estimator: workload-aware | logical tokens/s: 195737.7 | valid tokens/s: 195737.7 | physical tokens/s: 196173.1 | loss tokens/s: 153740.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.467161E-01 | lm-global-supervised-tokens: 1.317753E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.592480E-02 | lm-token-buffer-mib: 5.804167E+01 | global_load_balancing_loss: 1.000253E+00 | loss scale: 1.0 | grad norm: 0.052 | params norm: 11281.634 | params rms: 0.061086 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 23:04:15.621672 | finish at 2026-10-10 18:56:48 + [2026-10-09 19:54:01] iteration 5887/ 6855 | consumed samples: 753536 | elapsed time per iteration (ms): 89138.7 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.68% | FLOPs estimator: workload-aware | logical tokens/s: 188214.4 | valid tokens/s: 188214.4 | physical tokens/s: 188599.6 | loss tokens/s: 135480.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.601126E-01 | lm-global-supervised-tokens: 1.207657E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.338794E-02 | lm-token-buffer-mib: 5.808846E+01 | global_load_balancing_loss: 9.993168E-01 | loss scale: 1.0 | grad norm: 0.073 | params norm: 11281.576 | params rms: 0.061086 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 23:58:06.250992 | finish at 2026-10-10 19:52:07 + [2026-10-09 19:55:27] iteration 5888/ 6855 | consumed samples: 753664 | elapsed time per iteration (ms): 85680.1 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.56% | FLOPs estimator: workload-aware | logical tokens/s: 195811.4 | valid tokens/s: 195811.4 | physical tokens/s: 196379.2 | loss tokens/s: 151549.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.837020E-01 | lm-global-supervised-tokens: 1.298474E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.765984E-02 | lm-token-buffer-mib: 5.806287E+01 | global_load_balancing_loss: 1.000007E+00 | loss scale: 1.0 | grad norm: 0.064 | params norm: 11281.518 | params rms: 0.061086 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 23:00:52.661737 | finish at 2026-10-10 18:56:19 + [2026-10-09 19:56:52] iteration 5889/ 6855 | consumed samples: 753792 | elapsed time per iteration (ms): 85285.2 | throughput per GPU (TFLOP/s/GPU): 42.8 | MFU 13.72% | FLOPs estimator: workload-aware | logical tokens/s: 196718.5 | valid tokens/s: 196718.5 | physical tokens/s: 197127.0 | loss tokens/s: 143712.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.435587E-01 | lm-global-supervised-tokens: 1.225656E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.073246E-02 | lm-token-buffer-mib: 5.804552E+01 | global_load_balancing_loss: 1.000193E+00 | loss scale: 1.0 | grad norm: 0.051 | params norm: 11281.460 | params rms: 0.061085 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 22:53:05.519897 | finish at 2026-10-10 18:49:58 + [2026-10-09 19:58:40] iteration 5890/ 6855 | consumed samples: 753920 | elapsed time per iteration (ms): 107768.1 | throughput per GPU (TFLOP/s/GPU): 34.0 | MFU 10.89% | FLOPs estimator: workload-aware | logical tokens/s: 155678.3 | valid tokens/s: 155678.3 | physical tokens/s: 156032.8 | loss tokens/s: 121571.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.262694E-01 | lm-global-supervised-tokens: 1.310157E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.121546E-02 | lm-token-buffer-mib: 5.814168E+01 | global_load_balancing_loss: 9.997776E-01 | loss scale: 1.0 | grad norm: 0.089 | params norm: 11281.401 | params rms: 0.061085 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 4:53:16.207089 | finish at 2026-10-11 00:51:56 + [2026-10-09 20:00:11] iteration 5891/ 6855 | consumed samples: 754048 | elapsed time per iteration (ms): 90969.4 | throughput per GPU (TFLOP/s/GPU): 40.9 | MFU 13.11% | FLOPs estimator: workload-aware | logical tokens/s: 184426.8 | valid tokens/s: 184426.8 | physical tokens/s: 184854.5 | loss tokens/s: 143419.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.556541E-01 | lm-global-supervised-tokens: 1.304679E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.129102E-02 | lm-token-buffer-mib: 5.817405E+01 | global_load_balancing_loss: 9.998410E-01 | loss scale: 1.0 | grad norm: 0.063 | params norm: 11281.341 | params rms: 0.061085 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 0:21:34.539684 | finish at 2026-10-10 20:21:45 + [2026-10-09 20:01:36] iteration 5892/ 6855 | consumed samples: 754176 | elapsed time per iteration (ms): 84889.9 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.67% | FLOPs estimator: workload-aware | logical tokens/s: 197635.0 | valid tokens/s: 197635.0 | physical tokens/s: 198098.3 | loss tokens/s: 149692.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.429500E-01 | lm-global-supervised-tokens: 1.270739E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.121723E-02 | lm-token-buffer-mib: 5.815375E+01 | global_load_balancing_loss: 1.000295E+00 | loss scale: 1.0 | grad norm: 0.036 | params norm: 11281.283 | params rms: 0.061084 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 22:42:28.966553 | finish at 2026-10-10 18:44:05 + [2026-10-09 20:03:04] iteration 5893/ 6855 | consumed samples: 754304 | elapsed time per iteration (ms): 88022.5 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.49% | FLOPs estimator: workload-aware | logical tokens/s: 190601.5 | valid tokens/s: 190601.5 | physical tokens/s: 191076.0 | loss tokens/s: 134668.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.249798E-01 | lm-global-supervised-tokens: 1.185382E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.093758E-02 | lm-token-buffer-mib: 5.819538E+01 | global_load_balancing_loss: 1.001885E+00 | loss scale: 1.0 | grad norm: 0.064 | params norm: 11281.226 | params rms: 0.061084 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 23:31:17.605358 | finish at 2026-10-10 19:34:21 + [2026-10-09 20:04:33] iteration 5894/ 6855 | consumed samples: 754432 | elapsed time per iteration (ms): 89039.7 | throughput per GPU (TFLOP/s/GPU): 41.7 | MFU 13.38% | FLOPs estimator: workload-aware | logical tokens/s: 188423.9 | valid tokens/s: 188423.9 | physical tokens/s: 189062.1 | loss tokens/s: 138865.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.485006E-01 | lm-global-supervised-tokens: 1.236451E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.626024E-02 | lm-token-buffer-mib: 5.803390E+01 | global_load_balancing_loss: 1.000807E+00 | loss scale: 1.0 | grad norm: 0.037 | params norm: 11281.168 | params rms: 0.061084 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 23:46:07.164790 | finish at 2026-10-10 19:50:40 + [2026-10-09 20:06:00] iteration 5895/ 6855 | consumed samples: 754560 | elapsed time per iteration (ms): 87087.7 | throughput per GPU (TFLOP/s/GPU): 43.6 | MFU 13.97% | FLOPs estimator: workload-aware | logical tokens/s: 192647.2 | valid tokens/s: 192647.2 | physical tokens/s: 192920.0 | loss tokens/s: 139663.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.290424E-01 | lm-global-supervised-tokens: 1.216297E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.331954E-02 | lm-token-buffer-mib: 5.804795E+01 | global_load_balancing_loss: 1.000011E+00 | loss scale: 1.0 | grad norm: 0.051 | params norm: 11281.110 | params rms: 0.061084 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 23:13:24.185944 | finish at 2026-10-10 19:19:24 + [2026-10-09 20:07:29] iteration 5896/ 6855 | consumed samples: 754688 | elapsed time per iteration (ms): 89022.9 | throughput per GPU (TFLOP/s/GPU): 40.6 | MFU 13.02% | FLOPs estimator: workload-aware | logical tokens/s: 188458.6 | valid tokens/s: 188458.6 | physical tokens/s: 188959.7 | loss tokens/s: 138711.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.082222E-01 | lm-global-supervised-tokens: 1.234848E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.472822E-02 | lm-token-buffer-mib: 5.806439E+01 | global_load_balancing_loss: 9.996895E-01 | loss scale: 1.0 | grad norm: 0.056 | params norm: 11281.052 | params rms: 0.061083 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 23:42:52.980864 | finish at 2026-10-10 19:50:22 + [2026-10-09 20:08:56] iteration 5897/ 6855 | consumed samples: 754816 | elapsed time per iteration (ms): 87461.7 | throughput per GPU (TFLOP/s/GPU): 41.6 | MFU 13.35% | FLOPs estimator: workload-aware | logical tokens/s: 191823.0 | valid tokens/s: 191823.0 | physical tokens/s: 192209.2 | loss tokens/s: 147672.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.743792E-01 | lm-global-supervised-tokens: 1.291567E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.470531E-02 | lm-token-buffer-mib: 5.816681E+01 | global_load_balancing_loss: 9.990886E-01 | loss scale: 1.0 | grad norm: 0.047 | params norm: 11280.994 | params rms: 0.061083 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 23:16:28.321812 | finish at 2026-10-10 19:25:25 + [2026-10-09 20:10:20] iteration 5898/ 6855 | consumed samples: 754944 | elapsed time per iteration (ms): 83409.8 | throughput per GPU (TFLOP/s/GPU): 43.8 | MFU 14.03% | FLOPs estimator: workload-aware | logical tokens/s: 201141.5 | valid tokens/s: 201141.5 | physical tokens/s: 201726.7 | loss tokens/s: 151914.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.636541E-01 | lm-global-supervised-tokens: 1.267113E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.124987E-02 | lm-token-buffer-mib: 5.822277E+01 | global_load_balancing_loss: 9.993528E-01 | loss scale: 1.0 | grad norm: 0.050 | params norm: 11280.937 | params rms: 0.061083 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 22:10:23.169296 | finish at 2026-10-10 18:20:43 + [2026-10-09 20:11:49] iteration 5899/ 6855 | consumed samples: 755072 | elapsed time per iteration (ms): 89341.6 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.42% | FLOPs estimator: workload-aware | logical tokens/s: 187787.1 | valid tokens/s: 187787.1 | physical tokens/s: 188141.9 | loss tokens/s: 141918.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.382954E-01 | lm-global-supervised-tokens: 1.267925E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.755050E-02 | lm-token-buffer-mib: 5.810012E+01 | global_load_balancing_loss: 1.000506E+00 | loss scale: 1.0 | grad norm: 0.042 | params norm: 11280.878 | params rms: 0.061082 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 23:43:30.573419 | finish at 2026-10-10 19:55:20 + [2026-10-09 20:13:37] iteration 5900/ 6855 | consumed samples: 755200 | elapsed time per iteration (ms): 107582.1 | throughput per GPU (TFLOP/s/GPU): 34.4 | MFU 11.02% | FLOPs estimator: workload-aware | logical tokens/s: 155947.5 | valid tokens/s: 155947.5 | physical tokens/s: 156368.5 | loss tokens/s: 115251.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.558849E-01 | lm-global-supervised-tokens: 1.239899E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.158391E-02 | lm-token-buffer-mib: 5.809665E+01 | global_load_balancing_loss: 1.001294E+00 | loss scale: 1.0 | grad norm: 0.034 | params norm: 11280.820 | params rms: 0.061082 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 4:32:20.894945 | finish at 2026-10-11 00:45:58 + [2026-10-09 20:15:02] iteration 5901/ 6855 | consumed samples: 755328 | elapsed time per iteration (ms): 85117.2 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.64% | FLOPs estimator: workload-aware | logical tokens/s: 197107.2 | valid tokens/s: 197107.2 | physical tokens/s: 197521.2 | loss tokens/s: 144956.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.371658E-01 | lm-global-supervised-tokens: 1.233830E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.436188E-02 | lm-token-buffer-mib: 5.803680E+01 | global_load_balancing_loss: 1.001722E+00 | loss scale: 1.0 | grad norm: 0.050 | params norm: 11280.762 | params rms: 0.061082 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 22:33:21.788232 | finish at 2026-10-10 18:48:24 + [2026-10-09 20:16:28] iteration 5902/ 6855 | consumed samples: 755456 | elapsed time per iteration (ms): 85802.3 | throughput per GPU (TFLOP/s/GPU): 42.8 | MFU 13.71% | FLOPs estimator: workload-aware | logical tokens/s: 195532.8 | valid tokens/s: 195532.8 | physical tokens/s: 196105.8 | loss tokens/s: 142538.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.141580E-01 | lm-global-supervised-tokens: 1.223012E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.240984E-02 | lm-token-buffer-mib: 5.809192E+01 | global_load_balancing_loss: 9.998993E-01 | loss scale: 1.0 | grad norm: 0.040 | params norm: 11280.703 | params rms: 0.061081 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 22:42:49.552570 | finish at 2026-10-10 18:59:17 + [2026-10-09 20:17:54] iteration 5903/ 6855 | consumed samples: 755584 | elapsed time per iteration (ms): 86359.8 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.53% | FLOPs estimator: workload-aware | logical tokens/s: 194270.9 | valid tokens/s: 194270.9 | physical tokens/s: 194832.6 | loss tokens/s: 147358.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.305701E-01 | lm-global-supervised-tokens: 1.272583E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.110726E-02 | lm-token-buffer-mib: 5.835217E+01 | global_load_balancing_loss: 9.998249E-01 | loss scale: 1.0 | grad norm: 0.038 | params norm: 11280.644 | params rms: 0.061081 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 22:50:14.497238 | finish at 2026-10-10 19:08:08 + [2026-10-09 20:19:21] iteration 5904/ 6855 | consumed samples: 755712 | elapsed time per iteration (ms): 86591.9 | throughput per GPU (TFLOP/s/GPU): 43.3 | MFU 13.87% | FLOPs estimator: workload-aware | logical tokens/s: 193749.9 | valid tokens/s: 193749.9 | physical tokens/s: 194109.8 | loss tokens/s: 139056.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.973820E-01 | lm-global-supervised-tokens: 1.204115E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.734738E-02 | lm-token-buffer-mib: 5.814304E+01 | global_load_balancing_loss: 9.993525E-01 | loss scale: 1.0 | grad norm: 0.043 | params norm: 11280.585 | params rms: 0.061081 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 22:52:28.896778 | finish at 2026-10-10 19:11:49 + [2026-10-09 20:20:49] iteration 5905/ 6855 | consumed samples: 755840 | elapsed time per iteration (ms): 88833.0 | throughput per GPU (TFLOP/s/GPU): 41.1 | MFU 13.19% | FLOPs estimator: workload-aware | logical tokens/s: 188862.2 | valid tokens/s: 188862.2 | physical tokens/s: 189379.8 | loss tokens/s: 145658.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.090971E-01 | lm-global-supervised-tokens: 1.293929E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.074710E-02 | lm-token-buffer-mib: 5.807835E+01 | global_load_balancing_loss: 9.989464E-01 | loss scale: 1.0 | grad norm: 0.040 | params norm: 11280.527 | params rms: 0.061080 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 23:26:31.397059 | finish at 2026-10-10 19:47:21 + [2026-10-09 20:22:20] iteration 5906/ 6855 | consumed samples: 755968 | elapsed time per iteration (ms): 90495.2 | throughput per GPU (TFLOP/s/GPU): 41.1 | MFU 13.17% | FLOPs estimator: workload-aware | logical tokens/s: 185393.2 | valid tokens/s: 185393.2 | physical tokens/s: 185758.0 | loss tokens/s: 138868.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.683633E-01 | lm-global-supervised-tokens: 1.256692E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.299712E-02 | lm-token-buffer-mib: 5.826965E+01 | global_load_balancing_loss: 9.996270E-01 | loss scale: 1.0 | grad norm: 0.052 | params norm: 11280.468 | params rms: 0.061080 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 23:51:19.968480 | finish at 2026-10-10 20:13:40 + [2026-10-09 20:23:47] iteration 5907/ 6855 | consumed samples: 756096 | elapsed time per iteration (ms): 87354.7 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.79% | FLOPs estimator: workload-aware | logical tokens/s: 192058.5 | valid tokens/s: 192058.5 | physical tokens/s: 192561.8 | loss tokens/s: 143123.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.320618E-01 | lm-global-supervised-tokens: 1.250252E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.365281E-02 | lm-token-buffer-mib: 5.832821E+01 | global_load_balancing_loss: 1.000276E+00 | loss scale: 1.0 | grad norm: 0.054 | params norm: 11280.412 | params rms: 0.061080 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 23:00:12.279416 | finish at 2026-10-10 19:24:00 + [2026-10-09 20:25:14] iteration 5908/ 6855 | consumed samples: 756224 | elapsed time per iteration (ms): 87170.4 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.64% | FLOPs estimator: workload-aware | logical tokens/s: 192464.5 | valid tokens/s: 192464.5 | physical tokens/s: 192893.9 | loss tokens/s: 140279.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.472870E-01 | lm-global-supervised-tokens: 1.222820E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.728659E-02 | lm-token-buffer-mib: 5.803220E+01 | global_load_balancing_loss: 1.001181E+00 | loss scale: 1.0 | grad norm: 0.056 | params norm: 11280.358 | params rms: 0.061079 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 22:55:50.379773 | finish at 2026-10-10 19:21:05 + [2026-10-09 20:26:40] iteration 5909/ 6855 | consumed samples: 756352 | elapsed time per iteration (ms): 85327.1 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.67% | FLOPs estimator: workload-aware | logical tokens/s: 196622.0 | valid tokens/s: 196622.0 | physical tokens/s: 197026.8 | loss tokens/s: 147719.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.458910E-01 | lm-global-supervised-tokens: 1.260447E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.708058E-02 | lm-token-buffer-mib: 5.806958E+01 | global_load_balancing_loss: 1.001258E+00 | loss scale: 1.0 | grad norm: 0.039 | params norm: 11280.303 | params rms: 0.061079 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 22:25:19.464604 | finish at 2026-10-10 18:51:59 + [2026-10-09 20:28:30] iteration 5910/ 6855 | consumed samples: 756480 | elapsed time per iteration (ms): 110437.5 | throughput per GPU (TFLOP/s/GPU): 33.9 | MFU 10.86% | FLOPs estimator: workload-aware | logical tokens/s: 151915.7 | valid tokens/s: 151915.7 | physical tokens/s: 152243.7 | loss tokens/s: 105730.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.301827E-01 | lm-global-supervised-tokens: 1.167660E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.481248E-02 | lm-token-buffer-mib: 5.801653E+01 | global_load_balancing_loss: 1.001363E+00 | loss scale: 1.0 | grad norm: 0.134 | params norm: 11280.248 | params rms: 0.061079 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 4:59:23.403028 | finish at 2026-10-11 01:27:54 + [2026-10-09 20:29:59] iteration 5911/ 6855 | consumed samples: 756608 | elapsed time per iteration (ms): 89178.2 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.61% | FLOPs estimator: workload-aware | logical tokens/s: 188130.8 | valid tokens/s: 188130.8 | physical tokens/s: 188502.1 | loss tokens/s: 136745.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.678501E-01 | lm-global-supervised-tokens: 1.219468E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.509727E-02 | lm-token-buffer-mib: 5.811647E+01 | global_load_balancing_loss: 9.999055E-01 | loss scale: 1.0 | grad norm: 0.280 | params norm: 11280.192 | params rms: 0.061079 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 23:23:04.179169 | finish at 2026-10-10 19:53:04 + [2026-10-09 20:31:25] iteration 5912/ 6855 | consumed samples: 756736 | elapsed time per iteration (ms): 85617.9 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.80% | FLOPs estimator: workload-aware | logical tokens/s: 195953.7 | valid tokens/s: 195953.6 | physical tokens/s: 196462.0 | loss tokens/s: 151135.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.709381E-01 | lm-global-supervised-tokens: 1.293990E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.105784E-02 | lm-token-buffer-mib: 5.805149E+01 | global_load_balancing_loss: 9.988765E-01 | loss scale: 1.0 | grad norm: 0.272 | params norm: 11280.137 | params rms: 0.061078 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 22:25:37.721419 | finish at 2026-10-10 18:57:03 + [2026-10-09 20:32:49] iteration 5913/ 6855 | consumed samples: 756864 | elapsed time per iteration (ms): 84379.6 | throughput per GPU (TFLOP/s/GPU): 43.5 | MFU 13.95% | FLOPs estimator: workload-aware | logical tokens/s: 198829.4 | valid tokens/s: 198829.4 | physical tokens/s: 199291.3 | loss tokens/s: 146652.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.649086E-01 | lm-global-supervised-tokens: 1.237448E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.144515E-02 | lm-token-buffer-mib: 5.823370E+01 | global_load_balancing_loss: 9.999400E-01 | loss scale: 1.0 | grad norm: 0.052 | params norm: 11280.081 | params rms: 0.061078 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 22:04:45.548209 | finish at 2026-10-10 18:37:35 + [2026-10-09 20:34:15] iteration 5914/ 6855 | consumed samples: 756992 | elapsed time per iteration (ms): 85514.0 | throughput per GPU (TFLOP/s/GPU): 43.4 | MFU 13.92% | FLOPs estimator: workload-aware | logical tokens/s: 196192.2 | valid tokens/s: 196192.2 | physical tokens/s: 196543.7 | loss tokens/s: 143491.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.631016E-01 | lm-global-supervised-tokens: 1.227053E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.149492E-02 | lm-token-buffer-mib: 5.827228E+01 | global_load_balancing_loss: 9.993324E-01 | loss scale: 1.0 | grad norm: 0.119 | params norm: 11280.026 | params rms: 0.061078 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 22:21:08.716569 | finish at 2026-10-10 18:55:24 + [2026-10-09 20:35:42] iteration 5915/ 6855 | consumed samples: 757120 | elapsed time per iteration (ms): 87206.6 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.52% | FLOPs estimator: workload-aware | logical tokens/s: 192383.7 | valid tokens/s: 192383.7 | physical tokens/s: 192842.4 | loss tokens/s: 143661.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.450037E-01 | lm-global-supervised-tokens: 1.252823E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.446294E-02 | lm-token-buffer-mib: 5.811544E+01 | global_load_balancing_loss: 1.000868E+00 | loss scale: 1.0 | grad norm: 0.150 | params norm: 11279.970 | params rms: 0.061077 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 22:46:14.242501 | finish at 2026-10-10 19:21:56 + [2026-10-09 20:37:10] iteration 5916/ 6855 | consumed samples: 757248 | elapsed time per iteration (ms): 87889.0 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.69% | FLOPs estimator: workload-aware | logical tokens/s: 190890.3 | valid tokens/s: 190890.3 | physical tokens/s: 191395.1 | loss tokens/s: 136291.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.276573E-01 | lm-global-supervised-tokens: 1.197855E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.123598E-02 | lm-token-buffer-mib: 5.828891E+01 | global_load_balancing_loss: 1.001026E+00 | loss scale: 1.0 | grad norm: 0.067 | params norm: 11279.915 | params rms: 0.061077 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 22:55:27.739824 | finish at 2026-10-10 19:32:38 + [2026-10-09 20:38:36] iteration 5917/ 6855 | consumed samples: 757376 | elapsed time per iteration (ms): 86184.4 | throughput per GPU (TFLOP/s/GPU): 43.9 | MFU 14.06% | FLOPs estimator: workload-aware | logical tokens/s: 194666.2 | valid tokens/s: 194666.2 | physical tokens/s: 195109.2 | loss tokens/s: 145881.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.933309E-01 | lm-global-supervised-tokens: 1.257274E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.102190E-02 | lm-token-buffer-mib: 5.814898E+01 | global_load_balancing_loss: 1.000379E+00 | loss scale: 1.0 | grad norm: 0.141 | params norm: 11279.860 | params rms: 0.061077 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 22:27:20.926351 | finish at 2026-10-10 19:05:57 + [2026-10-09 20:40:04] iteration 5918/ 6855 | consumed samples: 757504 | elapsed time per iteration (ms): 88035.4 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.68% | FLOPs estimator: workload-aware | logical tokens/s: 190573.6 | valid tokens/s: 190573.6 | physical tokens/s: 191359.3 | loss tokens/s: 136616.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.431797E-01 | lm-global-supervised-tokens: 1.202708E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.327296E-02 | lm-token-buffer-mib: 5.810697E+01 | global_load_balancing_loss: 1.000366E+00 | loss scale: 1.0 | grad norm: 0.092 | params norm: 11279.805 | params rms: 0.061076 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 22:54:49.137103 | finish at 2026-10-10 19:34:53 + [2026-10-09 20:41:32] iteration 5919/ 6855 | consumed samples: 757632 | elapsed time per iteration (ms): 87894.1 | throughput per GPU (TFLOP/s/GPU): 43.1 | MFU 13.81% | FLOPs estimator: workload-aware | logical tokens/s: 190879.2 | valid tokens/s: 190879.2 | physical tokens/s: 191253.0 | loss tokens/s: 133961.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.643538E-01 | lm-global-supervised-tokens: 1.177444E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.737948E-02 | lm-token-buffer-mib: 5.812543E+01 | global_load_balancing_loss: 1.000346E+00 | loss scale: 1.0 | grad norm: 0.092 | params norm: 11279.751 | params rms: 0.061076 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 22:51:08.837385 | finish at 2026-10-10 19:32:41 + [2026-10-09 20:43:24] iteration 5920/ 6855 | consumed samples: 757760 | elapsed time per iteration (ms): 111732.2 | throughput per GPU (TFLOP/s/GPU): 33.5 | MFU 10.73% | FLOPs estimator: workload-aware | logical tokens/s: 150154.7 | valid tokens/s: 150154.7 | physical tokens/s: 150517.5 | loss tokens/s: 107655.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.957499E-01 | lm-global-supervised-tokens: 1.202860E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.225443E-02 | lm-token-buffer-mib: 5.814805E+01 | global_load_balancing_loss: 1.001032E+00 | loss scale: 1.0 | grad norm: 0.158 | params norm: 11279.698 | params rms: 0.061076 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 5:01:09.605799 | finish at 2026-10-11 01:44:33 +times across ranks (ms): + save-checkpoint: + rank 0: 25522.35 + rank 1: 25522.35 + rank 2: 25522.34 + rank 3: 25522.30 + rank 4: 25522.31 + rank 5: 25522.31 + rank 6: 25522.30 + rank 7: 25522.32 + rank 8: 25522.29 + rank 9: 25522.23 + rank 10: 25522.24 + rank 11: 25522.27 + rank 12: 25522.28 + rank 13: 25522.26 + rank 14: 25522.27 + rank 15: 25522.28 + rank 16: 25522.57 + rank 17: 25522.55 + rank 18: 25522.51 + rank 19: 25522.54 + rank 20: 25522.52 + rank 21: 25522.55 + rank 22: 25522.50 + rank 23: 25522.51 + rank 24: 25522.68 + rank 25: 25522.62 + rank 26: 25522.69 + rank 27: 25522.70 + rank 28: 25523.31 + rank 29: 25522.72 + rank 30: 25522.68 + rank 31: 25522.70 + rank 32: 25523.28 + rank 33: 25523.27 + rank 34: 25523.18 + rank 35: 25523.27 + rank 36: 25523.22 + rank 37: 25523.23 + rank 38: 25523.22 + rank 39: 25523.23 + rank 40: 25522.40 + rank 41: 25522.39 + rank 42: 25522.45 + rank 43: 25522.39 + rank 44: 25523.00 + rank 45: 25522.39 + rank 46: 25522.38 + rank 47: 25522.38 + rank 48: 25522.27 + rank 49: 25522.43 + rank 50: 25522.44 + rank 51: 25522.38 + rank 52: 25522.41 + rank 53: 25522.42 + rank 54: 25522.43 + rank 55: 25522.34 + rank 56: 25521.80 + rank 57: 25521.78 + rank 58: 25521.60 + rank 59: 25521.78 + rank 60: 25521.75 + rank 61: 25521.76 + rank 62: 25521.77 + rank 63: 25521.79 + [2026-10-09 20:45:19] iteration 5921/ 6855 | consumed samples: 757888 | elapsed time per iteration (ms): 89404.5 | throughput per GPU (TFLOP/s/GPU): 41.1 | MFU 13.17% | FLOPs estimator: workload-aware | logical tokens/s: 187654.4 | valid tokens/s: 187654.4 | physical tokens/s: 188068.3 | loss tokens/s: 138360.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.388689E-01 | lm-global-supervised-tokens: 1.237004E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.596286E+00 | lm-token-buffer-mib: 5.810365E+01 | global_load_balancing_loss: 9.991698E-01 | loss scale: 1.0 | grad norm: 0.048 | params norm: 11279.645 | params rms: 0.061076 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 23:11:43.827057 | finish at 2026-10-10 19:57:03 + [2026-10-09 20:46:51] iteration 5922/ 6855 | consumed samples: 758016 | elapsed time per iteration (ms): 91997.9 | throughput per GPU (TFLOP/s/GPU): 39.4 | MFU 12.63% | FLOPs estimator: workload-aware | logical tokens/s: 182364.5 | valid tokens/s: 182364.5 | physical tokens/s: 182933.9 | loss tokens/s: 138258.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.742452E-01 | lm-global-supervised-tokens: 1.271947E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.669960E-02 | lm-token-buffer-mib: 5.805840E+01 | global_load_balancing_loss: 9.998764E-01 | loss scale: 1.0 | grad norm: 0.134 | params norm: 11279.593 | params rms: 0.061075 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 23:50:34.038707 | finish at 2026-10-10 20:37:25 + [2026-10-09 20:48:15] iteration 5923/ 6855 | consumed samples: 758144 | elapsed time per iteration (ms): 84096.5 | throughput per GPU (TFLOP/s/GPU): 43.5 | MFU 13.93% | FLOPs estimator: workload-aware | logical tokens/s: 199499.6 | valid tokens/s: 199499.6 | physical tokens/s: 200229.9 | loss tokens/s: 142214.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.373588E-01 | lm-global-supervised-tokens: 1.195969E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.596085E-02 | lm-token-buffer-mib: 5.867177E+01 | global_load_balancing_loss: 9.993344E-01 | loss scale: 1.0 | grad norm: 0.074 | params norm: 11279.541 | params rms: 0.061075 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 21:46:17.900595 | finish at 2026-10-10 18:34:33 + [2026-10-09 20:49:42] iteration 5924/ 6855 | consumed samples: 758272 | elapsed time per iteration (ms): 86917.3 | throughput per GPU (TFLOP/s/GPU): 43.4 | MFU 13.91% | FLOPs estimator: workload-aware | logical tokens/s: 193024.7 | valid tokens/s: 193024.7 | physical tokens/s: 193464.1 | loss tokens/s: 145235.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.840641E-01 | lm-global-supervised-tokens: 1.262351E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.334729E-02 | lm-token-buffer-mib: 5.817641E+01 | global_load_balancing_loss: 9.989007E-01 | loss scale: 1.0 | grad norm: 0.110 | params norm: 11279.489 | params rms: 0.061075 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 22:28:39.983424 | finish at 2026-10-10 19:18:22 + [2026-10-09 20:51:06] iteration 5925/ 6855 | consumed samples: 758400 | elapsed time per iteration (ms): 84324.1 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.79% | FLOPs estimator: workload-aware | logical tokens/s: 198960.7 | valid tokens/s: 198960.7 | physical tokens/s: 199504.3 | loss tokens/s: 148486.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.403959E-01 | lm-global-supervised-tokens: 1.252097E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.155591E-02 | lm-token-buffer-mib: 5.822870E+01 | global_load_balancing_loss: 1.000694E+00 | loss scale: 1.0 | grad norm: 0.070 | params norm: 11279.434 | params rms: 0.061074 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 21:47:01.409690 | finish at 2026-10-10 18:38:08 + [2026-10-09 20:52:30] iteration 5926/ 6855 | consumed samples: 758528 | elapsed time per iteration (ms): 84175.5 | throughput per GPU (TFLOP/s/GPU): 43.3 | MFU 13.88% | FLOPs estimator: workload-aware | logical tokens/s: 199311.3 | valid tokens/s: 199311.3 | physical tokens/s: 199760.9 | loss tokens/s: 145271.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.387948E-01 | lm-global-supervised-tokens: 1.222829E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.461141E-02 | lm-token-buffer-mib: 5.807614E+01 | global_load_balancing_loss: 1.002152E+00 | loss scale: 1.0 | grad norm: 0.111 | params norm: 11279.381 | params rms: 0.061074 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 21:43:19.051383 | finish at 2026-10-10 18:35:49 + [2026-10-09 20:54:00] iteration 5927/ 6855 | consumed samples: 758656 | elapsed time per iteration (ms): 89505.6 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.45% | FLOPs estimator: workload-aware | logical tokens/s: 187443.3 | valid tokens/s: 187443.3 | physical tokens/s: 187876.4 | loss tokens/s: 130574.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.233062E-01 | lm-global-supervised-tokens: 1.168710E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.287552E-02 | lm-token-buffer-mib: 5.817688E+01 | global_load_balancing_loss: 1.001984E+00 | loss scale: 1.0 | grad norm: 0.057 | params norm: 11279.330 | params rms: 0.061074 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 23:04:21.156509 | finish at 2026-10-10 19:58:21 + [2026-10-09 20:55:26] iteration 5928/ 6855 | consumed samples: 758784 | elapsed time per iteration (ms): 86155.8 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.65% | FLOPs estimator: workload-aware | logical tokens/s: 194730.1 | valid tokens/s: 194730.1 | physical tokens/s: 195280.7 | loss tokens/s: 144154.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.311978E-01 | lm-global-supervised-tokens: 1.241972E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.436468E-02 | lm-token-buffer-mib: 5.818870E+01 | global_load_balancing_loss: 1.000214E+00 | loss scale: 1.0 | grad norm: 0.109 | params norm: 11279.280 | params rms: 0.061074 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 22:11:06.463827 | finish at 2026-10-10 19:06:33 + [2026-10-09 20:56:51] iteration 5929/ 6855 | consumed samples: 758912 | elapsed time per iteration (ms): 84557.7 | throughput per GPU (TFLOP/s/GPU): 43.3 | MFU 13.86% | FLOPs estimator: workload-aware | logical tokens/s: 198411.5 | valid tokens/s: 198411.5 | physical tokens/s: 198901.2 | loss tokens/s: 144250.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.614203E-01 | lm-global-supervised-tokens: 1.219745E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.461469E-02 | lm-token-buffer-mib: 5.816936E+01 | global_load_balancing_loss: 9.994580E-01 | loss scale: 1.0 | grad norm: 0.051 | params norm: 11279.226 | params rms: 0.061073 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 21:45:00.402528 | finish at 2026-10-10 18:41:51 + [2026-10-09 20:58:43] iteration 5930/ 6855 | consumed samples: 759040 | elapsed time per iteration (ms): 112189.9 | throughput per GPU (TFLOP/s/GPU): 33.3 | MFU 10.67% | FLOPs estimator: workload-aware | logical tokens/s: 149542.7 | valid tokens/s: 149542.7 | physical tokens/s: 149882.7 | loss tokens/s: 109105.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.822661E-01 | lm-global-supervised-tokens: 1.224053E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.132658E-02 | lm-token-buffer-mib: 5.827258E+01 | global_load_balancing_loss: 9.983923E-01 | loss scale: 1.0 | grad norm: 0.077 | params norm: 11279.172 | params rms: 0.061073 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 4:49:35.649709 | finish at 2026-10-11 01:48:18 + [2026-10-09 21:00:10] iteration 5931/ 6855 | consumed samples: 759168 | elapsed time per iteration (ms): 86792.6 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.44% | FLOPs estimator: workload-aware | logical tokens/s: 193302.0 | valid tokens/s: 193302.0 | physical tokens/s: 193876.8 | loss tokens/s: 143697.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.447318E-01 | lm-global-supervised-tokens: 1.247185E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.197238E-02 | lm-token-buffer-mib: 5.814742E+01 | global_load_balancing_loss: 9.996775E-01 | loss scale: 1.0 | grad norm: 0.083 | params norm: 11279.119 | params rms: 0.061073 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 22:16:36.369152 | finish at 2026-10-10 19:16:46 + [2026-10-09 21:01:34] iteration 5932/ 6855 | consumed samples: 759296 | elapsed time per iteration (ms): 84570.5 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.74% | FLOPs estimator: workload-aware | logical tokens/s: 198380.4 | valid tokens/s: 198380.4 | physical tokens/s: 198798.2 | loss tokens/s: 152023.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.545992E-01 | lm-global-supervised-tokens: 1.285673E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.148133E-02 | lm-token-buffer-mib: 5.822871E+01 | global_load_balancing_loss: 9.996698E-01 | loss scale: 1.0 | grad norm: 0.042 | params norm: 11279.067 | params rms: 0.061072 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 21:40:58.560182 | finish at 2026-10-10 18:42:33 + [2026-10-09 21:02:59] iteration 5933/ 6855 | consumed samples: 759424 | elapsed time per iteration (ms): 84414.8 | throughput per GPU (TFLOP/s/GPU): 43.4 | MFU 13.91% | FLOPs estimator: workload-aware | logical tokens/s: 198747.0 | valid tokens/s: 198747.0 | physical tokens/s: 199299.6 | loss tokens/s: 149832.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.526817E-01 | lm-global-supervised-tokens: 1.264809E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.352984E-02 | lm-token-buffer-mib: 5.812896E+01 | global_load_balancing_loss: 1.000407E+00 | loss scale: 1.0 | grad norm: 0.097 | params norm: 11279.014 | params rms: 0.061072 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 21:37:10.400251 | finish at 2026-10-10 18:40:09 + [2026-10-09 21:04:24] iteration 5934/ 6855 | consumed samples: 759552 | elapsed time per iteration (ms): 85822.9 | throughput per GPU (TFLOP/s/GPU): 41.7 | MFU 13.37% | FLOPs estimator: workload-aware | logical tokens/s: 195486.6 | valid tokens/s: 195486.6 | physical tokens/s: 196093.6 | loss tokens/s: 153257.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.543848E-01 | lm-global-supervised-tokens: 1.315298E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.153104E-02 | lm-token-buffer-mib: 5.804380E+01 | global_load_balancing_loss: 1.001387E+00 | loss scale: 1.0 | grad norm: 0.093 | params norm: 11278.960 | params rms: 0.061072 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 21:57:22.847036 | finish at 2026-10-10 19:01:47 + [2026-10-09 21:05:50] iteration 5935/ 6855 | consumed samples: 759680 | elapsed time per iteration (ms): 85167.8 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.69% | FLOPs estimator: workload-aware | logical tokens/s: 196989.6 | valid tokens/s: 196989.6 | physical tokens/s: 197423.3 | loss tokens/s: 144938.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 5.863086E-01 | lm-global-supervised-tokens: 1.234412E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.491466E-02 | lm-token-buffer-mib: 5.825282E+01 | global_load_balancing_loss: 1.000753E+00 | loss scale: 1.0 | grad norm: 0.057 | params norm: 11278.907 | params rms: 0.061072 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 21:45:54.403372 | finish at 2026-10-10 18:51:44 + [2026-10-09 21:07:18] iteration 5936/ 6855 | consumed samples: 759808 | elapsed time per iteration (ms): 88835.8 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.63% | FLOPs estimator: workload-aware | logical tokens/s: 188855.6 | valid tokens/s: 188855.6 | physical tokens/s: 189331.2 | loss tokens/s: 132632.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.410494E-01 | lm-global-supervised-tokens: 1.178251E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.251940E-02 | lm-token-buffer-mib: 5.820295E+01 | global_load_balancing_loss: 1.000752E+00 | loss scale: 1.0 | grad norm: 0.037 | params norm: 11278.856 | params rms: 0.061071 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 22:40:40.109998 | finish at 2026-10-10 19:47:59 + [2026-10-09 21:08:44] iteration 5937/ 6855 | consumed samples: 759936 | elapsed time per iteration (ms): 85198.4 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.78% | FLOPs estimator: workload-aware | logical tokens/s: 196918.9 | valid tokens/s: 196918.9 | physical tokens/s: 197341.5 | loss tokens/s: 142535.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.404496E-01 | lm-global-supervised-tokens: 1.214383E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.148631E-02 | lm-token-buffer-mib: 5.815622E+01 | global_load_balancing_loss: 1.001015E+00 | loss scale: 1.0 | grad norm: 0.059 | params norm: 11278.804 | params rms: 0.061071 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 21:43:32.136691 | finish at 2026-10-10 18:52:16 + [2026-10-09 21:10:14] iteration 5938/ 6855 | consumed samples: 760064 | elapsed time per iteration (ms): 90062.7 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.48% | FLOPs estimator: workload-aware | logical tokens/s: 186283.4 | valid tokens/s: 186283.4 | physical tokens/s: 186577.7 | loss tokens/s: 136052.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.438069E-01 | lm-global-supervised-tokens: 1.225322E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.152673E-02 | lm-token-buffer-mib: 5.810977E+01 | global_load_balancing_loss: 9.991544E-01 | loss scale: 1.0 | grad norm: 0.041 | params norm: 11278.751 | params rms: 0.061071 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 22:56:27.504238 | finish at 2026-10-10 20:06:41 + [2026-10-09 21:11:39] iteration 5939/ 6855 | consumed samples: 760192 | elapsed time per iteration (ms): 85607.3 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.75% | FLOPs estimator: workload-aware | logical tokens/s: 195977.9 | valid tokens/s: 195977.9 | physical tokens/s: 196442.5 | loss tokens/s: 147562.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.236645E-01 | lm-global-supervised-tokens: 1.263242E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.287091E-02 | lm-token-buffer-mib: 5.806733E+01 | global_load_balancing_loss: 9.987671E-01 | loss scale: 1.0 | grad norm: 0.066 | params norm: 11278.698 | params rms: 0.061070 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 21:46:56.271115 | finish at 2026-10-10 18:58:36 + [2026-10-09 21:13:25] iteration 5940/ 6855 | consumed samples: 760320 | elapsed time per iteration (ms): 106068.2 | throughput per GPU (TFLOP/s/GPU): 34.2 | MFU 10.95% | FLOPs estimator: workload-aware | logical tokens/s: 158173.7 | valid tokens/s: 158173.7 | physical tokens/s: 158527.1 | loss tokens/s: 117060.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.212812E-01 | lm-global-supervised-tokens: 1.241641E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.147003E-02 | lm-token-buffer-mib: 5.808597E+01 | global_load_balancing_loss: 1.000436E+00 | loss scale: 1.0 | grad norm: 0.047 | params norm: 11278.647 | params rms: 0.061070 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 2:57:32.400048 | finish at 2026-10-11 00:10:58 + [2026-10-09 21:14:50] iteration 5941/ 6855 | consumed samples: 760448 | elapsed time per iteration (ms): 84364.1 | throughput per GPU (TFLOP/s/GPU): 43.6 | MFU 13.97% | FLOPs estimator: workload-aware | logical tokens/s: 198866.6 | valid tokens/s: 198866.6 | physical tokens/s: 199276.5 | loss tokens/s: 149879.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.878584E-01 | lm-global-supervised-tokens: 1.264446E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.681326E-02 | lm-token-buffer-mib: 5.813268E+01 | global_load_balancing_loss: 1.000071E+00 | loss scale: 1.0 | grad norm: 0.065 | params norm: 11278.597 | params rms: 0.061070 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 21:25:08.750118 | finish at 2026-10-10 18:39:58 + [2026-10-09 21:16:17] iteration 5942/ 6855 | consumed samples: 760576 | elapsed time per iteration (ms): 87140.0 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.50% | FLOPs estimator: workload-aware | logical tokens/s: 192530.6 | valid tokens/s: 192530.6 | physical tokens/s: 192816.6 | loss tokens/s: 140089.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.461827E-01 | lm-global-supervised-tokens: 1.220737E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.030864E-02 | lm-token-buffer-mib: 5.806985E+01 | global_load_balancing_loss: 1.000671E+00 | loss scale: 1.0 | grad norm: 0.062 | params norm: 11278.543 | params rms: 0.061070 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 22:05:58.789621 | finish at 2026-10-10 19:22:16 + [2026-10-09 21:17:44] iteration 5943/ 6855 | consumed samples: 760704 | elapsed time per iteration (ms): 86667.9 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.49% | FLOPs estimator: workload-aware | logical tokens/s: 193580.2 | valid tokens/s: 193580.2 | physical tokens/s: 194364.6 | loss tokens/s: 145798.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.913803E-01 | lm-global-supervised-tokens: 1.263606E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.439533E-02 | lm-token-buffer-mib: 5.830914E+01 | global_load_balancing_loss: 1.000723E+00 | loss scale: 1.0 | grad norm: 0.041 | params norm: 11278.488 | params rms: 0.061069 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 21:57:21.098785 | finish at 2026-10-10 19:15:05 + [2026-10-09 21:19:10] iteration 5944/ 6855 | consumed samples: 760832 | elapsed time per iteration (ms): 86361.1 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.54% | FLOPs estimator: workload-aware | logical tokens/s: 194267.9 | valid tokens/s: 194267.9 | physical tokens/s: 194666.7 | loss tokens/s: 147443.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.841967E-01 | lm-global-supervised-tokens: 1.273341E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.148212E-02 | lm-token-buffer-mib: 5.808784E+01 | global_load_balancing_loss: 1.001019E+00 | loss scale: 1.0 | grad norm: 0.052 | params norm: 11278.435 | params rms: 0.061069 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 21:51:14.951202 | finish at 2026-10-10 19:10:25 + [2026-10-09 21:20:38] iteration 5945/ 6855 | consumed samples: 760960 | elapsed time per iteration (ms): 88590.8 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.51% | FLOPs estimator: workload-aware | logical tokens/s: 189378.4 | valid tokens/s: 189378.4 | physical tokens/s: 189758.2 | loss tokens/s: 140752.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.245974E-01 | lm-global-supervised-tokens: 1.246934E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.108160E-02 | lm-token-buffer-mib: 5.807508E+01 | global_load_balancing_loss: 1.000378E+00 | loss scale: 1.0 | grad norm: 0.044 | params norm: 11278.383 | params rms: 0.061069 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 22:23:37.616978 | finish at 2026-10-10 19:44:16 + [2026-10-09 21:22:04] iteration 5946/ 6855 | consumed samples: 761088 | elapsed time per iteration (ms): 85611.4 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.59% | FLOPs estimator: workload-aware | logical tokens/s: 195969.4 | valid tokens/s: 195969.4 | physical tokens/s: 196372.6 | loss tokens/s: 147392.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.634752E-01 | lm-global-supervised-tokens: 1.261846E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.170497E-02 | lm-token-buffer-mib: 5.805454E+01 | global_load_balancing_loss: 9.991375E-01 | loss scale: 1.0 | grad norm: 0.038 | params norm: 11278.332 | params rms: 0.061068 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 21:37:00.756648 | finish at 2026-10-10 18:59:05 + [2026-10-09 21:23:33] iteration 5947/ 6855 | consumed samples: 761216 | elapsed time per iteration (ms): 88989.4 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.43% | FLOPs estimator: workload-aware | logical tokens/s: 188529.9 | valid tokens/s: 188529.9 | physical tokens/s: 188818.6 | loss tokens/s: 138381.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.507518E-01 | lm-global-supervised-tokens: 1.231448E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.160876E-02 | lm-token-buffer-mib: 5.818745E+01 | global_load_balancing_loss: 1.000319E+00 | loss scale: 1.0 | grad norm: 0.036 | params norm: 11278.280 | params rms: 0.061068 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 22:26:42.360397 | finish at 2026-10-10 19:50:15 + [2026-10-09 21:25:00] iteration 5948/ 6855 | consumed samples: 761344 | elapsed time per iteration (ms): 87263.1 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.50% | FLOPs estimator: workload-aware | logical tokens/s: 192260.1 | valid tokens/s: 192260.1 | physical tokens/s: 192612.4 | loss tokens/s: 144227.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.528409E-01 | lm-global-supervised-tokens: 1.258570E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.348083E-02 | lm-token-buffer-mib: 5.812187E+01 | global_load_balancing_loss: 1.000272E+00 | loss scale: 1.0 | grad norm: 0.055 | params norm: 11278.229 | params rms: 0.061068 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 21:59:07.626211 | finish at 2026-10-10 19:24:08 + [2026-10-09 21:26:27] iteration 5949/ 6855 | consumed samples: 761472 | elapsed time per iteration (ms): 87044.5 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.54% | FLOPs estimator: workload-aware | logical tokens/s: 192742.5 | valid tokens/s: 192742.5 | physical tokens/s: 193142.1 | loss tokens/s: 143307.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.335776E-01 | lm-global-supervised-tokens: 1.247415E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.706352E-02 | lm-token-buffer-mib: 5.815099E+01 | global_load_balancing_loss: 9.997120E-01 | loss scale: 1.0 | grad norm: 0.039 | params norm: 11278.179 | params rms: 0.061068 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 21:54:22.281893 | finish at 2026-10-10 19:20:50 + [2026-10-09 21:34:43] iteration 5950/ 6855 | consumed samples: 761600 | elapsed time per iteration (ms): 495415.5 | throughput per GPU (TFLOP/s/GPU): 7.5 | MFU 2.40% | FLOPs estimator: workload-aware | logical tokens/s: 33864.9 | valid tokens/s: 33864.9 | physical tokens/s: 33925.1 | loss tokens/s: 23927.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.633945E-01 | lm-global-supervised-tokens: 1.185398E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 3.858341E+02 | lm-token-buffer-mib: 5.813420E+01 | global_load_balancing_loss: 1.000147E+00 | loss scale: 1.0 | grad norm: 0.045 | params norm: 11278.127 | params rms: 0.061067 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5 days, 4:32:31.038653 | finish at 2026-10-15 02:07:14 + [2026-10-09 21:36:12] iteration 5951/ 6855 | consumed samples: 761728 | elapsed time per iteration (ms): 89418.1 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.57% | FLOPs estimator: workload-aware | logical tokens/s: 187626.3 | valid tokens/s: 187626.3 | physical tokens/s: 188054.1 | loss tokens/s: 134073.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.226506E-01 | lm-global-supervised-tokens: 1.198855E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.143470E-02 | lm-token-buffer-mib: 5.816193E+01 | global_load_balancing_loss: 9.998791E-01 | loss scale: 1.0 | grad norm: 0.040 | params norm: 11278.074 | params rms: 0.061067 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 22:27:13.944618 | finish at 2026-10-10 20:03:26 + [2026-10-09 21:37:40] iteration 5952/ 6855 | consumed samples: 761856 | elapsed time per iteration (ms): 87922.8 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.61% | FLOPs estimator: workload-aware | logical tokens/s: 190816.7 | valid tokens/s: 190816.7 | physical tokens/s: 191236.4 | loss tokens/s: 138485.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.551937E-01 | lm-global-supervised-tokens: 1.217604E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.598513E-02 | lm-token-buffer-mib: 5.808341E+01 | global_load_balancing_loss: 1.001886E+00 | loss scale: 1.0 | grad norm: 0.084 | params norm: 11278.023 | params rms: 0.061067 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 22:03:14.251643 | finish at 2026-10-10 19:40:54 + [2026-10-09 21:39:07] iteration 5953/ 6855 | consumed samples: 761984 | elapsed time per iteration (ms): 87038.7 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.48% | FLOPs estimator: workload-aware | logical tokens/s: 192755.4 | valid tokens/s: 192755.4 | physical tokens/s: 193384.5 | loss tokens/s: 139727.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.435050E-01 | lm-global-supervised-tokens: 1.216166E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.158173E-02 | lm-token-buffer-mib: 5.810842E+01 | global_load_balancing_loss: 1.000924E+00 | loss scale: 1.0 | grad norm: 0.113 | params norm: 11277.971 | params rms: 0.061067 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 21:48:28.916526 | finish at 2026-10-10 19:27:36 + [2026-10-09 21:40:36] iteration 5954/ 6855 | consumed samples: 762112 | elapsed time per iteration (ms): 88803.9 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.39% | FLOPs estimator: workload-aware | logical tokens/s: 188924.2 | valid tokens/s: 188924.2 | physical tokens/s: 189260.1 | loss tokens/s: 132978.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.084497E-01 | lm-global-supervised-tokens: 1.180898E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.135983E-02 | lm-token-buffer-mib: 5.801961E+01 | global_load_balancing_loss: 9.999533E-01 | loss scale: 1.0 | grad norm: 0.108 | params norm: 11277.922 | params rms: 0.061066 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 22:13:32.333881 | finish at 2026-10-10 19:54:08 + [2026-10-09 21:42:03] iteration 5955/ 6855 | consumed samples: 762240 | elapsed time per iteration (ms): 86817.8 | throughput per GPU (TFLOP/s/GPU): 43.3 | MFU 13.89% | FLOPs estimator: workload-aware | logical tokens/s: 193245.9 | valid tokens/s: 193245.9 | physical tokens/s: 193703.8 | loss tokens/s: 145968.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.931022E-01 | lm-global-supervised-tokens: 1.267264E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.097804E-02 | lm-token-buffer-mib: 5.811462E+01 | global_load_balancing_loss: 9.987431E-01 | loss scale: 1.0 | grad norm: 0.078 | params norm: 11277.873 | params rms: 0.061066 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 21:42:16.029053 | finish at 2026-10-10 19:24:19 + [2026-10-09 21:43:31] iteration 5956/ 6855 | consumed samples: 762368 | elapsed time per iteration (ms): 88136.6 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.38% | FLOPs estimator: workload-aware | logical tokens/s: 190354.7 | valid tokens/s: 190354.7 | physical tokens/s: 190665.0 | loss tokens/s: 139425.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.052954E-01 | lm-global-supervised-tokens: 1.228844E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.153284E-02 | lm-token-buffer-mib: 5.805938E+01 | global_load_balancing_loss: 9.992549E-01 | loss scale: 1.0 | grad norm: 0.033 | params norm: 11277.826 | params rms: 0.061066 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 22:00:34.792485 | finish at 2026-10-10 19:44:06 + [2026-10-09 21:44:54] iteration 5957/ 6855 | consumed samples: 762496 | elapsed time per iteration (ms): 83362.8 | throughput per GPU (TFLOP/s/GPU): 43.7 | MFU 14.01% | FLOPs estimator: workload-aware | logical tokens/s: 201255.1 | valid tokens/s: 201255.1 | physical tokens/s: 201735.5 | loss tokens/s: 152892.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.476234E-01 | lm-global-supervised-tokens: 1.274552E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.163340E-02 | lm-token-buffer-mib: 5.814278E+01 | global_load_balancing_loss: 1.000241E+00 | loss scale: 1.0 | grad norm: 0.075 | params norm: 11277.777 | params rms: 0.061065 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 20:47:39.766890 | finish at 2026-10-10 18:32:34 + [2026-10-09 21:46:19] iteration 5958/ 6855 | consumed samples: 762624 | elapsed time per iteration (ms): 84991.9 | throughput per GPU (TFLOP/s/GPU): 43.2 | MFU 13.85% | FLOPs estimator: workload-aware | logical tokens/s: 197397.7 | valid tokens/s: 197397.7 | physical tokens/s: 197814.4 | loss tokens/s: 140833.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.355133E-01 | lm-global-supervised-tokens: 1.196972E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.208570E-02 | lm-token-buffer-mib: 5.804041E+01 | global_load_balancing_loss: 1.000717E+00 | loss scale: 1.0 | grad norm: 0.054 | params norm: 11277.729 | params rms: 0.061065 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 21:10:37.748599 | finish at 2026-10-10 18:56:57 + [2026-10-09 21:47:46] iteration 5959/ 6855 | consumed samples: 762752 | elapsed time per iteration (ms): 86529.9 | throughput per GPU (TFLOP/s/GPU): 43.1 | MFU 13.83% | FLOPs estimator: workload-aware | logical tokens/s: 193888.8 | valid tokens/s: 193888.8 | physical tokens/s: 194335.5 | loss tokens/s: 143272.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.781094E-01 | lm-global-supervised-tokens: 1.239734E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.851839E-02 | lm-token-buffer-mib: 5.805634E+01 | global_load_balancing_loss: 1.001157E+00 | loss scale: 1.0 | grad norm: 0.055 | params norm: 11277.681 | params rms: 0.061065 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 21:32:10.822723 | finish at 2026-10-10 19:19:57 + [2026-10-09 21:49:35] iteration 5960/ 6855 | consumed samples: 762880 | elapsed time per iteration (ms): 109007.2 | throughput per GPU (TFLOP/s/GPU): 33.3 | MFU 10.68% | FLOPs estimator: workload-aware | logical tokens/s: 153909.2 | valid tokens/s: 153909.2 | physical tokens/s: 154205.8 | loss tokens/s: 117573.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.840813E-01 | lm-global-supervised-tokens: 1.281634E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.276471E-02 | lm-token-buffer-mib: 5.810228E+01 | global_load_balancing_loss: 1.000803E+00 | loss scale: 1.0 | grad norm: 0.080 | params norm: 11277.632 | params rms: 0.061065 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 3:06:01.475583 | finish at 2026-10-11 00:55:36 +times across ranks (ms): + save-checkpoint: + rank 0: 24132.12 + rank 1: 24132.14 + rank 2: 24132.16 + rank 3: 24132.14 + rank 4: 24132.15 + rank 5: 24132.12 + rank 6: 24132.12 + rank 7: 24132.12 + rank 8: 24131.39 + rank 9: 24131.38 + rank 10: 24131.35 + rank 11: 24131.35 + rank 12: 24131.35 + rank 13: 24131.35 + rank 14: 24131.34 + rank 15: 24131.34 + rank 16: 24131.94 + rank 17: 24131.92 + rank 18: 24131.89 + rank 19: 24131.90 + rank 20: 24131.90 + rank 21: 24131.91 + rank 22: 24131.89 + rank 23: 24131.89 + rank 24: 24132.50 + rank 25: 24132.71 + rank 26: 24132.51 + rank 27: 24132.54 + rank 28: 24132.52 + rank 29: 24132.47 + rank 30: 24132.46 + rank 31: 24132.51 + rank 32: 24132.44 + rank 33: 24132.46 + rank 34: 24132.44 + rank 35: 24132.44 + rank 36: 24132.40 + rank 37: 24132.41 + rank 38: 24132.42 + rank 39: 24132.41 + rank 40: 24132.38 + rank 41: 24132.32 + rank 42: 24132.36 + rank 43: 24132.35 + rank 44: 24132.33 + rank 45: 24132.32 + rank 46: 24132.39 + rank 47: 24132.32 + rank 48: 24132.39 + rank 49: 24132.36 + rank 50: 24132.34 + rank 51: 24132.35 + rank 52: 24132.36 + rank 53: 24132.35 + rank 54: 24132.35 + rank 55: 24132.36 + rank 56: 24133.79 + rank 57: 24133.77 + rank 58: 24133.77 + rank 59: 24133.80 + rank 60: 24133.81 + rank 61: 24133.77 + rank 62: 24133.79 + rank 63: 24133.79 + [2026-10-09 21:51:54] iteration 5961/ 6855 | consumed samples: 763008 | elapsed time per iteration (ms): 114671.0 | throughput per GPU (TFLOP/s/GPU): 31.7 | MFU 10.16% | FLOPs estimator: workload-aware | logical tokens/s: 146306.6 | valid tokens/s: 146306.6 | physical tokens/s: 146708.1 | loss tokens/s: 107699.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.119427E-01 | lm-global-supervised-tokens: 1.235002E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.982722E-01 | lm-token-buffer-mib: 5.810420E+01 | global_load_balancing_loss: 1.001157E+00 | loss scale: 1.0 | grad norm: 0.046 | params norm: 11277.582 | params rms: 0.061064 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 4:28:35.897663 | finish at 2026-10-11 02:20:30 + [2026-10-09 21:53:27] iteration 5962/ 6855 | consumed samples: 763136 | elapsed time per iteration (ms): 92675.8 | throughput per GPU (TFLOP/s/GPU): 39.9 | MFU 12.78% | FLOPs estimator: workload-aware | logical tokens/s: 181030.7 | valid tokens/s: 181030.7 | physical tokens/s: 181581.4 | loss tokens/s: 131723.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.917888E-01 | lm-global-supervised-tokens: 1.220761E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.286246E-02 | lm-token-buffer-mib: 5.814926E+01 | global_load_balancing_loss: 1.000710E+00 | loss scale: 1.0 | grad norm: 0.219 | params norm: 11277.531 | params rms: 0.061064 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 22:59:19.445830 | finish at 2026-10-10 20:52:46 + [2026-10-09 21:54:56] iteration 5963/ 6855 | consumed samples: 763264 | elapsed time per iteration (ms): 89480.8 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.38% | FLOPs estimator: workload-aware | logical tokens/s: 187494.7 | valid tokens/s: 187494.7 | physical tokens/s: 187995.0 | loss tokens/s: 133129.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.083340E-01 | lm-global-supervised-tokens: 1.191251E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.302078E-02 | lm-token-buffer-mib: 5.816293E+01 | global_load_balancing_loss: 9.990925E-01 | loss scale: 1.0 | grad norm: 0.308 | params norm: 11277.481 | params rms: 0.061064 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 22:10:16.849278 | finish at 2026-10-10 20:05:13 + [2026-10-09 21:56:23] iteration 5964/ 6855 | consumed samples: 763392 | elapsed time per iteration (ms): 87016.7 | throughput per GPU (TFLOP/s/GPU): 41.4 | MFU 13.27% | FLOPs estimator: workload-aware | logical tokens/s: 192804.6 | valid tokens/s: 192804.6 | physical tokens/s: 193521.2 | loss tokens/s: 143660.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.311870E-01 | lm-global-supervised-tokens: 1.250089E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.233893E-02 | lm-token-buffer-mib: 5.811034E+01 | global_load_balancing_loss: 9.985924E-01 | loss scale: 1.0 | grad norm: 0.157 | params norm: 11277.429 | params rms: 0.061064 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 21:32:11.857846 | finish at 2026-10-10 19:28:35 + [2026-10-09 21:57:53] iteration 5965/ 6855 | consumed samples: 763520 | elapsed time per iteration (ms): 89618.0 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.77% | FLOPs estimator: workload-aware | logical tokens/s: 187207.8 | valid tokens/s: 187207.8 | physical tokens/s: 187648.7 | loss tokens/s: 142223.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.352755E-01 | lm-global-supervised-tokens: 1.274578E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.810808E-02 | lm-token-buffer-mib: 5.813820E+01 | global_load_balancing_loss: 9.987815E-01 | loss scale: 1.0 | grad norm: 0.146 | params norm: 11277.377 | params rms: 0.061063 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 22:09:20.020027 | finish at 2026-10-10 20:07:13 + [2026-10-09 21:59:21] iteration 5966/ 6855 | consumed samples: 763648 | elapsed time per iteration (ms): 88074.1 | throughput per GPU (TFLOP/s/GPU): 41.7 | MFU 13.35% | FLOPs estimator: workload-aware | logical tokens/s: 190489.5 | valid tokens/s: 190489.5 | physical tokens/s: 190943.7 | loss tokens/s: 138574.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.171944E-01 | lm-global-supervised-tokens: 1.220483E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.611962E-02 | lm-token-buffer-mib: 5.806813E+01 | global_load_balancing_loss: 9.990396E-01 | loss scale: 1.0 | grad norm: 0.095 | params norm: 11277.321 | params rms: 0.061063 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 21:44:57.896323 | finish at 2026-10-10 19:44:19 + [2026-10-09 22:00:47] iteration 5967/ 6855 | consumed samples: 763776 | elapsed time per iteration (ms): 86449.0 | throughput per GPU (TFLOP/s/GPU): 42.8 | MFU 13.71% | FLOPs estimator: workload-aware | logical tokens/s: 194070.2 | valid tokens/s: 194070.2 | physical tokens/s: 194481.6 | loss tokens/s: 145262.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.787035E-01 | lm-global-supervised-tokens: 1.255777E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.250102E-02 | lm-token-buffer-mib: 5.811883E+01 | global_load_balancing_loss: 9.993019E-01 | loss scale: 1.0 | grad norm: 0.188 | params norm: 11277.270 | params rms: 0.061063 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 21:19:26.755932 | finish at 2026-10-10 19:20:14 + [2026-10-09 22:02:15] iteration 5968/ 6855 | consumed samples: 763904 | elapsed time per iteration (ms): 87643.9 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.55% | FLOPs estimator: workload-aware | logical tokens/s: 191424.8 | valid tokens/s: 191424.8 | physical tokens/s: 191739.1 | loss tokens/s: 142682.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.784843E-01 | lm-global-supervised-tokens: 1.250521E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.260857E-02 | lm-token-buffer-mib: 5.810369E+01 | global_load_balancing_loss: 1.001304E+00 | loss scale: 1.0 | grad norm: 0.039 | params norm: 11277.220 | params rms: 0.061062 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 21:35:40.101795 | finish at 2026-10-10 19:37:55 + [2026-10-09 22:03:40] iteration 5969/ 6855 | consumed samples: 764032 | elapsed time per iteration (ms): 85162.5 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.64% | FLOPs estimator: workload-aware | logical tokens/s: 197001.6 | valid tokens/s: 197001.6 | physical tokens/s: 197502.3 | loss tokens/s: 147925.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.705732E-01 | lm-global-supervised-tokens: 1.259774E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.121291E-02 | lm-token-buffer-mib: 5.805908E+01 | global_load_balancing_loss: 1.002955E+00 | loss scale: 1.0 | grad norm: 0.046 | params norm: 11277.171 | params rms: 0.061062 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 20:57:34.001954 | finish at 2026-10-10 19:01:14 + [2026-10-09 22:05:29] iteration 5970/ 6855 | consumed samples: 764160 | elapsed time per iteration (ms): 108401.0 | throughput per GPU (TFLOP/s/GPU): 34.0 | MFU 10.90% | FLOPs estimator: workload-aware | logical tokens/s: 154769.4 | valid tokens/s: 154769.4 | physical tokens/s: 155037.7 | loss tokens/s: 110874.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.673351E-01 | lm-global-supervised-tokens: 1.201892E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.629262E-02 | lm-token-buffer-mib: 5.807451E+01 | global_load_balancing_loss: 1.002923E+00 | loss scale: 1.0 | grad norm: 0.072 | params norm: 11277.122 | params rms: 0.061062 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 2:38:54.882066 | finish at 2026-10-11 00:44:23 + [2026-10-09 22:07:00] iteration 5971/ 6855 | consumed samples: 764288 | elapsed time per iteration (ms): 91307.2 | throughput per GPU (TFLOP/s/GPU): 40.8 | MFU 13.07% | FLOPs estimator: workload-aware | logical tokens/s: 183743.8 | valid tokens/s: 183743.8 | physical tokens/s: 184268.6 | loss tokens/s: 135184.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.559610E-01 | lm-global-supervised-tokens: 1.234333E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.241628E-02 | lm-token-buffer-mib: 5.813538E+01 | global_load_balancing_loss: 1.000765E+00 | loss scale: 1.0 | grad norm: 0.038 | params norm: 11277.074 | params rms: 0.061062 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 22:25:15.579303 | finish at 2026-10-10 20:32:15 + [2026-10-09 22:08:31] iteration 5972/ 6855 | consumed samples: 764416 | elapsed time per iteration (ms): 90791.5 | throughput per GPU (TFLOP/s/GPU): 41.2 | MFU 13.21% | FLOPs estimator: workload-aware | logical tokens/s: 184788.5 | valid tokens/s: 184788.5 | physical tokens/s: 185367.9 | loss tokens/s: 133902.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.281662E-01 | lm-global-supervised-tokens: 1.215720E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.380324E-02 | lm-token-buffer-mib: 5.816711E+01 | global_load_balancing_loss: 9.990342E-01 | loss scale: 1.0 | grad norm: 0.037 | params norm: 11277.026 | params rms: 0.061061 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 22:16:08.855423 | finish at 2026-10-10 20:24:39 + [2026-10-09 22:09:57] iteration 5973/ 6855 | consumed samples: 764544 | elapsed time per iteration (ms): 85955.6 | throughput per GPU (TFLOP/s/GPU): 42.8 | MFU 13.71% | FLOPs estimator: workload-aware | logical tokens/s: 195184.6 | valid tokens/s: 195184.6 | physical tokens/s: 195562.9 | loss tokens/s: 137174.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.263724E-01 | lm-global-supervised-tokens: 1.179090E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.271356E-02 | lm-token-buffer-mib: 5.806721E+01 | global_load_balancing_loss: 9.987050E-01 | loss scale: 1.0 | grad norm: 0.089 | params norm: 11276.979 | params rms: 0.061061 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 21:03:32.827655 | finish at 2026-10-10 19:13:29 + [2026-10-09 22:11:23] iteration 5974/ 6855 | consumed samples: 764672 | elapsed time per iteration (ms): 86469.1 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.74% | FLOPs estimator: workload-aware | logical tokens/s: 194025.5 | valid tokens/s: 194025.5 | physical tokens/s: 194507.8 | loss tokens/s: 141090.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.748182E-01 | lm-global-supervised-tokens: 1.219995E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.112790E-02 | lm-token-buffer-mib: 5.812210E+01 | global_load_balancing_loss: 9.992286E-01 | loss scale: 1.0 | grad norm: 0.238 | params norm: 11276.932 | params rms: 0.061061 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 21:09:39.265756 | finish at 2026-10-10 19:21:02 + [2026-10-09 22:12:53] iteration 5975/ 6855 | consumed samples: 764800 | elapsed time per iteration (ms): 90151.0 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.42% | FLOPs estimator: workload-aware | logical tokens/s: 186100.9 | valid tokens/s: 186100.9 | physical tokens/s: 186500.9 | loss tokens/s: 135235.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.661443E-01 | lm-global-supervised-tokens: 1.219162E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.506513E-02 | lm-token-buffer-mib: 5.807765E+01 | global_load_balancing_loss: 9.995404E-01 | loss scale: 1.0 | grad norm: 0.375 | params norm: 11276.885 | params rms: 0.061061 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 22:02:12.912331 | finish at 2026-10-10 20:15:06 + [2026-10-09 22:14:23] iteration 5976/ 6855 | consumed samples: 764928 | elapsed time per iteration (ms): 89472.0 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.45% | FLOPs estimator: workload-aware | logical tokens/s: 187513.3 | valid tokens/s: 187513.3 | physical tokens/s: 188072.1 | loss tokens/s: 141409.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.529075E-01 | lm-global-supervised-tokens: 1.265221E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.468335E-02 | lm-token-buffer-mib: 5.809798E+01 | global_load_balancing_loss: 1.000370E+00 | loss scale: 1.0 | grad norm: 0.373 | params norm: 11276.839 | params rms: 0.061060 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 21:50:45.855414 | finish at 2026-10-10 20:05:09 + [2026-10-09 22:15:50] iteration 5977/ 6855 | consumed samples: 765056 | elapsed time per iteration (ms): 87517.7 | throughput per GPU (TFLOP/s/GPU): 43.1 | MFU 13.80% | FLOPs estimator: workload-aware | logical tokens/s: 191700.7 | valid tokens/s: 191700.7 | physical tokens/s: 192002.0 | loss tokens/s: 141522.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.787249E-01 | lm-global-supervised-tokens: 1.238574E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.138340E-02 | lm-token-buffer-mib: 5.806445E+01 | global_load_balancing_loss: 1.000687E+00 | loss scale: 1.0 | grad norm: 0.044 | params norm: 11276.791 | params rms: 0.061060 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 21:20:40.580963 | finish at 2026-10-10 19:36:31 + [2026-10-09 22:17:17] iteration 5978/ 6855 | consumed samples: 765184 | elapsed time per iteration (ms): 87257.9 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.66% | FLOPs estimator: workload-aware | logical tokens/s: 192271.0 | valid tokens/s: 192271.0 | physical tokens/s: 192718.5 | loss tokens/s: 137297.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.292241E-01 | lm-global-supervised-tokens: 1.198033E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.036260E-02 | lm-token-buffer-mib: 5.814140E+01 | global_load_balancing_loss: 1.000986E+00 | loss scale: 1.0 | grad norm: 0.331 | params norm: 11276.743 | params rms: 0.061060 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 21:15:25.171818 | finish at 2026-10-10 19:32:43 + [2026-10-09 22:18:45] iteration 5979/ 6855 | consumed samples: 765312 | elapsed time per iteration (ms): 87719.8 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.50% | FLOPs estimator: workload-aware | logical tokens/s: 191258.7 | valid tokens/s: 191258.7 | physical tokens/s: 191677.3 | loss tokens/s: 142845.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.617606E-01 | lm-global-supervised-tokens: 1.253033E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.519236E-02 | lm-token-buffer-mib: 5.812870E+01 | global_load_balancing_loss: 1.000323E+00 | loss scale: 1.0 | grad norm: 0.129 | params norm: 11276.694 | params rms: 0.061060 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 21:20:42.516183 | finish at 2026-10-10 19:39:28 + [2026-10-09 22:20:30] iteration 5980/ 6855 | consumed samples: 765440 | elapsed time per iteration (ms): 104618.8 | throughput per GPU (TFLOP/s/GPU): 34.2 | MFU 10.96% | FLOPs estimator: workload-aware | logical tokens/s: 160364.9 | valid tokens/s: 160364.8 | physical tokens/s: 160772.8 | loss tokens/s: 119283.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.318485E-01 | lm-global-supervised-tokens: 1.247932E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.462016E-02 | lm-token-buffer-mib: 5.814926E+01 | global_load_balancing_loss: 1.000646E+00 | loss scale: 1.0 | grad norm: 0.217 | params norm: 11276.647 | params rms: 0.061059 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 1:25:41.469336 | finish at 2026-10-10 23:46:11 + [2026-10-09 22:21:57] iteration 5981/ 6855 | consumed samples: 765568 | elapsed time per iteration (ms): 86908.1 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.61% | FLOPs estimator: workload-aware | logical tokens/s: 193044.8 | valid tokens/s: 193044.8 | physical tokens/s: 193510.1 | loss tokens/s: 139652.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.249164E-01 | lm-global-supervised-tokens: 1.213692E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.131444E-02 | lm-token-buffer-mib: 5.842176E+01 | global_load_balancing_loss: 1.001226E+00 | loss scale: 1.0 | grad norm: 0.047 | params norm: 11276.600 | params rms: 0.061059 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 21:05:57.685555 | finish at 2026-10-10 19:27:54 + [2026-10-09 22:23:24] iteration 5982/ 6855 | consumed samples: 765696 | elapsed time per iteration (ms): 87255.2 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.65% | FLOPs estimator: workload-aware | logical tokens/s: 192277.1 | valid tokens/s: 192277.1 | physical tokens/s: 192668.6 | loss tokens/s: 138172.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.192752E-01 | lm-global-supervised-tokens: 1.205628E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.461259E-02 | lm-token-buffer-mib: 5.810820E+01 | global_load_balancing_loss: 1.000383E+00 | loss scale: 1.0 | grad norm: 0.121 | params norm: 11276.553 | params rms: 0.061059 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 21:09:33.812000 | finish at 2026-10-10 19:32:58 + [2026-10-09 22:24:51] iteration 5983/ 6855 | consumed samples: 765824 | elapsed time per iteration (ms): 87228.7 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.49% | FLOPs estimator: workload-aware | logical tokens/s: 192335.4 | valid tokens/s: 192335.4 | physical tokens/s: 192788.4 | loss tokens/s: 145873.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.930442E-01 | lm-global-supervised-tokens: 1.272432E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.111668E-02 | lm-token-buffer-mib: 5.811170E+01 | global_load_balancing_loss: 9.987976E-01 | loss scale: 1.0 | grad norm: 0.116 | params norm: 11276.506 | params rms: 0.061059 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 21:07:43.441717 | finish at 2026-10-10 19:32:35 + [2026-10-09 22:26:13] iteration 5984/ 6855 | consumed samples: 765952 | elapsed time per iteration (ms): 82028.3 | throughput per GPU (TFLOP/s/GPU): 43.8 | MFU 14.04% | FLOPs estimator: workload-aware | logical tokens/s: 204528.8 | valid tokens/s: 204528.8 | physical tokens/s: 204962.0 | loss tokens/s: 154318.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.449518E-01 | lm-global-supervised-tokens: 1.265851E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.355395E-02 | lm-token-buffer-mib: 5.811491E+01 | global_load_balancing_loss: 9.993641E-01 | loss scale: 1.0 | grad norm: 0.040 | params norm: 11276.460 | params rms: 0.061058 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 19:50:46.660139 | finish at 2026-10-10 18:17:00 + [2026-10-09 22:27:41] iteration 5985/ 6855 | consumed samples: 766080 | elapsed time per iteration (ms): 87672.4 | throughput per GPU (TFLOP/s/GPU): 41.4 | MFU 13.28% | FLOPs estimator: workload-aware | logical tokens/s: 191362.3 | valid tokens/s: 191362.3 | physical tokens/s: 192087.1 | loss tokens/s: 142327.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.291944E-01 | lm-global-supervised-tokens: 1.247815E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.259084E-02 | lm-token-buffer-mib: 5.813733E+01 | global_load_balancing_loss: 1.000624E+00 | loss scale: 1.0 | grad norm: 0.155 | params norm: 11276.413 | params rms: 0.061058 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 21:11:14.996917 | finish at 2026-10-10 19:38:56 + [2026-10-09 22:29:13] iteration 5986/ 6855 | consumed samples: 766208 | elapsed time per iteration (ms): 91702.4 | throughput per GPU (TFLOP/s/GPU): 40.5 | MFU 12.97% | FLOPs estimator: workload-aware | logical tokens/s: 182952.6 | valid tokens/s: 182952.6 | physical tokens/s: 183483.1 | loss tokens/s: 134384.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.709286E-01 | lm-global-supervised-tokens: 1.232338E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.486754E-02 | lm-token-buffer-mib: 5.817568E+01 | global_load_balancing_loss: 1.000717E+00 | loss scale: 1.0 | grad norm: 0.174 | params norm: 11276.366 | params rms: 0.061058 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 22:08:09.374592 | finish at 2026-10-10 20:37:22 + [2026-10-09 22:30:44] iteration 5987/ 6855 | consumed samples: 766336 | elapsed time per iteration (ms): 91040.2 | throughput per GPU (TFLOP/s/GPU): 41.6 | MFU 13.34% | FLOPs estimator: workload-aware | logical tokens/s: 184283.6 | valid tokens/s: 184283.6 | physical tokens/s: 184546.2 | loss tokens/s: 133514.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.499969E-01 | lm-global-supervised-tokens: 1.215520E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.145113E-02 | lm-token-buffer-mib: 5.804491E+01 | global_load_balancing_loss: 9.999701E-01 | loss scale: 1.0 | grad norm: 0.101 | params norm: 11276.318 | params rms: 0.061058 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 21:57:02.884904 | finish at 2026-10-10 20:27:47 + [2026-10-09 22:32:12] iteration 5988/ 6855 | consumed samples: 766464 | elapsed time per iteration (ms): 88689.1 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.60% | FLOPs estimator: workload-aware | logical tokens/s: 189168.0 | valid tokens/s: 189168.0 | physical tokens/s: 189688.5 | loss tokens/s: 141641.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.292460E-01 | lm-global-supervised-tokens: 1.256202E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.716043E-02 | lm-token-buffer-mib: 5.817542E+01 | global_load_balancing_loss: 1.001304E+00 | loss scale: 1.0 | grad norm: 0.414 | params norm: 11276.271 | params rms: 0.061057 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 21:21:33.477836 | finish at 2026-10-10 19:53:46 + [2026-10-09 22:33:41] iteration 5989/ 6855 | consumed samples: 766592 | elapsed time per iteration (ms): 88394.3 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.43% | FLOPs estimator: workload-aware | logical tokens/s: 189798.9 | valid tokens/s: 189798.9 | physical tokens/s: 190217.8 | loss tokens/s: 138041.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.927260E-01 | lm-global-supervised-tokens: 1.220211E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.454951E-02 | lm-token-buffer-mib: 5.806280E+01 | global_load_balancing_loss: 9.996427E-01 | loss scale: 1.0 | grad norm: 0.145 | params norm: 11276.223 | params rms: 0.061057 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 21:15:49.431577 | finish at 2026-10-10 19:49:30 + [2026-10-09 22:35:33] iteration 5990/ 6855 | consumed samples: 766720 | elapsed time per iteration (ms): 111954.5 | throughput per GPU (TFLOP/s/GPU): 34.3 | MFU 10.98% | FLOPs estimator: workload-aware | logical tokens/s: 149857.4 | valid tokens/s: 149857.4 | physical tokens/s: 150193.1 | loss tokens/s: 110109.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.701135E-01 | lm-global-supervised-tokens: 1.232720E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.261627E-02 | lm-token-buffer-mib: 5.816691E+01 | global_load_balancing_loss: 9.999688E-01 | loss scale: 1.0 | grad norm: 0.210 | params norm: 11276.173 | params rms: 0.061057 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 2:54:00.649477 | finish at 2026-10-11 01:29:33 + [2026-10-09 22:36:59] iteration 5991/ 6855 | consumed samples: 766848 | elapsed time per iteration (ms): 86290.0 | throughput per GPU (TFLOP/s/GPU): 43.2 | MFU 13.85% | FLOPs estimator: workload-aware | logical tokens/s: 194428.0 | valid tokens/s: 194428.0 | physical tokens/s: 194904.5 | loss tokens/s: 143617.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.109494E-01 | lm-global-supervised-tokens: 1.239276E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.389135E-02 | lm-token-buffer-mib: 5.823890E+01 | global_load_balancing_loss: 1.000688E+00 | loss scale: 1.0 | grad norm: 0.067 | params norm: 11276.124 | params rms: 0.061057 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 20:42:34.533394 | finish at 2026-10-10 19:19:34 + [2026-10-09 22:38:27] iteration 5992/ 6855 | consumed samples: 766976 | elapsed time per iteration (ms): 88151.2 | throughput per GPU (TFLOP/s/GPU): 43.1 | MFU 13.80% | FLOPs estimator: workload-aware | logical tokens/s: 190322.3 | valid tokens/s: 190322.3 | physical tokens/s: 190744.0 | loss tokens/s: 132492.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.174254E-01 | lm-global-supervised-tokens: 1.167934E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.157571E-02 | lm-token-buffer-mib: 5.803961E+01 | global_load_balancing_loss: 1.000868E+00 | loss scale: 1.0 | grad norm: 0.104 | params norm: 11276.076 | params rms: 0.061056 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 21:07:54.443263 | finish at 2026-10-10 19:46:22 + [2026-10-09 22:39:53] iteration 5993/ 6855 | consumed samples: 767104 | elapsed time per iteration (ms): 85546.4 | throughput per GPU (TFLOP/s/GPU): 43.7 | MFU 13.99% | FLOPs estimator: workload-aware | logical tokens/s: 196118.0 | valid tokens/s: 196118.0 | physical tokens/s: 196637.6 | loss tokens/s: 144791.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.450874E-01 | lm-global-supervised-tokens: 1.238640E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.150251E-02 | lm-token-buffer-mib: 5.806947E+01 | global_load_balancing_loss: 9.996635E-01 | loss scale: 1.0 | grad norm: 0.124 | params norm: 11276.027 | params rms: 0.061056 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 20:29:00.955346 | finish at 2026-10-10 19:08:54 + [2026-10-09 22:41:16] iteration 5994/ 6855 | consumed samples: 767232 | elapsed time per iteration (ms): 83816.1 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.68% | FLOPs estimator: workload-aware | logical tokens/s: 200167.1 | valid tokens/s: 200167.1 | physical tokens/s: 200729.8 | loss tokens/s: 154126.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.505173E-01 | lm-global-supervised-tokens: 1.291828E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.519236E-02 | lm-token-buffer-mib: 5.811874E+01 | global_load_balancing_loss: 9.995832E-01 | loss scale: 1.0 | grad norm: 0.076 | params norm: 11275.978 | params rms: 0.061056 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 20:02:45.622996 | finish at 2026-10-10 18:44:02 + [2026-10-09 22:42:42] iteration 5995/ 6855 | consumed samples: 767360 | elapsed time per iteration (ms): 85862.8 | throughput per GPU (TFLOP/s/GPU): 43.2 | MFU 13.84% | FLOPs estimator: workload-aware | logical tokens/s: 195395.2 | valid tokens/s: 195395.2 | physical tokens/s: 196012.9 | loss tokens/s: 140698.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.067485E-01 | lm-global-supervised-tokens: 1.208074E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.395713E-02 | lm-token-buffer-mib: 5.820152E+01 | global_load_balancing_loss: 9.998651E-01 | loss scale: 1.0 | grad norm: 0.198 | params norm: 11275.930 | params rms: 0.061055 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 20:30:42.035375 | finish at 2026-10-10 19:13:24 + [2026-10-09 22:44:12] iteration 5996/ 6855 | consumed samples: 767488 | elapsed time per iteration (ms): 89779.8 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.62% | FLOPs estimator: workload-aware | logical tokens/s: 186869.3 | valid tokens/s: 186869.3 | physical tokens/s: 187477.8 | loss tokens/s: 132056.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.535216E-01 | lm-global-supervised-tokens: 1.185600E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.180357E-02 | lm-token-buffer-mib: 5.818159E+01 | global_load_balancing_loss: 1.000636E+00 | loss scale: 1.0 | grad norm: 0.188 | params norm: 11275.881 | params rms: 0.061055 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 21:25:20.889722 | finish at 2026-10-10 20:09:33 + [2026-10-09 22:45:36] iteration 5997/ 6855 | consumed samples: 767616 | elapsed time per iteration (ms): 83961.0 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.76% | FLOPs estimator: workload-aware | logical tokens/s: 199821.0 | valid tokens/s: 199821.0 | physical tokens/s: 200450.9 | loss tokens/s: 152759.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.013661E-01 | lm-global-supervised-tokens: 1.282582E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.685117E-02 | lm-token-buffer-mib: 5.834594E+01 | global_load_balancing_loss: 1.000667E+00 | loss scale: 1.0 | grad norm: 0.067 | params norm: 11275.834 | params rms: 0.061055 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 20:00:38.563745 | finish at 2026-10-10 18:46:15 + [2026-10-09 22:47:08] iteration 5998/ 6855 | consumed samples: 767744 | elapsed time per iteration (ms): 91472.0 | throughput per GPU (TFLOP/s/GPU): 41.4 | MFU 13.28% | FLOPs estimator: workload-aware | logical tokens/s: 183413.6 | valid tokens/s: 183413.6 | physical tokens/s: 183789.4 | loss tokens/s: 135913.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.323006E-01 | lm-global-supervised-tokens: 1.243224E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.296746E-02 | lm-token-buffer-mib: 5.806845E+01 | global_load_balancing_loss: 1.000612E+00 | loss scale: 1.0 | grad norm: 0.200 | params norm: 11275.786 | params rms: 0.061055 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 21:46:31.486941 | finish at 2026-10-10 20:33:39 + [2026-10-09 22:48:37] iteration 5999/ 6855 | consumed samples: 767872 | elapsed time per iteration (ms): 89034.2 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.41% | FLOPs estimator: workload-aware | logical tokens/s: 188435.6 | valid tokens/s: 188435.6 | physical tokens/s: 188994.8 | loss tokens/s: 140695.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.741840E-01 | lm-global-supervised-tokens: 1.252668E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.538309E-02 | lm-token-buffer-mib: 5.823652E+01 | global_load_balancing_loss: 9.998597E-01 | loss scale: 1.0 | grad norm: 0.300 | params norm: 11275.738 | params rms: 0.061054 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 21:10:13.288017 | finish at 2026-10-10 19:58:50 + [2026-10-09 22:50:27] iteration 6000/ 6855 | consumed samples: 768000 | elapsed time per iteration (ms): 110477.8 | throughput per GPU (TFLOP/s/GPU): 33.3 | MFU 10.68% | FLOPs estimator: workload-aware | logical tokens/s: 151860.4 | valid tokens/s: 151860.4 | physical tokens/s: 152257.7 | loss tokens/s: 112655.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.184631E-01 | lm-global-supervised-tokens: 1.244593E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.516150E-02 | lm-token-buffer-mib: 5.819425E+01 | global_load_balancing_loss: 1.000097E+00 | loss scale: 1.0 | grad norm: 0.219 | params norm: 11275.692 | params rms: 0.061054 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 2:14:18.536235 | finish at 2026-10-11 01:04:46 +times across ranks (ms): + save-checkpoint: + rank 0: 23757.10 + rank 1: 23757.11 + rank 2: 23757.10 + rank 3: 23757.10 + rank 4: 23757.13 + rank 5: 23757.12 + rank 6: 23757.08 + rank 7: 23757.13 + rank 8: 23755.71 + rank 9: 23755.73 + rank 10: 23755.70 + rank 11: 23755.72 + rank 12: 23755.72 + rank 13: 23755.69 + rank 14: 23755.73 + rank 15: 23755.71 + rank 16: 23757.87 + rank 17: 23757.87 + rank 18: 23757.87 + rank 19: 23757.93 + rank 20: 23757.84 + rank 21: 23757.90 + rank 22: 23757.83 + rank 23: 23757.90 + rank 24: 23757.56 + rank 25: 23757.54 + rank 26: 23757.56 + rank 27: 23757.55 + rank 28: 23757.53 + rank 29: 23757.55 + rank 30: 23757.53 + rank 31: 23757.55 + rank 32: 23757.58 + rank 33: 23757.60 + rank 34: 23757.56 + rank 35: 23757.55 + rank 36: 23757.53 + rank 37: 23757.54 + rank 38: 23757.53 + rank 39: 23757.51 + rank 40: 23757.67 + rank 41: 23757.66 + rank 42: 23757.67 + rank 43: 23757.68 + rank 44: 23757.64 + rank 45: 23757.66 + rank 46: 23757.65 + rank 47: 23757.64 + rank 48: 23757.42 + rank 49: 23757.40 + rank 50: 23757.41 + rank 51: 23757.40 + rank 52: 23757.39 + rank 53: 23757.37 + rank 54: 23757.42 + rank 55: 23757.39 + rank 56: 23757.47 + rank 57: 23757.45 + rank 58: 23757.42 + rank 59: 23757.41 + rank 60: 23757.41 + rank 61: 23757.42 + rank 62: 23757.42 + rank 63: 23757.42 + [2026-10-09 22:52:22] iteration 6001/ 6855 | consumed samples: 768128 | elapsed time per iteration (ms): 91081.7 | throughput per GPU (TFLOP/s/GPU): 41.2 | MFU 13.20% | FLOPs estimator: workload-aware | logical tokens/s: 184199.2 | valid tokens/s: 184199.2 | physical tokens/s: 184527.3 | loss tokens/s: 136306.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 5.965908E-01 | lm-global-supervised-tokens: 1.241497E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 5.482863E-01 | lm-token-buffer-mib: 5.814483E+01 | global_load_balancing_loss: 1.000252E+00 | loss scale: 1.0 | grad norm: 0.057 | params norm: 11275.645 | params rms: 0.061054 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 21:36:23.744183 | finish at 2026-10-10 20:28:46 +times across ranks (ms): + forward-backward: + rank 0: 90400.99 + rank 1: 90403.03 + rank 2: 90403.02 + rank 3: 90402.44 + rank 4: 90404.48 + rank 5: 90403.12 + rank 6: 90400.16 + rank 7: 90403.08 + rank 8: 90392.44 + rank 9: 90394.82 + rank 10: 90395.39 + rank 11: 90394.66 + rank 12: 90396.27 + rank 13: 90395.44 + rank 14: 90390.23 + rank 15: 90394.62 + rank 16: 90407.52 + rank 17: 90399.20 + rank 18: 90400.22 + rank 19: 90399.17 + rank 20: 90400.12 + rank 21: 90409.22 + rank 22: 90395.18 + rank 23: 90399.21 + rank 24: 90393.95 + rank 25: 90396.19 + rank 26: 90396.69 + rank 27: 90395.45 + rank 28: 90398.92 + rank 29: 90396.63 + rank 30: 90393.32 + rank 31: 90395.91 + rank 32: 90391.97 + rank 33: 90394.26 + rank 34: 90391.59 + rank 35: 90391.48 + rank 36: 90393.97 + rank 37: 90391.61 + rank 38: 90389.45 + rank 39: 90393.98 + rank 40: 90394.80 + rank 41: 90395.27 + rank 42: 90395.16 + rank 43: 90392.88 + rank 44: 90394.70 + rank 45: 90391.25 + rank 46: 90394.19 + rank 47: 90397.13 + rank 48: 90400.08 + rank 49: 90400.21 + rank 50: 90398.37 + rank 51: 90398.53 + rank 52: 90398.73 + rank 53: 90397.17 + rank 54: 90399.99 + rank 55: 90403.57 + rank 56: 90403.17 + rank 57: 90404.10 + rank 58: 90403.57 + rank 59: 90401.09 + rank 60: 90403.16 + rank 61: 90400.02 + rank 62: 90402.34 + rank 63: 90405.69 + forward-compute: + rank 0: 34509.66 + rank 1: 34516.32 + rank 2: 34519.90 + rank 3: 34519.12 + rank 4: 34527.85 + rank 5: 34522.54 + rank 6: 34518.07 + rank 7: 34530.50 + rank 8: 36042.49 + rank 9: 36048.75 + rank 10: 36048.39 + rank 11: 36048.44 + rank 12: 36051.78 + rank 13: 36050.23 + rank 14: 36049.97 + rank 15: 36048.74 + rank 16: 36844.84 + rank 17: 36847.65 + rank 18: 36848.07 + rank 19: 36846.49 + rank 20: 36849.98 + rank 21: 36847.43 + rank 22: 36846.84 + rank 23: 36851.85 + rank 24: 31175.66 + rank 25: 31184.19 + rank 26: 31183.18 + rank 27: 31180.69 + rank 28: 31196.08 + rank 29: 31190.08 + rank 30: 31193.38 + rank 31: 31197.59 + rank 32: 34724.24 + rank 33: 34726.93 + rank 34: 34727.63 + rank 35: 34726.29 + rank 36: 34730.59 + rank 37: 34729.88 + rank 38: 34728.20 + rank 39: 34731.25 + rank 40: 38084.69 + rank 41: 38086.85 + rank 42: 38086.44 + rank 43: 38085.00 + rank 44: 38086.52 + rank 45: 38084.80 + rank 46: 38084.07 + rank 47: 38086.69 + rank 48: 35533.45 + rank 49: 35532.69 + rank 50: 35535.28 + rank 51: 35535.64 + rank 52: 35539.62 + rank 53: 35538.16 + rank 54: 35535.55 + rank 55: 35543.30 + rank 56: 34069.32 + rank 57: 34067.41 + rank 58: 34063.30 + rank 59: 34063.07 + rank 60: 34070.70 + rank 61: 34068.84 + rank 62: 34064.30 + rank 63: 34067.99 + backward-compute: + rank 0: 55855.60 + rank 1: 55850.44 + rank 2: 55846.86 + rank 3: 55847.23 + rank 4: 55840.76 + rank 5: 55845.07 + rank 6: 55845.49 + rank 7: 55836.94 + rank 8: 54314.24 + rank 9: 54313.82 + rank 10: 54312.50 + rank 11: 54308.14 + rank 12: 54308.41 + rank 13: 54310.12 + rank 14: 54303.72 + rank 15: 54309.76 + rank 16: 53514.23 + rank 17: 53515.66 + rank 18: 53510.30 + rank 19: 53514.86 + rank 20: 53516.59 + rank 21: 53513.21 + rank 22: 53508.89 + rank 23: 53511.04 + rank 24: 58812.16 + rank 25: 58807.86 + rank 26: 58808.70 + rank 27: 58808.40 + rank 28: 58795.66 + rank 29: 58797.58 + rank 30: 58794.11 + rank 31: 58795.38 + rank 32: 55626.11 + rank 33: 55624.99 + rank 34: 55624.29 + rank 35: 55624.42 + rank 36: 55622.61 + rank 37: 55620.59 + rank 38: 55619.65 + rank 39: 55619.71 + rank 40: 52270.91 + rank 41: 52272.49 + rank 42: 52272.57 + rank 43: 52271.27 + rank 44: 52269.93 + rank 45: 52269.93 + rank 46: 52274.28 + rank 47: 52274.09 + rank 48: 54831.57 + rank 49: 54831.56 + rank 50: 54830.13 + rank 51: 54827.88 + rank 52: 54820.80 + rank 53: 54824.77 + rank 54: 54825.75 + rank 55: 54824.26 + rank 56: 56299.16 + rank 57: 56300.79 + rank 58: 56304.05 + rank 59: 56301.74 + rank 60: 56295.33 + rank 61: 56294.77 + rank 62: 56303.97 + rank 63: 56300.53 + batch-generator: + rank 0: 26.04 + rank 1: 25.30 + rank 2: 24.61 + rank 3: 25.34 + rank 4: 24.29 + rank 5: 26.27 + rank 6: 24.97 + rank 7: 24.89 + rank 8: 24.23 + rank 9: 22.54 + rank 10: 24.01 + rank 11: 24.34 + rank 12: 25.56 + rank 13: 24.05 + rank 14: 24.97 + rank 15: 24.37 + rank 16: 24.81 + rank 17: 24.45 + rank 18: 23.73 + rank 19: 25.85 + rank 20: 24.42 + rank 21: 27.13 + rank 22: 23.63 + rank 23: 25.19 + rank 24: 706.95 + rank 25: 705.74 + rank 26: 710.26 + rank 27: 707.60 + rank 28: 708.33 + rank 29: 709.78 + rank 30: 705.26 + rank 31: 709.05 + rank 32: 36.77 + rank 33: 33.72 + rank 34: 35.13 + rank 35: 33.59 + rank 36: 32.28 + rank 37: 33.42 + rank 38: 34.91 + rank 39: 32.28 + rank 40: 24.40 + rank 41: 24.30 + rank 42: 25.40 + rank 43: 24.22 + rank 44: 26.09 + rank 45: 24.19 + rank 46: 23.62 + rank 47: 23.96 + rank 48: 29.22 + rank 49: 29.80 + rank 50: 25.28 + rank 51: 30.30 + rank 52: 32.72 + rank 53: 27.20 + rank 54: 29.74 + rank 55: 31.41 + rank 56: 26.47 + rank 57: 27.45 + rank 58: 24.91 + rank 59: 24.14 + rank 60: 25.17 + rank 61: 25.65 + rank 62: 27.01 + rank 63: 28.27 + embedding-grads-all-reduce: + rank 0: 0.03 + rank 1: 0.03 + rank 2: 0.03 + rank 3: 0.03 + rank 4: 0.02 + rank 5: 0.03 + rank 6: 0.03 + rank 7: 0.03 + rank 8: 0.02 + rank 9: 0.03 + rank 10: 0.02 + rank 11: 0.03 + rank 12: 0.03 + rank 13: 0.03 + rank 14: 0.02 + rank 15: 0.02 + rank 16: 0.03 + rank 17: 0.03 + rank 18: 0.03 + rank 19: 0.03 + rank 20: 0.03 + rank 21: 0.04 + rank 22: 0.03 + rank 23: 0.03 + rank 24: 0.04 + rank 25: 0.04 + rank 26: 0.04 + rank 27: 0.05 + rank 28: 0.04 + rank 29: 0.04 + rank 30: 0.04 + rank 31: 0.04 + rank 32: 0.04 + rank 33: 0.03 + rank 34: 0.04 + rank 35: 0.03 + rank 36: 0.03 + rank 37: 0.03 + rank 38: 0.03 + rank 39: 0.03 + rank 40: 0.03 + rank 41: 0.03 + rank 42: 0.02 + rank 43: 0.02 + rank 44: 0.03 + rank 45: 0.03 + rank 46: 0.03 + rank 47: 0.03 + rank 48: 0.03 + rank 49: 0.03 + rank 50: 0.03 + rank 51: 0.03 + rank 52: 0.03 + rank 53: 0.03 + rank 54: 0.03 + rank 55: 0.03 + rank 56: 0.03 + rank 57: 0.03 + rank 58: 0.03 + rank 59: 0.03 + rank 60: 0.03 + rank 61: 0.03 + rank 62: 0.03 + rank 63: 0.03 + all-grads-sync: + rank 0: 0.80 + rank 1: 0.77 + rank 2: 0.83 + rank 3: 0.91 + rank 4: 0.77 + rank 5: 1.20 + rank 6: 1.41 + rank 7: 0.79 + rank 8: 0.79 + rank 9: 0.77 + rank 10: 0.81 + rank 11: 0.80 + rank 12: 0.80 + rank 13: 0.94 + rank 14: 0.84 + rank 15: 0.82 + rank 16: 1.47 + rank 17: 0.76 + rank 18: 0.79 + rank 19: 1.34 + rank 20: 0.79 + rank 21: 1.21 + rank 22: 0.97 + rank 23: 0.83 + rank 24: 1.24 + rank 25: 1.11 + rank 26: 1.20 + rank 27: 1.28 + rank 28: 1.21 + rank 29: 1.28 + rank 30: 1.23 + rank 31: 1.21 + rank 32: 1.24 + rank 33: 1.43 + rank 34: 1.23 + rank 35: 1.22 + rank 36: 1.12 + rank 37: 1.20 + rank 38: 1.24 + rank 39: 1.17 + rank 40: 0.80 + rank 41: 0.75 + rank 42: 0.78 + rank 43: 0.81 + rank 44: 0.78 + rank 45: 0.86 + rank 46: 0.82 + rank 47: 0.87 + rank 48: 0.97 + rank 49: 1.06 + rank 50: 0.82 + rank 51: 1.04 + rank 52: 0.82 + rank 53: 0.82 + rank 54: 1.24 + rank 55: 1.11 + rank 56: 0.80 + rank 57: 1.17 + rank 58: 0.78 + rank 59: 0.75 + rank 60: 0.80 + rank 61: 0.81 + rank 62: 0.84 + rank 63: 0.95 + params-all-gather: + rank 0: 0.02 + rank 1: 0.02 + rank 2: 0.02 + rank 3: 0.02 + rank 4: 0.02 + rank 5: 0.02 + rank 6: 0.02 + rank 7: 0.02 + rank 8: 0.02 + rank 9: 0.02 + rank 10: 0.02 + rank 11: 0.02 + rank 12: 0.02 + rank 13: 0.02 + rank 14: 0.02 + rank 15: 0.02 + rank 16: 0.03 + rank 17: 0.02 + rank 18: 0.02 + rank 19: 0.02 + rank 20: 0.02 + rank 21: 0.04 + rank 22: 0.02 + rank 23: 0.02 + rank 24: 0.03 + rank 25: 0.03 + rank 26: 0.03 + rank 27: 0.06 + rank 28: 0.03 + rank 29: 0.03 + rank 30: 0.03 + rank 31: 0.04 + rank 32: 0.02 + rank 33: 0.03 + rank 34: 0.02 + rank 35: 0.03 + rank 36: 0.02 + rank 37: 0.02 + rank 38: 0.03 + rank 39: 0.02 + rank 40: 0.02 + rank 41: 0.02 + rank 42: 0.02 + rank 43: 0.02 + rank 44: 0.02 + rank 45: 0.02 + rank 46: 0.02 + rank 47: 0.02 + rank 48: 0.03 + rank 49: 0.03 + rank 50: 0.03 + rank 51: 0.03 + rank 52: 0.03 + rank 53: 0.03 + rank 54: 0.03 + rank 55: 0.03 + rank 56: 0.02 + rank 57: 0.02 + rank 58: 0.02 + rank 59: 0.02 + rank 60: 0.02 + rank 61: 0.02 + rank 62: 0.02 + rank 63: 0.02 + optimizer-copy-to-main-grad: + rank 0: 2.19 + rank 1: 1.79 + rank 2: 1.67 + rank 3: 2.24 + rank 4: 1.77 + rank 5: 1.84 + rank 6: 1.93 + rank 7: 1.92 + rank 8: 1.75 + rank 9: 2.17 + rank 10: 1.79 + rank 11: 1.76 + rank 12: 1.97 + rank 13: 1.81 + rank 14: 1.87 + rank 15: 1.98 + rank 16: 3.95 + rank 17: 2.11 + rank 18: 2.23 + rank 19: 2.22 + rank 20: 2.20 + rank 21: 3.78 + rank 22: 2.66 + rank 23: 2.32 + rank 24: 2.13 + rank 25: 2.27 + rank 26: 1.93 + rank 27: 2.12 + rank 28: 2.34 + rank 29: 2.45 + rank 30: 2.25 + rank 31: 1.96 + rank 32: 2.03 + rank 33: 2.02 + rank 34: 1.98 + rank 35: 2.22 + rank 36: 2.10 + rank 37: 2.00 + rank 38: 2.18 + rank 39: 1.85 + rank 40: 1.69 + rank 41: 1.91 + rank 42: 1.84 + rank 43: 1.90 + rank 44: 1.98 + rank 45: 2.11 + rank 46: 1.74 + rank 47: 1.72 + rank 48: 1.95 + rank 49: 1.75 + rank 50: 2.06 + rank 51: 1.82 + rank 52: 1.99 + rank 53: 1.76 + rank 54: 1.80 + rank 55: 1.78 + rank 56: 1.77 + rank 57: 2.48 + rank 58: 1.86 + rank 59: 1.70 + rank 60: 1.75 + rank 61: 1.68 + rank 62: 1.86 + rank 63: 1.75 + optimizer-inner-step: + rank 0: 49.53 + rank 1: 49.37 + rank 2: 49.23 + rank 3: 49.29 + rank 4: 49.13 + rank 5: 49.28 + rank 6: 49.58 + rank 7: 49.63 + rank 8: 49.36 + rank 9: 49.35 + rank 10: 49.24 + rank 11: 49.34 + rank 12: 49.28 + rank 13: 49.55 + rank 14: 49.28 + rank 15: 49.20 + rank 16: 62.58 + rank 17: 49.45 + rank 18: 49.47 + rank 19: 49.36 + rank 20: 49.44 + rank 21: 54.44 + rank 22: 49.74 + rank 23: 49.51 + rank 24: 49.13 + rank 25: 49.32 + rank 26: 49.25 + rank 27: 49.47 + rank 28: 49.79 + rank 29: 49.22 + rank 30: 49.22 + rank 31: 49.50 + rank 32: 49.38 + rank 33: 49.22 + rank 34: 49.30 + rank 35: 49.86 + rank 36: 49.44 + rank 37: 49.42 + rank 38: 49.38 + rank 39: 49.26 + rank 40: 49.30 + rank 41: 49.42 + rank 42: 49.44 + rank 43: 49.49 + rank 44: 49.09 + rank 45: 49.74 + rank 46: 49.36 + rank 47: 49.01 + rank 48: 49.40 + rank 49: 49.25 + rank 50: 53.40 + rank 51: 49.02 + rank 52: 49.40 + rank 53: 49.28 + rank 54: 49.45 + rank 55: 49.33 + rank 56: 48.98 + rank 57: 50.27 + rank 58: 49.29 + rank 59: 49.17 + rank 60: 49.55 + rank 61: 49.11 + rank 62: 49.22 + rank 63: 49.06 + optimizer-copy-main-to-model-params: + rank 0: 4.40 + rank 1: 3.88 + rank 2: 3.78 + rank 3: 4.38 + rank 4: 3.75 + rank 5: 3.93 + rank 6: 3.95 + rank 7: 4.04 + rank 8: 3.97 + rank 9: 4.42 + rank 10: 3.77 + rank 11: 3.92 + rank 12: 4.06 + rank 13: 3.76 + rank 14: 4.09 + rank 15: 4.24 + rank 16: 5.72 + rank 17: 4.00 + rank 18: 4.18 + rank 19: 4.06 + rank 20: 4.08 + rank 21: 6.20 + rank 22: 4.62 + rank 23: 4.24 + rank 24: 3.97 + rank 25: 4.20 + rank 26: 4.01 + rank 27: 4.20 + rank 28: 4.36 + rank 29: 4.43 + rank 30: 4.33 + rank 31: 4.06 + rank 32: 4.29 + rank 33: 4.86 + rank 34: 4.22 + rank 35: 5.60 + rank 36: 4.37 + rank 37: 4.23 + rank 38: 6.02 + rank 39: 4.07 + rank 40: 3.74 + rank 41: 3.98 + rank 42: 3.95 + rank 43: 3.99 + rank 44: 4.01 + rank 45: 4.22 + rank 46: 3.73 + rank 47: 3.70 + rank 48: 4.10 + rank 49: 3.88 + rank 50: 4.92 + rank 51: 3.95 + rank 52: 4.24 + rank 53: 3.90 + rank 54: 3.87 + rank 55: 3.87 + rank 56: 3.81 + rank 57: 4.72 + rank 58: 3.93 + rank 59: 3.77 + rank 60: 3.78 + rank 61: 3.73 + rank 62: 3.95 + rank 63: 3.66 + optimizer: + rank 0: 63.23 + rank 1: 62.36 + rank 2: 62.08 + rank 3: 63.01 + rank 4: 61.93 + rank 5: 62.35 + rank 6: 62.71 + rank 7: 62.83 + rank 8: 62.49 + rank 9: 63.10 + rank 10: 62.10 + rank 11: 62.37 + rank 12: 62.53 + rank 13: 62.42 + rank 14: 62.58 + rank 15: 62.69 + rank 16: 78.85 + rank 17: 62.85 + rank 18: 63.07 + rank 19: 62.88 + rank 20: 62.88 + rank 21: 71.04 + rank 22: 64.04 + rank 23: 63.22 + rank 24: 62.20 + rank 25: 62.73 + rank 26: 62.37 + rank 27: 62.91 + rank 28: 63.39 + rank 29: 62.90 + rank 30: 62.79 + rank 31: 62.70 + rank 32: 62.81 + rank 33: 63.18 + rank 34: 62.61 + rank 35: 64.76 + rank 36: 62.98 + rank 37: 62.77 + rank 38: 64.76 + rank 39: 62.36 + rank 40: 62.16 + rank 41: 62.58 + rank 42: 62.58 + rank 43: 62.68 + rank 44: 62.30 + rank 45: 63.24 + rank 46: 62.17 + rank 47: 61.82 + rank 48: 62.68 + rank 49: 62.21 + rank 50: 67.79 + rank 51: 62.05 + rank 52: 62.81 + rank 53: 62.23 + rank 54: 62.39 + rank 55: 62.27 + rank 56: 61.88 + rank 57: 64.47 + rank 58: 62.34 + rank 59: 61.98 + rank 60: 62.42 + rank 61: 61.89 + rank 62: 62.29 + rank 63: 61.80 + interval-time: + rank 0: 0.83 + rank 1: 0.84 + rank 2: 0.87 + rank 3: 0.78 + rank 4: 0.87 + rank 5: 0.81 + rank 6: 0.86 + rank 7: 0.88 + rank 8: 0.77 + rank 9: 0.73 + rank 10: 0.82 + rank 11: 0.74 + rank 12: 0.86 + rank 13: 0.80 + rank 14: 0.86 + rank 15: 0.82 + rank 16: 0.74 + rank 17: 0.82 + rank 18: 0.84 + rank 19: 0.74 + rank 20: 0.81 + rank 21: 0.84 + rank 22: 0.74 + rank 23: 0.71 + rank 24: 0.86 + rank 25: 0.97 + rank 26: 0.97 + rank 27: 0.98 + rank 28: 0.83 + rank 29: 0.84 + rank 30: 0.94 + rank 31: 0.97 + rank 32: 0.92 + rank 33: 0.83 + rank 34: 1.15 + rank 35: 0.81 + rank 36: 0.79 + rank 37: 0.88 + rank 38: 0.80 + rank 39: 0.91 + rank 40: 0.75 + rank 41: 0.81 + rank 42: 0.73 + rank 43: 0.75 + rank 44: 0.81 + rank 45: 0.72 + rank 46: 0.83 + rank 47: 0.73 + rank 48: 0.96 + rank 49: 0.93 + rank 50: 0.81 + rank 51: 0.91 + rank 52: 0.86 + rank 53: 0.81 + rank 54: 0.84 + rank 55: 0.95 + rank 56: 0.84 + rank 57: 0.83 + rank 58: 0.85 + rank 59: 0.82 + rank 60: 0.75 + rank 61: 0.83 + rank 62: 0.82 + rank 63: 2.99 + [2026-10-09 22:54:00] iteration 6002/ 6855 | consumed samples: 768256 | elapsed time per iteration (ms): 97962.3 | throughput per GPU (TFLOP/s/GPU): 38.7 | MFU 12.39% | FLOPs estimator: workload-aware | logical tokens/s: 171262.0 | valid tokens/s: 171262.0 | physical tokens/s: 171553.2 | loss tokens/s: 123991.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.509331E-01 | lm-global-supervised-tokens: 1.214648E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.337786E-02 | lm-token-buffer-mib: 5.811837E+01 | global_load_balancing_loss: 1.000386E+00 | loss scale: 1.0 | grad norm: 0.225 | params norm: 11275.599 | params rms: 0.061054 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 23:12:41.825073 | finish at 2026-10-10 22:06:42 +times across ranks (ms): + forward-backward: + rank 0: 90812.39 + rank 1: 90894.55 + rank 2: 90893.84 + rank 3: 90894.33 + rank 4: 90895.13 + rank 5: 90893.49 + rank 6: 90891.60 + rank 7: 90893.95 + rank 8: 90805.75 + rank 9: 90886.97 + rank 10: 90886.52 + rank 11: 90887.98 + rank 12: 90888.39 + rank 13: 90886.07 + rank 14: 90883.60 + rank 15: 90885.96 + rank 16: 90808.19 + rank 17: 90890.07 + rank 18: 90889.02 + rank 19: 90890.25 + rank 20: 90890.39 + rank 21: 90890.93 + rank 22: 90886.97 + rank 23: 90889.05 + rank 24: 90805.16 + rank 25: 90887.27 + rank 26: 90887.80 + rank 27: 90887.02 + rank 28: 90889.58 + rank 29: 90886.41 + rank 30: 90885.48 + rank 31: 90886.57 + rank 32: 90802.11 + rank 33: 90883.58 + rank 34: 90883.17 + rank 35: 90883.14 + rank 36: 90884.46 + rank 37: 90882.23 + rank 38: 90881.50 + rank 39: 90884.62 + rank 40: 90806.38 + rank 41: 90886.64 + rank 42: 90886.23 + rank 43: 90885.25 + rank 44: 90885.17 + rank 45: 90883.49 + rank 46: 90886.65 + rank 47: 90886.93 + rank 48: 90811.27 + rank 49: 90891.44 + rank 50: 90890.27 + rank 51: 90890.45 + rank 52: 90889.47 + rank 53: 90889.20 + rank 54: 90891.48 + rank 55: 90894.20 + rank 56: 90814.82 + rank 57: 90894.37 + rank 58: 90894.28 + rank 59: 90893.17 + rank 60: 90893.20 + rank 61: 90891.39 + rank 62: 90895.19 + rank 63: 90895.92 + forward-compute: + rank 0: 34506.64 + rank 1: 34578.40 + rank 2: 34578.12 + rank 3: 34579.91 + rank 4: 34577.69 + rank 5: 34573.11 + rank 6: 34571.06 + rank 7: 34578.31 + rank 8: 37950.67 + rank 9: 38033.38 + rank 10: 38032.93 + rank 11: 38031.41 + rank 12: 38033.32 + rank 13: 38031.09 + rank 14: 38031.48 + rank 15: 38031.99 + rank 16: 39450.32 + rank 17: 39532.18 + rank 18: 39535.39 + rank 19: 39538.61 + rank 20: 39538.95 + rank 21: 39536.58 + rank 22: 39533.45 + rank 23: 39537.06 + rank 24: 36086.89 + rank 25: 36170.56 + rank 26: 36172.71 + rank 27: 36169.12 + rank 28: 36178.72 + rank 29: 36174.10 + rank 30: 36174.78 + rank 31: 36174.73 + rank 32: 28256.62 + rank 33: 28344.38 + rank 34: 28338.41 + rank 35: 28341.34 + rank 36: 28345.48 + rank 37: 28355.10 + rank 38: 28363.20 + rank 39: 28340.93 + rank 40: 37477.04 + rank 41: 37561.67 + rank 42: 37562.42 + rank 43: 37562.98 + rank 44: 37562.19 + rank 45: 37559.83 + rank 46: 37560.91 + rank 47: 37560.32 + rank 48: 38862.35 + rank 49: 38945.04 + rank 50: 38945.45 + rank 51: 38943.43 + rank 52: 38946.59 + rank 53: 38946.64 + rank 54: 38944.78 + rank 55: 38946.26 + rank 56: 35613.83 + rank 57: 35693.55 + rank 58: 35693.84 + rank 59: 35690.27 + rank 60: 35695.96 + rank 61: 35690.30 + rank 62: 35688.32 + rank 63: 35693.74 + backward-compute: + rank 0: 56272.84 + rank 1: 56281.56 + rank 2: 56280.67 + rank 3: 56279.31 + rank 4: 56283.02 + rank 5: 56287.37 + rank 6: 56286.78 + rank 7: 56280.31 + rank 8: 52816.78 + rank 9: 52816.51 + rank 10: 52810.46 + rank 11: 52819.05 + rank 12: 52815.57 + rank 13: 52817.29 + rank 14: 52814.34 + rank 15: 52817.75 + rank 16: 51321.47 + rank 17: 51322.73 + rank 18: 51318.83 + rank 19: 51313.17 + rank 20: 51316.12 + rank 21: 51314.13 + rank 22: 51318.52 + rank 23: 51314.64 + rank 24: 54678.30 + rank 25: 54676.72 + rank 26: 54673.98 + rank 27: 54675.52 + rank 28: 54665.78 + rank 29: 54669.62 + rank 30: 54666.74 + rank 31: 54670.18 + rank 32: 62509.50 + rank 33: 62503.44 + rank 34: 62508.98 + rank 35: 62505.88 + rank 36: 62504.23 + rank 37: 62492.84 + rank 38: 62482.36 + rank 39: 62509.09 + rank 40: 53292.36 + rank 41: 53288.85 + rank 42: 53288.63 + rank 43: 53286.09 + rank 44: 53287.23 + rank 45: 53289.86 + rank 46: 53290.26 + rank 47: 53292.12 + rank 48: 51914.36 + rank 49: 51910.22 + rank 50: 51910.82 + rank 51: 51913.06 + rank 52: 51905.34 + rank 53: 51909.54 + rank 54: 51910.88 + rank 55: 51911.36 + rank 56: 55164.36 + rank 57: 55164.26 + rank 58: 55164.88 + rank 59: 55168.46 + rank 60: 55163.59 + rank 61: 55166.66 + rank 62: 55172.07 + rank 63: 55165.23 + batch-generator: + rank 0: 24.46 + rank 1: 24.51 + rank 2: 25.62 + rank 3: 24.50 + rank 4: 26.09 + rank 5: 24.09 + rank 6: 24.66 + rank 7: 24.68 + rank 8: 23.28 + rank 9: 22.42 + rank 10: 23.19 + rank 11: 24.27 + rank 12: 23.52 + rank 13: 23.25 + rank 14: 23.07 + rank 15: 22.94 + rank 16: 22.41 + rank 17: 23.89 + rank 18: 23.47 + rank 19: 23.20 + rank 20: 22.33 + rank 21: 23.44 + rank 22: 23.55 + rank 23: 22.78 + rank 24: 27.39 + rank 25: 32.05 + rank 26: 33.38 + rank 27: 31.29 + rank 28: 34.46 + rank 29: 28.20 + rank 30: 34.89 + rank 31: 32.10 + rank 32: 34.81 + rank 33: 31.12 + rank 34: 35.97 + rank 35: 34.34 + rank 36: 33.73 + rank 37: 32.22 + rank 38: 33.90 + rank 39: 32.68 + rank 40: 23.45 + rank 41: 22.71 + rank 42: 23.62 + rank 43: 24.82 + rank 44: 24.84 + rank 45: 23.14 + rank 46: 22.96 + rank 47: 23.04 + rank 48: 23.57 + rank 49: 27.97 + rank 50: 26.77 + rank 51: 23.01 + rank 52: 25.81 + rank 53: 22.57 + rank 54: 24.31 + rank 55: 28.76 + rank 56: 25.57 + rank 57: 23.90 + rank 58: 23.05 + rank 59: 22.98 + rank 60: 22.76 + rank 61: 24.09 + rank 62: 22.77 + rank 63: 25.49 + embedding-grads-all-reduce: + rank 0: 0.03 + rank 1: 0.03 + rank 2: 0.03 + rank 3: 0.03 + rank 4: 0.02 + rank 5: 0.03 + rank 6: 0.03 + rank 7: 0.02 + rank 8: 0.02 + rank 9: 0.03 + rank 10: 0.03 + rank 11: 0.03 + rank 12: 0.03 + rank 13: 0.02 + rank 14: 0.02 + rank 15: 0.03 + rank 16: 0.03 + rank 17: 0.03 + rank 18: 0.03 + rank 19: 0.03 + rank 20: 0.03 + rank 21: 0.03 + rank 22: 0.03 + rank 23: 0.03 + rank 24: 0.04 + rank 25: 0.04 + rank 26: 0.04 + rank 27: 0.03 + rank 28: 0.04 + rank 29: 0.04 + rank 30: 0.04 + rank 31: 0.03 + rank 32: 0.03 + rank 33: 0.03 + rank 34: 0.03 + rank 35: 0.03 + rank 36: 0.03 + rank 37: 0.03 + rank 38: 0.03 + rank 39: 0.03 + rank 40: 0.03 + rank 41: 0.03 + rank 42: 0.03 + rank 43: 0.03 + rank 44: 0.03 + rank 45: 0.03 + rank 46: 0.03 + rank 47: 0.03 + rank 48: 0.03 + rank 49: 0.03 + rank 50: 0.03 + rank 51: 0.03 + rank 52: 0.03 + rank 53: 0.03 + rank 54: 0.03 + rank 55: 0.03 + rank 56: 0.03 + rank 57: 0.02 + rank 58: 0.02 + rank 59: 0.03 + rank 60: 0.02 + rank 61: 0.02 + rank 62: 0.03 + rank 63: 0.03 + all-grads-sync: + rank 0: 0.90 + rank 1: 1.02 + rank 2: 0.86 + rank 3: 0.77 + rank 4: 0.76 + rank 5: 0.79 + rank 6: 1.05 + rank 7: 0.86 + rank 8: 0.88 + rank 9: 0.79 + rank 10: 0.78 + rank 11: 0.82 + rank 12: 1.09 + rank 13: 0.78 + rank 14: 0.78 + rank 15: 0.78 + rank 16: 0.86 + rank 17: 0.81 + rank 18: 0.80 + rank 19: 0.84 + rank 20: 0.79 + rank 21: 0.82 + rank 22: 0.78 + rank 23: 0.82 + rank 24: 1.38 + rank 25: 1.18 + rank 26: 1.19 + rank 27: 1.29 + rank 28: 1.28 + rank 29: 1.29 + rank 30: 1.40 + rank 31: 1.27 + rank 32: 1.20 + rank 33: 1.10 + rank 34: 1.24 + rank 35: 1.19 + rank 36: 1.22 + rank 37: 1.10 + rank 38: 1.16 + rank 39: 1.42 + rank 40: 0.81 + rank 41: 0.85 + rank 42: 0.84 + rank 43: 0.94 + rank 44: 0.92 + rank 45: 0.78 + rank 46: 0.78 + rank 47: 0.90 + rank 48: 0.84 + rank 49: 0.95 + rank 50: 1.06 + rank 51: 0.82 + rank 52: 0.82 + rank 53: 1.10 + rank 54: 1.02 + rank 55: 1.09 + rank 56: 0.79 + rank 57: 0.78 + rank 58: 0.78 + rank 59: 0.78 + rank 60: 0.87 + rank 61: 0.80 + rank 62: 1.11 + rank 63: 1.03 + params-all-gather: + rank 0: 0.03 + rank 1: 0.02 + rank 2: 0.02 + rank 3: 0.02 + rank 4: 0.02 + rank 5: 0.02 + rank 6: 0.02 + rank 7: 0.02 + rank 8: 0.02 + rank 9: 0.02 + rank 10: 0.02 + rank 11: 0.02 + rank 12: 0.02 + rank 13: 0.02 + rank 14: 0.02 + rank 15: 0.02 + rank 16: 0.03 + rank 17: 0.02 + rank 18: 0.02 + rank 19: 0.02 + rank 20: 0.02 + rank 21: 0.02 + rank 22: 0.02 + rank 23: 0.02 + rank 24: 0.03 + rank 25: 0.04 + rank 26: 0.03 + rank 27: 0.04 + rank 28: 0.04 + rank 29: 0.04 + rank 30: 0.05 + rank 31: 0.03 + rank 32: 0.02 + rank 33: 0.03 + rank 34: 0.03 + rank 35: 0.03 + rank 36: 0.02 + rank 37: 0.02 + rank 38: 0.02 + rank 39: 0.02 + rank 40: 0.02 + rank 41: 0.02 + rank 42: 0.02 + rank 43: 0.02 + rank 44: 0.02 + rank 45: 0.02 + rank 46: 0.02 + rank 47: 0.02 + rank 48: 0.03 + rank 49: 0.03 + rank 50: 0.03 + rank 51: 0.02 + rank 52: 0.03 + rank 53: 0.02 + rank 54: 0.03 + rank 55: 0.03 + rank 56: 0.02 + rank 57: 0.02 + rank 58: 0.02 + rank 59: 0.02 + rank 60: 0.02 + rank 61: 0.02 + rank 62: 0.02 + rank 63: 0.02 + optimizer-copy-to-main-grad: + rank 0: 2.19 + rank 1: 1.82 + rank 2: 1.69 + rank 3: 2.09 + rank 4: 1.68 + rank 5: 1.93 + rank 6: 1.86 + rank 7: 1.88 + rank 8: 1.82 + rank 9: 2.07 + rank 10: 1.72 + rank 11: 1.71 + rank 12: 1.82 + rank 13: 1.72 + rank 14: 1.88 + rank 15: 2.02 + rank 16: 2.44 + rank 17: 2.11 + rank 18: 2.12 + rank 19: 2.14 + rank 20: 2.16 + rank 21: 2.39 + rank 22: 2.51 + rank 23: 2.29 + rank 24: 1.93 + rank 25: 2.16 + rank 26: 1.93 + rank 27: 2.09 + rank 28: 2.34 + rank 29: 2.39 + rank 30: 2.36 + rank 31: 1.92 + rank 32: 2.79 + rank 33: 1.85 + rank 34: 2.72 + rank 35: 2.78 + rank 36: 2.31 + rank 37: 1.99 + rank 38: 3.20 + rank 39: 2.04 + rank 40: 1.73 + rank 41: 1.89 + rank 42: 1.85 + rank 43: 1.85 + rank 44: 1.86 + rank 45: 2.05 + rank 46: 1.80 + rank 47: 1.72 + rank 48: 1.92 + rank 49: 1.75 + rank 50: 2.28 + rank 51: 1.79 + rank 52: 1.96 + rank 53: 1.77 + rank 54: 1.75 + rank 55: 1.81 + rank 56: 1.75 + rank 57: 2.37 + rank 58: 1.84 + rank 59: 1.70 + rank 60: 1.70 + rank 61: 1.66 + rank 62: 1.84 + rank 63: 1.72 + optimizer-inner-step: + rank 0: 43.33 + rank 1: 41.36 + rank 2: 41.14 + rank 3: 41.34 + rank 4: 41.16 + rank 5: 41.27 + rank 6: 41.64 + rank 7: 41.58 + rank 8: 41.36 + rank 9: 41.42 + rank 10: 41.23 + rank 11: 41.35 + rank 12: 41.30 + rank 13: 41.55 + rank 14: 41.36 + rank 15: 41.23 + rank 16: 41.66 + rank 17: 41.43 + rank 18: 41.50 + rank 19: 41.35 + rank 20: 41.45 + rank 21: 41.31 + rank 22: 41.81 + rank 23: 41.54 + rank 24: 41.02 + rank 25: 41.45 + rank 26: 41.25 + rank 27: 41.59 + rank 28: 45.61 + rank 29: 41.24 + rank 30: 41.28 + rank 31: 41.51 + rank 32: 41.39 + rank 33: 41.21 + rank 34: 41.33 + rank 35: 48.29 + rank 36: 41.79 + rank 37: 41.44 + rank 38: 41.28 + rank 39: 41.24 + rank 40: 41.35 + rank 41: 41.46 + rank 42: 41.53 + rank 43: 41.60 + rank 44: 41.17 + rank 45: 41.71 + rank 46: 41.41 + rank 47: 41.05 + rank 48: 42.06 + rank 49: 41.25 + rank 50: 52.83 + rank 51: 41.04 + rank 52: 41.37 + rank 53: 41.24 + rank 54: 41.46 + rank 55: 41.30 + rank 56: 40.94 + rank 57: 47.16 + rank 58: 41.33 + rank 59: 41.17 + rank 60: 41.50 + rank 61: 41.06 + rank 62: 41.24 + rank 63: 41.21 + optimizer-copy-main-to-model-params: + rank 0: 4.46 + rank 1: 3.87 + rank 2: 3.71 + rank 3: 4.38 + rank 4: 3.72 + rank 5: 3.93 + rank 6: 3.91 + rank 7: 4.05 + rank 8: 3.80 + rank 9: 4.42 + rank 10: 3.77 + rank 11: 3.77 + rank 12: 3.89 + rank 13: 3.77 + rank 14: 4.02 + rank 15: 4.20 + rank 16: 4.37 + rank 17: 4.02 + rank 18: 3.96 + rank 19: 3.99 + rank 20: 3.99 + rank 21: 4.29 + rank 22: 4.53 + rank 23: 4.17 + rank 24: 3.92 + rank 25: 4.20 + rank 26: 3.93 + rank 27: 4.11 + rank 28: 4.36 + rank 29: 4.34 + rank 30: 4.30 + rank 31: 4.01 + rank 32: 4.12 + rank 33: 3.86 + rank 34: 4.06 + rank 35: 4.54 + rank 36: 4.20 + rank 37: 4.09 + rank 38: 4.48 + rank 39: 3.87 + rank 40: 3.83 + rank 41: 4.13 + rank 42: 3.97 + rank 43: 4.04 + rank 44: 4.02 + rank 45: 4.33 + rank 46: 3.81 + rank 47: 3.77 + rank 48: 4.07 + rank 49: 3.89 + rank 50: 4.88 + rank 51: 3.98 + rank 52: 4.20 + rank 53: 3.85 + rank 54: 3.83 + rank 55: 3.86 + rank 56: 3.78 + rank 57: 4.77 + rank 58: 3.89 + rank 59: 3.77 + rank 60: 3.73 + rank 61: 3.68 + rank 62: 3.83 + rank 63: 3.66 + optimizer: + rank 0: 56.04 + rank 1: 53.30 + rank 2: 52.86 + rank 3: 53.96 + rank 4: 52.87 + rank 5: 53.28 + rank 6: 53.64 + rank 7: 53.77 + rank 8: 53.22 + rank 9: 54.08 + rank 10: 53.05 + rank 11: 53.11 + rank 12: 53.27 + rank 13: 53.35 + rank 14: 53.47 + rank 15: 53.59 + rank 16: 54.54 + rank 17: 53.78 + rank 18: 53.77 + rank 19: 53.72 + rank 20: 53.72 + rank 21: 54.02 + rank 22: 54.89 + rank 23: 54.09 + rank 24: 52.99 + rank 25: 53.79 + rank 26: 53.22 + rank 27: 53.81 + rank 28: 58.16 + rank 29: 53.77 + rank 30: 53.77 + rank 31: 53.60 + rank 32: 53.57 + rank 33: 53.07 + rank 34: 53.41 + rank 35: 61.07 + rank 36: 54.11 + rank 37: 53.61 + rank 38: 53.95 + rank 39: 53.09 + rank 40: 53.20 + rank 41: 53.72 + rank 42: 53.61 + rank 43: 53.77 + rank 44: 53.30 + rank 45: 54.28 + rank 46: 53.20 + rank 47: 52.86 + rank 48: 54.24 + rank 49: 53.15 + rank 50: 66.14 + rank 51: 53.04 + rank 52: 53.69 + rank 53: 53.07 + rank 54: 53.31 + rank 55: 53.17 + rank 56: 52.77 + rank 57: 60.37 + rank 58: 53.28 + rank 59: 52.92 + rank 60: 53.26 + rank 61: 52.73 + rank 62: 53.14 + rank 63: 52.83 + interval-time: + rank 0: 0.84 + rank 1: 0.85 + rank 2: 0.87 + rank 3: 0.98 + rank 4: 0.83 + rank 5: 0.80 + rank 6: 0.81 + rank 7: 0.84 + rank 8: 0.73 + rank 9: 0.73 + rank 10: 0.81 + rank 11: 0.71 + rank 12: 0.81 + rank 13: 0.79 + rank 14: 0.81 + rank 15: 0.80 + rank 16: 0.73 + rank 17: 0.82 + rank 18: 0.82 + rank 19: 0.75 + rank 20: 0.79 + rank 21: 0.84 + rank 22: 0.71 + rank 23: 0.72 + rank 24: 0.87 + rank 25: 0.95 + rank 26: 0.97 + rank 27: 0.98 + rank 28: 0.82 + rank 29: 0.83 + rank 30: 0.90 + rank 31: 0.96 + rank 32: 0.89 + rank 33: 0.84 + rank 34: 0.92 + rank 35: 0.81 + rank 36: 0.80 + rank 37: 0.90 + rank 38: 0.80 + rank 39: 0.89 + rank 40: 1.01 + rank 41: 1.06 + rank 42: 0.98 + rank 43: 1.00 + rank 44: 1.03 + rank 45: 0.90 + rank 46: 1.09 + rank 47: 0.97 + rank 48: 0.97 + rank 49: 0.89 + rank 50: 0.80 + rank 51: 0.89 + rank 52: 0.82 + rank 53: 0.83 + rank 54: 0.82 + rank 55: 0.92 + rank 56: 0.84 + rank 57: 0.83 + rank 58: 0.83 + rank 59: 0.81 + rank 60: 0.71 + rank 61: 0.82 + rank 62: 0.80 + rank 63: 2.81 + [2026-10-09 22:55:27] iteration 6003/ 6855 | consumed samples: 768384 | elapsed time per iteration (ms): 86788.6 | throughput per GPU (TFLOP/s/GPU): 42.8 | MFU 13.72% | FLOPs estimator: workload-aware | logical tokens/s: 193311.4 | valid tokens/s: 193311.4 | physical tokens/s: 193718.2 | loss tokens/s: 145346.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.756006E-01 | lm-global-supervised-tokens: 1.261438E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.381501E-02 | lm-token-buffer-mib: 5.810233E+01 | global_load_balancing_loss: 1.000778E+00 | loss scale: 1.0 | grad norm: 0.111 | params norm: 11275.553 | params rms: 0.061053 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 20:32:23.854671 | finish at 2026-10-10 19:27:51 +times across ranks (ms): + forward-backward: + rank 0: 86558.32 + rank 1: 86559.04 + rank 2: 86559.98 + rank 3: 86560.09 + rank 4: 86560.51 + rank 5: 86560.96 + rank 6: 86557.95 + rank 7: 86558.51 + rank 8: 86555.36 + rank 9: 86555.51 + rank 10: 86554.77 + rank 11: 86555.26 + rank 12: 86555.91 + rank 13: 86554.52 + rank 14: 86555.03 + rank 15: 86554.66 + rank 16: 86560.73 + rank 17: 86560.68 + rank 18: 86560.42 + rank 19: 86560.26 + rank 20: 86560.48 + rank 21: 86561.72 + rank 22: 86561.54 + rank 23: 86561.43 + rank 24: 86568.60 + rank 25: 86568.47 + rank 26: 86568.31 + rank 27: 86568.37 + rank 28: 86568.38 + rank 29: 86568.71 + rank 30: 86568.60 + rank 31: 86568.62 + rank 32: 86556.32 + rank 33: 86558.07 + rank 34: 86556.97 + rank 35: 86559.17 + rank 36: 86556.90 + rank 37: 86556.15 + rank 38: 86557.06 + rank 39: 86557.89 + rank 40: 86551.57 + rank 41: 86551.95 + rank 42: 86551.72 + rank 43: 86550.76 + rank 44: 86552.45 + rank 45: 86551.20 + rank 46: 86552.80 + rank 47: 86552.91 + rank 48: 86556.77 + rank 49: 86557.38 + rank 50: 86555.63 + rank 51: 86555.65 + rank 52: 86555.62 + rank 53: 86555.54 + rank 54: 86557.65 + rank 55: 86558.27 + rank 56: 86560.76 + rank 57: 86560.09 + rank 58: 86560.16 + rank 59: 86558.86 + rank 60: 86557.78 + rank 61: 86557.55 + rank 62: 86561.49 + rank 63: 86560.52 + forward-compute: + rank 0: 32865.55 + rank 1: 32872.58 + rank 2: 32878.26 + rank 3: 32876.43 + rank 4: 32880.70 + rank 5: 32878.91 + rank 6: 32876.53 + rank 7: 32887.42 + rank 8: 32976.98 + rank 9: 32975.12 + rank 10: 32977.31 + rank 11: 32973.87 + rank 12: 32974.76 + rank 13: 32980.68 + rank 14: 32976.22 + rank 15: 32977.05 + rank 16: 30037.91 + rank 17: 30034.80 + rank 18: 30033.83 + rank 19: 30039.54 + rank 20: 30035.65 + rank 21: 30039.38 + rank 22: 30036.23 + rank 23: 30038.76 + rank 24: 31621.71 + rank 25: 31621.26 + rank 26: 31622.76 + rank 27: 31623.05 + rank 28: 31623.15 + rank 29: 31624.37 + rank 30: 31625.25 + rank 31: 31622.55 + rank 32: 32666.89 + rank 33: 32671.06 + rank 34: 32672.68 + rank 35: 32669.99 + rank 36: 32678.15 + rank 37: 32684.56 + rank 38: 32684.94 + rank 39: 32671.32 + rank 40: 32053.78 + rank 41: 32060.62 + rank 42: 32062.09 + rank 43: 32065.74 + rank 44: 32064.21 + rank 45: 32069.28 + rank 46: 32068.15 + rank 47: 32070.78 + rank 48: 31899.70 + rank 49: 31900.43 + rank 50: 31902.15 + rank 51: 31904.13 + rank 52: 31921.48 + rank 53: 31920.53 + rank 54: 31903.75 + rank 55: 31908.03 + rank 56: 32124.77 + rank 57: 32122.65 + rank 58: 32120.08 + rank 59: 32119.70 + rank 60: 32125.22 + rank 61: 32129.88 + rank 62: 32126.16 + rank 63: 32127.66 + backward-compute: + rank 0: 53660.95 + rank 1: 53652.80 + rank 2: 53646.05 + rank 3: 53650.58 + rank 4: 53643.20 + rank 5: 53648.26 + rank 6: 53647.91 + rank 7: 53635.61 + rank 8: 53544.26 + rank 9: 53545.90 + rank 10: 53543.76 + rank 11: 53546.34 + rank 12: 53545.88 + rank 13: 53539.38 + rank 14: 53545.03 + rank 15: 53544.85 + rank 16: 56490.09 + rank 17: 56492.63 + rank 18: 56494.48 + rank 19: 56487.06 + rank 20: 56492.68 + rank 21: 56487.50 + rank 22: 56491.12 + rank 23: 56488.91 + rank 24: 54900.17 + rank 25: 54897.92 + rank 26: 54897.62 + rank 27: 54898.28 + rank 28: 54896.95 + rank 29: 54894.36 + rank 30: 54890.89 + rank 31: 54896.60 + rank 32: 53850.95 + rank 33: 53849.21 + rank 34: 53843.71 + rank 35: 53848.80 + rank 36: 53840.78 + rank 37: 53833.84 + rank 38: 53830.83 + rank 39: 53849.43 + rank 40: 54462.00 + rank 41: 54455.01 + rank 42: 54452.90 + rank 43: 54448.68 + rank 44: 54452.21 + rank 45: 54447.61 + rank 46: 54451.23 + rank 47: 54446.74 + rank 48: 54623.01 + rank 49: 54621.67 + rank 50: 54619.95 + rank 51: 54617.82 + rank 52: 54598.66 + rank 53: 54599.74 + rank 54: 54619.64 + rank 55: 54615.50 + rank 56: 54399.66 + rank 57: 54402.99 + rank 58: 54404.66 + rank 59: 54404.56 + rank 60: 54399.15 + rank 61: 54392.19 + rank 62: 54400.44 + rank 63: 54396.47 + batch-generator: + rank 0: 24.13 + rank 1: 24.14 + rank 2: 24.28 + rank 3: 23.91 + rank 4: 24.20 + rank 5: 22.15 + rank 6: 23.63 + rank 7: 23.86 + rank 8: 22.78 + rank 9: 21.89 + rank 10: 22.31 + rank 11: 23.62 + rank 12: 23.41 + rank 13: 22.13 + rank 14: 23.16 + rank 15: 21.83 + rank 16: 23.71 + rank 17: 23.47 + rank 18: 22.75 + rank 19: 23.93 + rank 20: 23.93 + rank 21: 23.49 + rank 22: 23.14 + rank 23: 22.79 + rank 24: 32.27 + rank 25: 31.03 + rank 26: 32.12 + rank 27: 35.96 + rank 28: 30.46 + rank 29: 33.06 + rank 30: 34.70 + rank 31: 29.92 + rank 32: 27.40 + rank 33: 31.19 + rank 34: 35.46 + rank 35: 33.93 + rank 36: 33.05 + rank 37: 32.83 + rank 38: 32.96 + rank 39: 33.45 + rank 40: 22.80 + rank 41: 21.96 + rank 42: 24.04 + rank 43: 23.97 + rank 44: 22.13 + rank 45: 22.71 + rank 46: 22.57 + rank 47: 22.30 + rank 48: 23.89 + rank 49: 27.95 + rank 50: 28.64 + rank 51: 22.05 + rank 52: 28.66 + rank 53: 23.89 + rank 54: 22.81 + rank 55: 29.37 + rank 56: 23.83 + rank 57: 22.38 + rank 58: 22.60 + rank 59: 21.37 + rank 60: 23.02 + rank 61: 21.33 + rank 62: 22.47 + rank 63: 23.65 + embedding-grads-all-reduce: + rank 0: 0.03 + rank 1: 0.02 + rank 2: 0.03 + rank 3: 0.03 + rank 4: 0.03 + rank 5: 0.03 + rank 6: 0.03 + rank 7: 0.03 + rank 8: 0.02 + rank 9: 0.03 + rank 10: 0.02 + rank 11: 0.03 + rank 12: 0.03 + rank 13: 0.02 + rank 14: 0.03 + rank 15: 0.03 + rank 16: 0.03 + rank 17: 0.03 + rank 18: 0.03 + rank 19: 0.03 + rank 20: 0.03 + rank 21: 0.03 + rank 22: 0.03 + rank 23: 0.03 + rank 24: 0.04 + rank 25: 0.04 + rank 26: 0.04 + rank 27: 0.04 + rank 28: 0.04 + rank 29: 0.03 + rank 30: 0.03 + rank 31: 0.04 + rank 32: 0.03 + rank 33: 0.03 + rank 34: 0.04 + rank 35: 0.03 + rank 36: 0.03 + rank 37: 0.03 + rank 38: 0.03 + rank 39: 0.03 + rank 40: 0.03 + rank 41: 0.02 + rank 42: 0.03 + rank 43: 0.03 + rank 44: 0.03 + rank 45: 0.03 + rank 46: 0.02 + rank 47: 0.03 + rank 48: 0.03 + rank 49: 0.03 + rank 50: 0.03 + rank 51: 0.03 + rank 52: 0.03 + rank 53: 0.05 + rank 54: 0.03 + rank 55: 0.03 + rank 56: 0.03 + rank 57: 0.03 + rank 58: 0.03 + rank 59: 0.03 + rank 60: 0.02 + rank 61: 0.03 + rank 62: 0.03 + rank 63: 0.03 + all-grads-sync: + rank 0: 0.76 + rank 1: 0.79 + rank 2: 0.84 + rank 3: 0.79 + rank 4: 0.80 + rank 5: 1.07 + rank 6: 0.80 + rank 7: 0.99 + rank 8: 0.82 + rank 9: 0.78 + rank 10: 0.78 + rank 11: 0.79 + rank 12: 1.05 + rank 13: 0.77 + rank 14: 0.94 + rank 15: 0.79 + rank 16: 0.84 + rank 17: 0.81 + rank 18: 0.91 + rank 19: 0.80 + rank 20: 0.81 + rank 21: 1.03 + rank 22: 1.04 + rank 23: 1.29 + rank 24: 1.60 + rank 25: 1.22 + rank 26: 1.53 + rank 27: 1.32 + rank 28: 1.57 + rank 29: 1.22 + rank 30: 1.45 + rank 31: 1.22 + rank 32: 1.05 + rank 33: 1.53 + rank 34: 1.65 + rank 35: 1.87 + rank 36: 1.22 + rank 37: 1.13 + rank 38: 1.42 + rank 39: 1.91 + rank 40: 0.84 + rank 41: 0.94 + rank 42: 0.84 + rank 43: 0.79 + rank 44: 1.75 + rank 45: 0.79 + rank 46: 0.78 + rank 47: 1.25 + rank 48: 0.83 + rank 49: 1.09 + rank 50: 1.09 + rank 51: 0.98 + rank 52: 1.21 + rank 53: 1.18 + rank 54: 0.88 + rank 55: 1.24 + rank 56: 1.13 + rank 57: 0.84 + rank 58: 1.28 + rank 59: 0.98 + rank 60: 0.86 + rank 61: 0.88 + rank 62: 0.79 + rank 63: 0.80 + params-all-gather: + rank 0: 0.02 + rank 1: 0.02 + rank 2: 0.02 + rank 3: 0.02 + rank 4: 0.02 + rank 5: 0.02 + rank 6: 0.02 + rank 7: 0.02 + rank 8: 0.02 + rank 9: 0.02 + rank 10: 0.02 + rank 11: 0.02 + rank 12: 0.02 + rank 13: 0.02 + rank 14: 0.02 + rank 15: 0.02 + rank 16: 0.02 + rank 17: 0.02 + rank 18: 0.02 + rank 19: 0.02 + rank 20: 0.02 + rank 21: 0.02 + rank 22: 0.02 + rank 23: 0.02 + rank 24: 0.03 + rank 25: 0.03 + rank 26: 0.03 + rank 27: 0.04 + rank 28: 0.04 + rank 29: 0.04 + rank 30: 0.03 + rank 31: 0.04 + rank 32: 0.03 + rank 33: 0.02 + rank 34: 0.03 + rank 35: 0.02 + rank 36: 0.02 + rank 37: 0.03 + rank 38: 0.02 + rank 39: 0.02 + rank 40: 0.02 + rank 41: 0.02 + rank 42: 0.02 + rank 43: 0.02 + rank 44: 0.02 + rank 45: 0.02 + rank 46: 0.02 + rank 47: 0.02 + rank 48: 0.03 + rank 49: 0.03 + rank 50: 0.03 + rank 51: 0.02 + rank 52: 0.03 + rank 53: 0.03 + rank 54: 0.03 + rank 55: 0.03 + rank 56: 0.02 + rank 57: 0.02 + rank 58: 0.02 + rank 59: 0.02 + rank 60: 0.02 + rank 61: 0.02 + rank 62: 0.02 + rank 63: 0.02 + optimizer-copy-to-main-grad: + rank 0: 2.13 + rank 1: 1.87 + rank 2: 1.84 + rank 3: 2.11 + rank 4: 1.66 + rank 5: 1.85 + rank 6: 1.86 + rank 7: 1.91 + rank 8: 1.76 + rank 9: 2.10 + rank 10: 1.71 + rank 11: 1.70 + rank 12: 1.76 + rank 13: 1.73 + rank 14: 1.85 + rank 15: 2.00 + rank 16: 2.42 + rank 17: 2.08 + rank 18: 2.11 + rank 19: 2.10 + rank 20: 2.15 + rank 21: 2.40 + rank 22: 2.59 + rank 23: 2.37 + rank 24: 1.93 + rank 25: 2.15 + rank 26: 1.84 + rank 27: 2.17 + rank 28: 2.20 + rank 29: 2.16 + rank 30: 2.35 + rank 31: 1.94 + rank 32: 1.95 + rank 33: 1.89 + rank 34: 1.98 + rank 35: 2.29 + rank 36: 2.23 + rank 37: 1.99 + rank 38: 2.21 + rank 39: 1.94 + rank 40: 1.75 + rank 41: 1.81 + rank 42: 1.86 + rank 43: 1.81 + rank 44: 1.87 + rank 45: 1.99 + rank 46: 1.78 + rank 47: 1.69 + rank 48: 1.91 + rank 49: 1.79 + rank 50: 2.35 + rank 51: 1.89 + rank 52: 2.02 + rank 53: 1.76 + rank 54: 1.74 + rank 55: 1.83 + rank 56: 1.73 + rank 57: 2.39 + rank 58: 1.83 + rank 59: 1.67 + rank 60: 1.69 + rank 61: 1.68 + rank 62: 1.83 + rank 63: 1.71 + optimizer-inner-step: + rank 0: 49.82 + rank 1: 49.68 + rank 2: 49.40 + rank 3: 49.56 + rank 4: 49.38 + rank 5: 49.49 + rank 6: 49.87 + rank 7: 49.85 + rank 8: 49.53 + rank 9: 49.62 + rank 10: 49.49 + rank 11: 49.58 + rank 12: 49.49 + rank 13: 49.75 + rank 14: 49.51 + rank 15: 49.46 + rank 16: 49.85 + rank 17: 49.61 + rank 18: 49.71 + rank 19: 49.58 + rank 20: 49.65 + rank 21: 49.54 + rank 22: 50.09 + rank 23: 49.75 + rank 24: 47.63 + rank 25: 51.54 + rank 26: 48.55 + rank 27: 52.47 + rank 28: 55.54 + rank 29: 50.86 + rank 30: 49.44 + rank 31: 51.75 + rank 32: 49.70 + rank 33: 49.50 + rank 34: 49.61 + rank 35: 50.00 + rank 36: 49.74 + rank 37: 49.70 + rank 38: 49.49 + rank 39: 49.51 + rank 40: 49.52 + rank 41: 49.61 + rank 42: 49.80 + rank 43: 49.74 + rank 44: 49.37 + rank 45: 49.96 + rank 46: 49.58 + rank 47: 49.22 + rank 48: 49.66 + rank 49: 49.49 + rank 50: 53.81 + rank 51: 49.28 + rank 52: 49.62 + rank 53: 49.44 + rank 54: 49.70 + rank 55: 49.53 + rank 56: 49.21 + rank 57: 50.48 + rank 58: 49.55 + rank 59: 49.37 + rank 60: 49.72 + rank 61: 49.34 + rank 62: 49.49 + rank 63: 49.38 + optimizer-copy-main-to-model-params: + rank 0: 4.35 + rank 1: 3.89 + rank 2: 3.74 + rank 3: 4.30 + rank 4: 3.71 + rank 5: 3.90 + rank 6: 3.90 + rank 7: 4.00 + rank 8: 3.91 + rank 9: 4.51 + rank 10: 3.78 + rank 11: 3.79 + rank 12: 3.90 + rank 13: 3.77 + rank 14: 4.02 + rank 15: 4.28 + rank 16: 4.39 + rank 17: 4.03 + rank 18: 3.97 + rank 19: 4.01 + rank 20: 4.04 + rank 21: 4.28 + rank 22: 4.50 + rank 23: 4.18 + rank 24: 3.96 + rank 25: 4.23 + rank 26: 3.94 + rank 27: 4.18 + rank 28: 4.37 + rank 29: 4.33 + rank 30: 4.27 + rank 31: 4.03 + rank 32: 4.08 + rank 33: 3.94 + rank 34: 4.03 + rank 35: 4.71 + rank 36: 4.29 + rank 37: 4.09 + rank 38: 4.52 + rank 39: 3.90 + rank 40: 3.74 + rank 41: 3.99 + rank 42: 3.96 + rank 43: 4.04 + rank 44: 3.98 + rank 45: 4.25 + rank 46: 3.80 + rank 47: 3.73 + rank 48: 4.09 + rank 49: 3.89 + rank 50: 5.14 + rank 51: 3.99 + rank 52: 4.17 + rank 53: 3.86 + rank 54: 3.86 + rank 55: 3.86 + rank 56: 3.78 + rank 57: 4.68 + rank 58: 3.90 + rank 59: 3.71 + rank 60: 3.73 + rank 61: 3.68 + rank 62: 3.90 + rank 63: 3.68 + optimizer: + rank 0: 65.10 + rank 1: 64.32 + rank 2: 63.86 + rank 3: 64.77 + rank 4: 63.76 + rank 5: 64.19 + rank 6: 64.55 + rank 7: 64.65 + rank 8: 64.25 + rank 9: 65.06 + rank 10: 63.99 + rank 11: 64.06 + rank 12: 64.23 + rank 13: 64.28 + rank 14: 64.34 + rank 15: 64.64 + rank 16: 65.49 + rank 17: 64.66 + rank 18: 64.69 + rank 19: 64.68 + rank 20: 64.67 + rank 21: 64.97 + rank 22: 65.82 + rank 23: 65.02 + rank 24: 65.51 + rank 25: 69.88 + rank 26: 66.45 + rank 27: 70.62 + rank 28: 73.98 + rank 29: 69.22 + rank 30: 67.77 + rank 31: 69.75 + rank 32: 64.52 + rank 33: 64.10 + rank 34: 64.34 + rank 35: 65.59 + rank 36: 64.78 + rank 37: 64.52 + rank 38: 64.87 + rank 39: 64.07 + rank 40: 64.00 + rank 41: 64.42 + rank 42: 64.53 + rank 43: 64.57 + rank 44: 64.16 + rank 45: 65.11 + rank 46: 64.13 + rank 47: 63.68 + rank 48: 64.53 + rank 49: 64.08 + rank 50: 70.14 + rank 51: 64.00 + rank 52: 64.59 + rank 53: 63.96 + rank 54: 64.24 + rank 55: 64.09 + rank 56: 63.69 + rank 57: 66.28 + rank 58: 64.19 + rank 59: 63.72 + rank 60: 64.13 + rank 61: 63.70 + rank 62: 64.11 + rank 63: 63.71 + interval-time: + rank 0: 0.71 + rank 1: 0.83 + rank 2: 0.83 + rank 3: 0.71 + rank 4: 0.83 + rank 5: 0.81 + rank 6: 0.81 + rank 7: 0.84 + rank 8: 0.74 + rank 9: 0.72 + rank 10: 0.81 + rank 11: 0.71 + rank 12: 0.81 + rank 13: 0.81 + rank 14: 0.81 + rank 15: 0.80 + rank 16: 0.74 + rank 17: 0.84 + rank 18: 0.80 + rank 19: 0.73 + rank 20: 0.79 + rank 21: 0.84 + rank 22: 0.71 + rank 23: 0.71 + rank 24: 0.90 + rank 25: 0.94 + rank 26: 0.97 + rank 27: 1.02 + rank 28: 0.83 + rank 29: 0.85 + rank 30: 1.05 + rank 31: 0.97 + rank 32: 0.93 + rank 33: 0.84 + rank 34: 0.95 + rank 35: 0.82 + rank 36: 0.79 + rank 37: 0.88 + rank 38: 0.78 + rank 39: 0.88 + rank 40: 0.80 + rank 41: 0.82 + rank 42: 0.72 + rank 43: 0.74 + rank 44: 0.82 + rank 45: 0.72 + rank 46: 0.82 + rank 47: 0.71 + rank 48: 0.93 + rank 49: 0.93 + rank 50: 0.82 + rank 51: 0.90 + rank 52: 0.87 + rank 53: 0.81 + rank 54: 0.82 + rank 55: 0.93 + rank 56: 0.82 + rank 57: 0.81 + rank 58: 0.82 + rank 59: 0.81 + rank 60: 0.70 + rank 61: 0.81 + rank 62: 0.80 + rank 63: 3.54 + [2026-10-09 22:56:55] iteration 6004/ 6855 | consumed samples: 768512 | elapsed time per iteration (ms): 87796.2 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.51% | FLOPs estimator: workload-aware | logical tokens/s: 191092.0 | valid tokens/s: 191092.0 | physical tokens/s: 191578.6 | loss tokens/s: 136686.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.230776E-01 | lm-global-supervised-tokens: 1.200058E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.334214E-02 | lm-token-buffer-mib: 5.809512E+01 | global_load_balancing_loss: 1.000504E+00 | loss scale: 1.0 | grad norm: 0.097 | params norm: 11275.507 | params rms: 0.061053 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 20:45:14.566029 | finish at 2026-10-10 19:42:09 +times across ranks (ms): + forward-backward: + rank 0: 87575.13 + rank 1: 87575.13 + rank 2: 87575.36 + rank 3: 87575.06 + rank 4: 87575.13 + rank 5: 87575.58 + rank 6: 87575.30 + rank 7: 87575.34 + rank 8: 87564.99 + rank 9: 87565.51 + rank 10: 87565.70 + rank 11: 87567.64 + rank 12: 87567.61 + rank 13: 87567.87 + rank 14: 87564.25 + rank 15: 87564.59 + rank 16: 87566.11 + rank 17: 87567.22 + rank 18: 87567.48 + rank 19: 87568.33 + rank 20: 87567.80 + rank 21: 87569.98 + rank 22: 87565.72 + rank 23: 87566.41 + rank 24: 87564.55 + rank 25: 87565.09 + rank 26: 87566.35 + rank 27: 87566.16 + rank 28: 87566.26 + rank 29: 87566.55 + rank 30: 87565.88 + rank 31: 87564.71 + rank 32: 87560.48 + rank 33: 87560.59 + rank 34: 87561.15 + rank 35: 87561.59 + rank 36: 87561.63 + rank 37: 87562.10 + rank 38: 87561.72 + rank 39: 87562.07 + rank 40: 87564.23 + rank 41: 87563.16 + rank 42: 87564.41 + rank 43: 87562.55 + rank 44: 87562.36 + rank 45: 87561.80 + rank 46: 87566.23 + rank 47: 87563.76 + rank 48: 87569.40 + rank 49: 87569.12 + rank 50: 87567.83 + rank 51: 87569.45 + rank 52: 87567.96 + rank 53: 87568.46 + rank 54: 87570.56 + rank 55: 87570.73 + rank 56: 87574.24 + rank 57: 87572.69 + rank 58: 87573.04 + rank 59: 87571.90 + rank 60: 87572.32 + rank 61: 87571.44 + rank 62: 87576.13 + rank 63: 87574.07 + forward-compute: + rank 0: 35220.86 + rank 1: 35230.27 + rank 2: 35226.59 + rank 3: 35222.01 + rank 4: 35224.74 + rank 5: 35230.15 + rank 6: 35222.99 + rank 7: 35232.15 + rank 8: 31804.79 + rank 9: 31803.45 + rank 10: 31800.10 + rank 11: 31802.10 + rank 12: 31796.06 + rank 13: 31806.30 + rank 14: 31802.12 + rank 15: 31804.59 + rank 16: 32046.00 + rank 17: 32042.13 + rank 18: 32050.68 + rank 19: 32051.19 + rank 20: 32055.87 + rank 21: 32053.52 + rank 22: 32055.06 + rank 23: 32055.63 + rank 24: 31321.20 + rank 25: 31328.91 + rank 26: 31333.94 + rank 27: 31330.02 + rank 28: 31339.83 + rank 29: 31689.84 + rank 30: 31339.49 + rank 31: 31339.54 + rank 32: 34503.51 + rank 33: 34500.33 + rank 34: 34500.01 + rank 35: 34494.61 + rank 36: 34498.32 + rank 37: 34501.05 + rank 38: 34500.11 + rank 39: 34497.72 + rank 40: 34483.19 + rank 41: 34483.66 + rank 42: 34483.33 + rank 43: 34480.23 + rank 44: 34482.10 + rank 45: 34481.11 + rank 46: 34480.76 + rank 47: 34477.51 + rank 48: 34318.98 + rank 49: 34322.25 + rank 50: 34323.03 + rank 51: 34320.15 + rank 52: 34327.78 + rank 53: 34328.44 + rank 54: 34324.05 + rank 55: 34325.13 + rank 56: 35215.15 + rank 57: 35219.02 + rank 58: 35218.93 + rank 59: 35215.61 + rank 60: 35218.41 + rank 61: 35219.89 + rank 62: 35218.50 + rank 63: 35218.32 + backward-compute: + rank 0: 52315.82 + rank 1: 52306.82 + rank 2: 52308.75 + rank 3: 52315.55 + rank 4: 52313.18 + rank 5: 52309.28 + rank 6: 52312.28 + rank 7: 52301.19 + rank 8: 55726.66 + rank 9: 55730.59 + rank 10: 55733.18 + rank 11: 55730.87 + rank 12: 55736.86 + rank 13: 55728.04 + rank 14: 55728.42 + rank 15: 55726.83 + rank 16: 55487.41 + rank 17: 55492.01 + rank 18: 55481.93 + rank 19: 55484.26 + rank 20: 55478.28 + rank 21: 55480.76 + rank 22: 55478.24 + rank 23: 55478.15 + rank 24: 56204.31 + rank 25: 56194.09 + rank 26: 56192.53 + rank 27: 56196.39 + rank 28: 56185.44 + rank 29: 55834.82 + rank 30: 56180.53 + rank 31: 56185.98 + rank 32: 53017.30 + rank 33: 53020.73 + rank 34: 53021.07 + rank 35: 53027.41 + rank 36: 53024.09 + rank 37: 53021.17 + rank 38: 53019.00 + rank 39: 53026.49 + rank 40: 53045.71 + rank 41: 53046.83 + rank 42: 53045.62 + rank 43: 53046.93 + rank 44: 53045.98 + rank 45: 53045.14 + rank 46: 53050.00 + rank 47: 53054.37 + rank 48: 53216.64 + rank 49: 53207.38 + rank 50: 53207.76 + rank 51: 53213.48 + rank 52: 53203.73 + rank 53: 53204.02 + rank 54: 53211.00 + rank 55: 53209.43 + rank 56: 52319.56 + rank 57: 52315.02 + rank 58: 52315.02 + rank 59: 52319.62 + rank 60: 52317.63 + rank 61: 52314.23 + rank 62: 52319.95 + rank 63: 52314.97 + batch-generator: + rank 0: 21.99 + rank 1: 22.96 + rank 2: 23.01 + rank 3: 22.56 + rank 4: 24.51 + rank 5: 22.74 + rank 6: 22.55 + rank 7: 24.49 + rank 8: 23.51 + rank 9: 22.06 + rank 10: 21.33 + rank 11: 22.23 + rank 12: 23.29 + rank 13: 21.69 + rank 14: 21.93 + rank 15: 21.24 + rank 16: 22.99 + rank 17: 21.97 + rank 18: 21.74 + rank 19: 22.97 + rank 20: 21.23 + rank 21: 22.45 + rank 22: 22.63 + rank 23: 22.81 + rank 24: 36.68 + rank 25: 35.61 + rank 26: 35.90 + rank 27: 37.95 + rank 28: 35.72 + rank 29: 31.81 + rank 30: 42.62 + rank 31: 34.31 + rank 32: 34.25 + rank 33: 32.65 + rank 34: 32.41 + rank 35: 34.14 + rank 36: 31.12 + rank 37: 28.19 + rank 38: 34.68 + rank 39: 33.53 + rank 40: 22.56 + rank 41: 22.91 + rank 42: 22.70 + rank 43: 24.37 + rank 44: 21.57 + rank 45: 23.10 + rank 46: 21.78 + rank 47: 22.68 + rank 48: 22.77 + rank 49: 30.51 + rank 50: 29.27 + rank 51: 27.03 + rank 52: 30.51 + rank 53: 25.58 + rank 54: 23.34 + rank 55: 28.34 + rank 56: 24.24 + rank 57: 23.25 + rank 58: 22.46 + rank 59: 21.67 + rank 60: 22.12 + rank 61: 22.78 + rank 62: 23.17 + rank 63: 23.23 + embedding-grads-all-reduce: + rank 0: 0.03 + rank 1: 0.02 + rank 2: 0.03 + rank 3: 0.03 + rank 4: 0.02 + rank 5: 0.03 + rank 6: 0.03 + rank 7: 0.03 + rank 8: 0.02 + rank 9: 0.03 + rank 10: 0.03 + rank 11: 0.03 + rank 12: 0.03 + rank 13: 0.02 + rank 14: 0.02 + rank 15: 0.02 + rank 16: 0.03 + rank 17: 0.03 + rank 18: 0.03 + rank 19: 0.03 + rank 20: 0.03 + rank 21: 0.03 + rank 22: 0.03 + rank 23: 0.03 + rank 24: 0.06 + rank 25: 0.04 + rank 26: 0.05 + rank 27: 0.05 + rank 28: 0.03 + rank 29: 0.04 + rank 30: 0.04 + rank 31: 0.05 + rank 32: 0.03 + rank 33: 0.03 + rank 34: 0.04 + rank 35: 0.03 + rank 36: 0.03 + rank 37: 0.03 + rank 38: 0.03 + rank 39: 0.03 + rank 40: 0.03 + rank 41: 0.03 + rank 42: 0.03 + rank 43: 0.03 + rank 44: 0.03 + rank 45: 0.03 + rank 46: 0.03 + rank 47: 0.03 + rank 48: 0.03 + rank 49: 0.03 + rank 50: 0.03 + rank 51: 0.03 + rank 52: 0.03 + rank 53: 0.03 + rank 54: 0.03 + rank 55: 0.03 + rank 56: 0.03 + rank 57: 0.03 + rank 58: 0.02 + rank 59: 0.03 + rank 60: 0.02 + rank 61: 0.03 + rank 62: 0.03 + rank 63: 0.03 + all-grads-sync: + rank 0: 0.75 + rank 1: 0.74 + rank 2: 0.78 + rank 3: 0.79 + rank 4: 0.87 + rank 5: 0.78 + rank 6: 0.84 + rank 7: 0.77 + rank 8: 0.78 + rank 9: 0.76 + rank 10: 0.82 + rank 11: 1.07 + rank 12: 0.94 + rank 13: 0.95 + rank 14: 0.76 + rank 15: 0.86 + rank 16: 0.85 + rank 17: 0.79 + rank 18: 0.76 + rank 19: 0.84 + rank 20: 0.81 + rank 21: 0.81 + rank 22: 0.86 + rank 23: 0.78 + rank 24: 1.44 + rank 25: 1.14 + rank 26: 1.53 + rank 27: 1.38 + rank 28: 1.13 + rank 29: 1.12 + rank 30: 2.09 + rank 31: 1.33 + rank 32: 1.27 + rank 33: 1.29 + rank 34: 1.33 + rank 35: 1.39 + rank 36: 1.18 + rank 37: 1.15 + rank 38: 1.23 + rank 39: 1.46 + rank 40: 0.76 + rank 41: 0.78 + rank 42: 1.32 + rank 43: 0.83 + rank 44: 0.77 + rank 45: 0.78 + rank 46: 1.26 + rank 47: 0.77 + rank 48: 0.84 + rank 49: 1.09 + rank 50: 1.10 + rank 51: 1.22 + rank 52: 1.17 + rank 53: 1.01 + rank 54: 1.11 + rank 55: 0.98 + rank 56: 0.80 + rank 57: 0.88 + rank 58: 0.78 + rank 59: 0.82 + rank 60: 0.83 + rank 61: 0.80 + rank 62: 1.22 + rank 63: 0.79 + params-all-gather: + rank 0: 0.02 + rank 1: 0.02 + rank 2: 0.02 + rank 3: 0.02 + rank 4: 0.02 + rank 5: 0.02 + rank 6: 0.02 + rank 7: 0.02 + rank 8: 0.02 + rank 9: 0.02 + rank 10: 0.02 + rank 11: 0.02 + rank 12: 0.02 + rank 13: 0.02 + rank 14: 0.02 + rank 15: 0.02 + rank 16: 0.03 + rank 17: 0.02 + rank 18: 0.02 + rank 19: 0.02 + rank 20: 0.02 + rank 21: 0.03 + rank 22: 0.02 + rank 23: 0.03 + rank 24: 0.04 + rank 25: 0.03 + rank 26: 0.03 + rank 27: 0.03 + rank 28: 0.04 + rank 29: 0.03 + rank 30: 0.03 + rank 31: 0.04 + rank 32: 0.02 + rank 33: 0.02 + rank 34: 0.02 + rank 35: 0.03 + rank 36: 0.02 + rank 37: 0.03 + rank 38: 0.02 + rank 39: 0.02 + rank 40: 0.02 + rank 41: 0.02 + rank 42: 0.02 + rank 43: 0.02 + rank 44: 0.02 + rank 45: 0.02 + rank 46: 0.02 + rank 47: 0.02 + rank 48: 0.03 + rank 49: 0.03 + rank 50: 0.02 + rank 51: 0.03 + rank 52: 0.03 + rank 53: 0.03 + rank 54: 0.03 + rank 55: 0.03 + rank 56: 0.02 + rank 57: 0.02 + rank 58: 0.03 + rank 59: 0.02 + rank 60: 0.02 + rank 61: 0.03 + rank 62: 0.02 + rank 63: 0.02 + optimizer-copy-to-main-grad: + rank 0: 2.14 + rank 1: 1.79 + rank 2: 1.68 + rank 3: 2.08 + rank 4: 1.67 + rank 5: 1.79 + rank 6: 1.89 + rank 7: 1.88 + rank 8: 1.76 + rank 9: 2.03 + rank 10: 1.72 + rank 11: 1.69 + rank 12: 1.77 + rank 13: 1.73 + rank 14: 1.83 + rank 15: 1.96 + rank 16: 2.37 + rank 17: 2.11 + rank 18: 2.15 + rank 19: 2.12 + rank 20: 2.13 + rank 21: 2.37 + rank 22: 2.54 + rank 23: 2.29 + rank 24: 2.42 + rank 25: 2.11 + rank 26: 1.80 + rank 27: 1.96 + rank 28: 2.16 + rank 29: 2.08 + rank 30: 2.34 + rank 31: 1.94 + rank 32: 2.04 + rank 33: 1.87 + rank 34: 1.97 + rank 35: 2.16 + rank 36: 2.04 + rank 37: 2.07 + rank 38: 2.31 + rank 39: 2.01 + rank 40: 1.78 + rank 41: 1.81 + rank 42: 1.88 + rank 43: 1.81 + rank 44: 1.85 + rank 45: 1.99 + rank 46: 1.79 + rank 47: 1.62 + rank 48: 1.96 + rank 49: 1.82 + rank 50: 2.23 + rank 51: 1.85 + rank 52: 1.96 + rank 53: 1.77 + rank 54: 1.77 + rank 55: 1.78 + rank 56: 1.75 + rank 57: 2.31 + rank 58: 1.82 + rank 59: 1.69 + rank 60: 1.76 + rank 61: 1.70 + rank 62: 1.85 + rank 63: 1.70 + optimizer-inner-step: + rank 0: 41.63 + rank 1: 41.19 + rank 2: 41.00 + rank 3: 41.16 + rank 4: 40.95 + rank 5: 41.10 + rank 6: 41.41 + rank 7: 41.39 + rank 8: 41.15 + rank 9: 41.15 + rank 10: 41.06 + rank 11: 41.18 + rank 12: 41.14 + rank 13: 41.39 + rank 14: 41.13 + rank 15: 41.06 + rank 16: 41.51 + rank 17: 41.21 + rank 18: 41.34 + rank 19: 41.28 + rank 20: 41.23 + rank 21: 41.30 + rank 22: 41.59 + rank 23: 41.42 + rank 24: 40.85 + rank 25: 41.35 + rank 26: 41.13 + rank 27: 41.55 + rank 28: 45.57 + rank 29: 41.05 + rank 30: 40.98 + rank 31: 41.31 + rank 32: 41.20 + rank 33: 40.94 + rank 34: 41.17 + rank 35: 47.69 + rank 36: 41.46 + rank 37: 41.22 + rank 38: 41.04 + rank 39: 41.03 + rank 40: 41.16 + rank 41: 41.27 + rank 42: 41.41 + rank 43: 41.32 + rank 44: 40.95 + rank 45: 41.56 + rank 46: 41.16 + rank 47: 40.86 + rank 48: 42.14 + rank 49: 41.05 + rank 50: 52.44 + rank 51: 40.84 + rank 52: 41.16 + rank 53: 41.08 + rank 54: 41.20 + rank 55: 41.11 + rank 56: 40.83 + rank 57: 47.01 + rank 58: 41.13 + rank 59: 41.01 + rank 60: 41.36 + rank 61: 40.90 + rank 62: 41.04 + rank 63: 40.94 + optimizer-copy-main-to-model-params: + rank 0: 4.44 + rank 1: 3.91 + rank 2: 3.77 + rank 3: 4.34 + rank 4: 3.72 + rank 5: 3.95 + rank 6: 3.91 + rank 7: 4.08 + rank 8: 3.84 + rank 9: 4.39 + rank 10: 3.73 + rank 11: 3.76 + rank 12: 3.90 + rank 13: 3.74 + rank 14: 4.02 + rank 15: 4.21 + rank 16: 5.34 + rank 17: 4.01 + rank 18: 4.03 + rank 19: 4.04 + rank 20: 4.04 + rank 21: 5.41 + rank 22: 5.89 + rank 23: 5.29 + rank 24: 3.88 + rank 25: 4.17 + rank 26: 3.85 + rank 27: 3.98 + rank 28: 4.33 + rank 29: 4.32 + rank 30: 4.48 + rank 31: 4.00 + rank 32: 4.11 + rank 33: 3.85 + rank 34: 3.99 + rank 35: 4.56 + rank 36: 4.22 + rank 37: 4.02 + rank 38: 4.51 + rank 39: 3.89 + rank 40: 3.81 + rank 41: 4.00 + rank 42: 3.93 + rank 43: 4.01 + rank 44: 3.98 + rank 45: 4.22 + rank 46: 3.78 + rank 47: 3.70 + rank 48: 4.08 + rank 49: 3.90 + rank 50: 4.99 + rank 51: 3.95 + rank 52: 4.19 + rank 53: 3.87 + rank 54: 3.83 + rank 55: 3.84 + rank 56: 3.85 + rank 57: 4.72 + rank 58: 3.91 + rank 59: 3.72 + rank 60: 3.75 + rank 61: 3.70 + rank 62: 3.89 + rank 63: 3.74 + optimizer: + rank 0: 54.21 + rank 1: 53.04 + rank 2: 52.64 + rank 3: 53.65 + rank 4: 52.56 + rank 5: 53.01 + rank 6: 53.31 + rank 7: 53.47 + rank 8: 52.92 + rank 9: 53.69 + rank 10: 52.70 + rank 11: 52.83 + rank 12: 53.00 + rank 13: 53.05 + rank 14: 53.11 + rank 15: 53.34 + rank 16: 55.28 + rank 17: 53.40 + rank 18: 53.55 + rank 19: 53.52 + rank 20: 53.46 + rank 21: 55.07 + rank 22: 55.91 + rank 23: 55.03 + rank 24: 52.65 + rank 25: 53.52 + rank 26: 52.87 + rank 27: 53.51 + rank 28: 57.96 + rank 29: 53.43 + rank 30: 53.54 + rank 31: 53.29 + rank 32: 53.25 + rank 33: 52.66 + rank 34: 53.06 + rank 35: 60.35 + rank 36: 53.66 + rank 37: 53.19 + rank 38: 53.62 + rank 39: 52.79 + rank 40: 52.87 + rank 41: 53.24 + rank 42: 53.28 + rank 43: 53.34 + rank 44: 52.90 + rank 45: 53.87 + rank 46: 52.81 + rank 47: 52.42 + rank 48: 54.20 + rank 49: 52.85 + rank 50: 65.76 + rank 51: 52.71 + rank 52: 53.34 + rank 53: 52.81 + rank 54: 52.93 + rank 55: 52.83 + rank 56: 52.59 + rank 57: 60.07 + rank 58: 52.98 + rank 59: 52.58 + rank 60: 52.98 + rank 61: 52.49 + rank 62: 52.87 + rank 63: 52.54 + interval-time: + rank 0: 0.91 + rank 1: 1.01 + rank 2: 1.03 + rank 3: 0.93 + rank 4: 0.99 + rank 5: 0.99 + rank 6: 0.98 + rank 7: 1.03 + rank 8: 0.74 + rank 9: 0.74 + rank 10: 0.81 + rank 11: 0.71 + rank 12: 0.81 + rank 13: 0.80 + rank 14: 0.80 + rank 15: 0.80 + rank 16: 0.74 + rank 17: 0.82 + rank 18: 0.81 + rank 19: 0.75 + rank 20: 0.81 + rank 21: 0.84 + rank 22: 0.73 + rank 23: 0.72 + rank 24: 0.89 + rank 25: 0.96 + rank 26: 1.00 + rank 27: 1.02 + rank 28: 0.86 + rank 29: 0.86 + rank 30: 1.03 + rank 31: 0.98 + rank 32: 0.94 + rank 33: 0.83 + rank 34: 0.93 + rank 35: 0.82 + rank 36: 0.79 + rank 37: 0.89 + rank 38: 0.81 + rank 39: 0.89 + rank 40: 0.75 + rank 41: 0.82 + rank 42: 0.74 + rank 43: 0.75 + rank 44: 0.80 + rank 45: 0.71 + rank 46: 0.81 + rank 47: 0.72 + rank 48: 0.97 + rank 49: 0.93 + rank 50: 0.82 + rank 51: 0.91 + rank 52: 0.84 + rank 53: 0.82 + rank 54: 0.82 + rank 55: 0.94 + rank 56: 1.18 + rank 57: 0.76 + rank 58: 0.84 + rank 59: 0.83 + rank 60: 0.73 + rank 61: 0.86 + rank 62: 0.83 + rank 63: 2.81 + [2026-10-09 22:58:25] iteration 6005/ 6855 | consumed samples: 768640 | elapsed time per iteration (ms): 90550.2 | throughput per GPU (TFLOP/s/GPU): 41.0 | MFU 13.13% | FLOPs estimator: workload-aware | logical tokens/s: 185280.5 | valid tokens/s: 185280.5 | physical tokens/s: 185764.8 | loss tokens/s: 133091.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.261193E-01 | lm-global-supervised-tokens: 1.205149E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.308670E-02 | lm-token-buffer-mib: 5.847932E+01 | global_load_balancing_loss: 1.000336E+00 | loss scale: 1.0 | grad norm: 0.167 | params norm: 11275.460 | params rms: 0.061053 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 21:22:47.690861 | finish at 2026-10-10 20:21:13 +times across ranks (ms): + forward-backward: + rank 0: 90222.58 + rank 1: 90223.48 + rank 2: 90223.28 + rank 3: 90223.86 + rank 4: 90224.23 + rank 5: 90224.70 + rank 6: 90222.64 + rank 7: 90222.86 + rank 8: 90216.76 + rank 9: 90216.75 + rank 10: 90216.91 + rank 11: 90218.30 + rank 12: 90218.19 + rank 13: 90218.73 + rank 14: 90215.42 + rank 15: 90216.36 + rank 16: 90216.64 + rank 17: 90217.94 + rank 18: 90217.71 + rank 19: 90218.55 + rank 20: 90218.33 + rank 21: 90220.06 + rank 22: 90216.37 + rank 23: 90216.94 + rank 24: 90214.42 + rank 25: 90216.65 + rank 26: 90215.83 + rank 27: 90215.72 + rank 28: 90217.24 + rank 29: 90337.65 + rank 30: 90215.08 + rank 31: 90214.83 + rank 32: 90211.76 + rank 33: 90212.14 + rank 34: 90210.84 + rank 35: 90212.60 + rank 36: 90212.59 + rank 37: 90211.36 + rank 38: 90211.24 + rank 39: 90211.27 + rank 40: 90215.16 + rank 41: 90214.39 + rank 42: 90214.19 + rank 43: 90214.21 + rank 44: 90213.84 + rank 45: 90212.12 + rank 46: 90215.66 + rank 47: 90215.68 + rank 48: 90220.62 + rank 49: 90219.98 + rank 50: 90219.09 + rank 51: 90219.94 + rank 52: 90218.46 + rank 53: 90218.39 + rank 54: 90221.10 + rank 55: 90221.78 + rank 56: 90227.81 + rank 57: 90227.06 + rank 58: 90226.70 + rank 59: 90226.12 + rank 60: 90225.66 + rank 61: 90224.57 + rank 62: 90229.39 + rank 63: 90227.91 + forward-compute: + rank 0: 34833.86 + rank 1: 34841.06 + rank 2: 34844.57 + rank 3: 34844.39 + rank 4: 34846.50 + rank 5: 34843.88 + rank 6: 34843.49 + rank 7: 34855.49 + rank 8: 38690.77 + rank 9: 38688.99 + rank 10: 38690.70 + rank 11: 38690.85 + rank 12: 38692.14 + rank 13: 38693.00 + rank 14: 38689.40 + rank 15: 38690.08 + rank 16: 38360.79 + rank 17: 38363.24 + rank 18: 38363.30 + rank 19: 38365.62 + rank 20: 38364.94 + rank 21: 38363.88 + rank 22: 38363.23 + rank 23: 38363.05 + rank 24: 33153.16 + rank 25: 33157.60 + rank 26: 33150.54 + rank 27: 33149.43 + rank 28: 33163.30 + rank 29: 33158.20 + rank 30: 33160.48 + rank 31: 33156.41 + rank 32: 33721.88 + rank 33: 33713.00 + rank 34: 33719.06 + rank 35: 33716.67 + rank 36: 33725.85 + rank 37: 33728.48 + rank 38: 33727.52 + rank 39: 33716.97 + rank 40: 37276.77 + rank 41: 37273.27 + rank 42: 37272.80 + rank 43: 37271.12 + rank 44: 37268.90 + rank 45: 37272.51 + rank 46: 37278.98 + rank 47: 37269.34 + rank 48: 37530.86 + rank 49: 37531.27 + rank 50: 37529.24 + rank 51: 37532.47 + rank 52: 37530.45 + rank 53: 37529.35 + rank 54: 37531.98 + rank 55: 37532.08 + rank 56: 32785.67 + rank 57: 32791.78 + rank 58: 32792.63 + rank 59: 32786.99 + rank 60: 32796.14 + rank 61: 32800.12 + rank 62: 32797.96 + rank 63: 32807.78 + backward-compute: + rank 0: 55353.57 + rank 1: 55344.11 + rank 2: 55339.89 + rank 3: 55343.67 + rank 4: 55341.27 + rank 5: 55343.01 + rank 6: 55343.67 + rank 7: 55330.74 + rank 8: 51489.36 + rank 9: 51495.28 + rank 10: 51492.42 + rank 11: 51490.81 + rank 12: 51489.88 + rank 13: 51490.95 + rank 14: 51491.55 + rank 15: 51491.67 + rank 16: 51820.15 + rank 17: 51819.17 + rank 18: 51819.65 + rank 19: 51817.36 + rank 20: 51819.80 + rank 21: 51820.19 + rank 22: 51817.77 + rank 23: 51819.11 + rank 24: 57022.65 + rank 25: 57017.04 + rank 26: 57027.66 + rank 27: 57022.99 + rank 28: 57012.71 + rank 29: 57138.77 + rank 30: 57010.84 + rank 31: 57016.58 + rank 32: 56449.54 + rank 33: 56460.96 + rank 34: 56453.70 + rank 35: 56457.45 + rank 36: 56449.12 + rank 37: 56442.08 + rank 38: 56441.38 + rank 39: 56458.58 + rank 40: 52902.40 + rank 41: 52900.64 + rank 42: 52906.02 + rank 43: 52904.61 + rank 44: 52908.31 + rank 45: 52905.41 + rank 46: 52903.50 + rank 47: 52909.20 + rank 48: 52652.87 + rank 49: 52641.04 + rank 50: 52652.38 + rank 51: 52651.41 + rank 52: 52649.93 + rank 53: 52655.11 + rank 54: 52652.35 + rank 55: 52652.17 + rank 56: 57408.01 + rank 57: 57399.81 + rank 58: 57400.29 + rank 59: 57406.28 + rank 60: 57395.67 + rank 61: 57391.75 + rank 62: 57397.91 + rank 63: 57384.14 + batch-generator: + rank 0: 27.56 + rank 1: 27.54 + rank 2: 27.87 + rank 3: 27.45 + rank 4: 29.37 + rank 5: 26.90 + rank 6: 28.42 + rank 7: 30.09 + rank 8: 22.39 + rank 9: 21.33 + rank 10: 21.02 + rank 11: 22.90 + rank 12: 22.62 + rank 13: 21.66 + rank 14: 22.64 + rank 15: 20.66 + rank 16: 22.25 + rank 17: 22.73 + rank 18: 21.02 + rank 19: 21.81 + rank 20: 21.10 + rank 21: 23.44 + rank 22: 22.00 + rank 23: 22.35 + rank 24: 32.93 + rank 25: 33.53 + rank 26: 34.66 + rank 27: 36.24 + rank 28: 33.09 + rank 29: 29.52 + rank 30: 31.07 + rank 31: 32.53 + rank 32: 33.21 + rank 33: 32.91 + rank 34: 32.49 + rank 35: 32.80 + rank 36: 29.14 + rank 37: 33.18 + rank 38: 34.48 + rank 39: 32.54 + rank 40: 22.41 + rank 41: 23.58 + rank 42: 22.88 + rank 43: 23.39 + rank 44: 22.43 + rank 45: 23.33 + rank 46: 22.05 + rank 47: 21.96 + rank 48: 29.38 + rank 49: 30.36 + rank 50: 26.87 + rank 51: 27.00 + rank 52: 32.33 + rank 53: 21.87 + rank 54: 30.18 + rank 55: 28.27 + rank 56: 24.64 + rank 57: 23.48 + rank 58: 24.22 + rank 59: 23.22 + rank 60: 22.46 + rank 61: 23.31 + rank 62: 23.13 + rank 63: 24.79 + embedding-grads-all-reduce: + rank 0: 0.03 + rank 1: 0.02 + rank 2: 0.02 + rank 3: 0.03 + rank 4: 0.02 + rank 5: 0.02 + rank 6: 0.03 + rank 7: 0.03 + rank 8: 0.02 + rank 9: 0.03 + rank 10: 0.02 + rank 11: 0.03 + rank 12: 0.03 + rank 13: 0.02 + rank 14: 0.03 + rank 15: 0.03 + rank 16: 0.03 + rank 17: 0.02 + rank 18: 0.03 + rank 19: 0.03 + rank 20: 0.03 + rank 21: 0.03 + rank 22: 0.03 + rank 23: 0.03 + rank 24: 0.04 + rank 25: 0.04 + rank 26: 0.03 + rank 27: 0.03 + rank 28: 0.03 + rank 29: 0.04 + rank 30: 0.04 + rank 31: 0.04 + rank 32: 0.03 + rank 33: 0.04 + rank 34: 0.03 + rank 35: 0.03 + rank 36: 0.03 + rank 37: 0.03 + rank 38: 0.03 + rank 39: 0.03 + rank 40: 0.03 + rank 41: 0.03 + rank 42: 0.02 + rank 43: 0.03 + rank 44: 0.03 + rank 45: 0.03 + rank 46: 0.02 + rank 47: 0.03 + rank 48: 0.03 + rank 49: 0.03 + rank 50: 0.04 + rank 51: 0.03 + rank 52: 0.03 + rank 53: 0.03 + rank 54: 0.03 + rank 55: 0.03 + rank 56: 0.03 + rank 57: 0.03 + rank 58: 0.02 + rank 59: 0.03 + rank 60: 0.03 + rank 61: 0.03 + rank 62: 0.03 + rank 63: 0.03 + all-grads-sync: + rank 0: 0.88 + rank 1: 0.83 + rank 2: 0.79 + rank 3: 0.78 + rank 4: 0.78 + rank 5: 0.78 + rank 6: 0.82 + rank 7: 0.84 + rank 8: 0.83 + rank 9: 0.74 + rank 10: 0.75 + rank 11: 0.77 + rank 12: 0.80 + rank 13: 0.95 + rank 14: 0.80 + rank 15: 0.77 + rank 16: 0.85 + rank 17: 0.76 + rank 18: 0.77 + rank 19: 0.78 + rank 20: 0.79 + rank 21: 0.84 + rank 22: 0.82 + rank 23: 0.93 + rank 24: 1.31 + rank 25: 1.27 + rank 26: 1.29 + rank 27: 1.31 + rank 28: 1.38 + rank 29: 1.31 + rank 30: 1.45 + rank 31: 1.21 + rank 32: 1.45 + rank 33: 1.21 + rank 34: 1.26 + rank 35: 1.24 + rank 36: 1.30 + rank 37: 1.22 + rank 38: 1.19 + rank 39: 1.08 + rank 40: 0.85 + rank 41: 0.84 + rank 42: 0.88 + rank 43: 0.76 + rank 44: 0.76 + rank 45: 0.76 + rank 46: 0.90 + rank 47: 1.36 + rank 48: 1.06 + rank 49: 1.07 + rank 50: 1.19 + rank 51: 1.02 + rank 52: 1.13 + rank 53: 1.01 + rank 54: 0.85 + rank 55: 1.11 + rank 56: 0.83 + rank 57: 0.81 + rank 58: 0.86 + rank 59: 0.79 + rank 60: 0.78 + rank 61: 0.88 + rank 62: 1.09 + rank 63: 0.84 + params-all-gather: + rank 0: 0.02 + rank 1: 0.02 + rank 2: 0.02 + rank 3: 0.02 + rank 4: 0.02 + rank 5: 0.02 + rank 6: 0.02 + rank 7: 0.02 + rank 8: 0.02 + rank 9: 0.02 + rank 10: 0.02 + rank 11: 0.02 + rank 12: 0.02 + rank 13: 0.02 + rank 14: 0.02 + rank 15: 0.02 + rank 16: 0.02 + rank 17: 0.02 + rank 18: 0.02 + rank 19: 0.02 + rank 20: 0.02 + rank 21: 0.02 + rank 22: 0.02 + rank 23: 0.02 + rank 24: 0.04 + rank 25: 0.03 + rank 26: 0.04 + rank 27: 0.04 + rank 28: 0.03 + rank 29: 0.03 + rank 30: 0.03 + rank 31: 0.03 + rank 32: 0.03 + rank 33: 0.02 + rank 34: 0.02 + rank 35: 0.02 + rank 36: 0.02 + rank 37: 0.02 + rank 38: 0.02 + rank 39: 0.02 + rank 40: 0.02 + rank 41: 0.02 + rank 42: 0.02 + rank 43: 0.02 + rank 44: 0.02 + rank 45: 0.02 + rank 46: 0.02 + rank 47: 0.02 + rank 48: 0.03 + rank 49: 0.03 + rank 50: 0.03 + rank 51: 0.02 + rank 52: 0.03 + rank 53: 0.02 + rank 54: 0.03 + rank 55: 0.03 + rank 56: 0.02 + rank 57: 0.02 + rank 58: 0.02 + rank 59: 0.02 + rank 60: 0.02 + rank 61: 0.02 + rank 62: 0.02 + rank 63: 0.02 + optimizer-copy-to-main-grad: + rank 0: 2.13 + rank 1: 1.93 + rank 2: 1.71 + rank 3: 2.08 + rank 4: 1.64 + rank 5: 1.85 + rank 6: 1.82 + rank 7: 1.93 + rank 8: 1.76 + rank 9: 2.05 + rank 10: 1.78 + rank 11: 1.68 + rank 12: 1.76 + rank 13: 1.72 + rank 14: 1.88 + rank 15: 2.01 + rank 16: 2.37 + rank 17: 2.11 + rank 18: 2.10 + rank 19: 2.25 + rank 20: 2.15 + rank 21: 2.32 + rank 22: 2.52 + rank 23: 2.33 + rank 24: 2.01 + rank 25: 2.33 + rank 26: 1.95 + rank 27: 2.06 + rank 28: 2.29 + rank 29: 2.11 + rank 30: 2.36 + rank 31: 2.01 + rank 32: 2.16 + rank 33: 2.00 + rank 34: 2.09 + rank 35: 2.20 + rank 36: 2.14 + rank 37: 2.13 + rank 38: 2.30 + rank 39: 1.92 + rank 40: 1.74 + rank 41: 1.87 + rank 42: 1.84 + rank 43: 1.88 + rank 44: 1.88 + rank 45: 2.05 + rank 46: 1.73 + rank 47: 1.66 + rank 48: 2.05 + rank 49: 1.82 + rank 50: 2.52 + rank 51: 1.94 + rank 52: 2.09 + rank 53: 1.79 + rank 54: 1.79 + rank 55: 1.78 + rank 56: 1.63 + rank 57: 2.32 + rank 58: 1.83 + rank 59: 1.73 + rank 60: 1.75 + rank 61: 1.67 + rank 62: 1.86 + rank 63: 1.72 + optimizer-inner-step: + rank 0: 43.95 + rank 1: 41.77 + rank 2: 41.54 + rank 3: 41.66 + rank 4: 41.53 + rank 5: 41.63 + rank 6: 42.04 + rank 7: 41.92 + rank 8: 41.72 + rank 9: 41.72 + rank 10: 41.57 + rank 11: 41.74 + rank 12: 41.66 + rank 13: 41.90 + rank 14: 41.67 + rank 15: 41.57 + rank 16: 41.97 + rank 17: 41.76 + rank 18: 41.83 + rank 19: 41.69 + rank 20: 41.77 + rank 21: 41.69 + rank 22: 42.15 + rank 23: 41.87 + rank 24: 41.48 + rank 25: 41.76 + rank 26: 41.62 + rank 27: 41.84 + rank 28: 45.53 + rank 29: 41.59 + rank 30: 41.59 + rank 31: 41.84 + rank 32: 41.71 + rank 33: 41.51 + rank 34: 41.69 + rank 35: 47.35 + rank 36: 41.79 + rank 37: 41.77 + rank 38: 41.59 + rank 39: 41.60 + rank 40: 41.71 + rank 41: 41.83 + rank 42: 41.91 + rank 43: 41.91 + rank 44: 41.43 + rank 45: 42.08 + rank 46: 41.73 + rank 47: 41.40 + rank 48: 42.70 + rank 49: 41.60 + rank 50: 53.24 + rank 51: 41.40 + rank 52: 41.73 + rank 53: 41.60 + rank 54: 41.77 + rank 55: 41.65 + rank 56: 41.35 + rank 57: 46.77 + rank 58: 41.66 + rank 59: 41.58 + rank 60: 41.89 + rank 61: 41.47 + rank 62: 41.58 + rank 63: 41.49 + optimizer-copy-main-to-model-params: + rank 0: 4.40 + rank 1: 3.95 + rank 2: 3.72 + rank 3: 4.37 + rank 4: 3.72 + rank 5: 3.91 + rank 6: 3.86 + rank 7: 4.04 + rank 8: 3.84 + rank 9: 4.40 + rank 10: 3.75 + rank 11: 3.74 + rank 12: 3.95 + rank 13: 3.78 + rank 14: 4.06 + rank 15: 4.25 + rank 16: 4.37 + rank 17: 4.00 + rank 18: 3.96 + rank 19: 4.03 + rank 20: 3.98 + rank 21: 4.20 + rank 22: 4.57 + rank 23: 4.16 + rank 24: 4.21 + rank 25: 4.19 + rank 26: 3.89 + rank 27: 4.03 + rank 28: 4.38 + rank 29: 4.39 + rank 30: 4.30 + rank 31: 4.00 + rank 32: 4.11 + rank 33: 3.87 + rank 34: 4.09 + rank 35: 4.54 + rank 36: 4.20 + rank 37: 4.07 + rank 38: 4.53 + rank 39: 3.87 + rank 40: 3.72 + rank 41: 3.98 + rank 42: 3.87 + rank 43: 3.98 + rank 44: 3.95 + rank 45: 4.22 + rank 46: 3.70 + rank 47: 3.71 + rank 48: 4.11 + rank 49: 3.89 + rank 50: 4.97 + rank 51: 3.96 + rank 52: 4.25 + rank 53: 3.86 + rank 54: 3.90 + rank 55: 3.86 + rank 56: 3.80 + rank 57: 4.72 + rank 58: 3.94 + rank 59: 3.76 + rank 60: 3.72 + rank 61: 3.67 + rank 62: 3.87 + rank 63: 3.68 + optimizer: + rank 0: 55.46 + rank 1: 52.65 + rank 2: 52.13 + rank 3: 53.15 + rank 4: 52.12 + rank 5: 52.49 + rank 6: 52.86 + rank 7: 52.95 + rank 8: 52.49 + rank 9: 53.24 + rank 10: 52.22 + rank 11: 52.34 + rank 12: 52.55 + rank 13: 52.56 + rank 14: 52.67 + rank 15: 52.87 + rank 16: 53.72 + rank 17: 52.95 + rank 18: 52.98 + rank 19: 52.99 + rank 20: 52.92 + rank 21: 53.17 + rank 22: 54.13 + rank 23: 53.27 + rank 24: 52.59 + rank 25: 52.96 + rank 26: 52.42 + rank 27: 52.85 + rank 28: 56.99 + rank 29: 53.05 + rank 30: 52.91 + rank 31: 52.79 + rank 32: 52.77 + rank 33: 52.24 + rank 34: 52.70 + rank 35: 58.98 + rank 36: 52.95 + rank 37: 52.76 + rank 38: 53.20 + rank 39: 52.32 + rank 40: 52.32 + rank 41: 52.77 + rank 42: 52.72 + rank 43: 52.86 + rank 44: 52.35 + rank 45: 53.38 + rank 46: 52.27 + rank 47: 51.98 + rank 48: 53.73 + rank 49: 52.39 + rank 50: 65.49 + rank 51: 52.26 + rank 52: 52.97 + rank 53: 52.31 + rank 54: 52.54 + rank 55: 52.41 + rank 56: 52.04 + rank 57: 58.77 + rank 58: 52.53 + rank 59: 52.19 + rank 60: 52.50 + rank 61: 52.00 + rank 62: 52.39 + rank 63: 52.00 + interval-time: + rank 0: 2.91 + rank 1: 2.94 + rank 2: 2.94 + rank 3: 2.91 + rank 4: 2.93 + rank 5: 2.91 + rank 6: 2.91 + rank 7: 2.93 + rank 8: 0.73 + rank 9: 0.73 + rank 10: 0.80 + rank 11: 0.71 + rank 12: 0.81 + rank 13: 0.80 + rank 14: 0.81 + rank 15: 0.80 + rank 16: 0.73 + rank 17: 0.83 + rank 18: 0.80 + rank 19: 0.74 + rank 20: 0.78 + rank 21: 0.82 + rank 22: 0.71 + rank 23: 0.71 + rank 24: 0.89 + rank 25: 0.92 + rank 26: 0.96 + rank 27: 0.99 + rank 28: 0.82 + rank 29: 0.84 + rank 30: 0.93 + rank 31: 0.96 + rank 32: 0.94 + rank 33: 0.86 + rank 34: 0.95 + rank 35: 0.83 + rank 36: 0.82 + rank 37: 0.89 + rank 38: 0.80 + rank 39: 0.90 + rank 40: 0.75 + rank 41: 0.82 + rank 42: 0.72 + rank 43: 0.74 + rank 44: 0.80 + rank 45: 0.74 + rank 46: 0.81 + rank 47: 0.72 + rank 48: 0.95 + rank 49: 0.91 + rank 50: 0.82 + rank 51: 0.89 + rank 52: 0.83 + rank 53: 0.81 + rank 54: 0.81 + rank 55: 0.94 + rank 56: 0.81 + rank 57: 0.81 + rank 58: 0.82 + rank 59: 0.73 + rank 60: 0.71 + rank 61: 0.83 + rank 62: 0.80 + rank 63: 2.81 + [2026-10-09 22:59:52] iteration 6006/ 6855 | consumed samples: 768768 | elapsed time per iteration (ms): 87156.6 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.74% | FLOPs estimator: workload-aware | logical tokens/s: 192494.6 | valid tokens/s: 192494.6 | physical tokens/s: 192899.4 | loss tokens/s: 141705.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.585058E-01 | lm-global-supervised-tokens: 1.235060E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.191626E-02 | lm-token-buffer-mib: 5.803757E+01 | global_load_balancing_loss: 1.000025E+00 | loss scale: 1.0 | grad norm: 0.068 | params norm: 11275.413 | params rms: 0.061053 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 20:33:15.925060 | finish at 2026-10-10 19:33:08 + [2026-10-09 23:01:20] iteration 6007/ 6855 | consumed samples: 768896 | elapsed time per iteration (ms): 87592.7 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.40% | FLOPs estimator: workload-aware | logical tokens/s: 191536.5 | valid tokens/s: 191536.5 | physical tokens/s: 192165.7 | loss tokens/s: 145342.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.672847E-01 | lm-global-supervised-tokens: 1.273092E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.189998E-02 | lm-token-buffer-mib: 5.823612E+01 | global_load_balancing_loss: 1.000224E+00 | loss scale: 1.0 | grad norm: 0.152 | params norm: 11275.365 | params rms: 0.061052 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 20:37:58.578266 | finish at 2026-10-10 19:39:19 + [2026-10-09 23:02:48] iteration 6008/ 6855 | consumed samples: 769024 | elapsed time per iteration (ms): 88376.8 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.60% | FLOPs estimator: workload-aware | logical tokens/s: 189837.0 | valid tokens/s: 189837.0 | physical tokens/s: 190242.8 | loss tokens/s: 143866.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.724234E-01 | lm-global-supervised-tokens: 1.271449E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.874377E-02 | lm-token-buffer-mib: 5.806255E+01 | global_load_balancing_loss: 9.994938E-01 | loss scale: 1.0 | grad norm: 0.160 | params norm: 11275.317 | params rms: 0.061052 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 20:47:35.168633 | finish at 2026-10-10 19:50:24 + [2026-10-09 23:04:15] iteration 6009/ 6855 | consumed samples: 769152 | elapsed time per iteration (ms): 86861.7 | throughput per GPU (TFLOP/s/GPU): 43.2 | MFU 13.84% | FLOPs estimator: workload-aware | logical tokens/s: 193148.6 | valid tokens/s: 193148.6 | physical tokens/s: 193546.0 | loss tokens/s: 140053.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.264434E-01 | lm-global-supervised-tokens: 1.216529E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.498904E-02 | lm-token-buffer-mib: 5.804319E+01 | global_load_balancing_loss: 9.997286E-01 | loss scale: 1.0 | grad norm: 0.093 | params norm: 11275.269 | params rms: 0.061052 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 20:24:44.967187 | finish at 2026-10-10 19:29:00 + [2026-10-09 23:06:08] iteration 6010/ 6855 | consumed samples: 769280 | elapsed time per iteration (ms): 112448.0 | throughput per GPU (TFLOP/s/GPU): 33.1 | MFU 10.61% | FLOPs estimator: workload-aware | logical tokens/s: 149199.7 | valid tokens/s: 149199.7 | physical tokens/s: 149604.1 | loss tokens/s: 107031.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.475440E-01 | lm-global-supervised-tokens: 1.203547E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.265747E-02 | lm-token-buffer-mib: 5.806338E+01 | global_load_balancing_loss: 1.000507E+00 | loss scale: 1.0 | grad norm: 0.093 | params norm: 11275.221 | params rms: 0.061052 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 2:23:38.572251 | finish at 2026-10-11 01:29:46 + [2026-10-09 23:07:36] iteration 6011/ 6855 | consumed samples: 769408 | elapsed time per iteration (ms): 88523.1 | throughput per GPU (TFLOP/s/GPU): 40.7 | MFU 13.05% | FLOPs estimator: workload-aware | logical tokens/s: 189523.6 | valid tokens/s: 189523.6 | physical tokens/s: 190114.9 | loss tokens/s: 141757.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.145995E-01 | lm-global-supervised-tokens: 1.254879E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.226715E-02 | lm-token-buffer-mib: 5.811944E+01 | global_load_balancing_loss: 1.000193E+00 | loss scale: 1.0 | grad norm: 0.184 | params norm: 11275.173 | params rms: 0.061051 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 20:45:13.471967 | finish at 2026-10-10 19:52:50 + [2026-10-09 23:09:07] iteration 6012/ 6855 | consumed samples: 769536 | elapsed time per iteration (ms): 90564.0 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.39% | FLOPs estimator: workload-aware | logical tokens/s: 185252.0 | valid tokens/s: 185252.0 | physical tokens/s: 185726.5 | loss tokens/s: 136020.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.984403E-01 | lm-global-supervised-tokens: 1.231851E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.739760E-02 | lm-token-buffer-mib: 5.819860E+01 | global_load_balancing_loss: 9.999195E-01 | loss scale: 1.0 | grad norm: 0.193 | params norm: 11275.125 | params rms: 0.061051 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 21:12:25.423569 | finish at 2026-10-10 20:21:32 + [2026-10-09 23:10:32] iteration 6013/ 6855 | consumed samples: 769664 | elapsed time per iteration (ms): 85338.5 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.66% | FLOPs estimator: workload-aware | logical tokens/s: 196594.9 | valid tokens/s: 196594.9 | physical tokens/s: 196992.7 | loss tokens/s: 147343.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.462269E-01 | lm-global-supervised-tokens: 1.257406E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.492414E-02 | lm-token-buffer-mib: 5.813519E+01 | global_load_balancing_loss: 1.000015E+00 | loss scale: 1.0 | grad norm: 0.084 | params norm: 11275.077 | params rms: 0.061051 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 19:57:35.008588 | finish at 2026-10-10 19:08:07 + [2026-10-09 23:12:01] iteration 6014/ 6855 | consumed samples: 769792 | elapsed time per iteration (ms): 88782.1 | throughput per GPU (TFLOP/s/GPU): 41.6 | MFU 13.34% | FLOPs estimator: workload-aware | logical tokens/s: 188970.8 | valid tokens/s: 188970.8 | physical tokens/s: 189433.4 | loss tokens/s: 138054.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.295294E-01 | lm-global-supervised-tokens: 1.225680E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.156481E-02 | lm-token-buffer-mib: 5.805460E+01 | global_load_balancing_loss: 1.000552E+00 | loss scale: 1.0 | grad norm: 0.114 | params norm: 11275.028 | params rms: 0.061051 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 20:44:25.709776 | finish at 2026-10-10 19:56:27 + [2026-10-09 23:13:28] iteration 6015/ 6855 | consumed samples: 769920 | elapsed time per iteration (ms): 87249.5 | throughput per GPU (TFLOP/s/GPU): 41.3 | MFU 13.23% | FLOPs estimator: workload-aware | logical tokens/s: 192289.0 | valid tokens/s: 192289.0 | physical tokens/s: 192916.0 | loss tokens/s: 144644.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.306236E-01 | lm-global-supervised-tokens: 1.262012E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.591917E-02 | lm-token-buffer-mib: 5.807635E+01 | global_load_balancing_loss: 1.000291E+00 | loss scale: 1.0 | grad norm: 0.175 | params norm: 11274.980 | params rms: 0.061050 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 20:21:29.584036 | finish at 2026-10-10 19:34:58 + [2026-10-09 23:14:58] iteration 6016/ 6855 | consumed samples: 770048 | elapsed time per iteration (ms): 90283.6 | throughput per GPU (TFLOP/s/GPU): 41.2 | MFU 13.21% | FLOPs estimator: workload-aware | logical tokens/s: 185827.8 | valid tokens/s: 185827.8 | physical tokens/s: 186447.4 | loss tokens/s: 140518.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.690863E-01 | lm-global-supervised-tokens: 1.268654E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.565195E-02 | lm-token-buffer-mib: 5.818427E+01 | global_load_balancing_loss: 9.999556E-01 | loss scale: 1.0 | grad norm: 0.069 | params norm: 11274.931 | params rms: 0.061050 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 21:02:27.908872 | finish at 2026-10-10 20:17:26 + [2026-10-09 23:16:26] iteration 6017/ 6855 | consumed samples: 770176 | elapsed time per iteration (ms): 87911.3 | throughput per GPU (TFLOP/s/GPU): 41.7 | MFU 13.38% | FLOPs estimator: workload-aware | logical tokens/s: 190842.2 | valid tokens/s: 190842.2 | physical tokens/s: 191303.9 | loss tokens/s: 140875.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 5.994803E-01 | lm-global-supervised-tokens: 1.238456E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.439482E-02 | lm-token-buffer-mib: 5.818253E+01 | global_load_balancing_loss: 1.000388E+00 | loss scale: 1.0 | grad norm: 0.106 | params norm: 11274.880 | params rms: 0.061050 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 20:27:49.662760 | finish at 2026-10-10 19:44:16 + [2026-10-09 23:17:55] iteration 6018/ 6855 | consumed samples: 770304 | elapsed time per iteration (ms): 88204.6 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.59% | FLOPs estimator: workload-aware | logical tokens/s: 190207.4 | valid tokens/s: 190207.4 | physical tokens/s: 190631.6 | loss tokens/s: 138763.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.773347E-01 | lm-global-supervised-tokens: 1.223955E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.101179E-02 | lm-token-buffer-mib: 5.802679E+01 | global_load_balancing_loss: 1.000250E+00 | loss scale: 1.0 | grad norm: 0.037 | params norm: 11274.829 | params rms: 0.061050 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 20:30:27.208772 | finish at 2026-10-10 19:48:22 + [2026-10-09 23:19:23] iteration 6019/ 6855 | consumed samples: 770432 | elapsed time per iteration (ms): 88432.5 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.44% | FLOPs estimator: workload-aware | logical tokens/s: 189717.7 | valid tokens/s: 189717.7 | physical tokens/s: 190232.3 | loss tokens/s: 135417.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.279386E-01 | lm-global-supervised-tokens: 1.197528E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.613383E-02 | lm-token-buffer-mib: 5.818952E+01 | global_load_balancing_loss: 1.000047E+00 | loss scale: 1.0 | grad norm: 0.067 | params norm: 11274.779 | params rms: 0.061049 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 20:32:09.573093 | finish at 2026-10-10 19:51:33 + [2026-10-09 23:21:11] iteration 6020/ 6855 | consumed samples: 770560 | elapsed time per iteration (ms): 108037.1 | throughput per GPU (TFLOP/s/GPU): 33.4 | MFU 10.69% | FLOPs estimator: workload-aware | logical tokens/s: 155290.7 | valid tokens/s: 155290.7 | physical tokens/s: 155669.8 | loss tokens/s: 118730.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.255853E-01 | lm-global-supervised-tokens: 1.282724E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.314520E-02 | lm-token-buffer-mib: 5.808737E+01 | global_load_balancing_loss: 1.001252E+00 | loss scale: 1.0 | grad norm: 0.074 | params norm: 11274.729 | params rms: 0.061049 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 1:03:30.953878 | finish at 2026-10-11 00:24:42 + [2026-10-09 23:22:38] iteration 6021/ 6855 | consumed samples: 770688 | elapsed time per iteration (ms): 87361.6 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.77% | FLOPs estimator: workload-aware | logical tokens/s: 192042.4 | valid tokens/s: 192042.4 | physical tokens/s: 192454.2 | loss tokens/s: 143539.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.348822E-01 | lm-global-supervised-tokens: 1.253984E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.214505E-02 | lm-token-buffer-mib: 5.810166E+01 | global_load_balancing_loss: 1.000474E+00 | loss scale: 1.0 | grad norm: 0.060 | params norm: 11274.680 | params rms: 0.061049 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 20:14:19.576920 | finish at 2026-10-10 19:36:58 + [2026-10-09 23:24:03] iteration 6022/ 6855 | consumed samples: 770816 | elapsed time per iteration (ms): 84155.9 | throughput per GPU (TFLOP/s/GPU): 43.3 | MFU 13.89% | FLOPs estimator: workload-aware | logical tokens/s: 199358.7 | valid tokens/s: 199358.7 | physical tokens/s: 199937.3 | loss tokens/s: 151772.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.798994E-01 | lm-global-supervised-tokens: 1.277256E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.557816E-02 | lm-token-buffer-mib: 5.813613E+01 | global_load_balancing_loss: 9.991984E-01 | loss scale: 1.0 | grad norm: 0.090 | params norm: 11274.630 | params rms: 0.061048 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 19:28:21.882377 | finish at 2026-10-10 18:52:24 + [2026-10-09 23:25:29] iteration 6023/ 6855 | consumed samples: 770944 | elapsed time per iteration (ms): 85988.5 | throughput per GPU (TFLOP/s/GPU): 43.1 | MFU 13.80% | FLOPs estimator: workload-aware | logical tokens/s: 195109.7 | valid tokens/s: 195109.7 | physical tokens/s: 195493.6 | loss tokens/s: 143849.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.700065E-01 | lm-global-supervised-tokens: 1.236940E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.126865E-02 | lm-token-buffer-mib: 5.814153E+01 | global_load_balancing_loss: 9.994962E-01 | loss scale: 1.0 | grad norm: 0.056 | params norm: 11274.580 | params rms: 0.061048 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 19:52:22.469986 | finish at 2026-10-10 19:17:51 + [2026-10-09 23:26:55] iteration 6024/ 6855 | consumed samples: 771072 | elapsed time per iteration (ms): 86574.2 | throughput per GPU (TFLOP/s/GPU): 43.1 | MFU 13.82% | FLOPs estimator: workload-aware | logical tokens/s: 193789.3 | valid tokens/s: 193789.3 | physical tokens/s: 194264.6 | loss tokens/s: 143332.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.496067E-01 | lm-global-supervised-tokens: 1.240887E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.280473E-02 | lm-token-buffer-mib: 5.802940E+01 | global_load_balancing_loss: 1.000587E+00 | loss scale: 1.0 | grad norm: 0.126 | params norm: 11274.531 | params rms: 0.061048 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 19:59:03.178753 | finish at 2026-10-10 19:25:58 + [2026-10-09 23:28:24] iteration 6025/ 6855 | consumed samples: 771200 | elapsed time per iteration (ms): 88759.0 | throughput per GPU (TFLOP/s/GPU): 41.5 | MFU 13.30% | FLOPs estimator: workload-aware | logical tokens/s: 189019.4 | valid tokens/s: 189019.4 | physical tokens/s: 189532.1 | loss tokens/s: 143201.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.027513E-01 | lm-global-supervised-tokens: 1.271047E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.147219E-02 | lm-token-buffer-mib: 5.810054E+01 | global_load_balancing_loss: 1.000758E+00 | loss scale: 1.0 | grad norm: 0.146 | params norm: 11274.482 | params rms: 0.061048 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 20:27:50.003891 | finish at 2026-10-10 19:56:14 + [2026-10-09 23:29:48] iteration 6026/ 6855 | consumed samples: 771328 | elapsed time per iteration (ms): 84004.7 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.58% | FLOPs estimator: workload-aware | logical tokens/s: 199717.1 | valid tokens/s: 199717.1 | physical tokens/s: 200321.6 | loss tokens/s: 153742.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.469488E-01 | lm-global-supervised-tokens: 1.291507E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.547299E-02 | lm-token-buffer-mib: 5.806450E+01 | global_load_balancing_loss: 1.001084E+00 | loss scale: 1.0 | grad norm: 0.061 | params norm: 11274.432 | params rms: 0.061047 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 19:20:39.891906 | finish at 2026-10-10 18:50:28 + [2026-10-09 23:31:13] iteration 6027/ 6855 | consumed samples: 771456 | elapsed time per iteration (ms): 85019.5 | throughput per GPU (TFLOP/s/GPU): 43.3 | MFU 13.87% | FLOPs estimator: workload-aware | logical tokens/s: 197333.8 | valid tokens/s: 197333.8 | physical tokens/s: 197695.5 | loss tokens/s: 149047.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.209135E-01 | lm-global-supervised-tokens: 1.267193E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.170295E-02 | lm-token-buffer-mib: 5.822176E+01 | global_load_balancing_loss: 1.000943E+00 | loss scale: 1.0 | grad norm: 0.077 | params norm: 11274.382 | params rms: 0.061047 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 19:33:16.119956 | finish at 2026-10-10 19:04:29 + [2026-10-09 23:32:43] iteration 6028/ 6855 | consumed samples: 771584 | elapsed time per iteration (ms): 89945.1 | throughput per GPU (TFLOP/s/GPU): 41.6 | MFU 13.35% | FLOPs estimator: workload-aware | logical tokens/s: 186526.5 | valid tokens/s: 186526.5 | physical tokens/s: 186897.3 | loss tokens/s: 135920.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.682957E-01 | lm-global-supervised-tokens: 1.222542E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.207317E-02 | lm-token-buffer-mib: 5.812969E+01 | global_load_balancing_loss: 9.998127E-01 | loss scale: 1.0 | grad norm: 0.050 | params norm: 11274.333 | params rms: 0.061047 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 20:39:44.591645 | finish at 2026-10-10 20:12:27 + [2026-10-09 23:34:13] iteration 6029/ 6855 | consumed samples: 771712 | elapsed time per iteration (ms): 90322.1 | throughput per GPU (TFLOP/s/GPU): 41.3 | MFU 13.24% | FLOPs estimator: workload-aware | logical tokens/s: 185748.4 | valid tokens/s: 185748.3 | physical tokens/s: 186228.9 | loss tokens/s: 138030.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.016847E-01 | lm-global-supervised-tokens: 1.246720E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.389312E-02 | lm-token-buffer-mib: 5.817797E+01 | global_load_balancing_loss: 9.994271E-01 | loss scale: 1.0 | grad norm: 0.160 | params norm: 11274.285 | params rms: 0.061047 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 20:43:26.064778 | finish at 2026-10-10 20:17:39 + [2026-10-09 23:36:01] iteration 6030/ 6855 | consumed samples: 771840 | elapsed time per iteration (ms): 107345.4 | throughput per GPU (TFLOP/s/GPU): 34.0 | MFU 10.91% | FLOPs estimator: workload-aware | logical tokens/s: 156291.8 | valid tokens/s: 156291.8 | physical tokens/s: 156712.7 | loss tokens/s: 116982.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.489136E-01 | lm-global-supervised-tokens: 1.255749E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.150266E-02 | lm-token-buffer-mib: 5.809724E+01 | global_load_balancing_loss: 9.998651E-01 | loss scale: 1.0 | grad norm: 0.302 | params norm: 11274.237 | params rms: 0.061046 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 0:35:59.950161 | finish at 2026-10-11 00:12:00 + [2026-10-09 23:37:33] iteration 6031/ 6855 | consumed samples: 771968 | elapsed time per iteration (ms): 92632.8 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.53% | FLOPs estimator: workload-aware | logical tokens/s: 181114.5 | valid tokens/s: 181114.5 | physical tokens/s: 181523.9 | loss tokens/s: 128398.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.243674E-01 | lm-global-supervised-tokens: 1.189387E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.355327E-02 | lm-token-buffer-mib: 5.803831E+01 | global_load_balancing_loss: 9.998217E-01 | loss scale: 1.0 | grad norm: 0.224 | params norm: 11274.189 | params rms: 0.061046 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 21:12:09.449484 | finish at 2026-10-10 20:49:43 + [2026-10-09 23:38:57] iteration 6032/ 6855 | consumed samples: 772096 | elapsed time per iteration (ms): 83563.3 | throughput per GPU (TFLOP/s/GPU): 43.5 | MFU 13.95% | FLOPs estimator: workload-aware | logical tokens/s: 200772.5 | valid tokens/s: 200772.5 | physical tokens/s: 201258.7 | loss tokens/s: 149251.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.724495E-01 | lm-global-supervised-tokens: 1.247196E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.146988E-02 | lm-token-buffer-mib: 5.817237E+01 | global_load_balancing_loss: 1.000155E+00 | loss scale: 1.0 | grad norm: 0.225 | params norm: 11274.140 | params rms: 0.061046 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 19:06:12.580312 | finish at 2026-10-10 18:45:09 + [2026-10-09 23:40:28] iteration 6033/ 6855 | consumed samples: 772224 | elapsed time per iteration (ms): 90821.5 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.41% | FLOPs estimator: workload-aware | logical tokens/s: 184726.7 | valid tokens/s: 184726.7 | physical tokens/s: 185126.7 | loss tokens/s: 134180.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.106315E-01 | lm-global-supervised-tokens: 1.218649E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.319008E-02 | lm-token-buffer-mib: 5.812064E+01 | global_load_balancing_loss: 1.000315E+00 | loss scale: 1.0 | grad norm: 0.291 | params norm: 11274.092 | params rms: 0.061046 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 20:44:15.299313 | finish at 2026-10-10 20:24:43 + [2026-10-09 23:41:56] iteration 6034/ 6855 | consumed samples: 772352 | elapsed time per iteration (ms): 88946.8 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.63% | FLOPs estimator: workload-aware | logical tokens/s: 188620.0 | valid tokens/s: 188620.0 | physical tokens/s: 188990.4 | loss tokens/s: 140350.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.084838E-01 | lm-global-supervised-tokens: 1.248370E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.584112E-02 | lm-token-buffer-mib: 5.813671E+01 | global_load_balancing_loss: 1.001122E+00 | loss scale: 1.0 | grad norm: 0.094 | params norm: 11274.046 | params rms: 0.061045 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 20:17:05.338454 | finish at 2026-10-10 19:59:02 + [2026-10-09 23:43:24] iteration 6035/ 6855 | consumed samples: 772480 | elapsed time per iteration (ms): 87419.7 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.55% | FLOPs estimator: workload-aware | logical tokens/s: 191915.1 | valid tokens/s: 191915.1 | physical tokens/s: 192384.4 | loss tokens/s: 143938.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.179727E-01 | lm-global-supervised-tokens: 1.258302E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.359471E-02 | lm-token-buffer-mib: 5.813586E+01 | global_load_balancing_loss: 1.000278E+00 | loss scale: 1.0 | grad norm: 0.082 | params norm: 11274.002 | params rms: 0.061045 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 19:54:44.163504 | finish at 2026-10-10 19:38:08 + [2026-10-09 23:44:53] iteration 6036/ 6855 | consumed samples: 772608 | elapsed time per iteration (ms): 89250.5 | throughput per GPU (TFLOP/s/GPU): 43.3 | MFU 13.87% | FLOPs estimator: workload-aware | logical tokens/s: 187978.6 | valid tokens/s: 187978.6 | physical tokens/s: 188433.6 | loss tokens/s: 136005.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.298500E-01 | lm-global-supervised-tokens: 1.213857E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.174047E-02 | lm-token-buffer-mib: 5.809156E+01 | global_load_balancing_loss: 1.000373E+00 | loss scale: 1.0 | grad norm: 0.142 | params norm: 11273.957 | params rms: 0.061045 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 20:18:16.154979 | finish at 2026-10-10 20:03:09 + [2026-10-09 23:46:22] iteration 6037/ 6855 | consumed samples: 772736 | elapsed time per iteration (ms): 89052.4 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.49% | FLOPs estimator: workload-aware | logical tokens/s: 188397.0 | valid tokens/s: 188397.0 | physical tokens/s: 188800.6 | loss tokens/s: 142804.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.232101E-01 | lm-global-supervised-tokens: 1.271711E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.584974E-02 | lm-token-buffer-mib: 5.825430E+01 | global_load_balancing_loss: 1.000802E+00 | loss scale: 1.0 | grad norm: 0.042 | params norm: 11273.912 | params rms: 0.061045 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 20:14:04.883574 | finish at 2026-10-10 20:00:27 + [2026-10-09 23:47:47] iteration 6038/ 6855 | consumed samples: 772864 | elapsed time per iteration (ms): 84858.9 | throughput per GPU (TFLOP/s/GPU): 43.5 | MFU 13.94% | FLOPs estimator: workload-aware | logical tokens/s: 197706.6 | valid tokens/s: 197706.6 | physical tokens/s: 198053.1 | loss tokens/s: 145425.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.343473E-01 | lm-global-supervised-tokens: 1.234066E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.631260E-02 | lm-token-buffer-mib: 5.807232E+01 | global_load_balancing_loss: 9.998789E-01 | loss scale: 1.0 | grad norm: 0.068 | params norm: 11273.868 | params rms: 0.061044 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 19:15:29.752718 | finish at 2026-10-10 19:03:17 + [2026-10-09 23:49:12] iteration 6039/ 6855 | consumed samples: 772992 | elapsed time per iteration (ms): 85185.6 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.70% | FLOPs estimator: workload-aware | logical tokens/s: 196948.3 | valid tokens/s: 196948.3 | physical tokens/s: 197358.3 | loss tokens/s: 150153.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.612452E-01 | lm-global-supervised-tokens: 1.279094E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.266075E-02 | lm-token-buffer-mib: 5.816642E+01 | global_load_balancing_loss: 9.996319E-01 | loss scale: 1.0 | grad norm: 0.085 | params norm: 11273.826 | params rms: 0.061044 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 19:18:31.428234 | finish at 2026-10-10 19:07:44 + [2026-10-09 23:51:03] iteration 6040/ 6855 | consumed samples: 773120 | elapsed time per iteration (ms): 111018.6 | throughput per GPU (TFLOP/s/GPU): 33.3 | MFU 10.68% | FLOPs estimator: workload-aware | logical tokens/s: 151119.9 | valid tokens/s: 151119.9 | physical tokens/s: 151554.5 | loss tokens/s: 112408.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.514490E-01 | lm-global-supervised-tokens: 1.247948E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.473261E-02 | lm-token-buffer-mib: 5.820985E+01 | global_load_balancing_loss: 1.000369E+00 | loss scale: 1.0 | grad norm: 0.045 | params norm: 11273.784 | params rms: 0.061044 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 1:08:00.171620 | finish at 2026-10-11 00:59:03 +times across ranks (ms): + save-checkpoint: + rank 0: 24271.07 + rank 1: 24271.12 + rank 2: 24271.10 + rank 3: 24270.92 + rank 4: 24271.07 + rank 5: 24271.10 + rank 6: 24271.05 + rank 7: 24271.06 + rank 8: 24271.82 + rank 9: 24271.82 + rank 10: 24271.80 + rank 11: 24271.80 + rank 12: 24271.80 + rank 13: 24271.81 + rank 14: 24271.79 + rank 15: 24271.78 + rank 16: 24271.37 + rank 17: 24271.37 + rank 18: 24271.34 + rank 19: 24271.48 + rank 20: 24271.32 + rank 21: 24271.38 + rank 22: 24271.33 + rank 23: 24271.34 + rank 24: 24271.75 + rank 25: 24271.65 + rank 26: 24271.75 + rank 27: 24271.73 + rank 28: 24271.61 + rank 29: 24271.71 + rank 30: 24271.65 + rank 31: 24271.70 + rank 32: 24271.51 + rank 33: 24271.51 + rank 34: 24271.63 + rank 35: 24271.46 + rank 36: 24271.44 + rank 37: 24271.31 + rank 38: 24271.45 + rank 39: 24271.50 + rank 40: 24271.36 + rank 41: 24271.33 + rank 42: 24271.39 + rank 43: 24271.37 + rank 44: 24271.32 + rank 45: 24271.32 + rank 46: 24271.34 + rank 47: 24271.33 + rank 48: 24271.31 + rank 49: 24271.29 + rank 50: 24271.35 + rank 51: 24271.28 + rank 52: 24271.47 + rank 53: 24271.42 + rank 54: 24271.41 + rank 55: 24271.38 + rank 56: 24270.85 + rank 57: 24271.00 + rank 58: 24270.98 + rank 59: 24270.97 + rank 60: 24270.96 + rank 61: 24270.99 + rank 62: 24270.97 + rank 63: 24270.99 + [2026-10-09 23:53:32] iteration 6041/ 6855 | consumed samples: 773248 | elapsed time per iteration (ms): 118815.1 | throughput per GPU (TFLOP/s/GPU): 31.8 | MFU 10.19% | FLOPs estimator: workload-aware | logical tokens/s: 141204.2 | valid tokens/s: 141204.2 | physical tokens/s: 141478.0 | loss tokens/s: 104945.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.068406E-01 | lm-global-supervised-tokens: 1.246914E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.230475E+00 | lm-token-buffer-mib: 5.808118E+01 | global_load_balancing_loss: 1.001150E+00 | loss scale: 1.0 | grad norm: 0.058 | params norm: 11273.742 | params rms: 0.061044 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 2:51:55.460894 | finish at 2026-10-11 02:45:28 + [2026-10-09 23:55:07] iteration 6042/ 6855 | consumed samples: 773376 | elapsed time per iteration (ms): 94933.8 | throughput per GPU (TFLOP/s/GPU): 39.1 | MFU 12.53% | FLOPs estimator: workload-aware | logical tokens/s: 176725.1 | valid tokens/s: 176725.1 | physical tokens/s: 177087.0 | loss tokens/s: 127861.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.553441E-01 | lm-global-supervised-tokens: 1.213841E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.435673E-02 | lm-token-buffer-mib: 5.810141E+01 | global_load_balancing_loss: 1.000659E+00 | loss scale: 1.0 | grad norm: 0.073 | params norm: 11273.699 | params rms: 0.061043 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 21:26:21.207685 | finish at 2026-10-10 21:21:28 + [2026-10-09 23:56:33] iteration 6043/ 6855 | consumed samples: 773504 | elapsed time per iteration (ms): 86069.1 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.58% | FLOPs estimator: workload-aware | logical tokens/s: 194927.0 | valid tokens/s: 194926.9 | physical tokens/s: 195365.4 | loss tokens/s: 144468.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.553150E-01 | lm-global-supervised-tokens: 1.243428E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.138096E-02 | lm-token-buffer-mib: 5.814075E+01 | global_load_balancing_loss: 1.000277E+00 | loss scale: 1.0 | grad norm: 0.035 | params norm: 11273.656 | params rms: 0.061043 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 19:24:48.128094 | finish at 2026-10-10 19:21:21 + [2026-10-09 23:58:00] iteration 6044/ 6855 | consumed samples: 773632 | elapsed time per iteration (ms): 86777.0 | throughput per GPU (TFLOP/s/GPU): 43.1 | MFU 13.80% | FLOPs estimator: workload-aware | logical tokens/s: 193336.7 | valid tokens/s: 193336.7 | physical tokens/s: 193768.8 | loss tokens/s: 142734.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.780929E-01 | lm-global-supervised-tokens: 1.238607E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.136793E-02 | lm-token-buffer-mib: 5.808266E+01 | global_load_balancing_loss: 9.992958E-01 | loss scale: 1.0 | grad norm: 0.057 | params norm: 11273.613 | params rms: 0.061043 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 19:32:56.146186 | finish at 2026-10-10 19:30:56 + [2026-10-09 23:59:26] iteration 6045/ 6855 | consumed samples: 773760 | elapsed time per iteration (ms): 86128.0 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.51% | FLOPs estimator: workload-aware | logical tokens/s: 194793.1 | valid tokens/s: 194793.0 | physical tokens/s: 195545.2 | loss tokens/s: 143827.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.878528E-01 | lm-global-supervised-tokens: 1.238762E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.094066E-02 | lm-token-buffer-mib: 5.810800E+01 | global_load_balancing_loss: 9.994011E-01 | loss scale: 1.0 | grad norm: 0.101 | params norm: 11273.570 | params rms: 0.061043 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 19:22:43.709950 | finish at 2026-10-10 19:22:10 + [2026-10-10 00:00:55] iteration 6046/ 6855 | consumed samples: 773888 | elapsed time per iteration (ms): 89099.1 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.39% | FLOPs estimator: workload-aware | logical tokens/s: 188297.4 | valid tokens/s: 188297.4 | physical tokens/s: 188603.3 | loss tokens/s: 137095.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.687031E-01 | lm-global-supervised-tokens: 1.221511E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.140385E-02 | lm-token-buffer-mib: 5.805958E+01 | global_load_balancing_loss: 1.000032E+00 | loss scale: 1.0 | grad norm: 0.073 | params norm: 11273.526 | params rms: 0.061042 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 20:01:21.151932 | finish at 2026-10-10 20:02:16 + [2026-10-10 00:02:25] iteration 6047/ 6855 | consumed samples: 774016 | elapsed time per iteration (ms): 89766.8 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.54% | FLOPs estimator: workload-aware | logical tokens/s: 186897.6 | valid tokens/s: 186897.6 | physical tokens/s: 187297.8 | loss tokens/s: 134318.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.158281E-01 | lm-global-supervised-tokens: 1.205737E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.369191E-02 | lm-token-buffer-mib: 5.805224E+01 | global_load_balancing_loss: 9.996701E-01 | loss scale: 1.0 | grad norm: 0.152 | params norm: 11273.481 | params rms: 0.061042 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 20:08:51.537931 | finish at 2026-10-10 20:11:16 + [2026-10-10 00:03:53] iteration 6048/ 6855 | consumed samples: 774144 | elapsed time per iteration (ms): 88572.4 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.41% | FLOPs estimator: workload-aware | logical tokens/s: 189418.0 | valid tokens/s: 189418.0 | physical tokens/s: 189951.3 | loss tokens/s: 137805.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.387376E-01 | lm-global-supervised-tokens: 1.220580E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.746436E-02 | lm-token-buffer-mib: 5.827090E+01 | global_load_balancing_loss: 1.001058E+00 | loss scale: 1.0 | grad norm: 0.134 | params norm: 11273.434 | params rms: 0.061042 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 19:51:17.894166 | finish at 2026-10-10 19:55:11 + [2026-10-10 00:05:21] iteration 6049/ 6855 | consumed samples: 774272 | elapsed time per iteration (ms): 87822.5 | throughput per GPU (TFLOP/s/GPU): 41.3 | MFU 13.25% | FLOPs estimator: workload-aware | logical tokens/s: 191035.0 | valid tokens/s: 191034.9 | physical tokens/s: 191754.4 | loss tokens/s: 144211.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.228247E-01 | lm-global-supervised-tokens: 1.266498E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.344266E-02 | lm-token-buffer-mib: 5.832936E+01 | global_load_balancing_loss: 1.002175E+00 | loss scale: 1.0 | grad norm: 0.131 | params norm: 11273.388 | params rms: 0.061042 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 19:39:44.957668 | finish at 2026-10-10 19:45:06 + [2026-10-10 00:07:11] iteration 6050/ 6855 | consumed samples: 774400 | elapsed time per iteration (ms): 109403.0 | throughput per GPU (TFLOP/s/GPU): 34.1 | MFU 10.92% | FLOPs estimator: workload-aware | logical tokens/s: 153352.1 | valid tokens/s: 153352.1 | physical tokens/s: 153695.1 | loss tokens/s: 115048.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.035524E-01 | lm-global-supervised-tokens: 1.258662E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.552819E-02 | lm-token-buffer-mib: 5.813200E+01 | global_load_balancing_loss: 1.000484E+00 | loss scale: 1.0 | grad norm: 0.242 | params norm: 11273.341 | params rms: 0.061041 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 0:27:49.410487 | finish at 2026-10-11 00:35:00 + [2026-10-10 00:08:38] iteration 6051/ 6855 | consumed samples: 774528 | elapsed time per iteration (ms): 86910.4 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.59% | FLOPs estimator: workload-aware | logical tokens/s: 193039.7 | valid tokens/s: 193039.6 | physical tokens/s: 193465.4 | loss tokens/s: 137677.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.403964E-01 | lm-global-supervised-tokens: 1.196561E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.163368E-02 | lm-token-buffer-mib: 5.805738E+01 | global_load_balancing_loss: 1.000716E+00 | loss scale: 1.0 | grad norm: 0.091 | params norm: 11273.294 | params rms: 0.061041 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 19:24:35.927218 | finish at 2026-10-10 19:33:14 + [2026-10-10 00:10:09] iteration 6052/ 6855 | consumed samples: 774656 | elapsed time per iteration (ms): 91310.9 | throughput per GPU (TFLOP/s/GPU): 41.2 | MFU 13.20% | FLOPs estimator: workload-aware | logical tokens/s: 183737.3 | valid tokens/s: 183737.3 | physical tokens/s: 184094.8 | loss tokens/s: 124054.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.185927E-01 | lm-global-supervised-tokens: 1.132753E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.114415E-02 | lm-token-buffer-mib: 5.805849E+01 | global_load_balancing_loss: 1.000334E+00 | loss scale: 1.0 | grad norm: 0.094 | params norm: 11273.248 | params rms: 0.061041 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 20:22:02.634108 | finish at 2026-10-10 20:32:12 + [2026-10-10 00:11:37] iteration 6053/ 6855 | consumed samples: 774784 | elapsed time per iteration (ms): 87816.4 | throughput per GPU (TFLOP/s/GPU): 41.6 | MFU 13.32% | FLOPs estimator: workload-aware | logical tokens/s: 191048.2 | valid tokens/s: 191048.2 | physical tokens/s: 191632.7 | loss tokens/s: 142723.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.487201E-01 | lm-global-supervised-tokens: 1.253343E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.180255E-02 | lm-token-buffer-mib: 5.813571E+01 | global_load_balancing_loss: 9.992318E-01 | loss scale: 1.0 | grad norm: 0.098 | params norm: 11273.202 | params rms: 0.061041 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 19:33:48.766417 | finish at 2026-10-10 19:45:26 + [2026-10-10 00:13:07] iteration 6054/ 6855 | consumed samples: 774912 | elapsed time per iteration (ms): 89788.9 | throughput per GPU (TFLOP/s/GPU): 41.1 | MFU 13.17% | FLOPs estimator: workload-aware | logical tokens/s: 186851.6 | valid tokens/s: 186851.6 | physical tokens/s: 187259.4 | loss tokens/s: 137311.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.237103E-01 | lm-global-supervised-tokens: 1.232907E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.574887E-02 | lm-token-buffer-mib: 5.824144E+01 | global_load_balancing_loss: 9.984604E-01 | loss scale: 1.0 | grad norm: 0.079 | params norm: 11273.154 | params rms: 0.061040 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 19:58:40.886093 | finish at 2026-10-10 20:11:47 + [2026-10-10 00:14:34] iteration 6055/ 6855 | consumed samples: 775040 | elapsed time per iteration (ms): 87073.1 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.43% | FLOPs estimator: workload-aware | logical tokens/s: 192679.6 | valid tokens/s: 192679.6 | physical tokens/s: 193271.3 | loss tokens/s: 138068.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.776758E-01 | lm-global-supervised-tokens: 1.202208E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.478389E-02 | lm-token-buffer-mib: 5.808975E+01 | global_load_balancing_loss: 1.000299E+00 | loss scale: 1.0 | grad norm: 0.065 | params norm: 11273.105 | params rms: 0.061040 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 19:20:58.488655 | finish at 2026-10-10 19:35:32 + [2026-10-10 00:15:58] iteration 6056/ 6855 | consumed samples: 775168 | elapsed time per iteration (ms): 84645.0 | throughput per GPU (TFLOP/s/GPU): 42.8 | MFU 13.73% | FLOPs estimator: workload-aware | logical tokens/s: 198206.6 | valid tokens/s: 198206.6 | physical tokens/s: 198694.6 | loss tokens/s: 152883.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.797202E-01 | lm-global-supervised-tokens: 1.294084E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.246346E-02 | lm-token-buffer-mib: 5.804360E+01 | global_load_balancing_loss: 9.996554E-01 | loss scale: 1.0 | grad norm: 0.042 | params norm: 11273.055 | params rms: 0.061040 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 18:47:11.385083 | finish at 2026-10-10 19:03:10 + [2026-10-10 00:17:29] iteration 6057/ 6855 | consumed samples: 775296 | elapsed time per iteration (ms): 90361.3 | throughput per GPU (TFLOP/s/GPU): 40.6 | MFU 13.01% | FLOPs estimator: workload-aware | logical tokens/s: 185667.1 | valid tokens/s: 185667.1 | physical tokens/s: 186031.4 | loss tokens/s: 131457.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.100440E-01 | lm-global-supervised-tokens: 1.187870E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.456429E-02 | lm-token-buffer-mib: 5.816663E+01 | global_load_balancing_loss: 1.000674E+00 | loss scale: 1.0 | grad norm: 0.126 | params norm: 11273.006 | params rms: 0.061040 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 20:01:48.281054 | finish at 2026-10-10 20:19:17 + [2026-10-10 00:18:54] iteration 6058/ 6855 | consumed samples: 775424 | elapsed time per iteration (ms): 84993.7 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.70% | FLOPs estimator: workload-aware | logical tokens/s: 197393.5 | valid tokens/s: 197393.5 | physical tokens/s: 197875.8 | loss tokens/s: 146934.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.313870E-01 | lm-global-supervised-tokens: 1.248848E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.165805E-02 | lm-token-buffer-mib: 5.809093E+01 | global_load_balancing_loss: 1.000919E+00 | loss scale: 1.0 | grad norm: 0.093 | params norm: 11272.955 | params rms: 0.061039 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 18:48:59.954409 | finish at 2026-10-10 19:07:54 + [2026-10-10 00:20:20] iteration 6059/ 6855 | consumed samples: 775552 | elapsed time per iteration (ms): 86698.5 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.77% | FLOPs estimator: workload-aware | logical tokens/s: 193512.2 | valid tokens/s: 193512.2 | physical tokens/s: 193881.3 | loss tokens/s: 140236.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.382016E-01 | lm-global-supervised-tokens: 1.215829E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.153248E-02 | lm-token-buffer-mib: 5.804349E+01 | global_load_balancing_loss: 1.001118E+00 | loss scale: 1.0 | grad norm: 0.080 | params norm: 11272.904 | params rms: 0.061039 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 19:10:11.985818 | finish at 2026-10-10 19:30:32 + [2026-10-10 00:22:07] iteration 6060/ 6855 | consumed samples: 775680 | elapsed time per iteration (ms): 106312.3 | throughput per GPU (TFLOP/s/GPU): 34.1 | MFU 10.92% | FLOPs estimator: workload-aware | logical tokens/s: 157810.0 | valid tokens/s: 157810.0 | physical tokens/s: 158155.3 | loss tokens/s: 119185.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.404085E-01 | lm-global-supervised-tokens: 1.267092E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.143932E-02 | lm-token-buffer-mib: 5.824171E+01 | global_load_balancing_loss: 1.000260E+00 | loss scale: 1.0 | grad norm: 0.085 | params norm: 11272.852 | params rms: 0.061039 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 23:28:38.268807 | finish at 2026-10-10 23:50:45 + [2026-10-10 00:23:35] iteration 6061/ 6855 | consumed samples: 775808 | elapsed time per iteration (ms): 88179.0 | throughput per GPU (TFLOP/s/GPU): 41.7 | MFU 13.38% | FLOPs estimator: workload-aware | logical tokens/s: 190262.4 | valid tokens/s: 190262.4 | physical tokens/s: 190757.5 | loss tokens/s: 150079.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.715562E-01 | lm-global-supervised-tokens: 1.323390E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.859247E-02 | lm-token-buffer-mib: 5.817670E+01 | global_load_balancing_loss: 9.995837E-01 | loss scale: 1.0 | grad norm: 0.155 | params norm: 11272.800 | params rms: 0.061039 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 19:26:54.151288 | finish at 2026-10-10 19:50:29 + [2026-10-10 00:25:04] iteration 6062/ 6855 | consumed samples: 775936 | elapsed time per iteration (ms): 89046.0 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.70% | FLOPs estimator: workload-aware | logical tokens/s: 188410.0 | valid tokens/s: 188410.0 | physical tokens/s: 188916.5 | loss tokens/s: 144742.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.834961E-01 | lm-global-supervised-tokens: 1.288876E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.423734E-02 | lm-token-buffer-mib: 5.803104E+01 | global_load_balancing_loss: 9.995636E-01 | loss scale: 1.0 | grad norm: 0.073 | params norm: 11272.750 | params rms: 0.061038 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 19:36:53.471764 | finish at 2026-10-10 20:01:57 + [2026-10-10 00:26:32] iteration 6063/ 6855 | consumed samples: 776064 | elapsed time per iteration (ms): 88123.4 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.64% | FLOPs estimator: workload-aware | logical tokens/s: 190382.7 | valid tokens/s: 190382.7 | physical tokens/s: 190907.7 | loss tokens/s: 137285.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.362969E-01 | lm-global-supervised-tokens: 1.209805E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.484578E-02 | lm-token-buffer-mib: 5.813516E+01 | global_load_balancing_loss: 1.001402E+00 | loss scale: 1.0 | grad norm: 0.287 | params norm: 11272.700 | params rms: 0.061038 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 19:23:13.742220 | finish at 2026-10-10 19:49:46 + [2026-10-10 00:27:54] iteration 6064/ 6855 | consumed samples: 776192 | elapsed time per iteration (ms): 81810.4 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.77% | FLOPs estimator: workload-aware | logical tokens/s: 205073.9 | valid tokens/s: 205073.9 | physical tokens/s: 205757.7 | loss tokens/s: 151788.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 5.865814E-01 | lm-global-supervised-tokens: 1.241790E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.447879E-02 | lm-token-buffer-mib: 5.810091E+01 | global_load_balancing_loss: 1.001578E+00 | loss scale: 1.0 | grad norm: 0.319 | params norm: 11272.650 | params rms: 0.061038 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 17:58:32.035648 | finish at 2026-10-10 18:26:26 + [2026-10-10 00:29:19] iteration 6065/ 6855 | consumed samples: 776320 | elapsed time per iteration (ms): 85138.2 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.53% | FLOPs estimator: workload-aware | logical tokens/s: 197058.0 | valid tokens/s: 197058.0 | physical tokens/s: 197558.8 | loss tokens/s: 152362.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.270031E-01 | lm-global-supervised-tokens: 1.297188E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.354178E-02 | lm-token-buffer-mib: 5.810138E+01 | global_load_balancing_loss: 9.999238E-01 | loss scale: 1.0 | grad norm: 0.063 | params norm: 11272.602 | params rms: 0.061037 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 18:40:59.210808 | finish at 2026-10-10 19:10:18 + [2026-10-10 00:30:47] iteration 6066/ 6855 | consumed samples: 776448 | elapsed time per iteration (ms): 87998.1 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.59% | FLOPs estimator: workload-aware | logical tokens/s: 190654.2 | valid tokens/s: 190654.2 | physical tokens/s: 191307.3 | loss tokens/s: 143708.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.331217E-01 | lm-global-supervised-tokens: 1.264610E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.454449E-02 | lm-token-buffer-mib: 5.806114E+01 | global_load_balancing_loss: 9.996835E-01 | loss scale: 1.0 | grad norm: 0.182 | params norm: 11272.554 | params rms: 0.061037 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 19:17:10.462182 | finish at 2026-10-10 19:47:57 + [2026-10-10 00:32:13] iteration 6067/ 6855 | consumed samples: 776576 | elapsed time per iteration (ms): 86129.8 | throughput per GPU (TFLOP/s/GPU): 43.1 | MFU 13.80% | FLOPs estimator: workload-aware | logical tokens/s: 194789.5 | valid tokens/s: 194789.5 | physical tokens/s: 195330.6 | loss tokens/s: 143102.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.583381E-01 | lm-global-supervised-tokens: 1.232537E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.182512E-02 | lm-token-buffer-mib: 5.805705E+01 | global_load_balancing_loss: 9.988654E-01 | loss scale: 1.0 | grad norm: 0.098 | params norm: 11272.504 | params rms: 0.061037 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 18:51:10.291482 | finish at 2026-10-10 19:23:23 + [2026-10-10 00:33:41] iteration 6068/ 6855 | consumed samples: 776704 | elapsed time per iteration (ms): 87620.2 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.54% | FLOPs estimator: workload-aware | logical tokens/s: 191476.4 | valid tokens/s: 191476.4 | physical tokens/s: 191941.9 | loss tokens/s: 139929.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.408516E-01 | lm-global-supervised-tokens: 1.226069E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.782265E-02 | lm-token-buffer-mib: 5.804118E+01 | global_load_balancing_loss: 9.992909E-01 | loss scale: 1.0 | grad norm: 0.085 | params norm: 11272.454 | params rms: 0.061037 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 19:09:17.094709 | finish at 2026-10-10 19:42:58 + [2026-10-10 00:35:11] iteration 6069/ 6855 | consumed samples: 776832 | elapsed time per iteration (ms): 90060.2 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.41% | FLOPs estimator: workload-aware | logical tokens/s: 186288.3 | valid tokens/s: 186288.3 | physical tokens/s: 187066.0 | loss tokens/s: 137914.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.838697E-01 | lm-global-supervised-tokens: 1.242064E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.321235E-02 | lm-token-buffer-mib: 5.815368E+01 | global_load_balancing_loss: 9.989254E-01 | loss scale: 1.0 | grad norm: 0.136 | params norm: 11272.406 | params rms: 0.061036 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 19:39:47.329924 | finish at 2026-10-10 20:14:58 + [2026-10-10 00:37:01] iteration 6070/ 6855 | consumed samples: 776960 | elapsed time per iteration (ms): 109935.6 | throughput per GPU (TFLOP/s/GPU): 33.8 | MFU 10.83% | FLOPs estimator: workload-aware | logical tokens/s: 152609.4 | valid tokens/s: 152609.4 | physical tokens/s: 152948.6 | loss tokens/s: 111238.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.337447E-01 | lm-global-supervised-tokens: 1.222906E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.288809E-02 | lm-token-buffer-mib: 5.804855E+01 | global_load_balancing_loss: 1.000806E+00 | loss scale: 1.0 | grad norm: 0.087 | params norm: 11272.356 | params rms: 0.061036 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 23:58:19.472610 | finish at 2026-10-11 00:35:20 + [2026-10-10 00:38:28] iteration 6071/ 6855 | consumed samples: 777088 | elapsed time per iteration (ms): 87307.3 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.39% | FLOPs estimator: workload-aware | logical tokens/s: 192162.7 | valid tokens/s: 192162.6 | physical tokens/s: 192587.0 | loss tokens/s: 142856.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.537344E-01 | lm-global-supervised-tokens: 1.247244E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.175362E-02 | lm-token-buffer-mib: 5.811745E+01 | global_load_balancing_loss: 1.001226E+00 | loss scale: 1.0 | grad norm: 0.061 | params norm: 11272.306 | params rms: 0.061036 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 19:00:48.884579 | finish at 2026-10-10 19:39:17 + [2026-10-10 00:39:55] iteration 6072/ 6855 | consumed samples: 777216 | elapsed time per iteration (ms): 87489.5 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.55% | FLOPs estimator: workload-aware | logical tokens/s: 191762.3 | valid tokens/s: 191762.3 | physical tokens/s: 192268.2 | loss tokens/s: 136390.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.122429E-01 | lm-global-supervised-tokens: 1.193274E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.132186E-02 | lm-token-buffer-mib: 5.811958E+01 | global_load_balancing_loss: 1.001710E+00 | loss scale: 1.0 | grad norm: 0.097 | params norm: 11272.255 | params rms: 0.061036 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 19:01:44.257441 | finish at 2026-10-10 19:41:40 + [2026-10-10 00:41:21] iteration 6073/ 6855 | consumed samples: 777344 | elapsed time per iteration (ms): 85641.2 | throughput per GPU (TFLOP/s/GPU): 43.1 | MFU 13.80% | FLOPs estimator: workload-aware | logical tokens/s: 195900.7 | valid tokens/s: 195900.7 | physical tokens/s: 196452.1 | loss tokens/s: 149915.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.620883E-01 | lm-global-supervised-tokens: 1.283895E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.410217E-02 | lm-token-buffer-mib: 5.815517E+01 | global_load_balancing_loss: 1.001281E+00 | loss scale: 1.0 | grad norm: 0.071 | params norm: 11272.206 | params rms: 0.061035 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 18:36:11.454621 | finish at 2026-10-10 19:17:33 + [2026-10-10 00:42:49] iteration 6074/ 6855 | consumed samples: 777472 | elapsed time per iteration (ms): 87957.0 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.43% | FLOPs estimator: workload-aware | logical tokens/s: 190743.0 | valid tokens/s: 190743.0 | physical tokens/s: 191200.3 | loss tokens/s: 146690.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.009895E-01 | lm-global-supervised-tokens: 1.290242E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.516078E-02 | lm-token-buffer-mib: 5.822232E+01 | global_load_balancing_loss: 9.995700E-01 | loss scale: 1.0 | grad norm: 0.074 | params norm: 11272.157 | params rms: 0.061035 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 19:04:54.440661 | finish at 2026-10-10 19:47:44 + [2026-10-10 00:44:16] iteration 6075/ 6855 | consumed samples: 777600 | elapsed time per iteration (ms): 86509.9 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.76% | FLOPs estimator: workload-aware | logical tokens/s: 193933.5 | valid tokens/s: 193933.5 | physical tokens/s: 194333.3 | loss tokens/s: 144095.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.722639E-01 | lm-global-supervised-tokens: 1.246564E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.117943E-02 | lm-token-buffer-mib: 5.841970E+01 | global_load_balancing_loss: 1.000424E+00 | loss scale: 1.0 | grad norm: 0.050 | params norm: 11272.108 | params rms: 0.061035 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 18:44:37.685623 | finish at 2026-10-10 19:28:53 + [2026-10-10 00:45:46] iteration 6076/ 6855 | consumed samples: 777728 | elapsed time per iteration (ms): 90359.8 | throughput per GPU (TFLOP/s/GPU): 41.6 | MFU 13.33% | FLOPs estimator: workload-aware | logical tokens/s: 185670.2 | valid tokens/s: 185670.2 | physical tokens/s: 186142.5 | loss tokens/s: 134333.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.548264E-01 | lm-global-supervised-tokens: 1.213831E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.137483E-02 | lm-token-buffer-mib: 5.820402E+01 | global_load_balancing_loss: 1.000034E+00 | loss scale: 1.0 | grad norm: 0.047 | params norm: 11272.058 | params rms: 0.061035 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 19:33:10.248247 | finish at 2026-10-10 20:18:56 + [2026-10-10 00:47:14] iteration 6077/ 6855 | consumed samples: 777856 | elapsed time per iteration (ms): 88298.2 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.49% | FLOPs estimator: workload-aware | logical tokens/s: 190006.2 | valid tokens/s: 190006.2 | physical tokens/s: 190478.5 | loss tokens/s: 138744.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.679996E-01 | lm-global-supervised-tokens: 1.225092E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.254948E-02 | lm-token-buffer-mib: 5.810594E+01 | global_load_balancing_loss: 9.993109E-01 | loss scale: 1.0 | grad norm: 0.069 | params norm: 11272.008 | params rms: 0.061034 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 19:04:55.997661 | finish at 2026-10-10 19:52:10 + [2026-10-10 00:48:43] iteration 6078/ 6855 | consumed samples: 777984 | elapsed time per iteration (ms): 88810.4 | throughput per GPU (TFLOP/s/GPU): 41.7 | MFU 13.36% | FLOPs estimator: workload-aware | logical tokens/s: 188910.2 | valid tokens/s: 188910.2 | physical tokens/s: 189440.6 | loss tokens/s: 137785.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.699421E-01 | lm-global-supervised-tokens: 1.223680E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.224055E-02 | lm-token-buffer-mib: 5.829284E+01 | global_load_balancing_loss: 9.985398E-01 | loss scale: 1.0 | grad norm: 0.055 | params norm: 11271.960 | params rms: 0.061034 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 19:10:05.663023 | finish at 2026-10-10 19:58:49 + [2026-10-10 00:50:08] iteration 6079/ 6855 | consumed samples: 778112 | elapsed time per iteration (ms): 85082.1 | throughput per GPU (TFLOP/s/GPU): 42.8 | MFU 13.71% | FLOPs estimator: workload-aware | logical tokens/s: 197188.4 | valid tokens/s: 197188.4 | physical tokens/s: 197608.8 | loss tokens/s: 147093.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.385322E-01 | lm-global-supervised-tokens: 1.251498E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.465255E-02 | lm-token-buffer-mib: 5.820516E+01 | global_load_balancing_loss: 9.991534E-01 | loss scale: 1.0 | grad norm: 0.057 | params norm: 11271.911 | params rms: 0.061034 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 18:20:23.723965 | finish at 2026-10-10 19:10:32 + [2026-10-10 00:52:01] iteration 6080/ 6855 | consumed samples: 778240 | elapsed time per iteration (ms): 113241.7 | throughput per GPU (TFLOP/s/GPU): 33.1 | MFU 10.61% | FLOPs estimator: workload-aware | logical tokens/s: 148153.7 | valid tokens/s: 148153.7 | physical tokens/s: 148546.8 | loss tokens/s: 104114.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.680149E-01 | lm-global-supervised-tokens: 1.179005E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.163259E-02 | lm-token-buffer-mib: 5.825719E+01 | global_load_balancing_loss: 1.000346E+00 | loss scale: 1.0 | grad norm: 0.056 | params norm: 11271.863 | params rms: 0.061033 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1 day, 0:22:42.352002 | finish at 2026-10-11 01:14:44 +times across ranks (ms): + save-checkpoint: + rank 0: 24998.01 + rank 1: 24998.05 + rank 2: 24998.06 + rank 3: 24998.01 + rank 4: 24998.06 + rank 5: 24998.01 + rank 6: 24998.03 + rank 7: 24998.03 + rank 8: 24998.33 + rank 9: 24998.28 + rank 10: 24998.27 + rank 11: 24998.28 + rank 12: 24998.30 + rank 13: 24998.27 + rank 14: 24998.30 + rank 15: 24998.29 + rank 16: 24999.04 + rank 17: 24998.99 + rank 18: 24998.98 + rank 19: 24999.03 + rank 20: 24998.98 + rank 21: 24999.00 + rank 22: 24999.00 + rank 23: 24998.97 + rank 24: 24998.46 + rank 25: 24998.44 + rank 26: 24998.49 + rank 27: 24998.48 + rank 28: 24998.47 + rank 29: 24998.45 + rank 30: 24998.46 + rank 31: 24998.46 + rank 32: 24998.43 + rank 33: 24998.43 + rank 34: 24998.44 + rank 35: 24998.42 + rank 36: 24998.42 + rank 37: 24998.42 + rank 38: 24998.42 + rank 39: 24998.40 + rank 40: 24998.04 + rank 41: 24998.01 + rank 42: 24998.00 + rank 43: 24998.02 + rank 44: 24998.00 + rank 45: 24998.03 + rank 46: 24998.01 + rank 47: 24998.03 + rank 48: 24997.01 + rank 49: 24996.98 + rank 50: 24996.96 + rank 51: 24996.99 + rank 52: 24996.98 + rank 53: 24996.98 + rank 54: 24996.99 + rank 55: 24997.02 + rank 56: 24997.54 + rank 57: 24997.52 + rank 58: 24997.52 + rank 59: 24997.53 + rank 60: 24997.52 + rank 61: 24997.52 + rank 62: 24997.54 + rank 63: 24997.52 + [2026-10-10 00:53:59] iteration 6081/ 6855 | consumed samples: 778368 | elapsed time per iteration (ms): 92266.4 | throughput per GPU (TFLOP/s/GPU): 40.4 | MFU 12.95% | FLOPs estimator: workload-aware | logical tokens/s: 181833.5 | valid tokens/s: 181833.5 | physical tokens/s: 182358.8 | loss tokens/s: 137832.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.079895E-01 | lm-global-supervised-tokens: 1.271730E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.567981E-01 | lm-token-buffer-mib: 5.814130E+01 | global_load_balancing_loss: 1.000682E+00 | loss scale: 1.0 | grad norm: 0.059 | params norm: 11271.814 | params rms: 0.061033 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 19:50:14.182420 | finish at 2026-10-10 20:44:13 + [2026-10-10 00:55:28] iteration 6082/ 6855 | consumed samples: 778496 | elapsed time per iteration (ms): 89166.7 | throughput per GPU (TFLOP/s/GPU): 41.4 | MFU 13.28% | FLOPs estimator: workload-aware | logical tokens/s: 188155.7 | valid tokens/s: 188155.7 | physical tokens/s: 188589.6 | loss tokens/s: 138478.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.839253E-01 | lm-global-supervised-tokens: 1.234764E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.444258E-02 | lm-token-buffer-mib: 5.813783E+01 | global_load_balancing_loss: 1.001521E+00 | loss scale: 1.0 | grad norm: 0.033 | params norm: 11271.765 | params rms: 0.061033 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 19:08:45.834316 | finish at 2026-10-10 20:04:14 + [2026-10-10 00:56:52] iteration 6083/ 6855 | consumed samples: 778624 | elapsed time per iteration (ms): 83397.3 | throughput per GPU (TFLOP/s/GPU): 43.5 | MFU 13.94% | FLOPs estimator: workload-aware | logical tokens/s: 201171.5 | valid tokens/s: 201171.5 | physical tokens/s: 201660.7 | loss tokens/s: 152528.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.611668E-01 | lm-global-supervised-tokens: 1.272042E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.268666E-02 | lm-token-buffer-mib: 5.810169E+01 | global_load_balancing_loss: 1.001016E+00 | loss scale: 1.0 | grad norm: 0.065 | params norm: 11271.715 | params rms: 0.061033 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 17:53:02.693333 | finish at 2026-10-10 18:49:54 + [2026-10-10 00:58:18] iteration 6084/ 6855 | consumed samples: 778752 | elapsed time per iteration (ms): 86151.0 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.69% | FLOPs estimator: workload-aware | logical tokens/s: 194741.8 | valid tokens/s: 194741.7 | physical tokens/s: 195193.7 | loss tokens/s: 146891.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.986453E-01 | lm-global-supervised-tokens: 1.265481E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.179015E-02 | lm-token-buffer-mib: 5.810265E+01 | global_load_balancing_loss: 1.002309E+00 | loss scale: 1.0 | grad norm: 0.067 | params norm: 11271.665 | params rms: 0.061032 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 18:27:02.393445 | finish at 2026-10-10 19:25:20 + [2026-10-10 00:59:42] iteration 6085/ 6855 | consumed samples: 778880 | elapsed time per iteration (ms): 84492.7 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.76% | FLOPs estimator: workload-aware | logical tokens/s: 198564.0 | valid tokens/s: 198564.0 | physical tokens/s: 198855.1 | loss tokens/s: 145131.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.404777E-01 | lm-global-supervised-tokens: 1.226257E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.620916E-02 | lm-token-buffer-mib: 5.807227E+01 | global_load_balancing_loss: 1.000778E+00 | loss scale: 1.0 | grad norm: 0.045 | params norm: 11271.616 | params rms: 0.061032 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 18:04:19.404411 | finish at 2026-10-10 19:04:02 + [2026-10-10 01:01:12] iteration 6086/ 6855 | consumed samples: 779008 | elapsed time per iteration (ms): 89436.2 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.60% | FLOPs estimator: workload-aware | logical tokens/s: 187588.1 | valid tokens/s: 187588.0 | physical tokens/s: 188140.9 | loss tokens/s: 135425.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.301267E-01 | lm-global-supervised-tokens: 1.211189E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.133795E-02 | lm-token-buffer-mib: 5.807531E+01 | global_load_balancing_loss: 1.000362E+00 | loss scale: 1.0 | grad norm: 0.096 | params norm: 11271.565 | params rms: 0.061032 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 19:06:16.435892 | finish at 2026-10-10 20:07:28 + [2026-10-10 01:02:36] iteration 6087/ 6855 | consumed samples: 779136 | elapsed time per iteration (ms): 84456.4 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.65% | FLOPs estimator: workload-aware | logical tokens/s: 198648.9 | valid tokens/s: 198648.9 | physical tokens/s: 199157.8 | loss tokens/s: 152626.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.763939E-01 | lm-global-supervised-tokens: 1.289023E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.670471E-02 | lm-token-buffer-mib: 5.829559E+01 | global_load_balancing_loss: 9.982945E-01 | loss scale: 1.0 | grad norm: 0.133 | params norm: 11271.516 | params rms: 0.061032 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 18:01:02.491150 | finish at 2026-10-10 19:03:39 + [2026-10-10 01:04:03] iteration 6088/ 6855 | consumed samples: 779264 | elapsed time per iteration (ms): 87238.7 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.61% | FLOPs estimator: workload-aware | logical tokens/s: 192313.2 | valid tokens/s: 192313.2 | physical tokens/s: 192749.3 | loss tokens/s: 139019.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.718638E-01 | lm-global-supervised-tokens: 1.212786E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.894942E-02 | lm-token-buffer-mib: 5.817058E+01 | global_load_balancing_loss: 9.988387E-01 | loss scale: 1.0 | grad norm: 0.149 | params norm: 11271.468 | params rms: 0.061031 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 18:35:12.079725 | finish at 2026-10-10 19:39:15 + [2026-10-10 01:05:31] iteration 6089/ 6855 | consumed samples: 779392 | elapsed time per iteration (ms): 87283.5 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.53% | FLOPs estimator: workload-aware | logical tokens/s: 192214.9 | valid tokens/s: 192214.9 | physical tokens/s: 192675.5 | loss tokens/s: 143036.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.400431E-01 | lm-global-supervised-tokens: 1.248472E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.174414E-02 | lm-token-buffer-mib: 5.808810E+01 | global_load_balancing_loss: 9.992332E-01 | loss scale: 1.0 | grad norm: 0.034 | params norm: 11271.420 | params rms: 0.061031 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 18:34:19.178316 | finish at 2026-10-10 19:39:50 + [2026-10-10 01:07:20] iteration 6090/ 6855 | consumed samples: 779520 | elapsed time per iteration (ms): 109449.2 | throughput per GPU (TFLOP/s/GPU): 33.2 | MFU 10.66% | FLOPs estimator: workload-aware | logical tokens/s: 153287.5 | valid tokens/s: 153287.5 | physical tokens/s: 153618.9 | loss tokens/s: 114434.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.293917E-01 | lm-global-supervised-tokens: 1.252475E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.316955E-02 | lm-token-buffer-mib: 5.822325E+01 | global_load_balancing_loss: 1.000929E+00 | loss scale: 1.0 | grad norm: 0.214 | params norm: 11271.372 | params rms: 0.061031 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 23:15:28.652344 | finish at 2026-10-11 00:22:49 + [2026-10-10 01:08:50] iteration 6091/ 6855 | consumed samples: 779648 | elapsed time per iteration (ms): 89892.4 | throughput per GPU (TFLOP/s/GPU): 41.1 | MFU 13.17% | FLOPs estimator: workload-aware | logical tokens/s: 186636.0 | valid tokens/s: 186636.0 | physical tokens/s: 186968.1 | loss tokens/s: 132715.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.387091E-01 | lm-global-supervised-tokens: 1.193007E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.143372E-02 | lm-token-buffer-mib: 5.802994E+01 | global_load_balancing_loss: 1.000600E+00 | loss scale: 1.0 | grad norm: 0.151 | params norm: 11271.324 | params rms: 0.061031 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 19:04:37.793438 | finish at 2026-10-10 20:13:28 + [2026-10-10 01:10:19] iteration 6092/ 6855 | consumed samples: 779776 | elapsed time per iteration (ms): 89011.3 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.66% | FLOPs estimator: workload-aware | logical tokens/s: 188483.5 | valid tokens/s: 188483.5 | physical tokens/s: 189013.5 | loss tokens/s: 139368.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.297998E-01 | lm-global-supervised-tokens: 1.240540E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.304176E-02 | lm-token-buffer-mib: 5.829591E+01 | global_load_balancing_loss: 1.001795E+00 | loss scale: 1.0 | grad norm: 0.128 | params norm: 11271.278 | params rms: 0.061030 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 18:51:55.634155 | finish at 2026-10-10 20:02:15 + [2026-10-10 01:11:48] iteration 6093/ 6855 | consumed samples: 779904 | elapsed time per iteration (ms): 89080.6 | throughput per GPU (TFLOP/s/GPU): 41.6 | MFU 13.35% | FLOPs estimator: workload-aware | logical tokens/s: 188337.4 | valid tokens/s: 188337.4 | physical tokens/s: 188670.2 | loss tokens/s: 136145.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.119311E-01 | lm-global-supervised-tokens: 1.212789E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.125415E-02 | lm-token-buffer-mib: 5.807133E+01 | global_load_balancing_loss: 1.001565E+00 | loss scale: 1.0 | grad norm: 0.046 | params norm: 11271.231 | params rms: 0.061030 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 18:51:19.445014 | finish at 2026-10-10 20:03:07 + [2026-10-10 01:13:16] iteration 6094/ 6855 | consumed samples: 780032 | elapsed time per iteration (ms): 87772.9 | throughput per GPU (TFLOP/s/GPU): 41.6 | MFU 13.33% | FLOPs estimator: workload-aware | logical tokens/s: 191142.8 | valid tokens/s: 191142.8 | physical tokens/s: 191658.7 | loss tokens/s: 149890.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.907051E-01 | lm-global-supervised-tokens: 1.315634E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.121062E-02 | lm-token-buffer-mib: 5.807225E+01 | global_load_balancing_loss: 9.997095E-01 | loss scale: 1.0 | grad norm: 0.189 | params norm: 11271.183 | params rms: 0.061030 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 18:33:15.211271 | finish at 2026-10-10 19:46:31 + [2026-10-10 01:14:45] iteration 6095/ 6855 | consumed samples: 780160 | elapsed time per iteration (ms): 89096.6 | throughput per GPU (TFLOP/s/GPU): 41.6 | MFU 13.33% | FLOPs estimator: workload-aware | logical tokens/s: 188303.1 | valid tokens/s: 188303.1 | physical tokens/s: 188673.6 | loss tokens/s: 135828.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.794543E-01 | lm-global-supervised-tokens: 1.210189E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.149214E-02 | lm-token-buffer-mib: 5.819440E+01 | global_load_balancing_loss: 9.997838E-01 | loss scale: 1.0 | grad norm: 0.040 | params norm: 11271.133 | params rms: 0.061030 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 18:48:33.418760 | finish at 2026-10-10 20:03:18 + [2026-10-10 01:16:13] iteration 6096/ 6855 | consumed samples: 780288 | elapsed time per iteration (ms): 87668.6 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.48% | FLOPs estimator: workload-aware | logical tokens/s: 191370.5 | valid tokens/s: 191370.4 | physical tokens/s: 191859.4 | loss tokens/s: 142123.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.706580E-01 | lm-global-supervised-tokens: 1.245975E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.137963E-02 | lm-token-buffer-mib: 5.805408E+01 | global_load_balancing_loss: 9.986072E-01 | loss scale: 1.0 | grad norm: 0.158 | params norm: 11271.084 | params rms: 0.061029 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 18:29:00.477234 | finish at 2026-10-10 19:45:13 + [2026-10-10 01:17:39] iteration 6097/ 6855 | consumed samples: 780416 | elapsed time per iteration (ms): 86836.0 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.57% | FLOPs estimator: workload-aware | logical tokens/s: 193205.1 | valid tokens/s: 193205.1 | physical tokens/s: 193675.7 | loss tokens/s: 142741.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.452215E-01 | lm-global-supervised-tokens: 1.239509E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.178989E-02 | lm-token-buffer-mib: 5.814443E+01 | global_load_balancing_loss: 1.000195E+00 | loss scale: 1.0 | grad norm: 0.103 | params norm: 11271.036 | params rms: 0.061029 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 18:17:01.676950 | finish at 2026-10-10 19:34:41 + [2026-10-10 01:19:04] iteration 6098/ 6855 | consumed samples: 780544 | elapsed time per iteration (ms): 84409.0 | throughput per GPU (TFLOP/s/GPU): 42.8 | MFU 13.72% | FLOPs estimator: workload-aware | logical tokens/s: 198760.0 | valid tokens/s: 198760.0 | physical tokens/s: 199353.3 | loss tokens/s: 153199.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.541131E-01 | lm-global-supervised-tokens: 1.293139E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.343522E-02 | lm-token-buffer-mib: 5.824873E+01 | global_load_balancing_loss: 9.986354E-01 | loss scale: 1.0 | grad norm: 0.209 | params norm: 11270.989 | params rms: 0.061029 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 17:44:57.614925 | finish at 2026-10-10 19:04:01 + [2026-10-10 01:20:28] iteration 6099/ 6855 | consumed samples: 780672 | elapsed time per iteration (ms): 84298.2 | throughput per GPU (TFLOP/s/GPU): 43.6 | MFU 13.97% | FLOPs estimator: workload-aware | logical tokens/s: 199021.5 | valid tokens/s: 199021.5 | physical tokens/s: 199373.6 | loss tokens/s: 150020.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.758194E-01 | lm-global-supervised-tokens: 1.264643E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.400731E-02 | lm-token-buffer-mib: 5.820201E+01 | global_load_balancing_loss: 1.000887E+00 | loss scale: 1.0 | grad norm: 0.126 | params norm: 11270.942 | params rms: 0.061028 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 17:42:09.420373 | finish at 2026-10-10 19:02:38 + [2026-10-10 01:22:16] iteration 6100/ 6855 | consumed samples: 780800 | elapsed time per iteration (ms): 107429.9 | throughput per GPU (TFLOP/s/GPU): 34.1 | MFU 10.94% | FLOPs estimator: workload-aware | logical tokens/s: 156168.3 | valid tokens/s: 156168.3 | physical tokens/s: 156394.1 | loss tokens/s: 116099.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.874264E-01 | lm-global-supervised-tokens: 1.247254E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.299550E-02 | lm-token-buffer-mib: 5.813271E+01 | global_load_balancing_loss: 1.001374E+00 | loss scale: 1.0 | grad norm: 0.106 | params norm: 11270.894 | params rms: 0.061028 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 22:31:49.538627 | finish at 2026-10-10 23:54:05 + [2026-10-10 01:23:45] iteration 6101/ 6855 | consumed samples: 780928 | elapsed time per iteration (ms): 88997.4 | throughput per GPU (TFLOP/s/GPU): 41.1 | MFU 13.18% | FLOPs estimator: workload-aware | logical tokens/s: 188513.1 | valid tokens/s: 188513.1 | physical tokens/s: 189050.9 | loss tokens/s: 143772.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.882437E-01 | lm-global-supervised-tokens: 1.279541E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.389715E-02 | lm-token-buffer-mib: 5.805899E+01 | global_load_balancing_loss: 1.000153E+00 | loss scale: 1.0 | grad norm: 0.046 | params norm: 11270.848 | params rms: 0.061028 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 18:38:24.057251 | finish at 2026-10-10 20:02:09 + [2026-10-10 01:25:12] iteration 6102/ 6855 | consumed samples: 781056 | elapsed time per iteration (ms): 87302.5 | throughput per GPU (TFLOP/s/GPU): 41.1 | MFU 13.17% | FLOPs estimator: workload-aware | logical tokens/s: 192173.0 | valid tokens/s: 192173.0 | physical tokens/s: 192731.7 | loss tokens/s: 148057.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.433749E-01 | lm-global-supervised-tokens: 1.292580E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.506300E-02 | lm-token-buffer-mib: 5.810914E+01 | global_load_balancing_loss: 1.000486E+00 | loss scale: 1.0 | grad norm: 0.093 | params norm: 11270.799 | params rms: 0.061028 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 18:15:38.753783 | finish at 2026-10-10 19:40:51 + [2026-10-10 01:26:34] iteration 6103/ 6855 | consumed samples: 781184 | elapsed time per iteration (ms): 82542.1 | throughput per GPU (TFLOP/s/GPU): 43.2 | MFU 13.86% | FLOPs estimator: workload-aware | logical tokens/s: 203256.3 | valid tokens/s: 203256.2 | physical tokens/s: 203826.5 | loss tokens/s: 156605.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.906799E-01 | lm-global-supervised-tokens: 1.292654E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.702682E-02 | lm-token-buffer-mib: 5.813731E+01 | global_load_balancing_loss: 1.000585E+00 | loss scale: 1.0 | grad norm: 0.068 | params norm: 11270.753 | params rms: 0.061027 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 17:14:31.665440 | finish at 2026-10-10 18:41:06 + [2026-10-10 01:28:03] iteration 6104/ 6855 | consumed samples: 781312 | elapsed time per iteration (ms): 88593.0 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.63% | FLOPs estimator: workload-aware | logical tokens/s: 189373.6 | valid tokens/s: 189373.6 | physical tokens/s: 189915.1 | loss tokens/s: 134492.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.286042E-01 | lm-global-supervised-tokens: 1.191506E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.143714E-02 | lm-token-buffer-mib: 5.810812E+01 | global_load_balancing_loss: 1.001110E+00 | loss scale: 1.0 | grad norm: 0.052 | params norm: 11270.707 | params rms: 0.061027 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 18:28:53.340803 | finish at 2026-10-10 19:56:56 + [2026-10-10 01:29:30] iteration 6105/ 6855 | consumed samples: 781440 | elapsed time per iteration (ms): 86975.1 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.51% | FLOPs estimator: workload-aware | logical tokens/s: 192896.8 | valid tokens/s: 192896.8 | physical tokens/s: 193338.5 | loss tokens/s: 145849.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.554139E-01 | lm-global-supervised-tokens: 1.268529E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.160657E-02 | lm-token-buffer-mib: 5.805333E+01 | global_load_balancing_loss: 9.997571E-01 | loss scale: 1.0 | grad norm: 0.032 | params norm: 11270.658 | params rms: 0.061027 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 18:07:11.299281 | finish at 2026-10-10 19:36:41 + [2026-10-10 01:31:01] iteration 6106/ 6855 | consumed samples: 781568 | elapsed time per iteration (ms): 90823.2 | throughput per GPU (TFLOP/s/GPU): 40.9 | MFU 13.10% | FLOPs estimator: workload-aware | logical tokens/s: 184723.5 | valid tokens/s: 184723.4 | physical tokens/s: 185080.3 | loss tokens/s: 134106.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.266898E-01 | lm-global-supervised-tokens: 1.217999E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.287084E-02 | lm-token-buffer-mib: 5.810472E+01 | global_load_balancing_loss: 9.997708E-01 | loss scale: 1.0 | grad norm: 0.055 | params norm: 11270.611 | params rms: 0.061027 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 18:53:46.585541 | finish at 2026-10-10 20:24:47 + [2026-10-10 01:32:24] iteration 6107/ 6855 | consumed samples: 781696 | elapsed time per iteration (ms): 83479.9 | throughput per GPU (TFLOP/s/GPU): 43.1 | MFU 13.82% | FLOPs estimator: workload-aware | logical tokens/s: 200973.2 | valid tokens/s: 200973.1 | physical tokens/s: 201387.4 | loss tokens/s: 152167.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.224071E-01 | lm-global-supervised-tokens: 1.270296E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.187981E-02 | lm-token-buffer-mib: 5.824578E+01 | global_load_balancing_loss: 9.993091E-01 | loss scale: 1.0 | grad norm: 0.076 | params norm: 11270.564 | params rms: 0.061026 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 17:20:42.946922 | finish at 2026-10-10 18:53:07 + [2026-10-10 01:33:53] iteration 6108/ 6855 | consumed samples: 781824 | elapsed time per iteration (ms): 88641.7 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.46% | FLOPs estimator: workload-aware | logical tokens/s: 189268.9 | valid tokens/s: 189268.9 | physical tokens/s: 189769.7 | loss tokens/s: 131246.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 5.777674E-01 | lm-global-supervised-tokens: 1.163388E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.181187E-02 | lm-token-buffer-mib: 5.830624E+01 | global_load_balancing_loss: 1.000315E+00 | loss scale: 1.0 | grad norm: 0.064 | params norm: 11270.515 | params rms: 0.061026 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 18:23:35.347250 | finish at 2026-10-10 19:57:28 + [2026-10-10 01:35:23] iteration 6109/ 6855 | consumed samples: 781952 | elapsed time per iteration (ms): 90178.3 | throughput per GPU (TFLOP/s/GPU): 41.3 | MFU 13.22% | FLOPs estimator: workload-aware | logical tokens/s: 186044.4 | valid tokens/s: 186044.4 | physical tokens/s: 186389.1 | loss tokens/s: 135188.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.257143E-01 | lm-global-supervised-tokens: 1.219105E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.738202E-02 | lm-token-buffer-mib: 5.809521E+01 | global_load_balancing_loss: 9.995531E-01 | loss scale: 1.0 | grad norm: 0.032 | params norm: 11270.466 | params rms: 0.061026 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 18:41:13.016886 | finish at 2026-10-10 20:16:36 + [2026-10-10 01:37:17] iteration 6110/ 6855 | consumed samples: 782080 | elapsed time per iteration (ms): 113645.5 | throughput per GPU (TFLOP/s/GPU): 32.9 | MFU 10.54% | FLOPs estimator: workload-aware | logical tokens/s: 147626.8 | valid tokens/s: 147626.7 | physical tokens/s: 147962.3 | loss tokens/s: 107879.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.682952E-01 | lm-global-supervised-tokens: 1.226006E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.491842E-02 | lm-token-buffer-mib: 5.819915E+01 | global_load_balancing_loss: 9.994221E-01 | loss scale: 1.0 | grad norm: 0.070 | params norm: 11270.416 | params rms: 0.061026 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 23:31:05.917352 | finish at 2026-10-11 01:08:23 + [2026-10-10 01:38:47] iteration 6111/ 6855 | consumed samples: 782208 | elapsed time per iteration (ms): 89907.4 | throughput per GPU (TFLOP/s/GPU): 41.0 | MFU 13.14% | FLOPs estimator: workload-aware | logical tokens/s: 186605.0 | valid tokens/s: 186605.0 | physical tokens/s: 186936.2 | loss tokens/s: 140275.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.611919E-01 | lm-global-supervised-tokens: 1.261181E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.319191E-02 | lm-token-buffer-mib: 5.810094E+01 | global_load_balancing_loss: 1.000548E+00 | loss scale: 1.0 | grad norm: 0.071 | params norm: 11270.365 | params rms: 0.061025 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 18:34:51.077139 | finish at 2026-10-10 20:13:38 + [2026-10-10 01:40:15] iteration 6112/ 6855 | consumed samples: 782336 | elapsed time per iteration (ms): 88337.0 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.59% | FLOPs estimator: workload-aware | logical tokens/s: 189922.0 | valid tokens/s: 189922.0 | physical tokens/s: 190274.3 | loss tokens/s: 137680.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.058086E-01 | lm-global-supervised-tokens: 1.216224E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.318527E-02 | lm-token-buffer-mib: 5.804832E+01 | global_load_balancing_loss: 9.998626E-01 | loss scale: 1.0 | grad norm: 0.090 | params norm: 11270.314 | params rms: 0.061025 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 18:13:54.420504 | finish at 2026-10-10 19:54:09 + [2026-10-10 01:41:36] iteration 6113/ 6855 | consumed samples: 782464 | elapsed time per iteration (ms): 80785.6 | throughput per GPU (TFLOP/s/GPU): 44.2 | MFU 14.18% | FLOPs estimator: workload-aware | logical tokens/s: 207675.7 | valid tokens/s: 207675.7 | physical tokens/s: 208237.8 | loss tokens/s: 156823.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.336153E-01 | lm-global-supervised-tokens: 1.266910E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.287097E-02 | lm-token-buffer-mib: 5.818799E+01 | global_load_balancing_loss: 1.001442E+00 | loss scale: 1.0 | grad norm: 0.113 | params norm: 11270.262 | params rms: 0.061025 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 16:39:02.949127 | finish at 2026-10-10 18:20:39 + [2026-10-10 01:42:59] iteration 6114/ 6855 | consumed samples: 782592 | elapsed time per iteration (ms): 83459.6 | throughput per GPU (TFLOP/s/GPU): 43.7 | MFU 13.99% | FLOPs estimator: workload-aware | logical tokens/s: 201021.7 | valid tokens/s: 201021.7 | physical tokens/s: 201497.7 | loss tokens/s: 149087.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.621791E-01 | lm-global-supervised-tokens: 1.244278E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.672007E-02 | lm-token-buffer-mib: 5.827736E+01 | global_load_balancing_loss: 1.000868E+00 | loss scale: 1.0 | grad norm: 0.160 | params norm: 11270.211 | params rms: 0.061025 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 17:10:43.538139 | finish at 2026-10-10 18:53:43 + [2026-10-10 01:44:24] iteration 6115/ 6855 | consumed samples: 782720 | elapsed time per iteration (ms): 84830.5 | throughput per GPU (TFLOP/s/GPU): 43.9 | MFU 14.07% | FLOPs estimator: workload-aware | logical tokens/s: 197772.3 | valid tokens/s: 197772.3 | physical tokens/s: 198310.5 | loss tokens/s: 144897.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.955081E-01 | lm-global-supervised-tokens: 1.229169E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.173632E-02 | lm-token-buffer-mib: 5.823384E+01 | global_load_balancing_loss: 1.001209E+00 | loss scale: 1.0 | grad norm: 0.225 | params norm: 11270.162 | params rms: 0.061024 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 17:26:14.596381 | finish at 2026-10-10 19:10:39 + [2026-10-10 01:45:52] iteration 6116/ 6855 | consumed samples: 782848 | elapsed time per iteration (ms): 88064.9 | throughput per GPU (TFLOP/s/GPU): 41.6 | MFU 13.33% | FLOPs estimator: workload-aware | logical tokens/s: 190509.4 | valid tokens/s: 190509.4 | physical tokens/s: 190924.0 | loss tokens/s: 147739.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.912467E-01 | lm-global-supervised-tokens: 1.301070E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.469020E-02 | lm-token-buffer-mib: 5.807399E+01 | global_load_balancing_loss: 9.992715E-01 | loss scale: 1.0 | grad norm: 0.158 | params norm: 11270.114 | params rms: 0.061024 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 18:04:39.983947 | finish at 2026-10-10 19:50:32 + [2026-10-10 01:47:21] iteration 6117/ 6855 | consumed samples: 782976 | elapsed time per iteration (ms): 88577.7 | throughput per GPU (TFLOP/s/GPU): 41.6 | MFU 13.35% | FLOPs estimator: workload-aware | logical tokens/s: 189406.1 | valid tokens/s: 189406.1 | physical tokens/s: 189790.5 | loss tokens/s: 136398.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.111392E-01 | lm-global-supervised-tokens: 1.208190E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.267457E-02 | lm-token-buffer-mib: 5.811143E+01 | global_load_balancing_loss: 1.000012E+00 | loss scale: 1.0 | grad norm: 0.076 | params norm: 11270.064 | params rms: 0.061024 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 18:09:30.312438 | finish at 2026-10-10 19:56:51 + [2026-10-10 01:48:51] iteration 6118/ 6855 | consumed samples: 783104 | elapsed time per iteration (ms): 89870.6 | throughput per GPU (TFLOP/s/GPU): 41.7 | MFU 13.35% | FLOPs estimator: workload-aware | logical tokens/s: 186681.9 | valid tokens/s: 186681.9 | physical tokens/s: 187199.3 | loss tokens/s: 137993.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.048657E-01 | lm-global-supervised-tokens: 1.240158E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.166101E-02 | lm-token-buffer-mib: 5.822102E+01 | global_load_balancing_loss: 1.000602E+00 | loss scale: 1.0 | grad norm: 0.218 | params norm: 11270.015 | params rms: 0.061023 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 18:23:54.637636 | finish at 2026-10-10 20:12:45 + [2026-10-10 01:50:16] iteration 6119/ 6855 | consumed samples: 783232 | elapsed time per iteration (ms): 85164.8 | throughput per GPU (TFLOP/s/GPU): 43.8 | MFU 14.02% | FLOPs estimator: workload-aware | logical tokens/s: 196996.7 | valid tokens/s: 196996.7 | physical tokens/s: 197372.6 | loss tokens/s: 151472.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.119868E-01 | lm-global-supervised-tokens: 1.290009E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.274693E-02 | lm-token-buffer-mib: 5.808808E+01 | global_load_balancing_loss: 9.993571E-01 | loss scale: 1.0 | grad norm: 0.180 | params norm: 11269.968 | params rms: 0.061023 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 17:24:41.301697 | finish at 2026-10-10 19:14:57 + [2026-10-10 01:52:07] iteration 6120/ 6855 | consumed samples: 783360 | elapsed time per iteration (ms): 110889.4 | throughput per GPU (TFLOP/s/GPU): 34.2 | MFU 10.96% | FLOPs estimator: workload-aware | logical tokens/s: 151296.6 | valid tokens/s: 151296.6 | physical tokens/s: 151644.7 | loss tokens/s: 114472.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.539470E-01 | lm-global-supervised-tokens: 1.269375E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.300153E-02 | lm-token-buffer-mib: 5.816931E+01 | global_load_balancing_loss: 9.995530E-01 | loss scale: 1.0 | grad norm: 0.071 | params norm: 11269.923 | params rms: 0.061023 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 22:38:23.699366 | finish at 2026-10-11 00:30:30 +times across ranks (ms): + save-checkpoint: + rank 0: 23980.46 + rank 1: 23980.50 + rank 2: 23980.47 + rank 3: 23980.44 + rank 4: 23980.44 + rank 5: 23980.47 + rank 6: 23980.43 + rank 7: 23980.47 + rank 8: 23979.59 + rank 9: 23979.57 + rank 10: 23979.57 + rank 11: 23979.55 + rank 12: 23979.58 + rank 13: 23979.57 + rank 14: 23979.60 + rank 15: 23979.57 + rank 16: 23977.70 + rank 17: 23977.70 + rank 18: 23977.68 + rank 19: 23977.74 + rank 20: 23977.70 + rank 21: 23977.75 + rank 22: 23977.69 + rank 23: 23977.71 + rank 24: 23978.99 + rank 25: 23979.00 + rank 26: 23979.03 + rank 27: 23979.01 + rank 28: 23978.96 + rank 29: 23979.05 + rank 30: 23979.02 + rank 31: 23979.04 + rank 32: 23979.47 + rank 33: 23979.44 + rank 34: 23979.47 + rank 35: 23979.46 + rank 36: 23979.41 + rank 37: 23979.39 + rank 38: 23979.41 + rank 39: 23979.39 + rank 40: 23979.39 + rank 41: 23979.35 + rank 42: 23979.38 + rank 43: 23979.36 + rank 44: 23979.38 + rank 45: 23979.37 + rank 46: 23979.39 + rank 47: 23979.38 + rank 48: 23979.20 + rank 49: 23979.17 + rank 50: 23979.13 + rank 51: 23979.15 + rank 52: 23979.15 + rank 53: 23979.14 + rank 54: 23979.12 + rank 55: 23979.15 + rank 56: 23978.97 + rank 57: 23978.94 + rank 58: 23978.92 + rank 59: 23978.93 + rank 60: 23978.92 + rank 61: 23979.00 + rank 62: 23978.92 + rank 63: 23978.93 + [2026-10-10 01:53:55] iteration 6121/ 6855 | consumed samples: 783488 | elapsed time per iteration (ms): 83694.7 | throughput per GPU (TFLOP/s/GPU): 43.5 | MFU 13.95% | FLOPs estimator: workload-aware | logical tokens/s: 200456.7 | valid tokens/s: 200456.7 | physical tokens/s: 201008.2 | loss tokens/s: 147895.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.463161E-01 | lm-global-supervised-tokens: 1.237808E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 3.476923E-01 | lm-token-buffer-mib: 5.805830E+01 | global_load_balancing_loss: 1.001056E+00 | loss scale: 1.0 | grad norm: 0.113 | params norm: 11269.876 | params rms: 0.061023 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 17:03:51.888277 | finish at 2026-10-10 18:57:46 + [2026-10-10 01:55:25] iteration 6122/ 6855 | consumed samples: 783616 | elapsed time per iteration (ms): 90740.9 | throughput per GPU (TFLOP/s/GPU): 40.8 | MFU 13.08% | FLOPs estimator: workload-aware | logical tokens/s: 184890.7 | valid tokens/s: 184890.7 | physical tokens/s: 185290.6 | loss tokens/s: 134839.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.045537E-01 | lm-global-supervised-tokens: 1.223549E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.771544E-02 | lm-token-buffer-mib: 5.812009E+01 | global_load_balancing_loss: 1.000893E+00 | loss scale: 1.0 | grad norm: 0.082 | params norm: 11269.831 | params rms: 0.061022 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 18:28:33.091613 | finish at 2026-10-10 20:23:58 + [2026-10-10 01:56:52] iteration 6123/ 6855 | consumed samples: 783744 | elapsed time per iteration (ms): 87127.8 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.67% | FLOPs estimator: workload-aware | logical tokens/s: 192557.9 | valid tokens/s: 192557.9 | physical tokens/s: 192992.3 | loss tokens/s: 138907.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.154688E-01 | lm-global-supervised-tokens: 1.210268E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.223612E-02 | lm-token-buffer-mib: 5.806023E+01 | global_load_balancing_loss: 1.000115E+00 | loss scale: 1.0 | grad norm: 0.071 | params norm: 11269.785 | params rms: 0.061022 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 17:42:57.540865 | finish at 2026-10-10 19:39:50 + [2026-10-10 01:58:19] iteration 6124/ 6855 | consumed samples: 783872 | elapsed time per iteration (ms): 86752.3 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.60% | FLOPs estimator: workload-aware | logical tokens/s: 193392.0 | valid tokens/s: 193392.0 | physical tokens/s: 193833.7 | loss tokens/s: 144875.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.885180E-01 | lm-global-supervised-tokens: 1.256825E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.161461E-02 | lm-token-buffer-mib: 5.808546E+01 | global_load_balancing_loss: 9.996259E-01 | loss scale: 1.0 | grad norm: 0.081 | params norm: 11269.739 | params rms: 0.061022 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 17:36:55.935500 | finish at 2026-10-10 19:35:15 + [2026-10-10 01:59:49] iteration 6125/ 6855 | consumed samples: 784000 | elapsed time per iteration (ms): 90288.3 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.44% | FLOPs estimator: workload-aware | logical tokens/s: 185818.1 | valid tokens/s: 185818.1 | physical tokens/s: 186103.4 | loss tokens/s: 137244.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.569055E-01 | lm-global-supervised-tokens: 1.239157E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.433872E-02 | lm-token-buffer-mib: 5.803271E+01 | global_load_balancing_loss: 9.993393E-01 | loss scale: 1.0 | grad norm: 0.049 | params norm: 11269.692 | params rms: 0.061022 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 18:18:30.454676 | finish at 2026-10-10 20:18:20 + [2026-10-10 02:01:16] iteration 6126/ 6855 | consumed samples: 784128 | elapsed time per iteration (ms): 86427.3 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.50% | FLOPs estimator: workload-aware | logical tokens/s: 194118.4 | valid tokens/s: 194118.4 | physical tokens/s: 194596.9 | loss tokens/s: 146570.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.602111E-01 | lm-global-supervised-tokens: 1.266772E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.163854E-02 | lm-token-buffer-mib: 5.817432E+01 | global_load_balancing_loss: 1.000444E+00 | loss scale: 1.0 | grad norm: 0.081 | params norm: 11269.646 | params rms: 0.061021 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 17:30:05.499423 | finish at 2026-10-10 19:31:21 + [2026-10-10 02:02:42] iteration 6127/ 6855 | consumed samples: 784256 | elapsed time per iteration (ms): 85958.5 | throughput per GPU (TFLOP/s/GPU): 41.7 | MFU 13.37% | FLOPs estimator: workload-aware | logical tokens/s: 195177.4 | valid tokens/s: 195177.4 | physical tokens/s: 195664.4 | loss tokens/s: 144829.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.823918E-01 | lm-global-supervised-tokens: 1.244931E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.491552E-02 | lm-token-buffer-mib: 5.808408E+01 | global_load_balancing_loss: 1.000907E+00 | loss scale: 1.0 | grad norm: 0.053 | params norm: 11269.598 | params rms: 0.061021 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 17:22:57.817614 | finish at 2026-10-10 19:25:40 + [2026-10-10 02:04:04] iteration 6128/ 6855 | consumed samples: 784384 | elapsed time per iteration (ms): 82541.3 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.68% | FLOPs estimator: workload-aware | logical tokens/s: 203258.5 | valid tokens/s: 203258.4 | physical tokens/s: 203971.4 | loss tokens/s: 154577.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.432049E-01 | lm-global-supervised-tokens: 1.275901E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.422046E-02 | lm-token-buffer-mib: 5.813252E+01 | global_load_balancing_loss: 1.000621E+00 | loss scale: 1.0 | grad norm: 0.100 | params norm: 11269.550 | params rms: 0.061021 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 16:40:07.522022 | finish at 2026-10-10 18:44:12 + [2026-10-10 02:05:33] iteration 6129/ 6855 | consumed samples: 784512 | elapsed time per iteration (ms): 88707.6 | throughput per GPU (TFLOP/s/GPU): 41.7 | MFU 13.37% | FLOPs estimator: workload-aware | logical tokens/s: 189129.0 | valid tokens/s: 189129.0 | physical tokens/s: 189674.1 | loss tokens/s: 138010.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.337277E-01 | lm-global-supervised-tokens: 1.224259E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.511836E-02 | lm-token-buffer-mib: 5.816203E+01 | global_load_balancing_loss: 1.000208E+00 | loss scale: 1.0 | grad norm: 0.058 | params norm: 11269.502 | params rms: 0.061021 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 17:53:21.734821 | finish at 2026-10-10 19:58:55 + [2026-10-10 02:07:23] iteration 6130/ 6855 | consumed samples: 784640 | elapsed time per iteration (ms): 109798.7 | throughput per GPU (TFLOP/s/GPU): 33.3 | MFU 10.68% | FLOPs estimator: workload-aware | logical tokens/s: 152799.8 | valid tokens/s: 152799.8 | physical tokens/s: 153084.8 | loss tokens/s: 110000.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.432350E-01 | lm-global-supervised-tokens: 1.207790E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.219356E-02 | lm-token-buffer-mib: 5.807229E+01 | global_load_balancing_loss: 1.001203E+00 | loss scale: 1.0 | grad norm: 0.091 | params norm: 11269.455 | params rms: 0.061020 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 22:06:44.036826 | finish at 2026-10-11 00:14:07 + [2026-10-10 02:08:55] iteration 6131/ 6855 | consumed samples: 784768 | elapsed time per iteration (ms): 91686.0 | throughput per GPU (TFLOP/s/GPU): 41.5 | MFU 13.29% | FLOPs estimator: workload-aware | logical tokens/s: 182984.3 | valid tokens/s: 182984.3 | physical tokens/s: 183229.6 | loss tokens/s: 132045.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.450223E-01 | lm-global-supervised-tokens: 1.210670E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.139315E-02 | lm-token-buffer-mib: 5.802634E+01 | global_load_balancing_loss: 1.000027E+00 | loss scale: 1.0 | grad norm: 0.037 | params norm: 11269.408 | params rms: 0.061020 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 18:26:20.689626 | finish at 2026-10-10 20:35:15 + [2026-10-10 02:10:25] iteration 6132/ 6855 | consumed samples: 784896 | elapsed time per iteration (ms): 90485.7 | throughput per GPU (TFLOP/s/GPU): 41.0 | MFU 13.14% | FLOPs estimator: workload-aware | logical tokens/s: 185412.6 | valid tokens/s: 185412.6 | physical tokens/s: 185758.1 | loss tokens/s: 131894.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.113656E-01 | lm-global-supervised-tokens: 1.193453E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.374360E-02 | lm-token-buffer-mib: 5.807721E+01 | global_load_balancing_loss: 1.000250E+00 | loss scale: 1.0 | grad norm: 0.042 | params norm: 11269.361 | params rms: 0.061020 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 18:10:21.153782 | finish at 2026-10-10 20:20:46 + [2026-10-10 02:11:55] iteration 6133/ 6855 | consumed samples: 785024 | elapsed time per iteration (ms): 90417.7 | throughput per GPU (TFLOP/s/GPU): 41.1 | MFU 13.16% | FLOPs estimator: workload-aware | logical tokens/s: 185552.1 | valid tokens/s: 185552.1 | physical tokens/s: 186032.5 | loss tokens/s: 136912.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.703935E-01 | lm-global-supervised-tokens: 1.237929E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.294042E-02 | lm-token-buffer-mib: 5.811422E+01 | global_load_balancing_loss: 9.997861E-01 | loss scale: 1.0 | grad norm: 0.096 | params norm: 11269.314 | params rms: 0.061020 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 18:08:01.550863 | finish at 2026-10-10 20:19:57 + [2026-10-10 02:13:23] iteration 6134/ 6855 | consumed samples: 785152 | elapsed time per iteration (ms): 87903.7 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.52% | FLOPs estimator: workload-aware | logical tokens/s: 190858.7 | valid tokens/s: 190858.7 | physical tokens/s: 191306.1 | loss tokens/s: 144418.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.164987E-01 | lm-global-supervised-tokens: 1.269492E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.499675E-02 | lm-token-buffer-mib: 5.805958E+01 | global_load_balancing_loss: 9.992715E-01 | loss scale: 1.0 | grad norm: 0.096 | params norm: 11269.267 | params rms: 0.061019 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 17:36:18.565375 | finish at 2026-10-10 19:49:42 + [2026-10-10 02:14:50] iteration 6135/ 6855 | consumed samples: 785280 | elapsed time per iteration (ms): 86765.4 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.42% | FLOPs estimator: workload-aware | logical tokens/s: 193362.9 | valid tokens/s: 193362.9 | physical tokens/s: 193851.1 | loss tokens/s: 142110.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.287375E-01 | lm-global-supervised-tokens: 1.233029E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.328362E-02 | lm-token-buffer-mib: 5.809635E+01 | global_load_balancing_loss: 1.000495E+00 | loss scale: 1.0 | grad norm: 0.098 | params norm: 11269.217 | params rms: 0.061019 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 17:21:11.096878 | finish at 2026-10-10 19:36:01 + [2026-10-10 02:16:17] iteration 6136/ 6855 | consumed samples: 785408 | elapsed time per iteration (ms): 86923.8 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.41% | FLOPs estimator: workload-aware | logical tokens/s: 193009.5 | valid tokens/s: 193009.5 | physical tokens/s: 193605.1 | loss tokens/s: 142555.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.150190E-01 | lm-global-supervised-tokens: 1.239144E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.219792E-02 | lm-token-buffer-mib: 5.825799E+01 | global_load_balancing_loss: 1.000784E+00 | loss scale: 1.0 | grad norm: 0.157 | params norm: 11269.167 | params rms: 0.061019 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 17:21:38.224365 | finish at 2026-10-10 19:37:55 + [2026-10-10 02:17:45] iteration 6137/ 6855 | consumed samples: 785536 | elapsed time per iteration (ms): 87934.6 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.67% | FLOPs estimator: workload-aware | logical tokens/s: 190792.1 | valid tokens/s: 190792.1 | physical tokens/s: 191266.6 | loss tokens/s: 141674.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.820877E-01 | lm-global-supervised-tokens: 1.245805E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.531638E-02 | lm-token-buffer-mib: 5.813134E+01 | global_load_balancing_loss: 1.000812E+00 | loss scale: 1.0 | grad norm: 0.215 | params norm: 11269.119 | params rms: 0.061019 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 17:32:17.008474 | finish at 2026-10-10 19:50:02 + [2026-10-10 02:19:12] iteration 6138/ 6855 | consumed samples: 785664 | elapsed time per iteration (ms): 86629.3 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.69% | FLOPs estimator: workload-aware | logical tokens/s: 193666.6 | valid tokens/s: 193666.6 | physical tokens/s: 194077.8 | loss tokens/s: 139446.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 5.879704E-01 | lm-global-supervised-tokens: 1.208011E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.407726E-02 | lm-token-buffer-mib: 5.815868E+01 | global_load_balancing_loss: 1.001194E+00 | loss scale: 1.0 | grad norm: 0.097 | params norm: 11269.068 | params rms: 0.061018 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 17:15:13.193996 | finish at 2026-10-10 19:34:25 + [2026-10-10 02:20:39] iteration 6139/ 6855 | consumed samples: 785792 | elapsed time per iteration (ms): 87793.8 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.45% | FLOPs estimator: workload-aware | logical tokens/s: 191097.2 | valid tokens/s: 191097.2 | physical tokens/s: 191560.7 | loss tokens/s: 141956.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.366074E-01 | lm-global-supervised-tokens: 1.246290E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.410075E-02 | lm-token-buffer-mib: 5.816075E+01 | global_load_balancing_loss: 9.991956E-01 | loss scale: 1.0 | grad norm: 0.095 | params norm: 11269.017 | params rms: 0.061018 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 17:27:40.352177 | finish at 2026-10-10 19:48:20 + [2026-10-10 02:22:27] iteration 6140/ 6855 | consumed samples: 785920 | elapsed time per iteration (ms): 107784.5 | throughput per GPU (TFLOP/s/GPU): 33.5 | MFU 10.74% | FLOPs estimator: workload-aware | logical tokens/s: 155654.5 | valid tokens/s: 155654.5 | physical tokens/s: 156073.1 | loss tokens/s: 111763.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.234268E-01 | lm-global-supervised-tokens: 1.204639E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.154851E-02 | lm-token-buffer-mib: 5.823882E+01 | global_load_balancing_loss: 9.996505E-01 | loss scale: 1.0 | grad norm: 0.186 | params norm: 11268.967 | params rms: 0.061018 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 21:24:25.946738 | finish at 2026-10-10 23:46:53 + [2026-10-10 02:23:56] iteration 6141/ 6855 | consumed samples: 786048 | elapsed time per iteration (ms): 89099.8 | throughput per GPU (TFLOP/s/GPU): 41.4 | MFU 13.28% | FLOPs estimator: workload-aware | logical tokens/s: 188296.3 | valid tokens/s: 188296.3 | physical tokens/s: 188851.6 | loss tokens/s: 140389.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.526209E-01 | lm-global-supervised-tokens: 1.250864E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.314012E-02 | lm-token-buffer-mib: 5.816795E+01 | global_load_balancing_loss: 9.987911E-01 | loss scale: 1.0 | grad norm: 0.166 | params norm: 11268.917 | params rms: 0.061018 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 17:40:17.259491 | finish at 2026-10-10 20:04:14 + [2026-10-10 02:25:21] iteration 6142/ 6855 | consumed samples: 786176 | elapsed time per iteration (ms): 84390.9 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.67% | FLOPs estimator: workload-aware | logical tokens/s: 198803.2 | valid tokens/s: 198803.2 | physical tokens/s: 199277.6 | loss tokens/s: 147368.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.350241E-01 | lm-global-supervised-tokens: 1.243658E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.179698E-02 | lm-token-buffer-mib: 5.807388E+01 | global_load_balancing_loss: 9.997418E-01 | loss scale: 1.0 | grad norm: 0.056 | params norm: 11268.868 | params rms: 0.061017 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 16:42:50.716556 | finish at 2026-10-10 19:08:11 + [2026-10-10 02:26:47] iteration 6143/ 6855 | consumed samples: 786304 | elapsed time per iteration (ms): 86792.0 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.63% | FLOPs estimator: workload-aware | logical tokens/s: 193303.7 | valid tokens/s: 193303.7 | physical tokens/s: 193791.6 | loss tokens/s: 139805.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.321052E-01 | lm-global-supervised-tokens: 1.213401E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.185716E-02 | lm-token-buffer-mib: 5.844540E+01 | global_load_balancing_loss: 1.001188E+00 | loss scale: 1.0 | grad norm: 0.126 | params norm: 11268.819 | params rms: 0.061017 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 17:09:55.887913 | finish at 2026-10-10 19:36:43 + [2026-10-10 02:28:17] iteration 6144/ 6855 | consumed samples: 786432 | elapsed time per iteration (ms): 89969.1 | throughput per GPU (TFLOP/s/GPU): 41.0 | MFU 13.14% | FLOPs estimator: workload-aware | logical tokens/s: 186477.0 | valid tokens/s: 186477.0 | physical tokens/s: 186995.1 | loss tokens/s: 140248.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.554341E-01 | lm-global-supervised-tokens: 1.261799E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.520407E-02 | lm-token-buffer-mib: 5.809901E+01 | global_load_balancing_loss: 1.000794E+00 | loss scale: 1.0 | grad norm: 0.081 | params norm: 11268.770 | params rms: 0.061017 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 17:46:08.019419 | finish at 2026-10-10 20:14:25 + [2026-10-10 02:29:44] iteration 6145/ 6855 | consumed samples: 786560 | elapsed time per iteration (ms): 86729.4 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.49% | FLOPs estimator: workload-aware | logical tokens/s: 193442.6 | valid tokens/s: 193442.5 | physical tokens/s: 193884.6 | loss tokens/s: 135213.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.226460E-01 | lm-global-supervised-tokens: 1.172702E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.297281E-02 | lm-token-buffer-mib: 5.829961E+01 | global_load_balancing_loss: 1.001952E+00 | loss scale: 1.0 | grad norm: 0.061 | params norm: 11268.721 | params rms: 0.061016 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 17:06:17.857354 | finish at 2026-10-10 19:36:02 + [2026-10-10 02:31:12] iteration 6146/ 6855 | consumed samples: 786688 | elapsed time per iteration (ms): 87926.7 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.39% | FLOPs estimator: workload-aware | logical tokens/s: 190809.0 | valid tokens/s: 190809.0 | physical tokens/s: 191177.7 | loss tokens/s: 143654.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.202567E-01 | lm-global-supervised-tokens: 1.263106E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.714010E-02 | lm-token-buffer-mib: 5.812704E+01 | global_load_balancing_loss: 1.000776E+00 | loss scale: 1.0 | grad norm: 0.038 | params norm: 11268.672 | params rms: 0.061016 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 17:19:00.050666 | finish at 2026-10-10 19:50:12 + [2026-10-10 02:32:42] iteration 6147/ 6855 | consumed samples: 786816 | elapsed time per iteration (ms): 89795.5 | throughput per GPU (TFLOP/s/GPU): 41.4 | MFU 13.27% | FLOPs estimator: workload-aware | logical tokens/s: 186837.9 | valid tokens/s: 186837.9 | physical tokens/s: 187365.9 | loss tokens/s: 141659.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.689779E-01 | lm-global-supervised-tokens: 1.272037E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.484715E-02 | lm-token-buffer-mib: 5.814128E+01 | global_load_balancing_loss: 9.999714E-01 | loss scale: 1.0 | grad norm: 0.065 | params norm: 11268.622 | params rms: 0.061016 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 17:39:35.244075 | finish at 2026-10-10 20:12:17 + [2026-10-10 02:34:08] iteration 6148/ 6855 | consumed samples: 786944 | elapsed time per iteration (ms): 86526.6 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.42% | FLOPs estimator: workload-aware | logical tokens/s: 193896.3 | valid tokens/s: 193896.2 | physical tokens/s: 194402.3 | loss tokens/s: 148173.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.577392E-01 | lm-global-supervised-tokens: 1.282097E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.218204E-02 | lm-token-buffer-mib: 5.814448E+01 | global_load_balancing_loss: 9.993713E-01 | loss scale: 1.0 | grad norm: 0.044 | params norm: 11268.573 | params rms: 0.061016 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 16:59:34.314715 | finish at 2026-10-10 19:33:43 + [2026-10-10 02:35:36] iteration 6149/ 6855 | consumed samples: 787072 | elapsed time per iteration (ms): 87817.4 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.61% | FLOPs estimator: workload-aware | logical tokens/s: 191046.5 | valid tokens/s: 191046.5 | physical tokens/s: 191487.7 | loss tokens/s: 140260.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.634974E-01 | lm-global-supervised-tokens: 1.231733E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.222129E-02 | lm-token-buffer-mib: 5.816757E+01 | global_load_balancing_loss: 9.988390E-01 | loss scale: 1.0 | grad norm: 0.041 | params norm: 11268.522 | params rms: 0.061015 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 17:13:19.071761 | finish at 2026-10-10 19:48:55 + [2026-10-10 02:37:29] iteration 6150/ 6855 | consumed samples: 787200 | elapsed time per iteration (ms): 112664.3 | throughput per GPU (TFLOP/s/GPU): 33.0 | MFU 10.58% | FLOPs estimator: workload-aware | logical tokens/s: 148912.8 | valid tokens/s: 148912.8 | physical tokens/s: 149171.8 | loss tokens/s: 111163.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.532008E-01 | lm-global-supervised-tokens: 1.252417E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.135181E-02 | lm-token-buffer-mib: 5.806820E+01 | global_load_balancing_loss: 9.995953E-01 | loss scale: 1.0 | grad norm: 0.037 | params norm: 11268.471 | params rms: 0.061015 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 22:03:48.304077 | finish at 2026-10-11 00:41:17 + [2026-10-10 02:38:58] iteration 6151/ 6855 | consumed samples: 787328 | elapsed time per iteration (ms): 88877.9 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.48% | FLOPs estimator: workload-aware | logical tokens/s: 188766.6 | valid tokens/s: 188766.6 | physical tokens/s: 189216.3 | loss tokens/s: 138937.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.469115E-01 | lm-global-supervised-tokens: 1.234848E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.571873E-02 | lm-token-buffer-mib: 5.806336E+01 | global_load_balancing_loss: 1.000001E+00 | loss scale: 1.0 | grad norm: 0.048 | params norm: 11268.418 | params rms: 0.061015 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 17:22:50.058472 | finish at 2026-10-10 20:01:48 + [2026-10-10 02:40:27] iteration 6152/ 6855 | consumed samples: 787456 | elapsed time per iteration (ms): 89094.8 | throughput per GPU (TFLOP/s/GPU): 41.7 | MFU 13.37% | FLOPs estimator: workload-aware | logical tokens/s: 188307.2 | valid tokens/s: 188307.2 | physical tokens/s: 188691.3 | loss tokens/s: 143423.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.099422E-01 | lm-global-supervised-tokens: 1.277832E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.422728E-02 | lm-token-buffer-mib: 5.807899E+01 | global_load_balancing_loss: 1.000326E+00 | loss scale: 1.0 | grad norm: 0.067 | params norm: 11268.367 | params rms: 0.061015 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 17:23:53.621099 | finish at 2026-10-10 20:04:20 + [2026-10-10 02:41:58] iteration 6153/ 6855 | consumed samples: 787584 | elapsed time per iteration (ms): 91341.1 | throughput per GPU (TFLOP/s/GPU): 41.7 | MFU 13.37% | FLOPs estimator: workload-aware | logical tokens/s: 183676.1 | valid tokens/s: 183676.1 | physical tokens/s: 183942.1 | loss tokens/s: 129996.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.766207E-01 | lm-global-supervised-tokens: 1.187403E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.425046E-02 | lm-token-buffer-mib: 5.803533E+01 | global_load_balancing_loss: 1.000826E+00 | loss scale: 1.0 | grad norm: 0.043 | params norm: 11268.315 | params rms: 0.061014 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 17:48:41.461390 | finish at 2026-10-10 20:30:40 + [2026-10-10 02:43:23] iteration 6154/ 6855 | consumed samples: 787712 | elapsed time per iteration (ms): 84946.7 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.48% | FLOPs estimator: workload-aware | logical tokens/s: 197501.8 | valid tokens/s: 197501.8 | physical tokens/s: 198120.9 | loss tokens/s: 149162.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 5.918461E-01 | lm-global-supervised-tokens: 1.267086E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.512195E-02 | lm-token-buffer-mib: 5.811334E+01 | global_load_balancing_loss: 1.001484E+00 | loss scale: 1.0 | grad norm: 0.075 | params norm: 11268.265 | params rms: 0.061014 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 16:32:27.608856 | finish at 2026-10-10 19:15:51 + [2026-10-10 02:44:51] iteration 6155/ 6855 | consumed samples: 787840 | elapsed time per iteration (ms): 87738.9 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.50% | FLOPs estimator: workload-aware | logical tokens/s: 191217.5 | valid tokens/s: 191217.5 | physical tokens/s: 191816.8 | loss tokens/s: 138286.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.476213E-01 | lm-global-supervised-tokens: 1.213307E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.151259E-02 | lm-token-buffer-mib: 5.818077E+01 | global_load_balancing_loss: 1.000409E+00 | loss scale: 1.0 | grad norm: 0.078 | params norm: 11268.213 | params rms: 0.061014 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 17:03:37.233634 | finish at 2026-10-10 19:48:28 + [2026-10-10 02:46:18] iteration 6156/ 6855 | consumed samples: 787968 | elapsed time per iteration (ms): 86967.0 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.76% | FLOPs estimator: workload-aware | logical tokens/s: 192914.6 | valid tokens/s: 192914.6 | physical tokens/s: 193533.9 | loss tokens/s: 145931.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.896281E-01 | lm-global-supervised-tokens: 1.269118E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.377541E-02 | lm-token-buffer-mib: 5.834566E+01 | global_load_balancing_loss: 1.000817E+00 | loss scale: 1.0 | grad norm: 0.276 | params norm: 11268.163 | params rms: 0.061013 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 16:53:09.962003 | finish at 2026-10-10 19:39:28 + [2026-10-10 02:47:45] iteration 6157/ 6855 | consumed samples: 788096 | elapsed time per iteration (ms): 86642.0 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.47% | FLOPs estimator: workload-aware | logical tokens/s: 193637.1 | valid tokens/s: 193637.0 | physical tokens/s: 194213.4 | loss tokens/s: 149558.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.698477E-01 | lm-global-supervised-tokens: 1.295805E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.264923E-02 | lm-token-buffer-mib: 5.817330E+01 | global_load_balancing_loss: 9.994316E-01 | loss scale: 1.0 | grad norm: 0.320 | params norm: 11268.112 | params rms: 0.061013 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 16:47:56.121464 | finish at 2026-10-10 19:35:41 + [2026-10-10 02:49:11] iteration 6158/ 6855 | consumed samples: 788224 | elapsed time per iteration (ms): 86936.8 | throughput per GPU (TFLOP/s/GPU): 42.8 | MFU 13.71% | FLOPs estimator: workload-aware | logical tokens/s: 192981.3 | valid tokens/s: 192981.3 | physical tokens/s: 193383.3 | loss tokens/s: 139602.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.516093E-01 | lm-global-supervised-tokens: 1.213658E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.514116E-02 | lm-token-buffer-mib: 5.804224E+01 | global_load_balancing_loss: 9.998536E-01 | loss scale: 1.0 | grad norm: 0.092 | params norm: 11268.062 | params rms: 0.061013 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 16:49:54.973366 | finish at 2026-10-10 19:39:06 + [2026-10-10 02:50:39] iteration 6159/ 6855 | consumed samples: 788352 | elapsed time per iteration (ms): 87480.0 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.54% | FLOPs estimator: workload-aware | logical tokens/s: 191782.5 | valid tokens/s: 191782.5 | physical tokens/s: 192343.6 | loss tokens/s: 142919.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.589028E-01 | lm-global-supervised-tokens: 1.250259E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.252247E-02 | lm-token-buffer-mib: 5.809206E+01 | global_load_balancing_loss: 9.995170E-01 | loss scale: 1.0 | grad norm: 0.368 | params norm: 11268.012 | params rms: 0.061013 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 16:54:46.058773 | finish at 2026-10-10 19:45:25 + [2026-10-10 02:52:27] iteration 6160/ 6855 | consumed samples: 788480 | elapsed time per iteration (ms): 108141.4 | throughput per GPU (TFLOP/s/GPU): 33.7 | MFU 10.82% | FLOPs estimator: workload-aware | logical tokens/s: 155141.1 | valid tokens/s: 155141.1 | physical tokens/s: 155458.5 | loss tokens/s: 115443.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.422581E-01 | lm-global-supervised-tokens: 1.248420E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.200897E-02 | lm-token-buffer-mib: 5.808252E+01 | global_load_balancing_loss: 1.000040E+00 | loss scale: 1.0 | grad norm: 0.194 | params norm: 11267.964 | params rms: 0.061012 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 20:52:38.302066 | finish at 2026-10-10 23:45:05 +times across ranks (ms): + save-checkpoint: + rank 0: 24633.50 + rank 1: 24633.51 + rank 2: 24633.56 + rank 3: 24633.51 + rank 4: 24633.49 + rank 5: 24633.46 + rank 6: 24633.50 + rank 7: 24633.51 + rank 8: 24633.75 + rank 9: 24633.69 + rank 10: 24633.70 + rank 11: 24633.72 + rank 12: 24633.71 + rank 13: 24633.70 + rank 14: 24633.74 + rank 15: 24633.71 + rank 16: 24633.59 + rank 17: 24633.62 + rank 18: 24633.55 + rank 19: 24633.65 + rank 20: 24633.55 + rank 21: 24633.61 + rank 22: 24633.55 + rank 23: 24633.56 + rank 24: 24633.54 + rank 25: 24633.43 + rank 26: 24633.53 + rank 27: 24633.55 + rank 28: 24633.49 + rank 29: 24633.67 + rank 30: 24633.46 + rank 31: 24633.52 + rank 32: 24633.27 + rank 33: 24633.25 + rank 34: 24633.26 + rank 35: 24633.23 + rank 36: 24633.20 + rank 37: 24633.14 + rank 38: 24633.21 + rank 39: 24633.14 + rank 40: 24633.59 + rank 41: 24633.57 + rank 42: 24633.59 + rank 43: 24633.71 + rank 44: 24633.57 + rank 45: 24633.59 + rank 46: 24633.58 + rank 47: 24633.56 + rank 48: 24633.38 + rank 49: 24633.43 + rank 50: 24633.42 + rank 51: 24633.31 + rank 52: 24633.46 + rank 53: 24633.41 + rank 54: 24633.45 + rank 55: 24633.43 + rank 56: 24633.71 + rank 57: 24633.69 + rank 58: 24633.66 + rank 59: 24633.67 + rank 60: 24633.66 + rank 61: 24633.68 + rank 62: 24633.66 + rank 63: 24633.51 + [2026-10-10 02:54:27] iteration 6161/ 6855 | consumed samples: 788608 | elapsed time per iteration (ms): 94866.4 | throughput per GPU (TFLOP/s/GPU): 39.3 | MFU 12.60% | FLOPs estimator: workload-aware | logical tokens/s: 176850.4 | valid tokens/s: 176850.4 | physical tokens/s: 177344.2 | loss tokens/s: 131547.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.914234E-01 | lm-global-supervised-tokens: 1.247944E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.568361E+00 | lm-token-buffer-mib: 5.824986E+01 | global_load_balancing_loss: 1.000063E+00 | loss scale: 1.0 | grad norm: 0.233 | params norm: 11267.918 | params rms: 0.061012 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 18:17:17.262905 | finish at 2026-10-10 21:11:44 + [2026-10-10 02:56:05] iteration 6162/ 6855 | consumed samples: 788736 | elapsed time per iteration (ms): 97887.6 | throughput per GPU (TFLOP/s/GPU): 39.3 | MFU 12.58% | FLOPs estimator: workload-aware | logical tokens/s: 171392.2 | valid tokens/s: 171392.2 | physical tokens/s: 171731.7 | loss tokens/s: 125897.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.167975E-01 | lm-global-supervised-tokens: 1.232382E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.344537E-02 | lm-token-buffer-mib: 5.813926E+01 | global_load_balancing_loss: 1.000376E+00 | loss scale: 1.0 | grad norm: 0.038 | params norm: 11267.870 | params rms: 0.061012 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 18:50:36.091396 | finish at 2026-10-10 21:46:41 + [2026-10-10 02:57:31] iteration 6163/ 6855 | consumed samples: 788864 | elapsed time per iteration (ms): 86577.1 | throughput per GPU (TFLOP/s/GPU): 42.8 | MFU 13.71% | FLOPs estimator: workload-aware | logical tokens/s: 193783.2 | valid tokens/s: 193783.2 | physical tokens/s: 194409.6 | loss tokens/s: 138845.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.144540E-01 | lm-global-supervised-tokens: 1.202079E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.451786E-02 | lm-token-buffer-mib: 5.806898E+01 | global_load_balancing_loss: 1.000923E+00 | loss scale: 1.0 | grad norm: 0.088 | params norm: 11267.823 | params rms: 0.061012 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 16:38:31.319735 | finish at 2026-10-10 19:36:02 + [2026-10-10 02:58:56] iteration 6164/ 6855 | consumed samples: 788992 | elapsed time per iteration (ms): 84685.2 | throughput per GPU (TFLOP/s/GPU): 43.4 | MFU 13.92% | FLOPs estimator: workload-aware | logical tokens/s: 198112.7 | valid tokens/s: 198112.7 | physical tokens/s: 198696.0 | loss tokens/s: 148438.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.008610E-01 | lm-global-supervised-tokens: 1.257055E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.501202E-02 | lm-token-buffer-mib: 5.822397E+01 | global_load_balancing_loss: 9.998582E-01 | loss scale: 1.0 | grad norm: 0.161 | params norm: 11267.776 | params rms: 0.061011 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 16:15:17.460827 | finish at 2026-10-10 19:14:13 + [2026-10-10 03:00:19] iteration 6165/ 6855 | consumed samples: 789120 | elapsed time per iteration (ms): 83479.4 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.77% | FLOPs estimator: workload-aware | logical tokens/s: 200974.0 | valid tokens/s: 200974.0 | physical tokens/s: 201433.2 | loss tokens/s: 150553.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.398037E-01 | lm-global-supervised-tokens: 1.256808E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.287691E-02 | lm-token-buffer-mib: 5.812878E+01 | global_load_balancing_loss: 9.996936E-01 | loss scale: 1.0 | grad norm: 0.063 | params norm: 11267.730 | params rms: 0.061011 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 16:00:00.790715 | finish at 2026-10-10 19:00:20 + [2026-10-10 03:01:44] iteration 6166/ 6855 | consumed samples: 789248 | elapsed time per iteration (ms): 84663.5 | throughput per GPU (TFLOP/s/GPU): 43.6 | MFU 13.97% | FLOPs estimator: workload-aware | logical tokens/s: 198163.0 | valid tokens/s: 198163.0 | physical tokens/s: 198771.2 | loss tokens/s: 149786.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.725750E-01 | lm-global-supervised-tokens: 1.268145E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.258641E-02 | lm-token-buffer-mib: 5.879016E+01 | global_load_balancing_loss: 1.000788E+00 | loss scale: 1.0 | grad norm: 0.048 | params norm: 11267.684 | params rms: 0.061011 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 16:12:13.169290 | finish at 2026-10-10 19:13:57 + [2026-10-10 03:03:11] iteration 6167/ 6855 | consumed samples: 789376 | elapsed time per iteration (ms): 87348.1 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.49% | FLOPs estimator: workload-aware | logical tokens/s: 192073.1 | valid tokens/s: 192073.1 | physical tokens/s: 192459.8 | loss tokens/s: 140709.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.315902E-01 | lm-global-supervised-tokens: 1.229071E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.780255E-02 | lm-token-buffer-mib: 5.808614E+01 | global_load_balancing_loss: 1.000658E+00 | loss scale: 1.0 | grad norm: 0.049 | params norm: 11267.639 | params rms: 0.061011 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 16:41:35.485874 | finish at 2026-10-10 19:44:47 + [2026-10-10 03:04:40] iteration 6168/ 6855 | consumed samples: 789504 | elapsed time per iteration (ms): 88289.0 | throughput per GPU (TFLOP/s/GPU): 41.6 | MFU 13.34% | FLOPs estimator: workload-aware | logical tokens/s: 190025.1 | valid tokens/s: 190025.1 | physical tokens/s: 190397.6 | loss tokens/s: 145989.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.669855E-01 | lm-global-supervised-tokens: 1.288928E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.294626E-02 | lm-token-buffer-mib: 5.814013E+01 | global_load_balancing_loss: 1.000619E+00 | loss scale: 1.0 | grad norm: 0.078 | params norm: 11267.594 | params rms: 0.061010 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 16:50:54.558584 | finish at 2026-10-10 19:55:34 + [2026-10-10 03:06:03] iteration 6169/ 6855 | consumed samples: 789632 | elapsed time per iteration (ms): 83255.0 | throughput per GPU (TFLOP/s/GPU): 43.3 | MFU 13.88% | FLOPs estimator: workload-aware | logical tokens/s: 201515.6 | valid tokens/s: 201515.6 | physical tokens/s: 201964.2 | loss tokens/s: 156235.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 5.724139E-01 | lm-global-supervised-tokens: 1.300741E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.156603E-02 | lm-token-buffer-mib: 5.822277E+01 | global_load_balancing_loss: 1.000044E+00 | loss scale: 1.0 | grad norm: 0.067 | params norm: 11267.548 | params rms: 0.061010 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 15:51:52.930569 | finish at 2026-10-10 18:57:56 + [2026-10-10 03:07:57] iteration 6170/ 6855 | consumed samples: 789760 | elapsed time per iteration (ms): 114562.9 | throughput per GPU (TFLOP/s/GPU): 32.5 | MFU 10.42% | FLOPs estimator: workload-aware | logical tokens/s: 146445.4 | valid tokens/s: 146445.4 | physical tokens/s: 146785.6 | loss tokens/s: 108520.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.775343E-01 | lm-global-supervised-tokens: 1.243246E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.138857E-02 | lm-token-buffer-mib: 5.814394E+01 | global_load_balancing_loss: 1.000462E+00 | loss scale: 1.0 | grad norm: 0.046 | params norm: 11267.506 | params rms: 0.061010 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 21:47:55.598958 | finish at 2026-10-11 00:55:53 + [2026-10-10 03:09:26] iteration 6171/ 6855 | consumed samples: 789888 | elapsed time per iteration (ms): 88785.9 | throughput per GPU (TFLOP/s/GPU): 41.2 | MFU 13.22% | FLOPs estimator: workload-aware | logical tokens/s: 188962.5 | valid tokens/s: 188962.5 | physical tokens/s: 189611.9 | loss tokens/s: 139246.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.646426E-01 | lm-global-supervised-tokens: 1.236312E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.219491E-02 | lm-token-buffer-mib: 5.817487E+01 | global_load_balancing_loss: 1.000594E+00 | loss scale: 1.0 | grad norm: 0.042 | params norm: 11267.464 | params rms: 0.061010 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 16:52:09.547688 | finish at 2026-10-10 20:01:36 + [2026-10-10 03:10:55] iteration 6172/ 6855 | consumed samples: 790016 | elapsed time per iteration (ms): 88519.2 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.46% | FLOPs estimator: workload-aware | logical tokens/s: 189531.2 | valid tokens/s: 189531.2 | physical tokens/s: 189934.1 | loss tokens/s: 138397.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.260501E-01 | lm-global-supervised-tokens: 1.225084E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.256749E-02 | lm-token-buffer-mib: 5.816290E+01 | global_load_balancing_loss: 1.001426E+00 | loss scale: 1.0 | grad norm: 0.056 | params norm: 11267.421 | params rms: 0.061009 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 16:47:38.605680 | finish at 2026-10-10 19:58:33 + [2026-10-10 03:12:25] iteration 6173/ 6855 | consumed samples: 790144 | elapsed time per iteration (ms): 90716.3 | throughput per GPU (TFLOP/s/GPU): 41.4 | MFU 13.26% | FLOPs estimator: workload-aware | logical tokens/s: 184941.6 | valid tokens/s: 184941.6 | physical tokens/s: 185408.5 | loss tokens/s: 135734.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.478018E-01 | lm-global-supervised-tokens: 1.231331E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.131018E-02 | lm-token-buffer-mib: 5.819344E+01 | global_load_balancing_loss: 1.000203E+00 | loss scale: 1.0 | grad norm: 0.058 | params norm: 11267.376 | params rms: 0.061009 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 17:11:08.482624 | finish at 2026-10-10 20:23:34 + [2026-10-10 03:13:52] iteration 6174/ 6855 | consumed samples: 790272 | elapsed time per iteration (ms): 86554.9 | throughput per GPU (TFLOP/s/GPU): 42.8 | MFU 13.73% | FLOPs estimator: workload-aware | logical tokens/s: 193832.6 | valid tokens/s: 193832.6 | physical tokens/s: 194366.4 | loss tokens/s: 138488.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.651936E-01 | lm-global-supervised-tokens: 1.198686E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.830837E-02 | lm-token-buffer-mib: 5.811024E+01 | global_load_balancing_loss: 9.999366E-01 | loss scale: 1.0 | grad norm: 0.045 | params norm: 11267.333 | params rms: 0.061009 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 16:22:23.883606 | finish at 2026-10-10 19:36:16 + [2026-10-10 03:15:21] iteration 6175/ 6855 | consumed samples: 790400 | elapsed time per iteration (ms): 88582.3 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.61% | FLOPs estimator: workload-aware | logical tokens/s: 189396.8 | valid tokens/s: 189396.8 | physical tokens/s: 189898.6 | loss tokens/s: 129878.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 5.828321E-01 | lm-global-supervised-tokens: 1.150495E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.144781E-02 | lm-token-buffer-mib: 5.814985E+01 | global_load_balancing_loss: 9.990938E-01 | loss scale: 1.0 | grad norm: 0.078 | params norm: 11267.288 | params rms: 0.061009 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 16:43:55.941916 | finish at 2026-10-10 19:59:17 + [2026-10-10 03:16:50] iteration 6176/ 6855 | consumed samples: 790528 | elapsed time per iteration (ms): 89486.4 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.55% | FLOPs estimator: workload-aware | logical tokens/s: 187483.2 | valid tokens/s: 187483.0 | physical tokens/s: 187986.7 | loss tokens/s: 135932.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.677969E-01 | lm-global-supervised-tokens: 1.216414E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.414846E-02 | lm-token-buffer-mib: 5.809759E+01 | global_load_balancing_loss: 9.975778E-01 | loss scale: 1.0 | grad norm: 0.035 | params norm: 11267.242 | params rms: 0.061008 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 16:52:41.289808 | finish at 2026-10-10 20:09:31 + [2026-10-10 03:18:18] iteration 6177/ 6855 | consumed samples: 790656 | elapsed time per iteration (ms): 87900.9 | throughput per GPU (TFLOP/s/GPU): 43.1 | MFU 13.80% | FLOPs estimator: workload-aware | logical tokens/s: 190865.0 | valid tokens/s: 190865.0 | physical tokens/s: 191297.3 | loss tokens/s: 136935.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.086885E-01 | lm-global-supervised-tokens: 1.203678E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.109547E-02 | lm-token-buffer-mib: 5.819363E+01 | global_load_balancing_loss: 9.985154E-01 | loss scale: 1.0 | grad norm: 0.036 | params norm: 11267.196 | params rms: 0.061008 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 16:33:16.796707 | finish at 2026-10-10 19:51:35 + [2026-10-10 03:19:43] iteration 6178/ 6855 | consumed samples: 790784 | elapsed time per iteration (ms): 85422.1 | throughput per GPU (TFLOP/s/GPU): 43.1 | MFU 13.81% | FLOPs estimator: workload-aware | logical tokens/s: 196402.7 | valid tokens/s: 196402.7 | physical tokens/s: 196977.6 | loss tokens/s: 147408.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.260721E-01 | lm-global-supervised-tokens: 1.259194E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.191386E-02 | lm-token-buffer-mib: 5.810955E+01 | global_load_balancing_loss: 1.000883E+00 | loss scale: 1.0 | grad norm: 0.055 | params norm: 11267.151 | params rms: 0.061008 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 16:03:50.774820 | finish at 2026-10-10 19:23:34 + [2026-10-10 03:21:13] iteration 6179/ 6855 | consumed samples: 790912 | elapsed time per iteration (ms): 90041.0 | throughput per GPU (TFLOP/s/GPU): 41.5 | MFU 13.31% | FLOPs estimator: workload-aware | logical tokens/s: 186328.2 | valid tokens/s: 186328.2 | physical tokens/s: 186896.4 | loss tokens/s: 141144.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.966661E-01 | lm-global-supervised-tokens: 1.270880E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.221788E-02 | lm-token-buffer-mib: 5.810513E+01 | global_load_balancing_loss: 1.000983E+00 | loss scale: 1.0 | grad norm: 0.038 | params norm: 11267.107 | params rms: 0.061008 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 16:54:27.702548 | finish at 2026-10-10 20:15:41 + [2026-10-10 03:23:02] iteration 6180/ 6855 | consumed samples: 791040 | elapsed time per iteration (ms): 108958.6 | throughput per GPU (TFLOP/s/GPU): 33.6 | MFU 10.77% | FLOPs estimator: workload-aware | logical tokens/s: 153977.9 | valid tokens/s: 153977.9 | physical tokens/s: 154369.9 | loss tokens/s: 111884.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.513527E-01 | lm-global-supervised-tokens: 1.219082E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.160477E-02 | lm-token-buffer-mib: 5.814742E+01 | global_load_balancing_loss: 1.003608E+00 | loss scale: 1.0 | grad norm: 0.042 | params norm: 11267.063 | params rms: 0.061007 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 20:25:47.075790 | finish at 2026-10-10 23:48:49 + [2026-10-10 03:24:29] iteration 6181/ 6855 | consumed samples: 791168 | elapsed time per iteration (ms): 87037.4 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.52% | FLOPs estimator: workload-aware | logical tokens/s: 192758.6 | valid tokens/s: 192758.6 | physical tokens/s: 193352.4 | loss tokens/s: 144033.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.662831E-01 | lm-global-supervised-tokens: 1.253632E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.575511E-02 | lm-token-buffer-mib: 5.808006E+01 | global_load_balancing_loss: 1.002502E+00 | loss scale: 1.0 | grad norm: 0.066 | params norm: 11267.017 | params rms: 0.061007 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 16:17:43.221264 | finish at 2026-10-10 19:42:13 + [2026-10-10 03:25:59] iteration 6182/ 6855 | consumed samples: 791296 | elapsed time per iteration (ms): 89734.2 | throughput per GPU (TFLOP/s/GPU): 41.4 | MFU 13.26% | FLOPs estimator: workload-aware | logical tokens/s: 186965.6 | valid tokens/s: 186965.6 | physical tokens/s: 187415.1 | loss tokens/s: 138806.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.828172E-01 | lm-global-supervised-tokens: 1.245566E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.852559E-02 | lm-token-buffer-mib: 5.810802E+01 | global_load_balancing_loss: 1.001530E+00 | loss scale: 1.0 | grad norm: 0.046 | params norm: 11266.971 | params rms: 0.061007 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 16:46:31.084509 | finish at 2026-10-10 20:12:30 + [2026-10-10 03:27:26] iteration 6183/ 6855 | consumed samples: 791424 | elapsed time per iteration (ms): 86570.3 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.79% | FLOPs estimator: workload-aware | logical tokens/s: 193798.4 | valid tokens/s: 193798.4 | physical tokens/s: 194257.8 | loss tokens/s: 146093.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.083510E-01 | lm-global-supervised-tokens: 1.264737E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.231350E-02 | lm-token-buffer-mib: 5.817764E+01 | global_load_balancing_loss: 9.997977E-01 | loss scale: 1.0 | grad norm: 0.121 | params norm: 11266.927 | params rms: 0.061007 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 16:09:35.244873 | finish at 2026-10-10 19:37:01 + [2026-10-10 03:28:54] iteration 6184/ 6855 | consumed samples: 791552 | elapsed time per iteration (ms): 88053.0 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.59% | FLOPs estimator: workload-aware | logical tokens/s: 190535.4 | valid tokens/s: 190535.4 | physical tokens/s: 190873.8 | loss tokens/s: 137321.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.863422E-01 | lm-global-supervised-tokens: 1.209159E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.536671E-02 | lm-token-buffer-mib: 5.814413E+01 | global_load_balancing_loss: 9.978667E-01 | loss scale: 1.0 | grad norm: 0.140 | params norm: 11266.881 | params rms: 0.061006 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 16:24:43.562182 | finish at 2026-10-10 19:53:37 + [2026-10-10 03:30:28] iteration 6185/ 6855 | consumed samples: 791680 | elapsed time per iteration (ms): 94273.2 | throughput per GPU (TFLOP/s/GPU): 40.2 | MFU 12.90% | FLOPs estimator: workload-aware | logical tokens/s: 177963.3 | valid tokens/s: 177963.3 | physical tokens/s: 178476.8 | loss tokens/s: 127867.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.675290E-01 | lm-global-supervised-tokens: 1.205451E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.566991E-02 | lm-token-buffer-mib: 5.814568E+01 | global_load_balancing_loss: 9.990416E-01 | loss scale: 1.0 | grad norm: 0.044 | params norm: 11266.837 | params rms: 0.061006 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 17:32:43.051212 | finish at 2026-10-10 21:03:11 + [2026-10-10 03:31:56] iteration 6186/ 6855 | consumed samples: 791808 | elapsed time per iteration (ms): 87586.9 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.59% | FLOPs estimator: workload-aware | logical tokens/s: 191548.8 | valid tokens/s: 191548.8 | physical tokens/s: 191948.8 | loss tokens/s: 141201.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.453972E-01 | lm-global-supervised-tokens: 1.236742E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.148087E-02 | lm-token-buffer-mib: 5.810815E+01 | global_load_balancing_loss: 9.989566E-01 | loss scale: 1.0 | grad norm: 0.118 | params norm: 11266.793 | params rms: 0.061006 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 16:16:35.653642 | finish at 2026-10-10 19:48:31 + [2026-10-10 03:33:24] iteration 6187/ 6855 | consumed samples: 791936 | elapsed time per iteration (ms): 88457.9 | throughput per GPU (TFLOP/s/GPU): 41.1 | MFU 13.17% | FLOPs estimator: workload-aware | logical tokens/s: 189663.1 | valid tokens/s: 189663.1 | physical tokens/s: 190118.4 | loss tokens/s: 139174.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.694410E-01 | lm-global-supervised-tokens: 1.231108E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.168272E-02 | lm-token-buffer-mib: 5.811584E+01 | global_load_balancing_loss: 9.997981E-01 | loss scale: 1.0 | grad norm: 0.079 | params norm: 11266.748 | params rms: 0.061006 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 16:24:49.860031 | finish at 2026-10-10 19:58:14 + [2026-10-10 03:34:53] iteration 6188/ 6855 | consumed samples: 792064 | elapsed time per iteration (ms): 88812.7 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.42% | FLOPs estimator: workload-aware | logical tokens/s: 188905.5 | valid tokens/s: 188905.5 | physical tokens/s: 189191.8 | loss tokens/s: 133503.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.878273E-01 | lm-global-supervised-tokens: 1.185683E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.540119E-02 | lm-token-buffer-mib: 5.807835E+01 | global_load_balancing_loss: 1.001315E+00 | loss scale: 1.0 | grad norm: 0.056 | params norm: 11266.705 | params rms: 0.061006 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 16:27:18.092868 | finish at 2026-10-10 20:02:11 + [2026-10-10 03:36:20] iteration 6189/ 6855 | consumed samples: 792192 | elapsed time per iteration (ms): 86820.1 | throughput per GPU (TFLOP/s/GPU): 42.8 | MFU 13.72% | FLOPs estimator: workload-aware | logical tokens/s: 193240.7 | valid tokens/s: 193240.7 | physical tokens/s: 193655.1 | loss tokens/s: 138665.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.442546E-01 | lm-global-supervised-tokens: 1.203896E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.816098E-02 | lm-token-buffer-mib: 5.813984E+01 | global_load_balancing_loss: 1.001862E+00 | loss scale: 1.0 | grad norm: 0.071 | params norm: 11266.661 | params rms: 0.061005 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 16:03:42.216022 | finish at 2026-10-10 19:40:02 + [2026-10-10 03:38:06] iteration 6190/ 6855 | consumed samples: 792320 | elapsed time per iteration (ms): 105798.6 | throughput per GPU (TFLOP/s/GPU): 34.0 | MFU 10.91% | FLOPs estimator: workload-aware | logical tokens/s: 158576.9 | valid tokens/s: 158576.9 | physical tokens/s: 159012.3 | loss tokens/s: 115603.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.169273E-01 | lm-global-supervised-tokens: 1.223064E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.230762E-02 | lm-token-buffer-mib: 5.813142E+01 | global_load_balancing_loss: 1.000721E+00 | loss scale: 1.0 | grad norm: 0.094 | params norm: 11266.619 | params rms: 0.061005 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 19:32:36.057081 | finish at 2026-10-10 23:10:42 + [2026-10-10 03:39:31] iteration 6191/ 6855 | consumed samples: 792448 | elapsed time per iteration (ms): 85325.7 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.68% | FLOPs estimator: workload-aware | logical tokens/s: 196625.2 | valid tokens/s: 196625.2 | physical tokens/s: 197151.2 | loss tokens/s: 149252.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.530926E-01 | lm-global-supervised-tokens: 1.273508E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.570452E-02 | lm-token-buffer-mib: 5.806274E+01 | global_load_balancing_loss: 1.000074E+00 | loss scale: 1.0 | grad norm: 0.128 | params norm: 11266.576 | params rms: 0.061005 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 15:44:16.271479 | finish at 2026-10-10 19:23:47 + [2026-10-10 03:40:57] iteration 6192/ 6855 | consumed samples: 792576 | elapsed time per iteration (ms): 86173.9 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.64% | FLOPs estimator: workload-aware | logical tokens/s: 194690.0 | valid tokens/s: 194690.0 | physical tokens/s: 194932.4 | loss tokens/s: 147104.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.613903E-01 | lm-global-supervised-tokens: 1.267658E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.632077E-02 | lm-token-buffer-mib: 5.806469E+01 | global_load_balancing_loss: 1.000695E+00 | loss scale: 1.0 | grad norm: 0.045 | params norm: 11266.530 | params rms: 0.061005 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 15:52:13.266067 | finish at 2026-10-10 19:33:10 + [2026-10-10 03:42:20] iteration 6193/ 6855 | consumed samples: 792704 | elapsed time per iteration (ms): 83342.6 | throughput per GPU (TFLOP/s/GPU): 43.1 | MFU 13.81% | FLOPs estimator: workload-aware | logical tokens/s: 201304.2 | valid tokens/s: 201304.2 | physical tokens/s: 201704.5 | loss tokens/s: 147859.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.422139E-01 | lm-global-supervised-tokens: 1.232300E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.673469E-02 | lm-token-buffer-mib: 5.810691E+01 | global_load_balancing_loss: 9.999866E-01 | loss scale: 1.0 | grad norm: 0.049 | params norm: 11266.484 | params rms: 0.061004 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 15:19:32.794484 | finish at 2026-10-10 19:01:53 + [2026-10-10 03:43:48] iteration 6194/ 6855 | consumed samples: 792832 | elapsed time per iteration (ms): 87127.0 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.75% | FLOPs estimator: workload-aware | logical tokens/s: 192559.7 | valid tokens/s: 192559.7 | physical tokens/s: 192950.5 | loss tokens/s: 136247.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.640214E-01 | lm-global-supervised-tokens: 1.187080E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.403195E-02 | lm-token-buffer-mib: 5.808238E+01 | global_load_balancing_loss: 9.991954E-01 | loss scale: 1.0 | grad norm: 0.066 | params norm: 11266.440 | params rms: 0.061004 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 15:59:50.952420 | finish at 2026-10-10 19:43:38 + [2026-10-10 03:45:14] iteration 6195/ 6855 | consumed samples: 792960 | elapsed time per iteration (ms): 86532.4 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.56% | FLOPs estimator: workload-aware | logical tokens/s: 193882.5 | valid tokens/s: 193882.5 | physical tokens/s: 194280.0 | loss tokens/s: 144946.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.585766E-01 | lm-global-supervised-tokens: 1.254256E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.578253E-02 | lm-token-buffer-mib: 5.811655E+01 | global_load_balancing_loss: 9.996632E-01 | loss scale: 1.0 | grad norm: 0.050 | params norm: 11266.395 | params rms: 0.061004 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 15:51:51.392741 | finish at 2026-10-10 19:37:05 + [2026-10-10 03:46:44] iteration 6196/ 6855 | consumed samples: 793088 | elapsed time per iteration (ms): 90082.9 | throughput per GPU (TFLOP/s/GPU): 41.0 | MFU 13.13% | FLOPs estimator: workload-aware | logical tokens/s: 186241.6 | valid tokens/s: 186241.5 | physical tokens/s: 186755.2 | loss tokens/s: 139492.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.347856E-01 | lm-global-supervised-tokens: 1.256592E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.796869E-02 | lm-token-buffer-mib: 5.825758E+01 | global_load_balancing_loss: 1.000572E+00 | loss scale: 1.0 | grad norm: 0.083 | params norm: 11266.352 | params rms: 0.061004 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 16:29:24.654699 | finish at 2026-10-10 20:16:09 + [2026-10-10 03:48:09] iteration 6197/ 6855 | consumed samples: 793216 | elapsed time per iteration (ms): 84782.0 | throughput per GPU (TFLOP/s/GPU): 43.4 | MFU 13.91% | FLOPs estimator: workload-aware | logical tokens/s: 197886.6 | valid tokens/s: 197886.6 | physical tokens/s: 198335.6 | loss tokens/s: 153876.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.894575E-01 | lm-global-supervised-tokens: 1.304598E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.154761E-02 | lm-token-buffer-mib: 5.816926E+01 | global_load_balancing_loss: 1.001536E+00 | loss scale: 1.0 | grad norm: 0.132 | params norm: 11266.307 | params rms: 0.061003 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 15:29:46.536119 | finish at 2026-10-10 19:17:55 + [2026-10-10 03:49:34] iteration 6198/ 6855 | consumed samples: 793344 | elapsed time per iteration (ms): 85104.3 | throughput per GPU (TFLOP/s/GPU): 43.2 | MFU 13.84% | FLOPs estimator: workload-aware | logical tokens/s: 197136.3 | valid tokens/s: 197136.2 | physical tokens/s: 197705.2 | loss tokens/s: 147089.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.430526E-01 | lm-global-supervised-tokens: 1.251794E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.178884E-02 | lm-token-buffer-mib: 5.826459E+01 | global_load_balancing_loss: 1.000528E+00 | loss scale: 1.0 | grad norm: 0.088 | params norm: 11266.262 | params rms: 0.061003 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 15:31:53.494100 | finish at 2026-10-10 19:21:28 + [2026-10-10 03:51:00] iteration 6199/ 6855 | consumed samples: 793472 | elapsed time per iteration (ms): 86040.4 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.51% | FLOPs estimator: workload-aware | logical tokens/s: 194991.5 | valid tokens/s: 194991.5 | physical tokens/s: 195417.9 | loss tokens/s: 147437.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.721843E-01 | lm-global-supervised-tokens: 1.268556E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.361579E-02 | lm-token-buffer-mib: 5.813779E+01 | global_load_balancing_loss: 9.989607E-01 | loss scale: 1.0 | grad norm: 0.049 | params norm: 11266.218 | params rms: 0.061003 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 15:40:42.533073 | finish at 2026-10-10 19:31:43 + [2026-10-10 03:52:56] iteration 6200/ 6855 | consumed samples: 793600 | elapsed time per iteration (ms): 115657.0 | throughput per GPU (TFLOP/s/GPU): 33.1 | MFU 10.61% | FLOPs estimator: workload-aware | logical tokens/s: 145060.1 | valid tokens/s: 145060.1 | physical tokens/s: 145365.1 | loss tokens/s: 107065.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.543306E-01 | lm-global-supervised-tokens: 1.238288E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.638209E-02 | lm-token-buffer-mib: 5.810778E+01 | global_load_balancing_loss: 9.995148E-01 | loss scale: 1.0 | grad norm: 0.075 | params norm: 11266.175 | params rms: 0.061003 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 21:02:35.306933 | finish at 2026-10-11 00:55:31 +times across ranks (ms): + save-checkpoint: + rank 0: 27988.26 + rank 1: 27988.27 + rank 2: 27988.29 + rank 3: 27988.23 + rank 4: 27988.25 + rank 5: 27988.25 + rank 6: 27988.22 + rank 7: 27988.25 + rank 8: 27988.47 + rank 9: 27988.42 + rank 10: 27988.42 + rank 11: 27988.43 + rank 12: 27988.44 + rank 13: 27988.43 + rank 14: 27988.45 + rank 15: 27988.44 + rank 16: 27988.37 + rank 17: 27988.30 + rank 18: 27988.28 + rank 19: 27988.31 + rank 20: 27988.28 + rank 21: 27988.31 + rank 22: 27988.31 + rank 23: 27988.28 + rank 24: 27988.91 + rank 25: 27989.59 + rank 26: 27988.94 + rank 27: 27988.96 + rank 28: 27988.90 + rank 29: 27988.92 + rank 30: 27988.94 + rank 31: 27988.93 + rank 32: 27988.30 + rank 33: 27988.39 + rank 34: 27988.35 + rank 35: 27988.36 + rank 36: 27988.30 + rank 37: 27988.31 + rank 38: 27988.29 + rank 39: 27988.32 + rank 40: 27988.26 + rank 41: 27988.24 + rank 42: 27988.23 + rank 43: 27988.25 + rank 44: 27988.21 + rank 45: 27988.23 + rank 46: 27988.25 + rank 47: 27988.21 + rank 48: 27988.95 + rank 49: 27988.92 + rank 50: 27988.61 + rank 51: 27988.86 + rank 52: 27989.03 + rank 53: 27988.91 + rank 54: 27988.90 + rank 55: 27988.87 + rank 56: 27988.62 + rank 57: 27988.59 + rank 58: 27988.57 + rank 59: 27988.57 + rank 60: 27988.59 + rank 61: 27988.58 + rank 62: 27988.57 + rank 63: 27988.58 + [2026-10-10 03:55:03] iteration 6201/ 6855 | consumed samples: 793728 | elapsed time per iteration (ms): 98840.8 | throughput per GPU (TFLOP/s/GPU): 37.3 | MFU 11.95% | FLOPs estimator: workload-aware | logical tokens/s: 169739.2 | valid tokens/s: 169739.2 | physical tokens/s: 170165.9 | loss tokens/s: 125254.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.329665E-01 | lm-global-supervised-tokens: 1.238023E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 3.397493E-01 | lm-token-buffer-mib: 5.828389E+01 | global_load_balancing_loss: 1.001397E+00 | loss scale: 1.0 | grad norm: 0.037 | params norm: 11266.127 | params rms: 0.061002 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 17:57:21.876370 | finish at 2026-10-10 21:52:25 + [2026-10-10 03:56:38] iteration 6202/ 6855 | consumed samples: 793856 | elapsed time per iteration (ms): 95120.1 | throughput per GPU (TFLOP/s/GPU): 39.0 | MFU 12.49% | FLOPs estimator: workload-aware | logical tokens/s: 176378.5 | valid tokens/s: 176378.4 | physical tokens/s: 176788.2 | loss tokens/s: 130044.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 5.913658E-01 | lm-global-supervised-tokens: 1.236983E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.622963E-02 | lm-token-buffer-mib: 5.831106E+01 | global_load_balancing_loss: 1.001340E+00 | loss scale: 1.0 | grad norm: 0.095 | params norm: 11266.080 | params rms: 0.061002 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 17:15:13.424847 | finish at 2026-10-10 21:11:51 + [2026-10-10 03:58:02] iteration 6203/ 6855 | consumed samples: 793984 | elapsed time per iteration (ms): 84085.9 | throughput per GPU (TFLOP/s/GPU): 43.9 | MFU 14.08% | FLOPs estimator: workload-aware | logical tokens/s: 199524.0 | valid tokens/s: 199524.0 | physical tokens/s: 200074.4 | loss tokens/s: 147863.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.856109E-01 | lm-global-supervised-tokens: 1.243319E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.127354E-02 | lm-token-buffer-mib: 5.811388E+01 | global_load_balancing_loss: 9.994958E-01 | loss scale: 1.0 | grad norm: 0.065 | params norm: 11266.036 | params rms: 0.061002 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 15:13:43.976066 | finish at 2026-10-10 19:11:46 + [2026-10-10 03:59:32] iteration 6204/ 6855 | consumed samples: 794112 | elapsed time per iteration (ms): 90481.3 | throughput per GPU (TFLOP/s/GPU): 41.7 | MFU 13.38% | FLOPs estimator: workload-aware | logical tokens/s: 185421.9 | valid tokens/s: 185421.9 | physical tokens/s: 185871.4 | loss tokens/s: 132242.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.581652E-01 | lm-global-supervised-tokens: 1.196547E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.184542E-02 | lm-token-buffer-mib: 5.817909E+01 | global_load_balancing_loss: 1.000046E+00 | loss scale: 1.0 | grad norm: 0.109 | params norm: 11265.990 | params rms: 0.061002 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 16:21:43.327307 | finish at 2026-10-10 20:21:16 + [2026-10-10 04:01:00] iteration 6205/ 6855 | consumed samples: 794240 | elapsed time per iteration (ms): 87374.2 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.67% | FLOPs estimator: workload-aware | logical tokens/s: 192015.1 | valid tokens/s: 192015.1 | physical tokens/s: 192404.1 | loss tokens/s: 144836.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.728468E-01 | lm-global-supervised-tokens: 1.265499E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.375085E-02 | lm-token-buffer-mib: 5.802444E+01 | global_load_balancing_loss: 1.000015E+00 | loss scale: 1.0 | grad norm: 0.217 | params norm: 11265.944 | params rms: 0.061001 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 15:46:33.224955 | finish at 2026-10-10 19:47:33 + [2026-10-10 04:02:27] iteration 6206/ 6855 | consumed samples: 794368 | elapsed time per iteration (ms): 87325.9 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.68% | FLOPs estimator: workload-aware | logical tokens/s: 192121.4 | valid tokens/s: 192121.4 | physical tokens/s: 192482.4 | loss tokens/s: 142722.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.571002E-01 | lm-global-supervised-tokens: 1.246333E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.404969E-02 | lm-token-buffer-mib: 5.806032E+01 | global_load_balancing_loss: 9.994571E-01 | loss scale: 1.0 | grad norm: 0.047 | params norm: 11265.898 | params rms: 0.061001 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 15:44:34.519363 | finish at 2026-10-10 19:47:02 + [2026-10-10 04:03:54] iteration 6207/ 6855 | consumed samples: 794496 | elapsed time per iteration (ms): 86521.5 | throughput per GPU (TFLOP/s/GPU): 43.2 | MFU 13.83% | FLOPs estimator: workload-aware | logical tokens/s: 193907.8 | valid tokens/s: 193907.8 | physical tokens/s: 194481.2 | loss tokens/s: 141885.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.377758E-01 | lm-global-supervised-tokens: 1.227618E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.438154E-02 | lm-token-buffer-mib: 5.807220E+01 | global_load_balancing_loss: 1.000065E+00 | loss scale: 1.0 | grad norm: 0.187 | params norm: 11265.852 | params rms: 0.061001 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 15:34:25.906116 | finish at 2026-10-10 19:38:20 + [2026-10-10 04:05:22] iteration 6208/ 6855 | consumed samples: 794624 | elapsed time per iteration (ms): 88675.9 | throughput per GPU (TFLOP/s/GPU): 41.4 | MFU 13.27% | FLOPs estimator: workload-aware | logical tokens/s: 189196.3 | valid tokens/s: 189196.3 | physical tokens/s: 189627.9 | loss tokens/s: 139432.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.657367E-01 | lm-global-supervised-tokens: 1.236426E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.338297E-02 | lm-token-buffer-mib: 5.803149E+01 | global_load_balancing_loss: 9.999614E-01 | loss scale: 1.0 | grad norm: 0.056 | params norm: 11265.807 | params rms: 0.061001 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 15:56:13.307134 | finish at 2026-10-10 20:01:36 + [2026-10-10 04:06:52] iteration 6209/ 6855 | consumed samples: 794752 | elapsed time per iteration (ms): 89329.0 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.47% | FLOPs estimator: workload-aware | logical tokens/s: 187813.4 | valid tokens/s: 187813.4 | physical tokens/s: 188309.5 | loss tokens/s: 141896.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.170900E-01 | lm-global-supervised-tokens: 1.267545E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.368203E-02 | lm-token-buffer-mib: 5.811009E+01 | global_load_balancing_loss: 9.999111E-01 | loss scale: 1.0 | grad norm: 0.150 | params norm: 11265.760 | params rms: 0.061000 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 16:01:46.535076 | finish at 2026-10-10 20:08:38 + [2026-10-10 04:08:36] iteration 6210/ 6855 | consumed samples: 794880 | elapsed time per iteration (ms): 104705.4 | throughput per GPU (TFLOP/s/GPU): 33.7 | MFU 10.79% | FLOPs estimator: workload-aware | logical tokens/s: 160232.4 | valid tokens/s: 160232.3 | physical tokens/s: 160711.1 | loss tokens/s: 121070.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.555426E-01 | lm-global-supervised-tokens: 1.267675E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.150521E-02 | lm-token-buffer-mib: 5.834184E+01 | global_load_balancing_loss: 9.997483E-01 | loss scale: 1.0 | grad norm: 0.068 | params norm: 11265.715 | params rms: 0.061000 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 18:45:35.002985 | finish at 2026-10-10 22:54:11 + [2026-10-10 04:10:04] iteration 6211/ 6855 | consumed samples: 795008 | elapsed time per iteration (ms): 87304.9 | throughput per GPU (TFLOP/s/GPU): 41.7 | MFU 13.36% | FLOPs estimator: workload-aware | logical tokens/s: 192167.9 | valid tokens/s: 192167.9 | physical tokens/s: 192699.0 | loss tokens/s: 150842.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.564326E-01 | lm-global-supervised-tokens: 1.316924E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.322640E-02 | lm-token-buffer-mib: 5.823681E+01 | global_load_balancing_loss: 9.998661E-01 | loss scale: 1.0 | grad norm: 0.165 | params norm: 11265.669 | params rms: 0.061000 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 15:37:04.347878 | finish at 2026-10-10 19:47:08 + [2026-10-10 04:11:32] iteration 6212/ 6855 | consumed samples: 795136 | elapsed time per iteration (ms): 88092.5 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.52% | FLOPs estimator: workload-aware | logical tokens/s: 190449.0 | valid tokens/s: 190449.0 | physical tokens/s: 190933.1 | loss tokens/s: 144041.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.881063E-01 | lm-global-supervised-tokens: 1.268899E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.377580E-02 | lm-token-buffer-mib: 5.827048E+01 | global_load_balancing_loss: 1.000621E+00 | loss scale: 1.0 | grad norm: 0.054 | params norm: 11265.624 | params rms: 0.061000 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 15:44:03.488213 | finish at 2026-10-10 19:55:35 + [2026-10-10 04:13:03] iteration 6213/ 6855 | consumed samples: 795264 | elapsed time per iteration (ms): 91106.7 | throughput per GPU (TFLOP/s/GPU): 41.5 | MFU 13.31% | FLOPs estimator: workload-aware | logical tokens/s: 184149.0 | valid tokens/s: 184149.0 | physical tokens/s: 184631.4 | loss tokens/s: 133281.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.191199E-01 | lm-global-supervised-tokens: 1.214282E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.325203E-02 | lm-token-buffer-mib: 5.824065E+01 | global_load_balancing_loss: 1.001119E+00 | loss scale: 1.0 | grad norm: 0.081 | params norm: 11265.577 | params rms: 0.060999 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 16:14:50.497843 | finish at 2026-10-10 20:27:53 + [2026-10-10 04:14:33] iteration 6214/ 6855 | consumed samples: 795392 | elapsed time per iteration (ms): 89825.3 | throughput per GPU (TFLOP/s/GPU): 41.7 | MFU 13.36% | FLOPs estimator: workload-aware | logical tokens/s: 186775.7 | valid tokens/s: 186775.7 | physical tokens/s: 187173.0 | loss tokens/s: 135326.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.396436E-01 | lm-global-supervised-tokens: 1.215571E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.462341E-02 | lm-token-buffer-mib: 5.821794E+01 | global_load_balancing_loss: 1.001029E+00 | loss scale: 1.0 | grad norm: 0.068 | params norm: 11265.532 | params rms: 0.060999 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 15:59:37.996807 | finish at 2026-10-10 20:14:11 + [2026-10-10 04:16:00] iteration 6215/ 6855 | consumed samples: 795520 | elapsed time per iteration (ms): 87828.9 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.48% | FLOPs estimator: workload-aware | logical tokens/s: 191021.6 | valid tokens/s: 191021.6 | physical tokens/s: 191388.2 | loss tokens/s: 137720.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.118478E-01 | lm-global-supervised-tokens: 1.209580E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.252057E-02 | lm-token-buffer-mib: 5.805180E+01 | global_load_balancing_loss: 1.001404E+00 | loss scale: 1.0 | grad norm: 0.093 | params norm: 11265.488 | params rms: 0.060999 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 15:36:50.479584 | finish at 2026-10-10 19:52:51 + [2026-10-10 04:17:25] iteration 6216/ 6855 | consumed samples: 795648 | elapsed time per iteration (ms): 84939.4 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.59% | FLOPs estimator: workload-aware | logical tokens/s: 197519.2 | valid tokens/s: 197519.2 | physical tokens/s: 197997.8 | loss tokens/s: 148391.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.455233E-01 | lm-global-supervised-tokens: 1.260432E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.576545E-02 | lm-token-buffer-mib: 5.806277E+01 | global_load_balancing_loss: 9.994500E-01 | loss scale: 1.0 | grad norm: 0.057 | params norm: 11265.443 | params rms: 0.060999 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 15:04:36.307043 | finish at 2026-10-10 19:22:02 + [2026-10-10 04:18:51] iteration 6217/ 6855 | consumed samples: 795776 | elapsed time per iteration (ms): 85862.0 | throughput per GPU (TFLOP/s/GPU): 43.2 | MFU 13.84% | FLOPs estimator: workload-aware | logical tokens/s: 195397.0 | valid tokens/s: 195396.9 | physical tokens/s: 195733.2 | loss tokens/s: 141537.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.272315E-01 | lm-global-supervised-tokens: 1.215271E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.137481E-02 | lm-token-buffer-mib: 5.813781E+01 | global_load_balancing_loss: 9.988208E-01 | loss scale: 1.0 | grad norm: 0.086 | params norm: 11265.398 | params rms: 0.060998 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 15:12:59.971356 | finish at 2026-10-10 19:31:51 + [2026-10-10 04:20:16] iteration 6218/ 6855 | consumed samples: 795904 | elapsed time per iteration (ms): 84905.4 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.59% | FLOPs estimator: workload-aware | logical tokens/s: 197598.8 | valid tokens/s: 197598.5 | physical tokens/s: 198224.7 | loss tokens/s: 151239.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.306050E-01 | lm-global-supervised-tokens: 1.284106E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.226668E-02 | lm-token-buffer-mib: 5.832175E+01 | global_load_balancing_loss: 9.996036E-01 | loss scale: 1.0 | grad norm: 0.074 | params norm: 11265.354 | params rms: 0.060998 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 15:01:24.754252 | finish at 2026-10-10 19:21:41 + [2026-10-10 04:21:43] iteration 6219/ 6855 | consumed samples: 796032 | elapsed time per iteration (ms): 86947.8 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.61% | FLOPs estimator: workload-aware | logical tokens/s: 192956.8 | valid tokens/s: 192956.8 | physical tokens/s: 193448.5 | loss tokens/s: 144675.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.727052E-01 | lm-global-supervised-tokens: 1.257922E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.146974E-02 | lm-token-buffer-mib: 5.812514E+01 | global_load_balancing_loss: 1.000711E+00 | loss scale: 1.0 | grad norm: 0.147 | params norm: 11265.310 | params rms: 0.060998 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 15:21:38.787558 | finish at 2026-10-10 19:43:22 + [2026-10-10 04:23:32] iteration 6220/ 6855 | consumed samples: 796160 | elapsed time per iteration (ms): 108702.9 | throughput per GPU (TFLOP/s/GPU): 34.7 | MFU 11.13% | FLOPs estimator: workload-aware | logical tokens/s: 154340.1 | valid tokens/s: 154340.1 | physical tokens/s: 154716.4 | loss tokens/s: 113723.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.616148E-01 | lm-global-supervised-tokens: 1.236201E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.275965E-02 | lm-token-buffer-mib: 5.841412E+01 | global_load_balancing_loss: 1.000203E+00 | loss scale: 1.0 | grad norm: 0.053 | params norm: 11265.265 | params rms: 0.060998 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 19:10:26.323290 | finish at 2026-10-10 23:33:58 + [2026-10-10 04:24:57] iteration 6221/ 6855 | consumed samples: 796288 | elapsed time per iteration (ms): 85051.2 | throughput per GPU (TFLOP/s/GPU): 43.7 | MFU 14.02% | FLOPs estimator: workload-aware | logical tokens/s: 197259.9 | valid tokens/s: 197259.9 | physical tokens/s: 197554.7 | loss tokens/s: 142996.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.443235E-01 | lm-global-supervised-tokens: 1.216203E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.391582E-02 | lm-token-buffer-mib: 5.808626E+01 | global_load_balancing_loss: 1.000799E+00 | loss scale: 1.0 | grad norm: 0.108 | params norm: 11265.220 | params rms: 0.060998 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 14:58:42.431723 | finish at 2026-10-10 19:23:39 + [2026-10-10 04:26:25] iteration 6222/ 6855 | consumed samples: 796416 | elapsed time per iteration (ms): 87928.6 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.69% | FLOPs estimator: workload-aware | logical tokens/s: 190804.9 | valid tokens/s: 190804.8 | physical tokens/s: 191143.9 | loss tokens/s: 138768.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.596830E-01 | lm-global-supervised-tokens: 1.220175E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.293148E-02 | lm-token-buffer-mib: 5.806199E+01 | global_load_balancing_loss: 1.001037E+00 | loss scale: 1.0 | grad norm: 0.049 | params norm: 11265.174 | params rms: 0.060997 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 15:27:38.827540 | finish at 2026-10-10 19:54:04 + [2026-10-10 04:27:54] iteration 6223/ 6855 | consumed samples: 796544 | elapsed time per iteration (ms): 88934.7 | throughput per GPU (TFLOP/s/GPU): 41.4 | MFU 13.26% | FLOPs estimator: workload-aware | logical tokens/s: 188645.7 | valid tokens/s: 188645.7 | physical tokens/s: 189211.3 | loss tokens/s: 141643.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 5.910103E-01 | lm-global-supervised-tokens: 1.259707E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.558870E-02 | lm-token-buffer-mib: 5.818431E+01 | global_load_balancing_loss: 9.996498E-01 | loss scale: 1.0 | grad norm: 0.227 | params norm: 11265.125 | params rms: 0.060997 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 15:36:46.758736 | finish at 2026-10-10 20:04:41 + [2026-10-10 04:29:24] iteration 6224/ 6855 | consumed samples: 796672 | elapsed time per iteration (ms): 90069.1 | throughput per GPU (TFLOP/s/GPU): 41.5 | MFU 13.30% | FLOPs estimator: workload-aware | logical tokens/s: 186269.9 | valid tokens/s: 186269.9 | physical tokens/s: 186650.9 | loss tokens/s: 133699.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.526852E-01 | lm-global-supervised-tokens: 1.204223E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.147508E-02 | lm-token-buffer-mib: 5.810667E+01 | global_load_balancing_loss: 1.000514E+00 | loss scale: 1.0 | grad norm: 0.088 | params norm: 11265.077 | params rms: 0.060997 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 15:47:13.583986 | finish at 2026-10-10 20:16:37 + [2026-10-10 04:30:51] iteration 6225/ 6855 | consumed samples: 796800 | elapsed time per iteration (ms): 86753.4 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.63% | FLOPs estimator: workload-aware | logical tokens/s: 193388.9 | valid tokens/s: 193388.8 | physical tokens/s: 193890.1 | loss tokens/s: 139644.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.393902E-01 | lm-global-supervised-tokens: 1.211465E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.582502E-02 | lm-token-buffer-mib: 5.804559E+01 | global_load_balancing_loss: 9.993446E-01 | loss scale: 1.0 | grad norm: 0.169 | params norm: 11265.030 | params rms: 0.060996 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 15:10:54.631240 | finish at 2026-10-10 19:41:45 + [2026-10-10 04:32:20] iteration 6226/ 6855 | consumed samples: 796928 | elapsed time per iteration (ms): 89328.6 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.43% | FLOPs estimator: workload-aware | logical tokens/s: 187814.2 | valid tokens/s: 187814.2 | physical tokens/s: 188211.7 | loss tokens/s: 139455.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.722149E-01 | lm-global-supervised-tokens: 1.245735E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.313378E-02 | lm-token-buffer-mib: 5.821741E+01 | global_load_balancing_loss: 9.994587E-01 | loss scale: 1.0 | grad norm: 0.070 | params norm: 11264.983 | params rms: 0.060996 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 15:36:27.702403 | finish at 2026-10-10 20:08:48 + [2026-10-10 04:33:50] iteration 6227/ 6855 | consumed samples: 797056 | elapsed time per iteration (ms): 89776.8 | throughput per GPU (TFLOP/s/GPU): 41.4 | MFU 13.28% | FLOPs estimator: workload-aware | logical tokens/s: 186876.1 | valid tokens/s: 186876.1 | physical tokens/s: 187281.8 | loss tokens/s: 132327.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.281742E-01 | lm-global-supervised-tokens: 1.187990E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.143476E-02 | lm-token-buffer-mib: 5.811548E+01 | global_load_balancing_loss: 1.000028E+00 | loss scale: 1.0 | grad norm: 0.129 | params norm: 11264.938 | params rms: 0.060996 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 15:39:39.849363 | finish at 2026-10-10 20:13:30 + [2026-10-10 04:35:16] iteration 6228/ 6855 | consumed samples: 797184 | elapsed time per iteration (ms): 85876.4 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.44% | FLOPs estimator: workload-aware | logical tokens/s: 195364.4 | valid tokens/s: 195364.4 | physical tokens/s: 195706.6 | loss tokens/s: 151078.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.544284E-01 | lm-global-supervised-tokens: 1.297410E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.502745E-02 | lm-token-buffer-mib: 5.811390E+01 | global_load_balancing_loss: 9.998034E-01 | loss scale: 1.0 | grad norm: 0.072 | params norm: 11264.893 | params rms: 0.060996 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 14:57:24.495023 | finish at 2026-10-10 19:32:40 + [2026-10-10 04:36:44] iteration 6229/ 6855 | consumed samples: 797312 | elapsed time per iteration (ms): 88842.9 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.51% | FLOPs estimator: workload-aware | logical tokens/s: 188840.9 | valid tokens/s: 188840.8 | physical tokens/s: 189106.5 | loss tokens/s: 133548.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.372024E-01 | lm-global-supervised-tokens: 1.186486E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.440368E-02 | lm-token-buffer-mib: 5.808631E+01 | global_load_balancing_loss: 1.001434E+00 | loss scale: 1.0 | grad norm: 0.076 | params norm: 11264.847 | params rms: 0.060995 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 15:26:55.649006 | finish at 2026-10-10 20:03:40 + [2026-10-10 04:38:32] iteration 6230/ 6855 | consumed samples: 797440 | elapsed time per iteration (ms): 107831.6 | throughput per GPU (TFLOP/s/GPU): 34.5 | MFU 11.04% | FLOPs estimator: workload-aware | logical tokens/s: 155587.1 | valid tokens/s: 155587.1 | physical tokens/s: 155900.5 | loss tokens/s: 114196.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.878158E-01 | lm-global-supervised-tokens: 1.231404E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.824955E-02 | lm-token-buffer-mib: 5.807487E+01 | global_load_balancing_loss: 1.001229E+00 | loss scale: 1.0 | grad norm: 0.098 | params norm: 11264.800 | params rms: 0.060995 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 18:43:14.779474 | finish at 2026-10-10 23:21:47 + [2026-10-10 04:39:58] iteration 6231/ 6855 | consumed samples: 797568 | elapsed time per iteration (ms): 85854.7 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.44% | FLOPs estimator: workload-aware | logical tokens/s: 195413.3 | valid tokens/s: 195413.3 | physical tokens/s: 196080.6 | loss tokens/s: 145958.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.273094E-01 | lm-global-supervised-tokens: 1.253120E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.155201E-02 | lm-token-buffer-mib: 5.805127E+01 | global_load_balancing_loss: 1.001091E+00 | loss scale: 1.0 | grad norm: 0.052 | params norm: 11264.751 | params rms: 0.060995 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 14:52:53.342972 | finish at 2026-10-10 19:32:51 + [2026-10-10 04:41:29] iteration 6232/ 6855 | consumed samples: 797696 | elapsed time per iteration (ms): 90839.4 | throughput per GPU (TFLOP/s/GPU): 41.5 | MFU 13.30% | FLOPs estimator: workload-aware | logical tokens/s: 184690.8 | valid tokens/s: 184690.8 | physical tokens/s: 185189.7 | loss tokens/s: 133944.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.656344E-01 | lm-global-supervised-tokens: 1.216742E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.280389E-02 | lm-token-buffer-mib: 5.810184E+01 | global_load_balancing_loss: 9.995002E-01 | loss scale: 1.0 | grad norm: 0.082 | params norm: 11264.705 | params rms: 0.060995 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 15:43:12.973118 | finish at 2026-10-10 20:24:42 + [2026-10-10 04:42:52] iteration 6233/ 6855 | consumed samples: 797824 | elapsed time per iteration (ms): 83386.8 | throughput per GPU (TFLOP/s/GPU): 43.8 | MFU 14.04% | FLOPs estimator: workload-aware | logical tokens/s: 201197.2 | valid tokens/s: 201197.1 | physical tokens/s: 201539.5 | loss tokens/s: 152226.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.436178E-01 | lm-global-supervised-tokens: 1.269368E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.370483E-02 | lm-token-buffer-mib: 5.805415E+01 | global_load_balancing_loss: 9.987669E-01 | loss scale: 1.0 | grad norm: 0.042 | params norm: 11264.659 | params rms: 0.060994 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 14:24:26.573319 | finish at 2026-10-10 19:07:19 + [2026-10-10 04:44:19] iteration 6234/ 6855 | consumed samples: 797952 | elapsed time per iteration (ms): 86819.3 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.80% | FLOPs estimator: workload-aware | logical tokens/s: 193242.2 | valid tokens/s: 193242.2 | physical tokens/s: 193737.4 | loss tokens/s: 137733.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.741961E-01 | lm-global-supervised-tokens: 1.195788E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.172094E-02 | lm-token-buffer-mib: 5.814214E+01 | global_load_balancing_loss: 9.991990E-01 | loss scale: 1.0 | grad norm: 0.086 | params norm: 11264.612 | params rms: 0.060994 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 14:58:34.768234 | finish at 2026-10-10 19:42:54 + [2026-10-10 04:45:48] iteration 6235/ 6855 | consumed samples: 798080 | elapsed time per iteration (ms): 88901.4 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.49% | FLOPs estimator: workload-aware | logical tokens/s: 188716.7 | valid tokens/s: 188716.7 | physical tokens/s: 189103.7 | loss tokens/s: 139173.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.917247E-01 | lm-global-supervised-tokens: 1.237270E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.394907E-02 | lm-token-buffer-mib: 5.804528E+01 | global_load_balancing_loss: 1.000572E+00 | loss scale: 1.0 | grad norm: 0.037 | params norm: 11264.564 | params rms: 0.060994 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 15:18:38.877368 | finish at 2026-10-10 20:04:27 + [2026-10-10 04:47:14] iteration 6236/ 6855 | consumed samples: 798208 | elapsed time per iteration (ms): 85848.1 | throughput per GPU (TFLOP/s/GPU): 43.1 | MFU 13.81% | FLOPs estimator: workload-aware | logical tokens/s: 195428.3 | valid tokens/s: 195428.3 | physical tokens/s: 196059.3 | loss tokens/s: 143463.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.136056E-01 | lm-global-supervised-tokens: 1.231609E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.239143E-02 | lm-token-buffer-mib: 5.817980E+01 | global_load_balancing_loss: 1.001204E+00 | loss scale: 1.0 | grad norm: 0.078 | params norm: 11264.517 | params rms: 0.060994 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 14:45:39.958371 | finish at 2026-10-10 19:32:54 + [2026-10-10 04:48:43] iteration 6237/ 6855 | consumed samples: 798336 | elapsed time per iteration (ms): 88611.8 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.53% | FLOPs estimator: workload-aware | logical tokens/s: 189333.1 | valid tokens/s: 189333.1 | physical tokens/s: 189670.2 | loss tokens/s: 134457.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.386898E-01 | lm-global-supervised-tokens: 1.191449E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.555030E-02 | lm-token-buffer-mib: 5.821110E+01 | global_load_balancing_loss: 1.001016E+00 | loss scale: 1.0 | grad norm: 0.035 | params norm: 11264.472 | params rms: 0.060993 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 15:12:42.072481 | finish at 2026-10-10 20:01:25 + [2026-10-10 04:50:15] iteration 6238/ 6855 | consumed samples: 798464 | elapsed time per iteration (ms): 92863.6 | throughput per GPU (TFLOP/s/GPU): 41.2 | MFU 13.21% | FLOPs estimator: workload-aware | logical tokens/s: 180665.1 | valid tokens/s: 180665.1 | physical tokens/s: 181005.0 | loss tokens/s: 131080.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.797516E-01 | lm-global-supervised-tokens: 1.217264E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.208083E-02 | lm-token-buffer-mib: 5.806017E+01 | global_load_balancing_loss: 9.992036E-01 | loss scale: 1.0 | grad norm: 0.043 | params norm: 11264.427 | params rms: 0.060993 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 15:54:56.816496 | finish at 2026-10-10 20:45:12 + [2026-10-10 04:51:45] iteration 6239/ 6855 | consumed samples: 798592 | elapsed time per iteration (ms): 89589.5 | throughput per GPU (TFLOP/s/GPU): 40.8 | MFU 13.07% | FLOPs estimator: workload-aware | logical tokens/s: 187267.5 | valid tokens/s: 187267.5 | physical tokens/s: 187728.4 | loss tokens/s: 142083.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.280184E-01 | lm-global-supervised-tokens: 1.272923E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.147115E-02 | lm-token-buffer-mib: 5.809805E+01 | global_load_balancing_loss: 9.998835E-01 | loss scale: 1.0 | grad norm: 0.055 | params norm: 11264.382 | params rms: 0.060993 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 15:19:47.133438 | finish at 2026-10-10 20:11:32 + [2026-10-10 04:53:34] iteration 6240/ 6855 | consumed samples: 798720 | elapsed time per iteration (ms): 109300.8 | throughput per GPU (TFLOP/s/GPU): 34.7 | MFU 11.12% | FLOPs estimator: workload-aware | logical tokens/s: 153495.6 | valid tokens/s: 153495.6 | physical tokens/s: 153776.1 | loss tokens/s: 111073.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.179925E-01 | lm-global-supervised-tokens: 1.214040E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.153728E-02 | lm-token-buffer-mib: 5.812665E+01 | global_load_balancing_loss: 9.999716E-01 | loss scale: 1.0 | grad norm: 0.037 | params norm: 11264.338 | params rms: 0.060993 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 18:40:20.001436 | finish at 2026-10-10 23:33:54 +times across ranks (ms): + save-checkpoint: + rank 0: 25952.79 + rank 1: 25952.89 + rank 2: 25952.84 + rank 3: 25952.81 + rank 4: 25952.82 + rank 5: 25952.80 + rank 6: 25952.77 + rank 7: 25952.81 + rank 8: 25952.49 + rank 9: 25952.48 + rank 10: 25952.44 + rank 11: 25952.44 + rank 12: 25952.46 + rank 13: 25952.45 + rank 14: 25952.49 + rank 15: 25952.44 + rank 16: 25952.78 + rank 17: 25952.77 + rank 18: 25952.73 + rank 19: 25952.76 + rank 20: 25952.73 + rank 21: 25952.61 + rank 22: 25952.72 + rank 23: 25953.67 + rank 24: 25953.62 + rank 25: 25953.61 + rank 26: 25953.64 + rank 27: 25953.63 + rank 28: 25953.54 + rank 29: 25953.60 + rank 30: 25953.54 + rank 31: 25953.65 + rank 32: 25952.61 + rank 33: 25952.64 + rank 34: 25952.61 + rank 35: 25952.57 + rank 36: 25952.51 + rank 37: 25952.49 + rank 38: 25952.55 + rank 39: 25952.53 + rank 40: 25953.13 + rank 41: 25953.10 + rank 42: 25953.11 + rank 43: 25953.15 + rank 44: 25953.12 + rank 45: 25953.13 + rank 46: 25953.11 + rank 47: 25953.11 + rank 48: 25952.69 + rank 49: 25952.68 + rank 50: 25952.66 + rank 51: 25952.60 + rank 52: 25952.69 + rank 53: 25952.67 + rank 54: 25952.64 + rank 55: 25952.66 + rank 56: 25952.97 + rank 57: 25952.97 + rank 58: 25952.93 + rank 59: 25952.92 + rank 60: 25952.93 + rank 61: 25952.93 + rank 62: 25952.92 + rank 63: 25952.99 + [2026-10-10 04:55:32] iteration 6241/ 6855 | consumed samples: 798848 | elapsed time per iteration (ms): 91383.6 | throughput per GPU (TFLOP/s/GPU): 41.0 | MFU 13.15% | FLOPs estimator: workload-aware | logical tokens/s: 183591.1 | valid tokens/s: 183591.1 | physical tokens/s: 183916.7 | loss tokens/s: 136122.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.105226E-01 | lm-global-supervised-tokens: 1.243939E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.345281E-01 | lm-token-buffer-mib: 5.806994E+01 | global_load_balancing_loss: 1.000345E+00 | loss scale: 1.0 | grad norm: 0.050 | params norm: 11264.294 | params rms: 0.060992 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 15:35:09.507561 | finish at 2026-10-10 20:30:41 + [2026-10-10 04:57:06] iteration 6242/ 6855 | consumed samples: 798976 | elapsed time per iteration (ms): 94440.4 | throughput per GPU (TFLOP/s/GPU): 39.4 | MFU 12.62% | FLOPs estimator: workload-aware | logical tokens/s: 177648.6 | valid tokens/s: 177648.6 | physical tokens/s: 178064.4 | loss tokens/s: 130553.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.324794E-01 | lm-global-supervised-tokens: 1.232955E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.625119E-02 | lm-token-buffer-mib: 5.812181E+01 | global_load_balancing_loss: 1.001022E+00 | loss scale: 1.0 | grad norm: 0.067 | params norm: 11264.249 | params rms: 0.060992 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 16:04:51.941453 | finish at 2026-10-10 21:01:58 + [2026-10-10 04:58:34] iteration 6243/ 6855 | consumed samples: 799104 | elapsed time per iteration (ms): 87450.3 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.41% | FLOPs estimator: workload-aware | logical tokens/s: 191847.8 | valid tokens/s: 191847.8 | physical tokens/s: 192282.1 | loss tokens/s: 146692.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.074581E-01 | lm-global-supervised-tokens: 1.282827E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.546055E-02 | lm-token-buffer-mib: 5.810125E+01 | global_load_balancing_loss: 9.994891E-01 | loss scale: 1.0 | grad norm: 0.137 | params norm: 11264.202 | params rms: 0.060992 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 14:51:59.555717 | finish at 2026-10-10 19:50:33 + [2026-10-10 05:00:01] iteration 6244/ 6855 | consumed samples: 799232 | elapsed time per iteration (ms): 87623.2 | throughput per GPU (TFLOP/s/GPU): 41.5 | MFU 13.32% | FLOPs estimator: workload-aware | logical tokens/s: 191469.5 | valid tokens/s: 191469.5 | physical tokens/s: 191929.6 | loss tokens/s: 141920.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.446083E-01 | lm-global-supervised-tokens: 1.243552E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.409908E-02 | lm-token-buffer-mib: 5.808392E+01 | global_load_balancing_loss: 1.001425E+00 | loss scale: 1.0 | grad norm: 0.108 | params norm: 11264.155 | params rms: 0.060992 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 14:52:17.772395 | finish at 2026-10-10 19:52:19 + [2026-10-10 05:01:30] iteration 6245/ 6855 | consumed samples: 799360 | elapsed time per iteration (ms): 88675.7 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.60% | FLOPs estimator: workload-aware | logical tokens/s: 189197.2 | valid tokens/s: 189197.2 | physical tokens/s: 189612.7 | loss tokens/s: 136486.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.422583E-01 | lm-global-supervised-tokens: 1.210303E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.408767E-02 | lm-token-buffer-mib: 5.816630E+01 | global_load_balancing_loss: 1.001369E+00 | loss scale: 1.0 | grad norm: 0.133 | params norm: 11264.109 | params rms: 0.060991 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 15:01:32.198057 | finish at 2026-10-10 20:03:02 + [2026-10-10 05:02:56] iteration 6246/ 6855 | consumed samples: 799488 | elapsed time per iteration (ms): 86042.9 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.63% | FLOPs estimator: workload-aware | logical tokens/s: 194986.2 | valid tokens/s: 194986.2 | physical tokens/s: 195408.8 | loss tokens/s: 143052.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.564559E-01 | lm-global-supervised-tokens: 1.230860E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.214820E-02 | lm-token-buffer-mib: 5.806751E+01 | global_load_balancing_loss: 9.990879E-01 | loss scale: 1.0 | grad norm: 0.203 | params norm: 11264.063 | params rms: 0.060991 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 14:33:20.109164 | finish at 2026-10-10 19:36:16 + [2026-10-10 05:04:25] iteration 6247/ 6855 | consumed samples: 799616 | elapsed time per iteration (ms): 89029.8 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.47% | FLOPs estimator: workload-aware | logical tokens/s: 188444.3 | valid tokens/s: 188444.3 | physical tokens/s: 188909.0 | loss tokens/s: 143448.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.230738E-01 | lm-global-supervised-tokens: 1.277116E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.335689E-02 | lm-token-buffer-mib: 5.813819E+01 | global_load_balancing_loss: 9.978358E-01 | loss scale: 1.0 | grad norm: 0.066 | params norm: 11264.018 | params rms: 0.060991 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 15:02:10.111549 | finish at 2026-10-10 20:06:35 + [2026-10-10 05:05:50] iteration 6248/ 6855 | consumed samples: 799744 | elapsed time per iteration (ms): 84533.2 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.68% | FLOPs estimator: workload-aware | logical tokens/s: 198467.2 | valid tokens/s: 198467.2 | physical tokens/s: 198941.9 | loss tokens/s: 152245.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.491771E-01 | lm-global-supervised-tokens: 1.286977E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.306139E-02 | lm-token-buffer-mib: 5.806999E+01 | global_load_balancing_loss: 9.995888E-01 | loss scale: 1.0 | grad norm: 0.333 | params norm: 11263.974 | params rms: 0.060991 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 14:15:11.666743 | finish at 2026-10-10 19:21:01 + [2026-10-10 05:07:18] iteration 6249/ 6855 | consumed samples: 799872 | elapsed time per iteration (ms): 88054.8 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.54% | FLOPs estimator: workload-aware | logical tokens/s: 190531.5 | valid tokens/s: 190531.5 | physical tokens/s: 190970.7 | loss tokens/s: 136584.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.688510E-01 | lm-global-supervised-tokens: 1.202693E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.265579E-02 | lm-token-buffer-mib: 5.811331E+01 | global_load_balancing_loss: 9.998086E-01 | loss scale: 1.0 | grad norm: 0.275 | params norm: 11263.928 | params rms: 0.060991 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 14:49:21.204052 | finish at 2026-10-10 19:56:39 + [2026-10-10 05:09:07] iteration 6250/ 6855 | consumed samples: 800000 | elapsed time per iteration (ms): 109302.4 | throughput per GPU (TFLOP/s/GPU): 34.4 | MFU 11.02% | FLOPs estimator: workload-aware | logical tokens/s: 153493.2 | valid tokens/s: 153493.2 | physical tokens/s: 153864.3 | loss tokens/s: 114785.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.078295E-01 | lm-global-supervised-tokens: 1.254637E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.170649E-02 | lm-token-buffer-mib: 5.822107E+01 | global_load_balancing_loss: 1.000329E+00 | loss scale: 1.0 | grad norm: 0.187 | params norm: 11263.882 | params rms: 0.060990 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 18:22:07.978464 | finish at 2026-10-10 23:31:15 + [2026-10-10 05:10:41] iteration 6251/ 6855 | consumed samples: 800128 | elapsed time per iteration (ms): 94487.9 | throughput per GPU (TFLOP/s/GPU): 39.1 | MFU 12.52% | FLOPs estimator: workload-aware | logical tokens/s: 177558.6 | valid tokens/s: 177558.6 | physical tokens/s: 177953.9 | loss tokens/s: 135365.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.025763E-01 | lm-global-supervised-tokens: 1.279044E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.230528E-02 | lm-token-buffer-mib: 5.812286E+01 | global_load_balancing_loss: 1.001700E+00 | loss scale: 1.0 | grad norm: 0.071 | params norm: 11263.836 | params rms: 0.060990 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 15:51:10.720412 | finish at 2026-10-10 21:01:52 + [2026-10-10 05:12:08] iteration 6252/ 6855 | consumed samples: 800256 | elapsed time per iteration (ms): 86901.7 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.51% | FLOPs estimator: workload-aware | logical tokens/s: 193059.1 | valid tokens/s: 193059.0 | physical tokens/s: 193509.7 | loss tokens/s: 148928.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.733400E-01 | lm-global-supervised-tokens: 1.294217E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.474221E-02 | lm-token-buffer-mib: 5.815838E+01 | global_load_balancing_loss: 1.002149E+00 | loss scale: 1.0 | grad norm: 0.187 | params norm: 11263.790 | params rms: 0.060990 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 14:33:21.711310 | finish at 2026-10-10 19:45:30 + [2026-10-10 05:13:38] iteration 6253/ 6855 | consumed samples: 800384 | elapsed time per iteration (ms): 89753.3 | throughput per GPU (TFLOP/s/GPU): 41.5 | MFU 13.30% | FLOPs estimator: workload-aware | logical tokens/s: 186925.3 | valid tokens/s: 186925.3 | physical tokens/s: 187426.7 | loss tokens/s: 143403.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.726807E-01 | lm-global-supervised-tokens: 1.287094E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.485058E-02 | lm-token-buffer-mib: 5.821265E+01 | global_load_balancing_loss: 1.000727E+00 | loss scale: 1.0 | grad norm: 0.164 | params norm: 11263.744 | params rms: 0.060990 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 15:00:31.515994 | finish at 2026-10-10 20:14:10 + [2026-10-10 05:15:04] iteration 6254/ 6855 | consumed samples: 800512 | elapsed time per iteration (ms): 86273.5 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.61% | FLOPs estimator: workload-aware | logical tokens/s: 194464.8 | valid tokens/s: 194464.8 | physical tokens/s: 195012.1 | loss tokens/s: 147814.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.673635E-01 | lm-global-supervised-tokens: 1.275250E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.230532E-02 | lm-token-buffer-mib: 5.813138E+01 | global_load_balancing_loss: 1.000615E+00 | loss scale: 1.0 | grad norm: 0.046 | params norm: 11263.698 | params rms: 0.060989 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 14:24:10.355711 | finish at 2026-10-10 19:39:15 + [2026-10-10 05:16:32] iteration 6255/ 6855 | consumed samples: 800640 | elapsed time per iteration (ms): 88006.7 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.46% | FLOPs estimator: workload-aware | logical tokens/s: 190635.3 | valid tokens/s: 190635.2 | physical tokens/s: 191045.6 | loss tokens/s: 137336.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.158010E-01 | lm-global-supervised-tokens: 1.208650E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.185886E-02 | lm-token-buffer-mib: 5.809676E+01 | global_load_balancing_loss: 9.998232E-01 | loss scale: 1.0 | grad norm: 0.153 | params norm: 11263.652 | params rms: 0.060989 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 14:40:04.028320 | finish at 2026-10-10 19:56:36 + [2026-10-10 05:18:00] iteration 6256/ 6855 | consumed samples: 800768 | elapsed time per iteration (ms): 88048.3 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.63% | FLOPs estimator: workload-aware | logical tokens/s: 190545.5 | valid tokens/s: 190545.5 | physical tokens/s: 190960.5 | loss tokens/s: 138135.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.612772E-01 | lm-global-supervised-tokens: 1.216257E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.764797E-02 | lm-token-buffer-mib: 5.814819E+01 | global_load_balancing_loss: 9.995972E-01 | loss scale: 1.0 | grad norm: 0.108 | params norm: 11263.607 | params rms: 0.060989 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 14:39:00.959565 | finish at 2026-10-10 19:57:01 + [2026-10-10 05:19:23] iteration 6257/ 6855 | consumed samples: 800896 | elapsed time per iteration (ms): 82207.6 | throughput per GPU (TFLOP/s/GPU): 43.3 | MFU 13.89% | FLOPs estimator: workload-aware | logical tokens/s: 204083.2 | valid tokens/s: 204083.2 | physical tokens/s: 204709.0 | loss tokens/s: 157017.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.039855E-01 | lm-global-supervised-tokens: 1.290799E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.843049E-02 | lm-token-buffer-mib: 5.811704E+01 | global_load_balancing_loss: 9.985351E-01 | loss scale: 1.0 | grad norm: 0.136 | params norm: 11263.560 | params rms: 0.060989 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 13:39:20.115642 | finish at 2026-10-10 18:58:43 + [2026-10-10 05:20:47] iteration 6258/ 6855 | consumed samples: 801024 | elapsed time per iteration (ms): 84738.7 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.58% | FLOPs estimator: workload-aware | logical tokens/s: 197986.5 | valid tokens/s: 197986.5 | physical tokens/s: 198559.1 | loss tokens/s: 146427.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.426818E-01 | lm-global-supervised-tokens: 1.240803E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.538425E-02 | lm-token-buffer-mib: 5.834231E+01 | global_load_balancing_loss: 9.995917E-01 | loss scale: 1.0 | grad norm: 0.177 | params norm: 11263.514 | params rms: 0.060988 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 14:03:08.979793 | finish at 2026-10-10 19:23:56 + [2026-10-10 05:22:16] iteration 6259/ 6855 | consumed samples: 801152 | elapsed time per iteration (ms): 88484.2 | throughput per GPU (TFLOP/s/GPU): 41.6 | MFU 13.33% | FLOPs estimator: workload-aware | logical tokens/s: 189606.6 | valid tokens/s: 189606.6 | physical tokens/s: 190131.7 | loss tokens/s: 132929.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.349731E-01 | lm-global-supervised-tokens: 1.176218E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.229340E-02 | lm-token-buffer-mib: 5.824991E+01 | global_load_balancing_loss: 1.001736E+00 | loss scale: 1.0 | grad norm: 0.077 | params norm: 11263.469 | params rms: 0.060988 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 14:38:56.610057 | finish at 2026-10-10 20:01:12 + [2026-10-10 05:24:02] iteration 6260/ 6855 | consumed samples: 801280 | elapsed time per iteration (ms): 106436.4 | throughput per GPU (TFLOP/s/GPU): 34.6 | MFU 11.09% | FLOPs estimator: workload-aware | logical tokens/s: 157626.7 | valid tokens/s: 157626.7 | physical tokens/s: 157954.4 | loss tokens/s: 118215.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.711869E-01 | lm-global-supervised-tokens: 1.258240E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.251949E-02 | lm-token-buffer-mib: 5.810611E+01 | global_load_balancing_loss: 9.995165E-01 | loss scale: 1.0 | grad norm: 0.065 | params norm: 11263.422 | params rms: 0.060988 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 17:35:29.655125 | finish at 2026-10-10 22:59:32 + [2026-10-10 05:25:28] iteration 6261/ 6855 | consumed samples: 801408 | elapsed time per iteration (ms): 85943.9 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.54% | FLOPs estimator: workload-aware | logical tokens/s: 195210.2 | valid tokens/s: 195210.2 | physical tokens/s: 195732.8 | loss tokens/s: 139861.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 5.882843E-01 | lm-global-supervised-tokens: 1.202023E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.633483E-02 | lm-token-buffer-mib: 5.808858E+01 | global_load_balancing_loss: 1.000169E+00 | loss scale: 1.0 | grad norm: 0.103 | params norm: 11263.373 | params rms: 0.060987 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 14:10:50.687569 | finish at 2026-10-10 19:36:19 + [2026-10-10 05:26:54] iteration 6262/ 6855 | consumed samples: 801536 | elapsed time per iteration (ms): 85802.1 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.58% | FLOPs estimator: workload-aware | logical tokens/s: 195533.1 | valid tokens/s: 195533.1 | physical tokens/s: 196079.5 | loss tokens/s: 145662.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.246365E-01 | lm-global-supervised-tokens: 1.249811E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.117454E-02 | lm-token-buffer-mib: 5.814486E+01 | global_load_balancing_loss: 9.997272E-01 | loss scale: 1.0 | grad norm: 0.063 | params norm: 11263.323 | params rms: 0.060987 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 14:08:00.636642 | finish at 2026-10-10 19:34:55 + [2026-10-10 05:28:20] iteration 6263/ 6855 | consumed samples: 801664 | elapsed time per iteration (ms): 85914.2 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.63% | FLOPs estimator: workload-aware | logical tokens/s: 195277.3 | valid tokens/s: 195277.3 | physical tokens/s: 195752.6 | loss tokens/s: 153318.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.474132E-01 | lm-global-supervised-tokens: 1.317226E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.312154E-02 | lm-token-buffer-mib: 5.822380E+01 | global_load_balancing_loss: 1.000248E+00 | loss scale: 1.0 | grad norm: 0.059 | params norm: 11263.274 | params rms: 0.060987 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 14:07:41.218014 | finish at 2026-10-10 19:36:01 + [2026-10-10 05:29:47] iteration 6264/ 6855 | consumed samples: 801792 | elapsed time per iteration (ms): 87354.2 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.65% | FLOPs estimator: workload-aware | logical tokens/s: 192059.3 | valid tokens/s: 192059.3 | physical tokens/s: 192651.1 | loss tokens/s: 140185.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.659105E-01 | lm-global-supervised-tokens: 1.224580E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.209261E-02 | lm-token-buffer-mib: 5.831246E+01 | global_load_balancing_loss: 1.000778E+00 | loss scale: 1.0 | grad norm: 0.061 | params norm: 11263.225 | params rms: 0.060987 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 14:20:26.310574 | finish at 2026-10-10 19:50:14 + [2026-10-10 05:31:14] iteration 6265/ 6855 | consumed samples: 801920 | elapsed time per iteration (ms): 86946.9 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.48% | FLOPs estimator: workload-aware | logical tokens/s: 192958.8 | valid tokens/s: 192958.7 | physical tokens/s: 193554.6 | loss tokens/s: 143796.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.242961E-01 | lm-global-supervised-tokens: 1.250262E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.102161E-02 | lm-token-buffer-mib: 5.822131E+01 | global_load_balancing_loss: 1.000275E+00 | loss scale: 1.0 | grad norm: 0.094 | params norm: 11263.174 | params rms: 0.060986 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 14:14:58.674452 | finish at 2026-10-10 19:46:13 + [2026-10-10 05:32:43] iteration 6266/ 6855 | consumed samples: 802048 | elapsed time per iteration (ms): 88941.9 | throughput per GPU (TFLOP/s/GPU): 41.4 | MFU 13.28% | FLOPs estimator: workload-aware | logical tokens/s: 188630.7 | valid tokens/s: 188630.7 | physical tokens/s: 188958.0 | loss tokens/s: 142031.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.742581E-01 | lm-global-supervised-tokens: 1.263251E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.232793E-02 | lm-token-buffer-mib: 5.806232E+01 | global_load_balancing_loss: 1.000333E+00 | loss scale: 1.0 | grad norm: 0.040 | params norm: 11263.123 | params rms: 0.060986 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 14:33:06.799611 | finish at 2026-10-10 20:05:50 + [2026-10-10 05:34:11] iteration 6267/ 6855 | consumed samples: 802176 | elapsed time per iteration (ms): 87711.8 | throughput per GPU (TFLOP/s/GPU): 41.7 | MFU 13.36% | FLOPs estimator: workload-aware | logical tokens/s: 191276.3 | valid tokens/s: 191276.3 | physical tokens/s: 191783.6 | loss tokens/s: 144426.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.887630E-01 | lm-global-supervised-tokens: 1.266792E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.310839E-02 | lm-token-buffer-mib: 5.812119E+01 | global_load_balancing_loss: 9.992974E-01 | loss scale: 1.0 | grad norm: 0.067 | params norm: 11263.073 | params rms: 0.060986 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 14:19:34.524719 | finish at 2026-10-10 19:53:45 + [2026-10-10 05:35:38] iteration 6268/ 6855 | consumed samples: 802304 | elapsed time per iteration (ms): 87247.3 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.69% | FLOPs estimator: workload-aware | logical tokens/s: 192294.5 | valid tokens/s: 192293.7 | physical tokens/s: 192786.2 | loss tokens/s: 137421.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.724490E-01 | lm-global-supervised-tokens: 1.198964E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.200802E-02 | lm-token-buffer-mib: 5.847629E+01 | global_load_balancing_loss: 1.002799E+00 | loss scale: 1.0 | grad norm: 0.110 | params norm: 11263.021 | params rms: 0.060986 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 14:13:34.166027 | finish at 2026-10-10 19:49:12 + [2026-10-10 05:37:05] iteration 6269/ 6855 | consumed samples: 802432 | elapsed time per iteration (ms): 87194.3 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.44% | FLOPs estimator: workload-aware | logical tokens/s: 192411.4 | valid tokens/s: 192411.4 | physical tokens/s: 193168.4 | loss tokens/s: 149567.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.519774E-01 | lm-global-supervised-tokens: 1.304143E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.134414E-02 | lm-token-buffer-mib: 5.804606E+01 | global_load_balancing_loss: 1.000761E+00 | loss scale: 1.0 | grad norm: 0.061 | params norm: 11262.969 | params rms: 0.060985 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 14:11:35.868844 | finish at 2026-10-10 19:48:41 + [2026-10-10 05:38:55] iteration 6270/ 6855 | consumed samples: 802560 | elapsed time per iteration (ms): 109314.4 | throughput per GPU (TFLOP/s/GPU): 32.9 | MFU 10.54% | FLOPs estimator: workload-aware | logical tokens/s: 153476.7 | valid tokens/s: 153476.7 | physical tokens/s: 153837.8 | loss tokens/s: 117252.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.540984E-01 | lm-global-supervised-tokens: 1.281735E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.156358E-02 | lm-token-buffer-mib: 5.807146E+01 | global_load_balancing_loss: 9.999344E-01 | loss scale: 1.0 | grad norm: 0.043 | params norm: 11262.916 | params rms: 0.060985 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 17:45:48.925370 | finish at 2026-10-10 23:24:44 + [2026-10-10 05:40:20] iteration 6271/ 6855 | consumed samples: 802688 | elapsed time per iteration (ms): 85788.5 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.76% | FLOPs estimator: workload-aware | logical tokens/s: 195564.7 | valid tokens/s: 195564.7 | physical tokens/s: 196115.0 | loss tokens/s: 150103.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.638781E-01 | lm-global-supervised-tokens: 1.287716E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.243616E-02 | lm-token-buffer-mib: 5.813636E+01 | global_load_balancing_loss: 9.991124E-01 | loss scale: 1.0 | grad norm: 0.061 | params norm: 11262.866 | params rms: 0.060985 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 13:55:00.501307 | finish at 2026-10-10 19:35:21 + [2026-10-10 05:41:46] iteration 6272/ 6855 | consumed samples: 802816 | elapsed time per iteration (ms): 85366.3 | throughput per GPU (TFLOP/s/GPU): 43.2 | MFU 13.85% | FLOPs estimator: workload-aware | logical tokens/s: 196531.5 | valid tokens/s: 196531.5 | physical tokens/s: 197069.9 | loss tokens/s: 145747.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.194088E-01 | lm-global-supervised-tokens: 1.244193E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.285185E-02 | lm-token-buffer-mib: 5.836286E+01 | global_load_balancing_loss: 9.994963E-01 | loss scale: 1.0 | grad norm: 0.098 | params norm: 11262.815 | params rms: 0.060984 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 13:49:28.530027 | finish at 2026-10-10 19:31:14 + [2026-10-10 05:43:12] iteration 6273/ 6855 | consumed samples: 802944 | elapsed time per iteration (ms): 85730.1 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.79% | FLOPs estimator: workload-aware | logical tokens/s: 195697.4 | valid tokens/s: 195697.4 | physical tokens/s: 196180.5 | loss tokens/s: 147089.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.553324E-01 | lm-global-supervised-tokens: 1.261003E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.150325E-02 | lm-token-buffer-mib: 5.817292E+01 | global_load_balancing_loss: 9.989762E-01 | loss scale: 1.0 | grad norm: 0.195 | params norm: 11262.765 | params rms: 0.060984 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 13:51:34.916764 | finish at 2026-10-10 19:34:46 + [2026-10-10 05:44:40] iteration 6274/ 6855 | consumed samples: 803072 | elapsed time per iteration (ms): 88684.8 | throughput per GPU (TFLOP/s/GPU): 41.4 | MFU 13.27% | FLOPs estimator: workload-aware | logical tokens/s: 189177.2 | valid tokens/s: 189177.2 | physical tokens/s: 189590.8 | loss tokens/s: 141792.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.766504E-01 | lm-global-supervised-tokens: 1.257488E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.146722E-02 | lm-token-buffer-mib: 5.826223E+01 | global_load_balancing_loss: 9.999706E-01 | loss scale: 1.0 | grad norm: 0.357 | params norm: 11262.719 | params rms: 0.060984 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 14:18:45.887171 | finish at 2026-10-10 20:03:26 + [2026-10-10 05:46:06] iteration 6275/ 6855 | consumed samples: 803200 | elapsed time per iteration (ms): 86170.9 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.63% | FLOPs estimator: workload-aware | logical tokens/s: 194696.2 | valid tokens/s: 194696.2 | physical tokens/s: 195104.3 | loss tokens/s: 146219.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.707099E-01 | lm-global-supervised-tokens: 1.259986E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.383414E-02 | lm-token-buffer-mib: 5.821446E+01 | global_load_balancing_loss: 9.994859E-01 | loss scale: 1.0 | grad norm: 0.303 | params norm: 11262.672 | params rms: 0.060984 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 13:52:59.105606 | finish at 2026-10-10 19:39:06 + [2026-10-10 05:47:34] iteration 6276/ 6855 | consumed samples: 803328 | elapsed time per iteration (ms): 87981.1 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.75% | FLOPs estimator: workload-aware | logical tokens/s: 190690.8 | valid tokens/s: 190690.7 | physical tokens/s: 191123.9 | loss tokens/s: 145562.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.508879E-01 | lm-global-supervised-tokens: 1.280672E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.293932E-02 | lm-token-buffer-mib: 5.818182E+01 | global_load_balancing_loss: 1.000341E+00 | loss scale: 1.0 | grad norm: 0.064 | params norm: 11262.624 | params rms: 0.060983 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 14:09:01.064816 | finish at 2026-10-10 19:56:35 + [2026-10-10 05:49:03] iteration 6277/ 6855 | consumed samples: 803456 | elapsed time per iteration (ms): 89067.9 | throughput per GPU (TFLOP/s/GPU): 41.5 | MFU 13.31% | FLOPs estimator: workload-aware | logical tokens/s: 188364.0 | valid tokens/s: 188364.0 | physical tokens/s: 188839.0 | loss tokens/s: 146679.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.894799E-01 | lm-global-supervised-tokens: 1.306440E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.145644E-02 | lm-token-buffer-mib: 5.806585E+01 | global_load_balancing_loss: 1.000828E+00 | loss scale: 1.0 | grad norm: 0.211 | params norm: 11262.577 | params rms: 0.060983 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 14:18:01.226047 | finish at 2026-10-10 20:07:05 + [2026-10-10 05:50:31] iteration 6278/ 6855 | consumed samples: 803584 | elapsed time per iteration (ms): 87199.4 | throughput per GPU (TFLOP/s/GPU): 43.2 | MFU 13.84% | FLOPs estimator: workload-aware | logical tokens/s: 192399.4 | valid tokens/s: 192399.4 | physical tokens/s: 192746.9 | loss tokens/s: 140360.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.939921E-01 | lm-global-supervised-tokens: 1.223936E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.908350E-02 | lm-token-buffer-mib: 5.806590E+01 | global_load_balancing_loss: 1.001464E+00 | loss scale: 1.0 | grad norm: 0.159 | params norm: 11262.529 | params rms: 0.060983 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 13:58:34.080596 | finish at 2026-10-10 19:49:05 + [2026-10-10 05:51:58] iteration 6279/ 6855 | consumed samples: 803712 | elapsed time per iteration (ms): 87350.9 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.48% | FLOPs estimator: workload-aware | logical tokens/s: 192065.7 | valid tokens/s: 192065.7 | physical tokens/s: 192425.2 | loss tokens/s: 139768.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.268427E-01 | lm-global-supervised-tokens: 1.220891E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.155323E-02 | lm-token-buffer-mib: 5.812589E+01 | global_load_balancing_loss: 1.001601E+00 | loss scale: 1.0 | grad norm: 0.127 | params norm: 11262.480 | params rms: 0.060983 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 13:58:34.116302 | finish at 2026-10-10 19:50:32 + [2026-10-10 05:53:44] iteration 6280/ 6855 | consumed samples: 803840 | elapsed time per iteration (ms): 105655.0 | throughput per GPU (TFLOP/s/GPU): 34.5 | MFU 11.05% | FLOPs estimator: workload-aware | logical tokens/s: 158791.5 | valid tokens/s: 158791.5 | physical tokens/s: 159082.6 | loss tokens/s: 115426.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.475050E-01 | lm-global-supervised-tokens: 1.219536E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.244682E-02 | lm-token-buffer-mib: 5.825853E+01 | global_load_balancing_loss: 1.000763E+00 | loss scale: 1.0 | grad norm: 0.048 | params norm: 11262.432 | params rms: 0.060982 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 16:52:31.620871 | finish at 2026-10-10 22:46:15 +times across ranks (ms): + save-checkpoint: + rank 0: 23854.04 + rank 1: 23854.04 + rank 2: 23854.10 + rank 3: 23854.03 + rank 4: 23854.06 + rank 5: 23854.03 + rank 6: 23854.03 + rank 7: 23854.06 + rank 8: 23854.31 + rank 9: 23854.30 + rank 10: 23854.31 + rank 11: 23854.31 + rank 12: 23854.28 + rank 13: 23854.28 + rank 14: 23854.31 + rank 15: 23854.31 + rank 16: 23853.14 + rank 17: 23853.18 + rank 18: 23853.11 + rank 19: 23853.22 + rank 20: 23853.10 + rank 21: 23853.15 + rank 22: 23853.12 + rank 23: 23853.10 + rank 24: 23854.15 + rank 25: 23854.03 + rank 26: 23854.21 + rank 27: 23854.06 + rank 28: 23854.03 + rank 29: 23854.02 + rank 30: 23854.02 + rank 31: 23854.07 + rank 32: 23853.82 + rank 33: 23853.84 + rank 34: 23853.82 + rank 35: 23853.81 + rank 36: 23853.78 + rank 37: 23853.79 + rank 38: 23853.78 + rank 39: 23853.80 + rank 40: 23854.37 + rank 41: 23854.35 + rank 42: 23854.35 + rank 43: 23854.35 + rank 44: 23854.35 + rank 45: 23854.34 + rank 46: 23854.35 + rank 47: 23854.35 + rank 48: 23853.79 + rank 49: 23853.78 + rank 50: 23853.78 + rank 51: 23853.77 + rank 52: 23853.77 + rank 53: 23853.78 + rank 54: 23853.77 + rank 55: 23853.77 + rank 56: 23854.43 + rank 57: 23854.42 + rank 58: 23854.40 + rank 59: 23854.39 + rank 60: 23854.41 + rank 61: 23854.38 + rank 62: 23854.42 + rank 63: 23854.44 + [2026-10-10 05:55:52] iteration 6281/ 6855 | consumed samples: 803968 | elapsed time per iteration (ms): 104565.8 | throughput per GPU (TFLOP/s/GPU): 34.5 | MFU 11.05% | FLOPs estimator: workload-aware | logical tokens/s: 160446.1 | valid tokens/s: 160446.1 | physical tokens/s: 160854.5 | loss tokens/s: 121388.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.224979E-01 | lm-global-supervised-tokens: 1.269304E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 4.804174E-01 | lm-token-buffer-mib: 5.818024E+01 | global_load_balancing_loss: 1.000064E+00 | loss scale: 1.0 | grad norm: 0.158 | params norm: 11262.386 | params rms: 0.060982 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 16:40:20.756992 | finish at 2026-10-10 22:36:13 + [2026-10-10 05:57:27] iteration 6282/ 6855 | consumed samples: 804096 | elapsed time per iteration (ms): 94250.9 | throughput per GPU (TFLOP/s/GPU): 40.1 | MFU 12.87% | FLOPs estimator: workload-aware | logical tokens/s: 178005.4 | valid tokens/s: 178005.4 | physical tokens/s: 178457.8 | loss tokens/s: 132993.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.187085E-01 | lm-global-supervised-tokens: 1.253474E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.727170E-02 | lm-token-buffer-mib: 5.812823E+01 | global_load_balancing_loss: 9.992816E-01 | loss scale: 1.0 | grad norm: 0.049 | params norm: 11262.340 | params rms: 0.060982 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 15:00:05.767220 | finish at 2026-10-10 20:57:32 + [2026-10-10 05:58:56] iteration 6283/ 6855 | consumed samples: 804224 | elapsed time per iteration (ms): 89294.8 | throughput per GPU (TFLOP/s/GPU): 41.3 | MFU 13.24% | FLOPs estimator: workload-aware | logical tokens/s: 187885.4 | valid tokens/s: 187885.4 | physical tokens/s: 188178.1 | loss tokens/s: 136128.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.341987E-01 | lm-global-supervised-tokens: 1.215553E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.297516E-02 | lm-token-buffer-mib: 5.811427E+01 | global_load_balancing_loss: 9.997248E-01 | loss scale: 1.0 | grad norm: 0.062 | params norm: 11262.292 | params rms: 0.060982 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 14:11:16.629443 | finish at 2026-10-10 20:10:12 + [2026-10-10 06:00:20] iteration 6284/ 6855 | consumed samples: 804352 | elapsed time per iteration (ms): 84039.0 | throughput per GPU (TFLOP/s/GPU): 44.0 | MFU 14.10% | FLOPs estimator: workload-aware | logical tokens/s: 199635.5 | valid tokens/s: 199635.5 | physical tokens/s: 200118.1 | loss tokens/s: 151626.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.699577E-01 | lm-global-supervised-tokens: 1.274250E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.144786E-02 | lm-token-buffer-mib: 5.812151E+01 | global_load_balancing_loss: 1.000143E+00 | loss scale: 1.0 | grad norm: 0.152 | params norm: 11262.244 | params rms: 0.060981 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 13:19:46.280591 | finish at 2026-10-10 19:20:06 + [2026-10-10 06:01:47] iteration 6285/ 6855 | consumed samples: 804480 | elapsed time per iteration (ms): 87097.6 | throughput per GPU (TFLOP/s/GPU): 43.2 | MFU 13.84% | FLOPs estimator: workload-aware | logical tokens/s: 192624.8 | valid tokens/s: 192624.8 | physical tokens/s: 193060.6 | loss tokens/s: 142946.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.443279E-01 | lm-global-supervised-tokens: 1.245027E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.267074E-02 | lm-token-buffer-mib: 5.811353E+01 | global_load_balancing_loss: 9.994115E-01 | loss scale: 1.0 | grad norm: 0.048 | params norm: 11262.197 | params rms: 0.060981 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 13:47:25.611606 | finish at 2026-10-10 19:49:13 + [2026-10-10 06:03:14] iteration 6286/ 6855 | consumed samples: 804608 | elapsed time per iteration (ms): 87542.3 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.63% | FLOPs estimator: workload-aware | logical tokens/s: 191646.1 | valid tokens/s: 191646.1 | physical tokens/s: 192022.6 | loss tokens/s: 145796.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.441458E-01 | lm-global-supervised-tokens: 1.276337E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.553659E-02 | lm-token-buffer-mib: 5.804108E+01 | global_load_balancing_loss: 1.000200E+00 | loss scale: 1.0 | grad norm: 0.086 | params norm: 11262.150 | params rms: 0.060981 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 13:50:11.553634 | finish at 2026-10-10 19:53:26 + [2026-10-10 06:04:40] iteration 6287/ 6855 | consumed samples: 804736 | elapsed time per iteration (ms): 85046.4 | throughput per GPU (TFLOP/s/GPU): 43.3 | MFU 13.89% | FLOPs estimator: workload-aware | logical tokens/s: 197271.1 | valid tokens/s: 197271.1 | physical tokens/s: 197717.5 | loss tokens/s: 142903.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.159930E-01 | lm-global-supervised-tokens: 1.215342E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.131502E-02 | lm-token-buffer-mib: 5.808144E+01 | global_load_balancing_loss: 9.998960E-01 | loss scale: 1.0 | grad norm: 0.089 | params norm: 11262.102 | params rms: 0.060981 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 13:25:06.382730 | finish at 2026-10-10 19:29:46 + [2026-10-10 06:06:05] iteration 6288/ 6855 | consumed samples: 804864 | elapsed time per iteration (ms): 84969.9 | throughput per GPU (TFLOP/s/GPU): 43.3 | MFU 13.88% | FLOPs estimator: workload-aware | logical tokens/s: 197448.5 | valid tokens/s: 197448.5 | physical tokens/s: 197958.8 | loss tokens/s: 144351.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.479120E-01 | lm-global-supervised-tokens: 1.226556E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.113916E-02 | lm-token-buffer-mib: 5.813078E+01 | global_load_balancing_loss: 1.001225E+00 | loss scale: 1.0 | grad norm: 0.060 | params norm: 11262.054 | params rms: 0.060980 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 13:22:57.942837 | finish at 2026-10-10 19:29:02 + [2026-10-10 06:07:33] iteration 6289/ 6855 | consumed samples: 804992 | elapsed time per iteration (ms): 88916.4 | throughput per GPU (TFLOP/s/GPU): 41.1 | MFU 13.17% | FLOPs estimator: workload-aware | logical tokens/s: 188684.7 | valid tokens/s: 188684.6 | physical tokens/s: 189088.6 | loss tokens/s: 139055.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.454508E-01 | lm-global-supervised-tokens: 1.236428E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.728978E-02 | lm-token-buffer-mib: 5.820307E+01 | global_load_balancing_loss: 1.001065E+00 | loss scale: 1.0 | grad norm: 0.082 | params norm: 11262.006 | params rms: 0.060980 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 13:58:46.708959 | finish at 2026-10-10 20:06:20 + [2026-10-10 06:09:23] iteration 6290/ 6855 | consumed samples: 805120 | elapsed time per iteration (ms): 109230.5 | throughput per GPU (TFLOP/s/GPU): 34.0 | MFU 10.90% | FLOPs estimator: workload-aware | logical tokens/s: 153594.7 | valid tokens/s: 153594.7 | physical tokens/s: 153962.2 | loss tokens/s: 113319.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.532201E-01 | lm-global-supervised-tokens: 1.237789E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.178568E-02 | lm-token-buffer-mib: 5.824471E+01 | global_load_balancing_loss: 1.000590E+00 | loss scale: 1.0 | grad norm: 0.049 | params norm: 11261.955 | params rms: 0.060980 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 17:08:35.211611 | finish at 2026-10-10 23:17:58 + [2026-10-10 06:10:51] iteration 6291/ 6855 | consumed samples: 805248 | elapsed time per iteration (ms): 88383.8 | throughput per GPU (TFLOP/s/GPU): 41.4 | MFU 13.26% | FLOPs estimator: workload-aware | logical tokens/s: 189821.5 | valid tokens/s: 189821.4 | physical tokens/s: 190277.6 | loss tokens/s: 139799.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.639565E-01 | lm-global-supervised-tokens: 1.235600E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.435569E-02 | lm-token-buffer-mib: 5.812478E+01 | global_load_balancing_loss: 9.995475E-01 | loss scale: 1.0 | grad norm: 0.037 | params norm: 11261.906 | params rms: 0.060980 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 13:50:48.461603 | finish at 2026-10-10 20:01:39 + [2026-10-10 06:12:20] iteration 6292/ 6855 | consumed samples: 805376 | elapsed time per iteration (ms): 88915.5 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.47% | FLOPs estimator: workload-aware | logical tokens/s: 188686.9 | valid tokens/s: 188686.9 | physical tokens/s: 189083.0 | loss tokens/s: 136077.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.731355E-01 | lm-global-supervised-tokens: 1.209939E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.155618E-02 | lm-token-buffer-mib: 5.811071E+01 | global_load_balancing_loss: 1.000018E+00 | loss scale: 1.0 | grad norm: 0.086 | params norm: 11261.858 | params rms: 0.060979 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 13:54:19.448874 | finish at 2026-10-10 20:06:39 + [2026-10-10 06:13:50] iteration 6293/ 6855 | consumed samples: 805504 | elapsed time per iteration (ms): 89601.2 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.57% | FLOPs estimator: workload-aware | logical tokens/s: 187242.4 | valid tokens/s: 187242.4 | physical tokens/s: 187646.9 | loss tokens/s: 135260.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.577270E-01 | lm-global-supervised-tokens: 1.211952E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.512444E-02 | lm-token-buffer-mib: 5.824476E+01 | global_load_balancing_loss: 1.000591E+00 | loss scale: 1.0 | grad norm: 0.033 | params norm: 11261.809 | params rms: 0.060979 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 13:59:15.849134 | finish at 2026-10-10 20:13:05 + [2026-10-10 06:15:16] iteration 6294/ 6855 | consumed samples: 805632 | elapsed time per iteration (ms): 86876.3 | throughput per GPU (TFLOP/s/GPU): 43.5 | MFU 13.93% | FLOPs estimator: workload-aware | logical tokens/s: 193115.8 | valid tokens/s: 193115.8 | physical tokens/s: 193568.6 | loss tokens/s: 143009.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.722124E-01 | lm-global-supervised-tokens: 1.242417E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.166512E-02 | lm-token-buffer-mib: 5.809669E+01 | global_load_balancing_loss: 9.990966E-01 | loss scale: 1.0 | grad norm: 0.074 | params norm: 11261.757 | params rms: 0.060979 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 13:32:17.619067 | finish at 2026-10-10 19:47:34 + [2026-10-10 06:16:45] iteration 6295/ 6855 | consumed samples: 805760 | elapsed time per iteration (ms): 88684.5 | throughput per GPU (TFLOP/s/GPU): 41.2 | MFU 13.21% | FLOPs estimator: workload-aware | logical tokens/s: 189178.4 | valid tokens/s: 189178.4 | physical tokens/s: 189585.9 | loss tokens/s: 143323.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.843717E-01 | lm-global-supervised-tokens: 1.271057E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.365141E-02 | lm-token-buffer-mib: 5.811947E+01 | global_load_balancing_loss: 1.000088E+00 | loss scale: 1.0 | grad norm: 0.044 | params norm: 11261.708 | params rms: 0.060978 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 13:47:43.308640 | finish at 2026-10-10 20:04:28 + [2026-10-10 06:18:13] iteration 6296/ 6855 | consumed samples: 805888 | elapsed time per iteration (ms): 87971.0 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.63% | FLOPs estimator: workload-aware | logical tokens/s: 190713.0 | valid tokens/s: 190713.0 | physical tokens/s: 191047.1 | loss tokens/s: 141976.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.802908E-01 | lm-global-supervised-tokens: 1.248985E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.142255E-02 | lm-token-buffer-mib: 5.818958E+01 | global_load_balancing_loss: 1.000342E+00 | loss scale: 1.0 | grad norm: 0.068 | params norm: 11261.661 | params rms: 0.060978 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 13:39:35.787776 | finish at 2026-10-10 19:57:49 + [2026-10-10 06:19:42] iteration 6297/ 6855 | consumed samples: 806016 | elapsed time per iteration (ms): 88979.8 | throughput per GPU (TFLOP/s/GPU): 41.2 | MFU 13.19% | FLOPs estimator: workload-aware | logical tokens/s: 188550.9 | valid tokens/s: 188550.9 | physical tokens/s: 188907.3 | loss tokens/s: 138908.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.164922E-01 | lm-global-supervised-tokens: 1.236000E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.473954E-02 | lm-token-buffer-mib: 5.801882E+01 | global_load_balancing_loss: 1.000528E+00 | loss scale: 1.0 | grad norm: 0.062 | params norm: 11261.613 | params rms: 0.060978 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 13:47:30.703248 | finish at 2026-10-10 20:07:13 + [2026-10-10 06:21:12] iteration 6298/ 6855 | consumed samples: 806144 | elapsed time per iteration (ms): 90152.6 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.52% | FLOPs estimator: workload-aware | logical tokens/s: 186097.3 | valid tokens/s: 186097.3 | physical tokens/s: 186485.4 | loss tokens/s: 134940.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.833784E-01 | lm-global-supervised-tokens: 1.216522E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.502624E-02 | lm-token-buffer-mib: 5.808212E+01 | global_load_balancing_loss: 1.000852E+00 | loss scale: 1.0 | grad norm: 0.042 | params norm: 11261.564 | params rms: 0.060978 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 13:56:55.004602 | finish at 2026-10-10 20:18:07 + [2026-10-10 06:22:43] iteration 6299/ 6855 | consumed samples: 806272 | elapsed time per iteration (ms): 90584.3 | throughput per GPU (TFLOP/s/GPU): 40.8 | MFU 13.09% | FLOPs estimator: workload-aware | logical tokens/s: 185211.2 | valid tokens/s: 185211.2 | physical tokens/s: 185523.1 | loss tokens/s: 140095.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.901490E-01 | lm-global-supervised-tokens: 1.269041E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.118964E-02 | lm-token-buffer-mib: 5.814058E+01 | global_load_balancing_loss: 1.000535E+00 | loss scale: 1.0 | grad norm: 0.103 | params norm: 11261.515 | params rms: 0.060977 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 13:59:24.844311 | finish at 2026-10-10 20:22:08 + [2026-10-10 06:24:31] iteration 6300/ 6855 | consumed samples: 806400 | elapsed time per iteration (ms): 108313.8 | throughput per GPU (TFLOP/s/GPU): 34.2 | MFU 10.96% | FLOPs estimator: workload-aware | logical tokens/s: 154894.2 | valid tokens/s: 154894.1 | physical tokens/s: 155212.5 | loss tokens/s: 114095.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.793184E-01 | lm-global-supervised-tokens: 1.235816E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.381898E-02 | lm-token-buffer-mib: 5.808569E+01 | global_load_balancing_loss: 1.000283E+00 | loss scale: 1.0 | grad norm: 0.039 | params norm: 11261.467 | params rms: 0.060977 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 16:41:54.171227 | finish at 2026-10-10 23:06:25 + [2026-10-10 06:25:57] iteration 6301/ 6855 | consumed samples: 806528 | elapsed time per iteration (ms): 85848.5 | throughput per GPU (TFLOP/s/GPU): 43.5 | MFU 13.96% | FLOPs estimator: workload-aware | logical tokens/s: 195428.3 | valid tokens/s: 195428.3 | physical tokens/s: 195843.9 | loss tokens/s: 133585.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.281322E-01 | lm-global-supervised-tokens: 1.146810E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.374408E-02 | lm-token-buffer-mib: 5.809878E+01 | global_load_balancing_loss: 1.000780E+00 | loss scale: 1.0 | grad norm: 0.125 | params norm: 11261.420 | params rms: 0.060977 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 13:12:40.043515 | finish at 2026-10-10 19:38:37 + [2026-10-10 06:27:24] iteration 6302/ 6855 | consumed samples: 806656 | elapsed time per iteration (ms): 86858.5 | throughput per GPU (TFLOP/s/GPU): 42.8 | MFU 13.71% | FLOPs estimator: workload-aware | logical tokens/s: 193155.8 | valid tokens/s: 193155.8 | physical tokens/s: 193679.3 | loss tokens/s: 142238.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.513639E-01 | lm-global-supervised-tokens: 1.235462E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.253986E-02 | lm-token-buffer-mib: 5.813061E+01 | global_load_balancing_loss: 1.000223E+00 | loss scale: 1.0 | grad norm: 0.041 | params norm: 11261.375 | params rms: 0.060977 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 13:20:32.728748 | finish at 2026-10-10 19:47:57 + [2026-10-10 06:28:51] iteration 6303/ 6855 | consumed samples: 806784 | elapsed time per iteration (ms): 87248.5 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.60% | FLOPs estimator: workload-aware | logical tokens/s: 192292.1 | valid tokens/s: 192292.1 | physical tokens/s: 192719.1 | loss tokens/s: 139499.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.571661E-01 | lm-global-supervised-tokens: 1.217109E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.611256E-02 | lm-token-buffer-mib: 5.810861E+01 | global_load_balancing_loss: 1.000746E+00 | loss scale: 1.0 | grad norm: 0.148 | params norm: 11261.330 | params rms: 0.060976 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 13:22:41.197987 | finish at 2026-10-10 19:51:32 + [2026-10-10 06:30:21] iteration 6304/ 6855 | consumed samples: 806912 | elapsed time per iteration (ms): 90072.2 | throughput per GPU (TFLOP/s/GPU): 40.7 | MFU 13.05% | FLOPs estimator: workload-aware | logical tokens/s: 186264.0 | valid tokens/s: 186264.0 | physical tokens/s: 186632.3 | loss tokens/s: 137102.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.203135E-01 | lm-global-supervised-tokens: 1.234910E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.135736E-02 | lm-token-buffer-mib: 5.831612E+01 | global_load_balancing_loss: 9.994733E-01 | loss scale: 1.0 | grad norm: 0.051 | params norm: 11261.286 | params rms: 0.060976 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 13:47:09.781445 | finish at 2026-10-10 20:17:31 + [2026-10-10 06:31:51] iteration 6305/ 6855 | consumed samples: 807040 | elapsed time per iteration (ms): 90082.9 | throughput per GPU (TFLOP/s/GPU): 41.5 | MFU 13.31% | FLOPs estimator: workload-aware | logical tokens/s: 186241.7 | valid tokens/s: 186241.7 | physical tokens/s: 186571.1 | loss tokens/s: 137993.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.434968E-01 | lm-global-supervised-tokens: 1.243084E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.147259E-02 | lm-token-buffer-mib: 5.818072E+01 | global_load_balancing_loss: 9.988309E-01 | loss scale: 1.0 | grad norm: 0.107 | params norm: 11261.241 | params rms: 0.060976 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 13:45:45.579159 | finish at 2026-10-10 20:17:37 + [2026-10-10 06:33:20] iteration 6306/ 6855 | consumed samples: 807168 | elapsed time per iteration (ms): 88722.6 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.61% | FLOPs estimator: workload-aware | logical tokens/s: 189097.4 | valid tokens/s: 189097.4 | physical tokens/s: 189467.9 | loss tokens/s: 136512.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.524908E-01 | lm-global-supervised-tokens: 1.211171E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.132229E-02 | lm-token-buffer-mib: 5.810590E+01 | global_load_balancing_loss: 9.998386E-01 | loss scale: 1.0 | grad norm: 0.038 | params norm: 11261.195 | params rms: 0.060976 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 13:31:48.696789 | finish at 2026-10-10 20:05:09 + [2026-10-10 06:34:46] iteration 6307/ 6855 | consumed samples: 807296 | elapsed time per iteration (ms): 86311.3 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.69% | FLOPs estimator: workload-aware | logical tokens/s: 194380.3 | valid tokens/s: 194380.3 | physical tokens/s: 194885.9 | loss tokens/s: 143779.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.710494E-01 | lm-global-supervised-tokens: 1.240979E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.853077E-02 | lm-token-buffer-mib: 5.835101E+01 | global_load_balancing_loss: 1.000709E+00 | loss scale: 1.0 | grad norm: 0.078 | params norm: 11261.150 | params rms: 0.060975 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 13:08:18.577527 | finish at 2026-10-10 19:43:05 + [2026-10-10 06:36:10] iteration 6308/ 6855 | consumed samples: 807424 | elapsed time per iteration (ms): 84081.5 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.76% | FLOPs estimator: workload-aware | logical tokens/s: 199534.5 | valid tokens/s: 199534.5 | physical tokens/s: 200331.6 | loss tokens/s: 148550.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.674932E-01 | lm-global-supervised-tokens: 1.249038E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.135538E-02 | lm-token-buffer-mib: 5.825835E+01 | global_load_balancing_loss: 1.000493E+00 | loss scale: 1.0 | grad norm: 0.097 | params norm: 11261.103 | params rms: 0.060975 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 12:46:32.570487 | finish at 2026-10-10 19:22:43 + [2026-10-10 06:37:39] iteration 6309/ 6855 | consumed samples: 807552 | elapsed time per iteration (ms): 88501.0 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.74% | FLOPs estimator: workload-aware | logical tokens/s: 189570.4 | valid tokens/s: 189570.4 | physical tokens/s: 189973.2 | loss tokens/s: 139184.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.800322E-01 | lm-global-supervised-tokens: 1.231794E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.183353E-02 | lm-token-buffer-mib: 5.804018E+01 | global_load_balancing_loss: 1.000425E+00 | loss scale: 1.0 | grad norm: 0.101 | params norm: 11261.058 | params rms: 0.060975 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 13:25:21.529037 | finish at 2026-10-10 20:03:00 + [2026-10-10 06:39:27] iteration 6310/ 6855 | consumed samples: 807680 | elapsed time per iteration (ms): 108539.4 | throughput per GPU (TFLOP/s/GPU): 33.7 | MFU 10.81% | FLOPs estimator: workload-aware | logical tokens/s: 154571.7 | valid tokens/s: 154571.7 | physical tokens/s: 154987.1 | loss tokens/s: 116521.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.853380E-01 | lm-global-supervised-tokens: 1.264713E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.196643E-02 | lm-token-buffer-mib: 5.835964E+01 | global_load_balancing_loss: 1.001058E+00 | loss scale: 1.0 | grad norm: 0.097 | params norm: 11261.012 | params rms: 0.060975 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 16:25:53.985529 | finish at 2026-10-10 23:05:21 + [2026-10-10 06:40:58] iteration 6311/ 6855 | consumed samples: 807808 | elapsed time per iteration (ms): 90496.8 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.55% | FLOPs estimator: workload-aware | logical tokens/s: 185389.7 | valid tokens/s: 185389.7 | physical tokens/s: 185803.4 | loss tokens/s: 130894.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.328442E-01 | lm-global-supervised-tokens: 1.184551E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.800855E-02 | lm-token-buffer-mib: 5.806666E+01 | global_load_balancing_loss: 1.001596E+00 | loss scale: 1.0 | grad norm: 0.085 | params norm: 11260.967 | params rms: 0.060974 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 13:40:30.240494 | finish at 2026-10-10 20:21:28 + [2026-10-10 06:42:24] iteration 6312/ 6855 | consumed samples: 807936 | elapsed time per iteration (ms): 85889.4 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.43% | FLOPs estimator: workload-aware | logical tokens/s: 195334.8 | valid tokens/s: 195334.8 | physical tokens/s: 195898.7 | loss tokens/s: 148653.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.571236E-01 | lm-global-supervised-tokens: 1.276775E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.170154E-02 | lm-token-buffer-mib: 5.810058E+01 | global_load_balancing_loss: 1.000131E+00 | loss scale: 1.0 | grad norm: 0.118 | params norm: 11260.922 | params rms: 0.060974 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 12:57:17.937471 | finish at 2026-10-10 19:39:42 + [2026-10-10 06:43:52] iteration 6313/ 6855 | consumed samples: 808064 | elapsed time per iteration (ms): 88230.8 | throughput per GPU (TFLOP/s/GPU): 41.3 | MFU 13.24% | FLOPs estimator: workload-aware | logical tokens/s: 190151.0 | valid tokens/s: 190151.0 | physical tokens/s: 190640.2 | loss tokens/s: 142834.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.771199E-01 | lm-global-supervised-tokens: 1.260242E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.140433E-02 | lm-token-buffer-mib: 5.811230E+01 | global_load_balancing_loss: 9.991351E-01 | loss scale: 1.0 | grad norm: 0.066 | params norm: 11260.876 | params rms: 0.060974 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 13:17:01.095491 | finish at 2026-10-10 20:00:53 + [2026-10-10 06:45:17] iteration 6314/ 6855 | consumed samples: 808192 | elapsed time per iteration (ms): 85422.3 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.52% | FLOPs estimator: workload-aware | logical tokens/s: 196402.9 | valid tokens/s: 196402.9 | physical tokens/s: 197118.6 | loss tokens/s: 146952.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.445676E-01 | lm-global-supervised-tokens: 1.255302E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.130713E-02 | lm-token-buffer-mib: 5.836829E+01 | global_load_balancing_loss: 9.995348E-01 | loss scale: 1.0 | grad norm: 0.172 | params norm: 11260.831 | params rms: 0.060974 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 12:50:13.457002 | finish at 2026-10-10 19:35:31 + [2026-10-10 06:46:43] iteration 6315/ 6855 | consumed samples: 808320 | elapsed time per iteration (ms): 85091.6 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.69% | FLOPs estimator: workload-aware | logical tokens/s: 197165.9 | valid tokens/s: 197165.9 | physical tokens/s: 197797.5 | loss tokens/s: 144940.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.154766E-01 | lm-global-supervised-tokens: 1.233322E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.265526E-02 | lm-token-buffer-mib: 5.819100E+01 | global_load_balancing_loss: 9.995667E-01 | loss scale: 1.0 | grad norm: 0.230 | params norm: 11260.786 | params rms: 0.060973 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 12:45:49.444914 | finish at 2026-10-10 19:32:32 + [2026-10-10 06:48:08] iteration 6316/ 6855 | consumed samples: 808448 | elapsed time per iteration (ms): 85147.0 | throughput per GPU (TFLOP/s/GPU): 43.6 | MFU 13.97% | FLOPs estimator: workload-aware | logical tokens/s: 197038.0 | valid tokens/s: 197038.0 | physical tokens/s: 197444.6 | loss tokens/s: 145520.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.041999E-01 | lm-global-supervised-tokens: 1.239062E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.453862E-02 | lm-token-buffer-mib: 5.810473E+01 | global_load_balancing_loss: 1.000532E+00 | loss scale: 1.0 | grad norm: 0.048 | params norm: 11260.741 | params rms: 0.060973 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 12:44:54.212993 | finish at 2026-10-10 19:33:02 + [2026-10-10 06:49:34] iteration 6317/ 6855 | consumed samples: 808576 | elapsed time per iteration (ms): 86524.3 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.60% | FLOPs estimator: workload-aware | logical tokens/s: 193901.3 | valid tokens/s: 193901.3 | physical tokens/s: 194265.6 | loss tokens/s: 144942.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.520994E-01 | lm-global-supervised-tokens: 1.254108E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.668941E-02 | lm-token-buffer-mib: 5.808394E+01 | global_load_balancing_loss: 1.000827E+00 | loss scale: 1.0 | grad norm: 0.252 | params norm: 11260.699 | params rms: 0.060973 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 12:55:50.062807 | finish at 2026-10-10 19:45:24 + [2026-10-10 06:51:00] iteration 6318/ 6855 | consumed samples: 808704 | elapsed time per iteration (ms): 85723.0 | throughput per GPU (TFLOP/s/GPU): 44.0 | MFU 14.10% | FLOPs estimator: workload-aware | logical tokens/s: 195714.3 | valid tokens/s: 195714.3 | physical tokens/s: 195946.9 | loss tokens/s: 135693.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.286698E-01 | lm-global-supervised-tokens: 1.163200E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.402524E-02 | lm-token-buffer-mib: 5.803574E+01 | global_load_balancing_loss: 1.001618E+00 | loss scale: 1.0 | grad norm: 0.044 | params norm: 11260.654 | params rms: 0.060973 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 12:47:13.229005 | finish at 2026-10-10 19:38:13 + [2026-10-10 06:52:23] iteration 6319/ 6855 | consumed samples: 808832 | elapsed time per iteration (ms): 83085.7 | throughput per GPU (TFLOP/s/GPU): 43.7 | MFU 14.02% | FLOPs estimator: workload-aware | logical tokens/s: 201925.8 | valid tokens/s: 201925.8 | physical tokens/s: 202362.6 | loss tokens/s: 149534.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.429479E-01 | lm-global-supervised-tokens: 1.242418E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.168749E-02 | lm-token-buffer-mib: 5.810476E+01 | global_load_balancing_loss: 9.999484E-01 | loss scale: 1.0 | grad norm: 0.360 | params norm: 11260.608 | params rms: 0.060973 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 12:22:13.947359 | finish at 2026-10-10 19:14:37 + [2026-10-10 06:54:11] iteration 6320/ 6855 | consumed samples: 808960 | elapsed time per iteration (ms): 108150.3 | throughput per GPU (TFLOP/s/GPU): 33.3 | MFU 10.67% | FLOPs estimator: workload-aware | logical tokens/s: 155128.4 | valid tokens/s: 155128.4 | physical tokens/s: 155673.3 | loss tokens/s: 119968.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.357540E-01 | lm-global-supervised-tokens: 1.297459E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.155198E-02 | lm-token-buffer-mib: 5.814014E+01 | global_load_balancing_loss: 9.994239E-01 | loss scale: 1.0 | grad norm: 0.080 | params norm: 11260.562 | params rms: 0.060972 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 16:04:20.387682 | finish at 2026-10-10 22:58:32 +times across ranks (ms): + save-checkpoint: + rank 0: 24579.29 + rank 1: 24579.34 + rank 2: 24579.31 + rank 3: 24579.29 + rank 4: 24579.30 + rank 5: 24579.29 + rank 6: 24579.30 + rank 7: 24579.29 + rank 8: 24578.75 + rank 9: 24578.74 + rank 10: 24578.75 + rank 11: 24578.72 + rank 12: 24578.77 + rank 13: 24578.75 + rank 14: 24578.73 + rank 15: 24578.74 + rank 16: 24579.38 + rank 17: 24579.36 + rank 18: 24579.35 + rank 19: 24579.39 + rank 20: 24579.35 + rank 21: 24579.39 + rank 22: 24579.47 + rank 23: 24579.36 + rank 24: 24579.16 + rank 25: 24579.12 + rank 26: 24579.17 + rank 27: 24579.15 + rank 28: 24579.10 + rank 29: 24579.10 + rank 30: 24579.12 + rank 31: 24579.12 + rank 32: 24578.86 + rank 33: 24578.86 + rank 34: 24578.85 + rank 35: 24578.84 + rank 36: 24578.82 + rank 37: 24578.83 + rank 38: 24578.83 + rank 39: 24578.81 + rank 40: 24579.38 + rank 41: 24579.36 + rank 42: 24579.37 + rank 43: 24579.38 + rank 44: 24579.35 + rank 45: 24579.34 + rank 46: 24579.43 + rank 47: 24579.34 + rank 48: 24579.19 + rank 49: 24579.21 + rank 50: 24579.19 + rank 51: 24579.17 + rank 52: 24579.17 + rank 53: 24579.16 + rank 54: 24579.23 + rank 55: 24579.15 + rank 56: 24579.34 + rank 57: 24579.32 + rank 58: 24579.33 + rank 59: 24579.33 + rank 60: 24579.31 + rank 61: 24579.34 + rank 62: 24579.32 + rank 63: 24579.33 + [2026-10-10 06:56:18] iteration 6321/ 6855 | consumed samples: 809088 | elapsed time per iteration (ms): 101257.0 | throughput per GPU (TFLOP/s/GPU): 36.0 | MFU 11.53% | FLOPs estimator: workload-aware | logical tokens/s: 165689.0 | valid tokens/s: 165689.0 | physical tokens/s: 166103.7 | loss tokens/s: 125823.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.703674E-01 | lm-global-supervised-tokens: 1.274055E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 9.537349E-01 | lm-token-buffer-mib: 5.811403E+01 | global_load_balancing_loss: 9.991459E-01 | loss scale: 1.0 | grad norm: 0.246 | params norm: 11260.516 | params rms: 0.060972 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 15:01:11.254917 | finish at 2026-10-10 21:57:29 + [2026-10-10 06:57:52] iteration 6322/ 6855 | consumed samples: 809216 | elapsed time per iteration (ms): 93504.2 | throughput per GPU (TFLOP/s/GPU): 40.3 | MFU 12.90% | FLOPs estimator: workload-aware | logical tokens/s: 179427.3 | valid tokens/s: 179427.3 | physical tokens/s: 179729.0 | loss tokens/s: 130509.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.628838E-01 | lm-global-supervised-tokens: 1.220323E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.652134E-02 | lm-token-buffer-mib: 5.816527E+01 | global_load_balancing_loss: 9.992446E-01 | loss scale: 1.0 | grad norm: 0.148 | params norm: 11260.470 | params rms: 0.060972 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 13:50:37.754475 | finish at 2026-10-10 20:48:29 + [2026-10-10 06:59:18] iteration 6323/ 6855 | consumed samples: 809344 | elapsed time per iteration (ms): 86556.0 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.69% | FLOPs estimator: workload-aware | logical tokens/s: 193829.8 | valid tokens/s: 193829.8 | physical tokens/s: 194219.0 | loss tokens/s: 138639.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.184456E-01 | lm-global-supervised-tokens: 1.200008E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.511440E-02 | lm-token-buffer-mib: 5.807388E+01 | global_load_balancing_loss: 1.000914E+00 | loss scale: 1.0 | grad norm: 0.097 | params norm: 11260.423 | params rms: 0.060972 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 12:47:27.788319 | finish at 2026-10-10 19:46:46 + [2026-10-10 07:00:45] iteration 6324/ 6855 | consumed samples: 809472 | elapsed time per iteration (ms): 86835.2 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.65% | FLOPs estimator: workload-aware | logical tokens/s: 193206.1 | valid tokens/s: 193206.1 | physical tokens/s: 193736.5 | loss tokens/s: 144652.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.874875E-01 | lm-global-supervised-tokens: 1.256089E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.293927E-02 | lm-token-buffer-mib: 5.813569E+01 | global_load_balancing_loss: 1.001247E+00 | loss scale: 1.0 | grad norm: 0.169 | params norm: 11260.376 | params rms: 0.060971 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 12:48:29.499847 | finish at 2026-10-10 19:49:15 + [2026-10-10 07:02:11] iteration 6325/ 6855 | consumed samples: 809600 | elapsed time per iteration (ms): 86044.4 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.59% | FLOPs estimator: workload-aware | logical tokens/s: 194983.3 | valid tokens/s: 194983.3 | physical tokens/s: 195449.0 | loss tokens/s: 149422.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.002731E-01 | lm-global-supervised-tokens: 1.285696E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.133541E-02 | lm-token-buffer-mib: 5.813399E+01 | global_load_balancing_loss: 9.997388E-01 | loss scale: 1.0 | grad norm: 0.077 | params norm: 11260.330 | params rms: 0.060971 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 12:40:03.507473 | finish at 2026-10-10 19:42:15 + [2026-10-10 07:03:36] iteration 6326/ 6855 | consumed samples: 809728 | elapsed time per iteration (ms): 85130.3 | throughput per GPU (TFLOP/s/GPU): 43.4 | MFU 13.90% | FLOPs estimator: workload-aware | logical tokens/s: 197076.2 | valid tokens/s: 197076.2 | physical tokens/s: 197536.5 | loss tokens/s: 143321.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.426178E-01 | lm-global-supervised-tokens: 1.220103E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.166484E-02 | lm-token-buffer-mib: 5.809092E+01 | global_load_balancing_loss: 9.997926E-01 | loss scale: 1.0 | grad norm: 0.100 | params norm: 11260.284 | params rms: 0.060971 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 12:30:33.903373 | finish at 2026-10-10 19:34:10 + [2026-10-10 07:05:02] iteration 6327/ 6855 | consumed samples: 809856 | elapsed time per iteration (ms): 85494.0 | throughput per GPU (TFLOP/s/GPU): 41.5 | MFU 13.31% | FLOPs estimator: workload-aware | logical tokens/s: 196237.9 | valid tokens/s: 196237.9 | physical tokens/s: 196733.3 | loss tokens/s: 149299.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.225739E-01 | lm-global-supervised-tokens: 1.276420E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.427395E-02 | lm-token-buffer-mib: 5.837383E+01 | global_load_balancing_loss: 1.000916E+00 | loss scale: 1.0 | grad norm: 0.197 | params norm: 11260.238 | params rms: 0.060971 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 12:32:20.831223 | finish at 2026-10-10 19:37:23 + [2026-10-10 07:06:26] iteration 6328/ 6855 | consumed samples: 809984 | elapsed time per iteration (ms): 84625.3 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.49% | FLOPs estimator: workload-aware | logical tokens/s: 198252.5 | valid tokens/s: 198252.5 | physical tokens/s: 198733.0 | loss tokens/s: 151867.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.313020E-01 | lm-global-supervised-tokens: 1.285182E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.181236E-02 | lm-token-buffer-mib: 5.815388E+01 | global_load_balancing_loss: 1.000939E+00 | loss scale: 1.0 | grad norm: 0.053 | params norm: 11260.191 | params rms: 0.060970 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 12:23:17.544623 | finish at 2026-10-10 19:29:44 + [2026-10-10 07:07:53] iteration 6329/ 6855 | consumed samples: 810112 | elapsed time per iteration (ms): 86868.5 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.44% | FLOPs estimator: workload-aware | logical tokens/s: 193132.4 | valid tokens/s: 193132.4 | physical tokens/s: 193707.6 | loss tokens/s: 139587.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.309963E-01 | lm-global-supervised-tokens: 1.212575E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.453483E-02 | lm-token-buffer-mib: 5.828270E+01 | global_load_balancing_loss: 1.000080E+00 | loss scale: 1.0 | grad norm: 0.220 | params norm: 11260.142 | params rms: 0.060970 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 12:41:32.823723 | finish at 2026-10-10 19:49:26 + [2026-10-10 07:09:43] iteration 6330/ 6855 | consumed samples: 810240 | elapsed time per iteration (ms): 110198.9 | throughput per GPU (TFLOP/s/GPU): 33.1 | MFU 10.62% | FLOPs estimator: workload-aware | logical tokens/s: 152244.4 | valid tokens/s: 152244.4 | physical tokens/s: 152664.7 | loss tokens/s: 116530.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.435559E-01 | lm-global-supervised-tokens: 1.284149E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.717675E-02 | lm-token-buffer-mib: 5.819750E+01 | global_load_balancing_loss: 1.000307E+00 | loss scale: 1.0 | grad norm: 0.075 | params norm: 11260.092 | params rms: 0.060970 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 16:04:14.437011 | finish at 2026-10-10 23:13:58 + [2026-10-10 07:11:12] iteration 6331/ 6855 | consumed samples: 810368 | elapsed time per iteration (ms): 88671.7 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.41% | FLOPs estimator: workload-aware | logical tokens/s: 189205.4 | valid tokens/s: 189205.4 | physical tokens/s: 189660.6 | loss tokens/s: 139813.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.080055E-01 | lm-global-supervised-tokens: 1.239754E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.380050E-02 | lm-token-buffer-mib: 5.822392E+01 | global_load_balancing_loss: 1.000272E+00 | loss scale: 1.0 | grad norm: 0.096 | params norm: 11260.042 | params rms: 0.060969 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 12:54:23.996037 | finish at 2026-10-10 20:05:36 + [2026-10-10 07:12:41] iteration 6332/ 6855 | consumed samples: 810496 | elapsed time per iteration (ms): 89402.0 | throughput per GPU (TFLOP/s/GPU): 41.2 | MFU 13.20% | FLOPs estimator: workload-aware | logical tokens/s: 187660.2 | valid tokens/s: 187660.2 | physical tokens/s: 188078.4 | loss tokens/s: 143182.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.647681E-01 | lm-global-supervised-tokens: 1.280079E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.177499E-02 | lm-token-buffer-mib: 5.822815E+01 | global_load_balancing_loss: 1.000118E+00 | loss scale: 1.0 | grad norm: 0.097 | params norm: 11259.991 | params rms: 0.060969 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 12:59:17.220038 | finish at 2026-10-10 20:11:59 + [2026-10-10 07:14:07] iteration 6333/ 6855 | consumed samples: 810624 | elapsed time per iteration (ms): 85382.3 | throughput per GPU (TFLOP/s/GPU): 42.8 | MFU 13.71% | FLOPs estimator: workload-aware | logical tokens/s: 196494.6 | valid tokens/s: 196494.6 | physical tokens/s: 197055.8 | loss tokens/s: 145764.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.211975E-01 | lm-global-supervised-tokens: 1.244567E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.536394E-02 | lm-token-buffer-mib: 5.814165E+01 | global_load_balancing_loss: 1.000247E+00 | loss scale: 1.0 | grad norm: 0.087 | params norm: 11259.938 | params rms: 0.060969 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 12:22:49.542253 | finish at 2026-10-10 19:36:56 + [2026-10-10 07:15:33] iteration 6334/ 6855 | consumed samples: 810752 | elapsed time per iteration (ms): 86085.5 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.70% | FLOPs estimator: workload-aware | logical tokens/s: 194889.7 | valid tokens/s: 194889.7 | physical tokens/s: 195399.4 | loss tokens/s: 142401.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.519986E-01 | lm-global-supervised-tokens: 1.225870E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.795508E-02 | lm-token-buffer-mib: 5.807932E+01 | global_load_balancing_loss: 9.998692E-01 | loss scale: 1.0 | grad norm: 0.079 | params norm: 11259.886 | params rms: 0.060969 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 12:27:30.543886 | finish at 2026-10-10 19:43:03 + [2026-10-10 07:17:02] iteration 6335/ 6855 | consumed samples: 810880 | elapsed time per iteration (ms): 88955.6 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.38% | FLOPs estimator: workload-aware | logical tokens/s: 188601.8 | valid tokens/s: 188601.8 | physical tokens/s: 188981.8 | loss tokens/s: 144967.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.786988E-01 | lm-global-supervised-tokens: 1.289567E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.197617E-02 | lm-token-buffer-mib: 5.806189E+01 | global_load_balancing_loss: 9.999464E-01 | loss scale: 1.0 | grad norm: 0.062 | params norm: 11259.834 | params rms: 0.060968 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 12:50:56.909161 | finish at 2026-10-10 20:07:59 + [2026-10-10 07:18:26] iteration 6336/ 6855 | consumed samples: 811008 | elapsed time per iteration (ms): 83951.4 | throughput per GPU (TFLOP/s/GPU): 43.1 | MFU 13.80% | FLOPs estimator: workload-aware | logical tokens/s: 199843.0 | valid tokens/s: 199843.0 | physical tokens/s: 200523.0 | loss tokens/s: 150687.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.840909E-01 | lm-global-supervised-tokens: 1.265044E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.214723E-02 | lm-token-buffer-mib: 5.816524E+01 | global_load_balancing_loss: 1.000327E+00 | loss scale: 1.0 | grad norm: 0.049 | params norm: 11259.783 | params rms: 0.060968 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 12:06:10.787016 | finish at 2026-10-10 19:24:37 + [2026-10-10 07:19:51] iteration 6337/ 6855 | consumed samples: 811136 | elapsed time per iteration (ms): 85552.4 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.79% | FLOPs estimator: workload-aware | logical tokens/s: 196104.1 | valid tokens/s: 196104.1 | physical tokens/s: 196421.2 | loss tokens/s: 141356.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.468036E-01 | lm-global-supervised-tokens: 1.209342E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.248921E-02 | lm-token-buffer-mib: 5.806313E+01 | global_load_balancing_loss: 1.000313E+00 | loss scale: 1.0 | grad norm: 0.040 | params norm: 11259.732 | params rms: 0.060968 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 12:18:36.144123 | finish at 2026-10-10 19:38:28 + [2026-10-10 07:21:19] iteration 6338/ 6855 | consumed samples: 811264 | elapsed time per iteration (ms): 87767.5 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.67% | FLOPs estimator: workload-aware | logical tokens/s: 191154.6 | valid tokens/s: 191154.6 | physical tokens/s: 191528.8 | loss tokens/s: 142147.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.710486E-01 | lm-global-supervised-tokens: 1.247597E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.288005E-02 | lm-token-buffer-mib: 5.806680E+01 | global_load_balancing_loss: 1.000640E+00 | loss scale: 1.0 | grad norm: 0.076 | params norm: 11259.682 | params rms: 0.060968 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 12:36:15.820141 | finish at 2026-10-10 19:57:35 + [2026-10-10 07:22:47] iteration 6339/ 6855 | consumed samples: 811392 | elapsed time per iteration (ms): 88099.2 | throughput per GPU (TFLOP/s/GPU): 42.8 | MFU 13.73% | FLOPs estimator: workload-aware | logical tokens/s: 190435.3 | valid tokens/s: 190435.3 | physical tokens/s: 190960.9 | loss tokens/s: 141164.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.628323E-01 | lm-global-supervised-tokens: 1.243652E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.606950E-02 | lm-token-buffer-mib: 5.805674E+01 | global_load_balancing_loss: 9.989986E-01 | loss scale: 1.0 | grad norm: 0.077 | params norm: 11259.631 | params rms: 0.060967 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 12:37:39.181177 | finish at 2026-10-10 20:00:26 + [2026-10-10 07:24:40] iteration 6340/ 6855 | consumed samples: 811520 | elapsed time per iteration (ms): 112360.3 | throughput per GPU (TFLOP/s/GPU): 34.0 | MFU 10.89% | FLOPs estimator: workload-aware | logical tokens/s: 149315.9 | valid tokens/s: 149315.9 | physical tokens/s: 149566.9 | loss tokens/s: 113197.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.628462E-01 | lm-global-supervised-tokens: 1.271892E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.209779E-02 | lm-token-buffer-mib: 5.805500E+01 | global_load_balancing_loss: 1.000130E+00 | loss scale: 1.0 | grad norm: 0.037 | params norm: 11259.581 | params rms: 0.060967 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 16:04:25.571600 | finish at 2026-10-10 23:29:05 + [2026-10-10 07:26:07] iteration 6341/ 6855 | consumed samples: 811648 | elapsed time per iteration (ms): 87666.2 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.54% | FLOPs estimator: workload-aware | logical tokens/s: 191375.8 | valid tokens/s: 191375.7 | physical tokens/s: 191863.2 | loss tokens/s: 142436.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.768985E-01 | lm-global-supervised-tokens: 1.248683E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.840192E-02 | lm-token-buffer-mib: 5.823676E+01 | global_load_balancing_loss: 1.000563E+00 | loss scale: 1.0 | grad norm: 0.054 | params norm: 11259.530 | params rms: 0.060967 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 12:31:00.425167 | finish at 2026-10-10 19:57:08 + [2026-10-10 07:27:33] iteration 6342/ 6855 | consumed samples: 811776 | elapsed time per iteration (ms): 85920.0 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.55% | FLOPs estimator: workload-aware | logical tokens/s: 195264.8 | valid tokens/s: 195264.8 | physical tokens/s: 195775.5 | loss tokens/s: 148961.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.514145E-01 | lm-global-supervised-tokens: 1.279879E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.200162E-02 | lm-token-buffer-mib: 5.835944E+01 | global_load_balancing_loss: 1.001165E+00 | loss scale: 1.0 | grad norm: 0.059 | params norm: 11259.479 | params rms: 0.060966 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 12:14:36.946341 | finish at 2026-10-10 19:42:10 + [2026-10-10 07:29:01] iteration 6343/ 6855 | consumed samples: 811904 | elapsed time per iteration (ms): 88126.6 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.39% | FLOPs estimator: workload-aware | logical tokens/s: 190375.7 | valid tokens/s: 190375.7 | physical tokens/s: 190833.5 | loss tokens/s: 139821.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.458301E-01 | lm-global-supervised-tokens: 1.232199E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.316290E-02 | lm-token-buffer-mib: 5.813113E+01 | global_load_balancing_loss: 1.000831E+00 | loss scale: 1.0 | grad norm: 0.068 | params norm: 11259.429 | params rms: 0.060966 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 12:32:00.802124 | finish at 2026-10-10 20:01:02 + [2026-10-10 07:30:27] iteration 6344/ 6855 | consumed samples: 812032 | elapsed time per iteration (ms): 85508.6 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.66% | FLOPs estimator: workload-aware | logical tokens/s: 196204.1 | valid tokens/s: 196204.1 | physical tokens/s: 196641.0 | loss tokens/s: 148474.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.214454E-01 | lm-global-supervised-tokens: 1.269580E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.218451E-02 | lm-token-buffer-mib: 5.822017E+01 | global_load_balancing_loss: 1.000551E+00 | loss scale: 1.0 | grad norm: 0.039 | params norm: 11259.382 | params rms: 0.060966 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 12:08:14.893989 | finish at 2026-10-10 19:38:42 + [2026-10-10 07:31:53] iteration 6345/ 6855 | consumed samples: 812160 | elapsed time per iteration (ms): 85906.1 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.60% | FLOPs estimator: workload-aware | logical tokens/s: 195296.8 | valid tokens/s: 195296.8 | physical tokens/s: 195673.3 | loss tokens/s: 144210.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.795509E-01 | lm-global-supervised-tokens: 1.238856E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.232998E-02 | lm-token-buffer-mib: 5.815059E+01 | global_load_balancing_loss: 9.998493E-01 | loss scale: 1.0 | grad norm: 0.047 | params norm: 11259.334 | params rms: 0.060966 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 12:10:12.089860 | finish at 2026-10-10 19:42:05 + [2026-10-10 07:33:19] iteration 6346/ 6855 | consumed samples: 812288 | elapsed time per iteration (ms): 86540.5 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.44% | FLOPs estimator: workload-aware | logical tokens/s: 193865.4 | valid tokens/s: 193865.4 | physical tokens/s: 194280.3 | loss tokens/s: 140923.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.140803E-01 | lm-global-supervised-tokens: 1.219560E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.330148E-02 | lm-token-buffer-mib: 5.826580E+01 | global_load_balancing_loss: 9.997810E-01 | loss scale: 1.0 | grad norm: 0.049 | params norm: 11259.286 | params rms: 0.060965 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 12:14:09.122593 | finish at 2026-10-10 19:47:28 + [2026-10-10 07:34:46] iteration 6347/ 6855 | consumed samples: 812416 | elapsed time per iteration (ms): 86994.7 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.41% | FLOPs estimator: workload-aware | logical tokens/s: 192852.3 | valid tokens/s: 192852.3 | physical tokens/s: 193243.9 | loss tokens/s: 146289.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.165407E-01 | lm-global-supervised-tokens: 1.272642E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.807160E-02 | lm-token-buffer-mib: 5.811375E+01 | global_load_balancing_loss: 9.986572E-01 | loss scale: 1.0 | grad norm: 0.114 | params norm: 11259.238 | params rms: 0.060965 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 12:16:33.331074 | finish at 2026-10-10 19:51:20 + [2026-10-10 07:36:17] iteration 6348/ 6855 | consumed samples: 812544 | elapsed time per iteration (ms): 91046.7 | throughput per GPU (TFLOP/s/GPU): 41.3 | MFU 13.23% | FLOPs estimator: workload-aware | logical tokens/s: 184270.4 | valid tokens/s: 184270.4 | physical tokens/s: 184549.0 | loss tokens/s: 129983.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.694137E-01 | lm-global-supervised-tokens: 1.183458E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.142335E-02 | lm-token-buffer-mib: 5.806218E+01 | global_load_balancing_loss: 9.995413E-01 | loss scale: 1.0 | grad norm: 0.089 | params norm: 11259.190 | params rms: 0.060965 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 12:49:20.671461 | finish at 2026-10-10 20:25:38 + [2026-10-10 07:37:43] iteration 6349/ 6855 | consumed samples: 812672 | elapsed time per iteration (ms): 85860.2 | throughput per GPU (TFLOP/s/GPU): 42.8 | MFU 13.70% | FLOPs estimator: workload-aware | logical tokens/s: 195401.1 | valid tokens/s: 195401.1 | physical tokens/s: 195773.0 | loss tokens/s: 146069.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 5.759329E-01 | lm-global-supervised-tokens: 1.254153E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.352638E-02 | lm-token-buffer-mib: 5.810873E+01 | global_load_balancing_loss: 1.001128E+00 | loss scale: 1.0 | grad norm: 0.204 | params norm: 11259.142 | params rms: 0.060965 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 12:04:05.244636 | finish at 2026-10-10 19:41:48 + [2026-10-10 07:39:33] iteration 6350/ 6855 | consumed samples: 812800 | elapsed time per iteration (ms): 109924.7 | throughput per GPU (TFLOP/s/GPU): 33.1 | MFU 10.61% | FLOPs estimator: workload-aware | logical tokens/s: 152624.1 | valid tokens/s: 152624.1 | physical tokens/s: 153086.5 | loss tokens/s: 118853.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.325817E-01 | lm-global-supervised-tokens: 1.306498E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.303834E-02 | lm-token-buffer-mib: 5.819869E+01 | global_load_balancing_loss: 1.000506E+00 | loss scale: 1.0 | grad norm: 0.252 | params norm: 11259.094 | params rms: 0.060964 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 15:25:11.948829 | finish at 2026-10-10 23:04:45 + [2026-10-10 07:41:00] iteration 6351/ 6855 | consumed samples: 812928 | elapsed time per iteration (ms): 86690.9 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.59% | FLOPs estimator: workload-aware | logical tokens/s: 193528.9 | valid tokens/s: 193528.9 | physical tokens/s: 193980.9 | loss tokens/s: 149996.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.894032E-01 | lm-global-supervised-tokens: 1.300333E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.262420E-02 | lm-token-buffer-mib: 5.810105E+01 | global_load_balancing_loss: 1.000296E+00 | loss scale: 1.0 | grad norm: 0.161 | params norm: 11259.047 | params rms: 0.060964 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 12:08:12.192976 | finish at 2026-10-10 19:49:12 + [2026-10-10 07:42:27] iteration 6352/ 6855 | consumed samples: 813056 | elapsed time per iteration (ms): 86682.8 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.75% | FLOPs estimator: workload-aware | logical tokens/s: 193545.7 | valid tokens/s: 193545.6 | physical tokens/s: 193996.6 | loss tokens/s: 142214.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.488687E-01 | lm-global-supervised-tokens: 1.232755E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.404585E-02 | lm-token-buffer-mib: 5.807236E+01 | global_load_balancing_loss: 1.001086E+00 | loss scale: 1.0 | grad norm: 0.364 | params norm: 11259.000 | params rms: 0.060964 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 12:06:41.473372 | finish at 2026-10-10 19:49:08 + [2026-10-10 07:43:52] iteration 6353/ 6855 | consumed samples: 813184 | elapsed time per iteration (ms): 85390.5 | throughput per GPU (TFLOP/s/GPU): 43.2 | MFU 13.84% | FLOPs estimator: workload-aware | logical tokens/s: 196476.4 | valid tokens/s: 196476.4 | physical tokens/s: 196794.4 | loss tokens/s: 144348.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.374683E-01 | lm-global-supervised-tokens: 1.232596E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.153726E-02 | lm-token-buffer-mib: 5.807938E+01 | global_load_balancing_loss: 1.000389E+00 | loss scale: 1.0 | grad norm: 0.093 | params norm: 11258.952 | params rms: 0.060964 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 11:54:26.015356 | finish at 2026-10-10 19:38:18 + [2026-10-10 07:45:16] iteration 6354/ 6855 | consumed samples: 813312 | elapsed time per iteration (ms): 83629.9 | throughput per GPU (TFLOP/s/GPU): 43.4 | MFU 13.92% | FLOPs estimator: workload-aware | logical tokens/s: 200612.2 | valid tokens/s: 200612.2 | physical tokens/s: 201219.7 | loss tokens/s: 148518.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.316925E-01 | lm-global-supervised-tokens: 1.242058E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.153085E-02 | lm-token-buffer-mib: 5.807884E+01 | global_load_balancing_loss: 9.993640E-01 | loss scale: 1.0 | grad norm: 0.310 | params norm: 11258.905 | params rms: 0.060963 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 11:38:18.577739 | finish at 2026-10-10 19:23:34 + [2026-10-10 07:46:42] iteration 6355/ 6855 | consumed samples: 813440 | elapsed time per iteration (ms): 86529.0 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.74% | FLOPs estimator: workload-aware | logical tokens/s: 193891.1 | valid tokens/s: 193891.1 | physical tokens/s: 194418.4 | loss tokens/s: 137848.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.440757E-01 | lm-global-supervised-tokens: 1.192792E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.882537E-02 | lm-token-buffer-mib: 5.815205E+01 | global_load_balancing_loss: 1.000135E+00 | loss scale: 1.0 | grad norm: 0.079 | params norm: 11258.856 | params rms: 0.060963 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 12:01:04.510393 | finish at 2026-10-10 19:47:47 + [2026-10-10 07:48:09] iteration 6356/ 6855 | consumed samples: 813568 | elapsed time per iteration (ms): 86688.4 | throughput per GPU (TFLOP/s/GPU): 42.8 | MFU 13.73% | FLOPs estimator: workload-aware | logical tokens/s: 193533.7 | valid tokens/s: 193533.7 | physical tokens/s: 194088.7 | loss tokens/s: 144963.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.045312E-01 | lm-global-supervised-tokens: 1.256663E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.166919E-02 | lm-token-buffer-mib: 5.813606E+01 | global_load_balancing_loss: 1.000667E+00 | loss scale: 1.0 | grad norm: 0.157 | params norm: 11258.807 | params rms: 0.060963 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 12:00:57.534814 | finish at 2026-10-10 19:49:06 + [2026-10-10 07:49:36] iteration 6357/ 6855 | consumed samples: 813696 | elapsed time per iteration (ms): 87313.2 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.61% | FLOPs estimator: workload-aware | logical tokens/s: 192149.1 | valid tokens/s: 192149.1 | physical tokens/s: 192470.6 | loss tokens/s: 141695.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.464139E-01 | lm-global-supervised-tokens: 1.237184E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.330249E-02 | lm-token-buffer-mib: 5.805018E+01 | global_load_balancing_loss: 9.994448E-01 | loss scale: 1.0 | grad norm: 0.147 | params norm: 11258.759 | params rms: 0.060963 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 12:04:41.953651 | finish at 2026-10-10 19:54:18 + [2026-10-10 07:51:01] iteration 6358/ 6855 | consumed samples: 813824 | elapsed time per iteration (ms): 84847.7 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.65% | FLOPs estimator: workload-aware | logical tokens/s: 197732.9 | valid tokens/s: 197732.8 | physical tokens/s: 198296.1 | loss tokens/s: 144437.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.252611E-01 | lm-global-supervised-tokens: 1.225516E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.497770E-02 | lm-token-buffer-mib: 5.802493E+01 | global_load_balancing_loss: 9.995351E-01 | loss scale: 1.0 | grad norm: 0.091 | params norm: 11258.713 | params rms: 0.060962 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 11:42:49.317387 | finish at 2026-10-10 19:33:50 + [2026-10-10 07:52:27] iteration 6359/ 6855 | consumed samples: 813952 | elapsed time per iteration (ms): 85984.5 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.41% | FLOPs estimator: workload-aware | logical tokens/s: 195118.5 | valid tokens/s: 195118.5 | physical tokens/s: 195495.3 | loss tokens/s: 140331.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.565621E-01 | lm-global-supervised-tokens: 1.206637E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.114099E-02 | lm-token-buffer-mib: 5.819544E+01 | global_load_balancing_loss: 1.000984E+00 | loss scale: 1.0 | grad norm: 0.070 | params norm: 11258.667 | params rms: 0.060962 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 11:50:48.324974 | finish at 2026-10-10 19:43:15 + [2026-10-10 07:54:13] iteration 6360/ 6855 | consumed samples: 814080 | elapsed time per iteration (ms): 106306.1 | throughput per GPU (TFLOP/s/GPU): 33.9 | MFU 10.86% | FLOPs estimator: workload-aware | logical tokens/s: 157819.5 | valid tokens/s: 157819.4 | physical tokens/s: 158209.3 | loss tokens/s: 121950.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.615589E-01 | lm-global-supervised-tokens: 1.296405E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.256409E-02 | lm-token-buffer-mib: 5.815572E+01 | global_load_balancing_loss: 9.994453E-01 | loss scale: 1.0 | grad norm: 0.195 | params norm: 11258.619 | params rms: 0.060962 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 14:37:01.504930 | finish at 2026-10-10 22:31:15 +times across ranks (ms): + save-checkpoint: + rank 0: 23871.17 + rank 1: 23871.12 + rank 2: 23871.13 + rank 3: 23871.12 + rank 4: 23871.12 + rank 5: 23871.08 + rank 6: 23871.07 + rank 7: 23871.11 + rank 8: 23871.25 + rank 9: 23871.26 + rank 10: 23871.23 + rank 11: 23871.23 + rank 12: 23871.23 + rank 13: 23871.24 + rank 14: 23871.22 + rank 15: 23871.23 + rank 16: 23870.93 + rank 17: 23871.13 + rank 18: 23870.93 + rank 19: 23870.94 + rank 20: 23870.95 + rank 21: 23870.94 + rank 22: 23870.93 + rank 23: 23870.93 + rank 24: 23871.16 + rank 25: 23871.14 + rank 26: 23871.16 + rank 27: 23871.17 + rank 28: 23871.11 + rank 29: 23871.17 + rank 30: 23871.12 + rank 31: 23871.13 + rank 32: 23871.21 + rank 33: 23871.20 + rank 34: 23871.23 + rank 35: 23871.17 + rank 36: 23873.60 + rank 37: 23871.16 + rank 38: 23871.19 + rank 39: 23871.16 + rank 40: 23871.28 + rank 41: 23871.24 + rank 42: 23871.27 + rank 43: 23871.25 + rank 44: 23871.40 + rank 45: 23871.21 + rank 46: 23871.25 + rank 47: 23871.24 + rank 48: 23871.47 + rank 49: 23871.45 + rank 50: 23871.41 + rank 51: 23871.41 + rank 52: 23871.43 + rank 53: 23871.45 + rank 54: 23871.46 + rank 55: 23871.56 + rank 56: 23871.37 + rank 57: 23871.36 + rank 58: 23871.35 + rank 59: 23871.34 + rank 60: 23871.35 + rank 61: 23871.33 + rank 62: 23871.35 + rank 63: 23871.36 + [2026-10-10 07:56:07] iteration 6361/ 6855 | consumed samples: 814208 | elapsed time per iteration (ms): 89350.2 | throughput per GPU (TFLOP/s/GPU): 41.7 | MFU 13.36% | FLOPs estimator: workload-aware | logical tokens/s: 187768.4 | valid tokens/s: 187768.4 | physical tokens/s: 188295.8 | loss tokens/s: 143430.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.136401E-01 | lm-global-supervised-tokens: 1.281551E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.574322E+00 | lm-token-buffer-mib: 5.811532E+01 | global_load_balancing_loss: 1.000582E+00 | loss scale: 1.0 | grad norm: 0.082 | params norm: 11258.569 | params rms: 0.060961 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 12:15:39.019263 | finish at 2026-10-10 20:11:46 + [2026-10-10 07:57:38] iteration 6362/ 6855 | consumed samples: 814336 | elapsed time per iteration (ms): 91485.1 | throughput per GPU (TFLOP/s/GPU): 40.1 | MFU 12.87% | FLOPs estimator: workload-aware | logical tokens/s: 183386.5 | valid tokens/s: 183386.5 | physical tokens/s: 183756.2 | loss tokens/s: 142241.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.012040E-01 | lm-global-supervised-tokens: 1.301302E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.448462E-02 | lm-token-buffer-mib: 5.806647E+01 | global_load_balancing_loss: 1.000041E+00 | loss scale: 1.0 | grad norm: 0.062 | params norm: 11258.520 | params rms: 0.060961 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 12:31:42.165482 | finish at 2026-10-10 20:29:20 + [2026-10-10 07:59:05] iteration 6363/ 6855 | consumed samples: 814464 | elapsed time per iteration (ms): 87054.9 | throughput per GPU (TFLOP/s/GPU): 43.3 | MFU 13.87% | FLOPs estimator: workload-aware | logical tokens/s: 192719.9 | valid tokens/s: 192719.9 | physical tokens/s: 193136.1 | loss tokens/s: 144030.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.632992E-01 | lm-global-supervised-tokens: 1.253859E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.144213E-02 | lm-token-buffer-mib: 5.809325E+01 | global_load_balancing_loss: 1.000631E+00 | loss scale: 1.0 | grad norm: 0.141 | params norm: 11258.471 | params rms: 0.060961 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 11:53:51.009124 | finish at 2026-10-10 19:52:56 + [2026-10-10 08:00:32] iteration 6364/ 6855 | consumed samples: 814592 | elapsed time per iteration (ms): 86745.9 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.47% | FLOPs estimator: workload-aware | logical tokens/s: 193405.9 | valid tokens/s: 193405.9 | physical tokens/s: 193845.0 | loss tokens/s: 143513.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.399261E-01 | lm-global-supervised-tokens: 1.244920E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.189519E-02 | lm-token-buffer-mib: 5.825214E+01 | global_load_balancing_loss: 1.001574E+00 | loss scale: 1.0 | grad norm: 0.059 | params norm: 11258.420 | params rms: 0.060961 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 11:49:52.259452 | finish at 2026-10-10 19:50:24 + [2026-10-10 08:01:59] iteration 6365/ 6855 | consumed samples: 814720 | elapsed time per iteration (ms): 86620.5 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.55% | FLOPs estimator: workload-aware | logical tokens/s: 193686.3 | valid tokens/s: 193686.2 | physical tokens/s: 194134.4 | loss tokens/s: 148170.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.243067E-01 | lm-global-supervised-tokens: 1.283459E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.296596E-02 | lm-token-buffer-mib: 5.816046E+01 | global_load_balancing_loss: 9.997748E-01 | loss scale: 1.0 | grad norm: 0.120 | params norm: 11258.370 | params rms: 0.060960 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 11:47:24.047730 | finish at 2026-10-10 19:49:23 + [2026-10-10 08:03:26] iteration 6366/ 6855 | consumed samples: 814848 | elapsed time per iteration (ms): 87763.7 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.46% | FLOPs estimator: workload-aware | logical tokens/s: 191163.4 | valid tokens/s: 191163.4 | physical tokens/s: 191491.9 | loss tokens/s: 140988.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.772062E-01 | lm-global-supervised-tokens: 1.237365E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.266624E-02 | lm-token-buffer-mib: 5.810262E+01 | global_load_balancing_loss: 9.996219E-01 | loss scale: 1.0 | grad norm: 0.080 | params norm: 11258.322 | params rms: 0.060960 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 11:55:16.471456 | finish at 2026-10-10 19:58:43 + [2026-10-10 08:04:57] iteration 6367/ 6855 | consumed samples: 814976 | elapsed time per iteration (ms): 90622.2 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.39% | FLOPs estimator: workload-aware | logical tokens/s: 185133.6 | valid tokens/s: 185133.5 | physical tokens/s: 185523.8 | loss tokens/s: 132153.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.454728E-01 | lm-global-supervised-tokens: 1.197603E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.325780E-02 | lm-token-buffer-mib: 5.811285E+01 | global_load_balancing_loss: 1.000553E+00 | loss scale: 1.0 | grad norm: 0.056 | params norm: 11258.272 | params rms: 0.060960 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 12:17:03.629185 | finish at 2026-10-10 20:22:01 + [2026-10-10 08:06:20] iteration 6368/ 6855 | consumed samples: 815104 | elapsed time per iteration (ms): 82985.6 | throughput per GPU (TFLOP/s/GPU): 43.9 | MFU 14.08% | FLOPs estimator: workload-aware | logical tokens/s: 202170.0 | valid tokens/s: 202170.0 | physical tokens/s: 202678.4 | loss tokens/s: 152405.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.547284E-01 | lm-global-supervised-tokens: 1.264745E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.446825E-02 | lm-token-buffer-mib: 5.823850E+01 | global_load_balancing_loss: 9.999812E-01 | loss scale: 1.0 | grad norm: 0.107 | params norm: 11258.222 | params rms: 0.060960 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 11:13:33.994164 | finish at 2026-10-10 19:19:54 + [2026-10-10 08:07:50] iteration 6369/ 6855 | consumed samples: 815232 | elapsed time per iteration (ms): 90138.7 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.43% | FLOPs estimator: workload-aware | logical tokens/s: 186126.3 | valid tokens/s: 186126.3 | physical tokens/s: 186614.2 | loss tokens/s: 141060.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.043731E-01 | lm-global-supervised-tokens: 1.271496E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.444221E-02 | lm-token-buffer-mib: 5.813129E+01 | global_load_balancing_loss: 9.988266E-01 | loss scale: 1.0 | grad norm: 0.064 | params norm: 11258.176 | params rms: 0.060959 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 12:10:07.388969 | finish at 2026-10-10 20:17:58 + [2026-10-10 08:09:44] iteration 6370/ 6855 | consumed samples: 815360 | elapsed time per iteration (ms): 113453.0 | throughput per GPU (TFLOP/s/GPU): 32.9 | MFU 10.56% | FLOPs estimator: workload-aware | logical tokens/s: 147877.8 | valid tokens/s: 147877.8 | physical tokens/s: 148294.1 | loss tokens/s: 107610.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.601093E-01 | lm-global-supervised-tokens: 1.220874E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.205507E-02 | lm-token-buffer-mib: 5.809505E+01 | global_load_balancing_loss: 1.000532E+00 | loss scale: 1.0 | grad norm: 0.067 | params norm: 11258.130 | params rms: 0.060959 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 15:17:04.711277 | finish at 2026-10-10 23:26:48 + [2026-10-10 08:11:07] iteration 6371/ 6855 | consumed samples: 815488 | elapsed time per iteration (ms): 82950.7 | throughput per GPU (TFLOP/s/GPU): 43.9 | MFU 14.06% | FLOPs estimator: workload-aware | logical tokens/s: 202255.2 | valid tokens/s: 202255.2 | physical tokens/s: 202829.7 | loss tokens/s: 147065.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.219749E-01 | lm-global-supervised-tokens: 1.219919E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.167157E-02 | lm-token-buffer-mib: 5.838332E+01 | global_load_balancing_loss: 1.000892E+00 | loss scale: 1.0 | grad norm: 0.091 | params norm: 11258.083 | params rms: 0.060959 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 11:09:08.130744 | finish at 2026-10-10 19:20:15 + [2026-10-10 08:12:38] iteration 6372/ 6855 | consumed samples: 815616 | elapsed time per iteration (ms): 90967.0 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.43% | FLOPs estimator: workload-aware | logical tokens/s: 184431.6 | valid tokens/s: 184431.6 | physical tokens/s: 184957.3 | loss tokens/s: 139597.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.026936E-01 | lm-global-supervised-tokens: 1.269880E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.232150E-02 | lm-token-buffer-mib: 5.814799E+01 | global_load_balancing_loss: 9.996151E-01 | loss scale: 1.0 | grad norm: 0.053 | params norm: 11258.036 | params rms: 0.060959 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 12:12:17.063652 | finish at 2026-10-10 20:24:55 + [2026-10-10 08:14:05] iteration 6373/ 6855 | consumed samples: 815744 | elapsed time per iteration (ms): 87630.1 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.43% | FLOPs estimator: workload-aware | logical tokens/s: 191454.5 | valid tokens/s: 191454.5 | physical tokens/s: 191858.6 | loss tokens/s: 145920.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.431130E-01 | lm-global-supervised-tokens: 1.278700E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.122870E-02 | lm-token-buffer-mib: 5.813017E+01 | global_load_balancing_loss: 1.000760E+00 | loss scale: 1.0 | grad norm: 0.079 | params norm: 11257.989 | params rms: 0.060958 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 11:43:57.686141 | finish at 2026-10-10 19:58:03 + [2026-10-10 08:15:36] iteration 6374/ 6855 | consumed samples: 815872 | elapsed time per iteration (ms): 90357.2 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.42% | FLOPs estimator: workload-aware | logical tokens/s: 185676.6 | valid tokens/s: 185676.6 | physical tokens/s: 186176.9 | loss tokens/s: 138584.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.837900E-01 | lm-global-supervised-tokens: 1.252213E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.232634E-02 | lm-token-buffer-mib: 5.820502E+01 | global_load_balancing_loss: 1.001048E+00 | loss scale: 1.0 | grad norm: 0.054 | params norm: 11257.945 | params rms: 0.060958 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 12:04:21.792398 | finish at 2026-10-10 20:19:57 + [2026-10-10 08:17:06] iteration 6375/ 6855 | consumed samples: 816000 | elapsed time per iteration (ms): 90742.6 | throughput per GPU (TFLOP/s/GPU): 41.2 | MFU 13.20% | FLOPs estimator: workload-aware | logical tokens/s: 184886.9 | valid tokens/s: 184886.9 | physical tokens/s: 185485.9 | loss tokens/s: 135508.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.711866E-01 | lm-global-supervised-tokens: 1.229635E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.234428E-02 | lm-token-buffer-mib: 5.805179E+01 | global_load_balancing_loss: 1.000590E+00 | loss scale: 1.0 | grad norm: 0.100 | params norm: 11257.899 | params rms: 0.060958 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 12:05:56.451874 | finish at 2026-10-10 20:23:03 + [2026-10-10 08:18:37] iteration 6376/ 6855 | consumed samples: 816128 | elapsed time per iteration (ms): 90422.0 | throughput per GPU (TFLOP/s/GPU): 40.8 | MFU 13.06% | FLOPs estimator: workload-aware | logical tokens/s: 185543.2 | valid tokens/s: 185543.2 | physical tokens/s: 186000.3 | loss tokens/s: 136575.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.341705E-01 | lm-global-supervised-tokens: 1.234939E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.470938E-02 | lm-token-buffer-mib: 5.821470E+01 | global_load_balancing_loss: 1.000275E+00 | loss scale: 1.0 | grad norm: 0.048 | params norm: 11257.857 | params rms: 0.060958 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 12:01:52.128717 | finish at 2026-10-10 20:20:29 + [2026-10-10 08:20:06] iteration 6377/ 6855 | consumed samples: 816256 | elapsed time per iteration (ms): 88934.4 | throughput per GPU (TFLOP/s/GPU): 41.6 | MFU 13.32% | FLOPs estimator: workload-aware | logical tokens/s: 188646.6 | valid tokens/s: 188646.6 | physical tokens/s: 189096.1 | loss tokens/s: 137656.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.723909E-01 | lm-global-supervised-tokens: 1.224237E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.925922E-02 | lm-token-buffer-mib: 5.807358E+01 | global_load_balancing_loss: 9.993399E-01 | loss scale: 1.0 | grad norm: 0.034 | params norm: 11257.818 | params rms: 0.060957 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 11:48:30.654977 | finish at 2026-10-10 20:08:36 + [2026-10-10 08:21:31] iteration 6378/ 6855 | consumed samples: 816384 | elapsed time per iteration (ms): 85411.8 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.62% | FLOPs estimator: workload-aware | logical tokens/s: 196426.5 | valid tokens/s: 196426.5 | physical tokens/s: 196961.6 | loss tokens/s: 145452.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.588877E-01 | lm-global-supervised-tokens: 1.242333E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.141683E-02 | lm-token-buffer-mib: 5.810065E+01 | global_load_balancing_loss: 9.990506E-01 | loss scale: 1.0 | grad norm: 0.061 | params norm: 11257.780 | params rms: 0.060957 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 11:19:01.438905 | finish at 2026-10-10 19:40:33 + [2026-10-10 08:22:59] iteration 6379/ 6855 | consumed samples: 816512 | elapsed time per iteration (ms): 88076.9 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.45% | FLOPs estimator: workload-aware | logical tokens/s: 190483.6 | valid tokens/s: 190483.6 | physical tokens/s: 190999.3 | loss tokens/s: 142535.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.705480E-01 | lm-global-supervised-tokens: 1.255411E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.214183E-02 | lm-token-buffer-mib: 5.818522E+01 | global_load_balancing_loss: 1.000517E+00 | loss scale: 1.0 | grad norm: 0.057 | params norm: 11257.740 | params rms: 0.060957 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 11:38:44.604289 | finish at 2026-10-10 20:01:44 + [2026-10-10 08:24:46] iteration 6380/ 6855 | consumed samples: 816640 | elapsed time per iteration (ms): 107229.1 | throughput per GPU (TFLOP/s/GPU): 33.2 | MFU 10.65% | FLOPs estimator: workload-aware | logical tokens/s: 156461.2 | valid tokens/s: 156461.2 | physical tokens/s: 157026.6 | loss tokens/s: 116484.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.495046E-01 | lm-global-supervised-tokens: 1.249049E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.138593E-02 | lm-token-buffer-mib: 5.834716E+01 | global_load_balancing_loss: 1.000983E+00 | loss scale: 1.0 | grad norm: 0.039 | params norm: 11257.698 | params rms: 0.060957 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 14:08:53.833367 | finish at 2026-10-10 22:33:40 + [2026-10-10 08:26:16] iteration 6381/ 6855 | consumed samples: 816768 | elapsed time per iteration (ms): 89389.3 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.69% | FLOPs estimator: workload-aware | logical tokens/s: 187686.5 | valid tokens/s: 187686.5 | physical tokens/s: 188110.1 | loss tokens/s: 130978.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.018589E-01 | lm-global-supervised-tokens: 1.170804E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.734429E-02 | lm-token-buffer-mib: 5.810831E+01 | global_load_balancing_loss: 1.000792E+00 | loss scale: 1.0 | grad norm: 0.072 | params norm: 11257.659 | params rms: 0.060957 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 11:46:10.519662 | finish at 2026-10-10 20:12:26 + [2026-10-10 08:27:42] iteration 6382/ 6855 | consumed samples: 816896 | elapsed time per iteration (ms): 86058.5 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.78% | FLOPs estimator: workload-aware | logical tokens/s: 194950.7 | valid tokens/s: 194950.7 | physical tokens/s: 195476.6 | loss tokens/s: 145463.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.696779E-01 | lm-global-supervised-tokens: 1.251833E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.114800E-02 | lm-token-buffer-mib: 5.808153E+01 | global_load_balancing_loss: 1.001189E+00 | loss scale: 1.0 | grad norm: 0.057 | params norm: 11257.621 | params rms: 0.060956 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 11:18:25.655751 | finish at 2026-10-10 19:46:08 + [2026-10-10 08:29:05] iteration 6383/ 6855 | consumed samples: 817024 | elapsed time per iteration (ms): 83445.2 | throughput per GPU (TFLOP/s/GPU): 43.4 | MFU 13.90% | FLOPs estimator: workload-aware | logical tokens/s: 201056.8 | valid tokens/s: 201056.8 | physical tokens/s: 201558.0 | loss tokens/s: 151848.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.187068E-01 | lm-global-supervised-tokens: 1.267099E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.371192E-02 | lm-token-buffer-mib: 5.815125E+01 | global_load_balancing_loss: 1.000807E+00 | loss scale: 1.0 | grad norm: 0.085 | params norm: 11257.583 | params rms: 0.060956 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 10:56:26.112215 | finish at 2026-10-10 19:25:31 + [2026-10-10 08:30:36] iteration 6384/ 6855 | consumed samples: 817152 | elapsed time per iteration (ms): 90529.8 | throughput per GPU (TFLOP/s/GPU): 40.6 | MFU 13.01% | FLOPs estimator: workload-aware | logical tokens/s: 185322.5 | valid tokens/s: 185322.5 | physical tokens/s: 185716.6 | loss tokens/s: 133319.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.224989E-01 | lm-global-supervised-tokens: 1.206941E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.519941E-02 | lm-token-buffer-mib: 5.817381E+01 | global_load_balancing_loss: 9.992971E-01 | loss scale: 1.0 | grad norm: 0.173 | params norm: 11257.545 | params rms: 0.060956 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 11:50:39.551941 | finish at 2026-10-10 20:21:15 + [2026-10-10 08:32:06] iteration 6385/ 6855 | consumed samples: 817280 | elapsed time per iteration (ms): 90395.9 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.51% | FLOPs estimator: workload-aware | logical tokens/s: 185596.7 | valid tokens/s: 185596.7 | physical tokens/s: 185935.8 | loss tokens/s: 129142.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.895027E-01 | lm-global-supervised-tokens: 1.167399E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.184647E-02 | lm-token-buffer-mib: 5.803357E+01 | global_load_balancing_loss: 9.994048E-01 | loss scale: 1.0 | grad norm: 0.083 | params norm: 11257.507 | params rms: 0.060956 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 11:48:06.057653 | finish at 2026-10-10 20:20:12 + [2026-10-10 08:33:33] iteration 6386/ 6855 | consumed samples: 817408 | elapsed time per iteration (ms): 86824.9 | throughput per GPU (TFLOP/s/GPU): 41.7 | MFU 13.37% | FLOPs estimator: workload-aware | logical tokens/s: 193230.3 | valid tokens/s: 193230.3 | physical tokens/s: 193710.1 | loss tokens/s: 142430.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.083924E-01 | lm-global-supervised-tokens: 1.236650E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.302374E-02 | lm-token-buffer-mib: 5.809943E+01 | global_load_balancing_loss: 9.988130E-01 | loss scale: 1.0 | grad norm: 0.119 | params norm: 11257.467 | params rms: 0.060956 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 11:18:40.857260 | finish at 2026-10-10 19:52:14 + [2026-10-10 08:35:03] iteration 6387/ 6855 | consumed samples: 817536 | elapsed time per iteration (ms): 89880.4 | throughput per GPU (TFLOP/s/GPU): 41.4 | MFU 13.27% | FLOPs estimator: workload-aware | logical tokens/s: 186661.1 | valid tokens/s: 186661.1 | physical tokens/s: 187175.6 | loss tokens/s: 137258.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.581019E-01 | lm-global-supervised-tokens: 1.233680E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.496538E-02 | lm-token-buffer-mib: 5.819573E+01 | global_load_balancing_loss: 1.000101E+00 | loss scale: 1.0 | grad norm: 0.127 | params norm: 11257.429 | params rms: 0.060955 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 11:41:04.012556 | finish at 2026-10-10 20:16:07 + [2026-10-10 08:36:31] iteration 6388/ 6855 | consumed samples: 817664 | elapsed time per iteration (ms): 87794.8 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.42% | FLOPs estimator: workload-aware | logical tokens/s: 191095.7 | valid tokens/s: 191095.6 | physical tokens/s: 191497.7 | loss tokens/s: 144240.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.988739E-01 | lm-global-supervised-tokens: 1.266359E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.150133E-02 | lm-token-buffer-mib: 5.808444E+01 | global_load_balancing_loss: 1.000498E+00 | loss scale: 1.0 | grad norm: 0.065 | params norm: 11257.390 | params rms: 0.060955 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 11:23:20.186206 | finish at 2026-10-10 19:59:51 + [2026-10-10 08:37:59] iteration 6389/ 6855 | consumed samples: 817792 | elapsed time per iteration (ms): 87820.6 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.52% | FLOPs estimator: workload-aware | logical tokens/s: 191039.6 | valid tokens/s: 191039.6 | physical tokens/s: 191617.6 | loss tokens/s: 144174.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.716095E-01 | lm-global-supervised-tokens: 1.266148E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.113392E-02 | lm-token-buffer-mib: 5.821613E+01 | global_load_balancing_loss: 9.994026E-01 | loss scale: 1.0 | grad norm: 0.131 | params norm: 11257.350 | params rms: 0.060955 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 11:22:04.398949 | finish at 2026-10-10 20:00:03 + [2026-10-10 08:39:48] iteration 6390/ 6855 | consumed samples: 817920 | elapsed time per iteration (ms): 109151.5 | throughput per GPU (TFLOP/s/GPU): 34.6 | MFU 11.09% | FLOPs estimator: workload-aware | logical tokens/s: 153705.3 | valid tokens/s: 153705.3 | physical tokens/s: 154089.0 | loss tokens/s: 110225.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.553698E-01 | lm-global-supervised-tokens: 1.203126E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.139037E-02 | lm-token-buffer-mib: 5.812816E+01 | global_load_balancing_loss: 1.002440E+00 | loss scale: 1.0 | grad norm: 0.046 | params norm: 11257.307 | params rms: 0.060955 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 14:05:55.447161 | finish at 2026-10-10 22:45:43 + [2026-10-10 08:41:17] iteration 6391/ 6855 | consumed samples: 818048 | elapsed time per iteration (ms): 89010.4 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.50% | FLOPs estimator: workload-aware | logical tokens/s: 188485.4 | valid tokens/s: 188485.4 | physical tokens/s: 188835.2 | loss tokens/s: 138265.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.280562E-01 | lm-global-supervised-tokens: 1.230707E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.122932E-02 | lm-token-buffer-mib: 5.812502E+01 | global_load_balancing_loss: 1.001251E+00 | loss scale: 1.0 | grad norm: 0.061 | params norm: 11257.265 | params rms: 0.060954 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 11:28:20.834698 | finish at 2026-10-10 20:09:38 + [2026-10-10 08:42:46] iteration 6392/ 6855 | consumed samples: 818176 | elapsed time per iteration (ms): 89415.0 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.54% | FLOPs estimator: workload-aware | logical tokens/s: 187633.0 | valid tokens/s: 187633.0 | physical tokens/s: 187968.6 | loss tokens/s: 137752.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.523027E-01 | lm-global-supervised-tokens: 1.231712E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.655421E-02 | lm-token-buffer-mib: 5.820547E+01 | global_load_balancing_loss: 9.998413E-01 | loss scale: 1.0 | grad norm: 0.101 | params norm: 11257.222 | params rms: 0.060954 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 11:29:59.163527 | finish at 2026-10-10 20:12:45 + [2026-10-10 08:44:12] iteration 6393/ 6855 | consumed samples: 818304 | elapsed time per iteration (ms): 85713.7 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.80% | FLOPs estimator: workload-aware | logical tokens/s: 195735.5 | valid tokens/s: 195735.4 | physical tokens/s: 196215.9 | loss tokens/s: 146736.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.544487E-01 | lm-global-supervised-tokens: 1.257731E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.207975E-02 | lm-token-buffer-mib: 5.811803E+01 | global_load_balancing_loss: 1.000367E+00 | loss scale: 1.0 | grad norm: 0.038 | params norm: 11257.179 | params rms: 0.060954 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 10:59:59.719072 | finish at 2026-10-10 19:44:12 + [2026-10-10 08:45:39] iteration 6394/ 6855 | consumed samples: 818432 | elapsed time per iteration (ms): 87370.8 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.67% | FLOPs estimator: workload-aware | logical tokens/s: 192022.6 | valid tokens/s: 192022.6 | physical tokens/s: 192496.3 | loss tokens/s: 139208.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.440006E-01 | lm-global-supervised-tokens: 1.216274E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.132713E-02 | lm-token-buffer-mib: 5.803848E+01 | global_load_balancing_loss: 9.989426E-01 | loss scale: 1.0 | grad norm: 0.108 | params norm: 11257.135 | params rms: 0.060954 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 11:11:17.959032 | finish at 2026-10-10 19:56:57 + [2026-10-10 08:47:06] iteration 6395/ 6855 | consumed samples: 818560 | elapsed time per iteration (ms): 86318.8 | throughput per GPU (TFLOP/s/GPU): 43.1 | MFU 13.80% | FLOPs estimator: workload-aware | logical tokens/s: 194363.2 | valid tokens/s: 194363.2 | physical tokens/s: 195036.3 | loss tokens/s: 146180.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.720538E-01 | lm-global-supervised-tokens: 1.261808E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.142441E-02 | lm-token-buffer-mib: 5.819325E+01 | global_load_balancing_loss: 9.988478E-01 | loss scale: 1.0 | grad norm: 0.120 | params norm: 11257.092 | params rms: 0.060953 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 11:01:46.635814 | finish at 2026-10-10 19:48:52 + [2026-10-10 08:48:33] iteration 6396/ 6855 | consumed samples: 818688 | elapsed time per iteration (ms): 87569.5 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.51% | FLOPs estimator: workload-aware | logical tokens/s: 191586.8 | valid tokens/s: 191586.8 | physical tokens/s: 192123.2 | loss tokens/s: 137405.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.081583E-01 | lm-global-supervised-tokens: 1.203254E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.180828E-02 | lm-token-buffer-mib: 5.809712E+01 | global_load_balancing_loss: 1.000266E+00 | loss scale: 1.0 | grad norm: 0.199 | params norm: 11257.049 | params rms: 0.060953 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 11:09:54.418652 | finish at 2026-10-10 19:58:28 + [2026-10-10 08:50:01] iteration 6397/ 6855 | consumed samples: 818816 | elapsed time per iteration (ms): 88227.9 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.64% | FLOPs estimator: workload-aware | logical tokens/s: 190157.1 | valid tokens/s: 190157.1 | physical tokens/s: 190596.0 | loss tokens/s: 142685.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.151403E-01 | lm-global-supervised-tokens: 1.258880E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.540860E-02 | lm-token-buffer-mib: 5.819884E+01 | global_load_balancing_loss: 9.992532E-01 | loss scale: 1.0 | grad norm: 0.049 | params norm: 11257.005 | params rms: 0.060953 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 11:13:28.356483 | finish at 2026-10-10 20:03:30 + [2026-10-10 08:51:30] iteration 6398/ 6855 | consumed samples: 818944 | elapsed time per iteration (ms): 89032.8 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.64% | FLOPs estimator: workload-aware | logical tokens/s: 188437.7 | valid tokens/s: 188437.7 | physical tokens/s: 188862.9 | loss tokens/s: 137429.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.846969E-01 | lm-global-supervised-tokens: 1.223571E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.144649E-02 | lm-token-buffer-mib: 5.813457E+01 | global_load_balancing_loss: 1.000429E+00 | loss scale: 1.0 | grad norm: 0.116 | params norm: 11256.962 | params rms: 0.060953 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 11:18:07.978141 | finish at 2026-10-10 20:09:38 + [2026-10-10 08:52:51] iteration 6399/ 6855 | consumed samples: 819072 | elapsed time per iteration (ms): 81005.2 | throughput per GPU (TFLOP/s/GPU): 43.9 | MFU 14.07% | FLOPs estimator: workload-aware | logical tokens/s: 207112.4 | valid tokens/s: 207112.4 | physical tokens/s: 207500.3 | loss tokens/s: 156304.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.176595E-01 | lm-global-supervised-tokens: 1.266144E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.469127E-02 | lm-token-buffer-mib: 5.805328E+01 | global_load_balancing_loss: 1.001888E+00 | loss scale: 1.0 | grad norm: 0.056 | params norm: 11256.916 | params rms: 0.060953 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 10:15:38.381161 | finish at 2026-10-10 19:08:30 + [2026-10-10 08:54:40] iteration 6400/ 6855 | consumed samples: 819200 | elapsed time per iteration (ms): 108855.6 | throughput per GPU (TFLOP/s/GPU): 33.5 | MFU 10.74% | FLOPs estimator: workload-aware | logical tokens/s: 154123.4 | valid tokens/s: 154123.4 | physical tokens/s: 154491.6 | loss tokens/s: 113511.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.504489E-01 | lm-global-supervised-tokens: 1.235635E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.161140E-02 | lm-token-buffer-mib: 5.834173E+01 | global_load_balancing_loss: 1.001350E+00 | loss scale: 1.0 | grad norm: 0.033 | params norm: 11256.872 | params rms: 0.060952 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 13:45:29.302502 | finish at 2026-10-10 22:40:10 +times across ranks (ms): + save-checkpoint: + rank 0: 25669.12 + rank 1: 25669.14 + rank 2: 25669.17 + rank 3: 25669.12 + rank 4: 25669.12 + rank 5: 25669.10 + rank 6: 25669.10 + rank 7: 25669.12 + rank 8: 25669.08 + rank 9: 25669.03 + rank 10: 25669.02 + rank 11: 25669.08 + rank 12: 25669.05 + rank 13: 25669.04 + rank 14: 25669.06 + rank 15: 25669.02 + rank 16: 25670.14 + rank 17: 25670.13 + rank 18: 25670.11 + rank 19: 25670.14 + rank 20: 25670.10 + rank 21: 25670.16 + rank 22: 25670.10 + rank 23: 25670.10 + rank 24: 25669.56 + rank 25: 25669.49 + rank 26: 25669.58 + rank 27: 25669.57 + rank 28: 25669.54 + rank 29: 25669.49 + rank 30: 25669.49 + rank 31: 25669.49 + rank 32: 25669.16 + rank 33: 25669.01 + rank 34: 25669.08 + rank 35: 25669.12 + rank 36: 25668.93 + rank 37: 25669.00 + rank 38: 25668.99 + rank 39: 25668.90 + rank 40: 25669.30 + rank 41: 25669.26 + rank 42: 25669.26 + rank 43: 25669.27 + rank 44: 25669.25 + rank 45: 25669.26 + rank 46: 25669.26 + rank 47: 25669.26 + rank 48: 25668.73 + rank 49: 25668.62 + rank 50: 25668.66 + rank 51: 25668.73 + rank 52: 25668.74 + rank 53: 25668.68 + rank 54: 25668.71 + rank 55: 25668.71 + rank 56: 25669.45 + rank 57: 25669.47 + rank 58: 25669.43 + rank 59: 25669.42 + rank 60: 25669.43 + rank 61: 25669.43 + rank 62: 25669.44 + rank 63: 25669.44 + [2026-10-10 08:56:45] iteration 6401/ 6855 | consumed samples: 819328 | elapsed time per iteration (ms): 98561.3 | throughput per GPU (TFLOP/s/GPU): 36.6 | MFU 11.73% | FLOPs estimator: workload-aware | logical tokens/s: 170220.9 | valid tokens/s: 170220.9 | physical tokens/s: 170659.5 | loss tokens/s: 128223.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.465533E-01 | lm-global-supervised-tokens: 1.263782E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.036322E+00 | lm-token-buffer-mib: 5.810061E+01 | global_load_balancing_loss: 1.001176E+00 | loss scale: 1.0 | grad norm: 0.079 | params norm: 11256.826 | params rms: 0.060952 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 12:25:46.807920 | finish at 2026-10-10 21:22:32 + [2026-10-10 08:58:20] iteration 6402/ 6855 | consumed samples: 819456 | elapsed time per iteration (ms): 95558.4 | throughput per GPU (TFLOP/s/GPU): 38.3 | MFU 12.29% | FLOPs estimator: workload-aware | logical tokens/s: 175570.1 | valid tokens/s: 175570.1 | physical tokens/s: 176010.0 | loss tokens/s: 127214.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.386722E-01 | lm-global-supervised-tokens: 1.215642E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.349751E-02 | lm-token-buffer-mib: 5.815118E+01 | global_load_balancing_loss: 1.000802E+00 | loss scale: 1.0 | grad norm: 0.043 | params norm: 11256.781 | params rms: 0.060952 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 12:01:27.949546 | finish at 2026-10-10 20:59:48 + [2026-10-10 08:59:48] iteration 6403/ 6855 | consumed samples: 819584 | elapsed time per iteration (ms): 87674.5 | throughput per GPU (TFLOP/s/GPU): 41.2 | MFU 13.22% | FLOPs estimator: workload-aware | logical tokens/s: 191357.6 | valid tokens/s: 191357.5 | physical tokens/s: 191801.3 | loss tokens/s: 142407.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.359597E-01 | lm-global-supervised-tokens: 1.248554E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.125448E-02 | lm-token-buffer-mib: 5.809955E+01 | global_load_balancing_loss: 9.999795E-01 | loss scale: 1.0 | grad norm: 0.037 | params norm: 11256.736 | params rms: 0.060952 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 11:00:28.859231 | finish at 2026-10-10 20:00:17 + [2026-10-10 09:01:16] iteration 6404/ 6855 | consumed samples: 819712 | elapsed time per iteration (ms): 88253.3 | throughput per GPU (TFLOP/s/GPU): 41.6 | MFU 13.33% | FLOPs estimator: workload-aware | logical tokens/s: 190102.3 | valid tokens/s: 190102.3 | physical tokens/s: 190675.4 | loss tokens/s: 140308.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.115330E-01 | lm-global-supervised-tokens: 1.238266E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.165762E-02 | lm-token-buffer-mib: 5.855700E+01 | global_load_balancing_loss: 9.998639E-01 | loss scale: 1.0 | grad norm: 0.117 | params norm: 11256.689 | params rms: 0.060951 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 11:03:22.254837 | finish at 2026-10-10 20:04:38 + [2026-10-10 09:02:46] iteration 6405/ 6855 | consumed samples: 819840 | elapsed time per iteration (ms): 89574.8 | throughput per GPU (TFLOP/s/GPU): 40.8 | MFU 13.09% | FLOPs estimator: workload-aware | logical tokens/s: 187298.0 | valid tokens/s: 187298.0 | physical tokens/s: 187779.0 | loss tokens/s: 141408.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.674629E-01 | lm-global-supervised-tokens: 1.266667E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.941609E-02 | lm-token-buffer-mib: 5.830939E+01 | global_load_balancing_loss: 9.988481E-01 | loss scale: 1.0 | grad norm: 0.082 | params norm: 11256.642 | params rms: 0.060951 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 11:11:48.664727 | finish at 2026-10-10 20:14:34 + [2026-10-10 09:04:11] iteration 6406/ 6855 | consumed samples: 819968 | elapsed time per iteration (ms): 85457.2 | throughput per GPU (TFLOP/s/GPU): 43.1 | MFU 13.81% | FLOPs estimator: workload-aware | logical tokens/s: 196322.8 | valid tokens/s: 196322.8 | physical tokens/s: 196717.4 | loss tokens/s: 141778.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.250086E-01 | lm-global-supervised-tokens: 1.211598E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.287525E-02 | lm-token-buffer-mib: 5.817629E+01 | global_load_balancing_loss: 9.986159E-01 | loss scale: 1.0 | grad norm: 0.082 | params norm: 11256.595 | params rms: 0.060951 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 10:39:30.277684 | finish at 2026-10-10 19:43:42 + [2026-10-10 09:05:40] iteration 6407/ 6855 | consumed samples: 820096 | elapsed time per iteration (ms): 89220.7 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.40% | FLOPs estimator: workload-aware | logical tokens/s: 188041.5 | valid tokens/s: 188041.5 | physical tokens/s: 188438.2 | loss tokens/s: 130450.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.237844E-01 | lm-global-supervised-tokens: 1.163887E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.344842E-02 | lm-token-buffer-mib: 5.804656E+01 | global_load_balancing_loss: 1.000652E+00 | loss scale: 1.0 | grad norm: 0.076 | params norm: 11256.547 | params rms: 0.060951 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 11:06:10.885574 | finish at 2026-10-10 20:11:51 + [2026-10-10 09:07:06] iteration 6408/ 6855 | consumed samples: 820224 | elapsed time per iteration (ms): 85560.8 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.61% | FLOPs estimator: workload-aware | logical tokens/s: 196084.7 | valid tokens/s: 196084.7 | physical tokens/s: 196589.9 | loss tokens/s: 142890.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.084364E-01 | lm-global-supervised-tokens: 1.222581E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.157629E-02 | lm-token-buffer-mib: 5.821191E+01 | global_load_balancing_loss: 1.000024E+00 | loss scale: 1.0 | grad norm: 0.105 | params norm: 11256.501 | params rms: 0.060950 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 10:37:25.698202 | finish at 2026-10-10 19:44:32 + [2026-10-10 09:08:33] iteration 6409/ 6855 | consumed samples: 820352 | elapsed time per iteration (ms): 87316.6 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.59% | FLOPs estimator: workload-aware | logical tokens/s: 192142.2 | valid tokens/s: 192142.2 | physical tokens/s: 192413.0 | loss tokens/s: 139941.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.964875E-01 | lm-global-supervised-tokens: 1.221923E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.153295E-02 | lm-token-buffer-mib: 5.806606E+01 | global_load_balancing_loss: 9.998940E-01 | loss scale: 1.0 | grad norm: 0.035 | params norm: 11256.455 | params rms: 0.060950 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 10:49:03.195343 | finish at 2026-10-10 19:57:37 + [2026-10-10 09:10:25] iteration 6410/ 6855 | consumed samples: 820480 | elapsed time per iteration (ms): 111400.5 | throughput per GPU (TFLOP/s/GPU): 33.9 | MFU 10.87% | FLOPs estimator: workload-aware | logical tokens/s: 150602.7 | valid tokens/s: 150602.7 | physical tokens/s: 150909.1 | loss tokens/s: 108698.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.064755E-01 | lm-global-supervised-tokens: 1.210907E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.556954E-02 | lm-token-buffer-mib: 5.810549E+01 | global_load_balancing_loss: 1.000947E+00 | loss scale: 1.0 | grad norm: 0.085 | params norm: 11256.409 | params rms: 0.060950 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 13:46:13.208203 | finish at 2026-10-10 22:56:38 + [2026-10-10 09:11:55] iteration 6411/ 6855 | consumed samples: 820608 | elapsed time per iteration (ms): 90140.7 | throughput per GPU (TFLOP/s/GPU): 41.7 | MFU 13.36% | FLOPs estimator: workload-aware | logical tokens/s: 186122.5 | valid tokens/s: 186122.5 | physical tokens/s: 186611.7 | loss tokens/s: 138284.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.164101E-01 | lm-global-supervised-tokens: 1.246508E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.332939E-02 | lm-token-buffer-mib: 5.808540E+01 | global_load_balancing_loss: 1.000786E+00 | loss scale: 1.0 | grad norm: 0.111 | params norm: 11256.362 | params rms: 0.060950 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 11:07:02.449780 | finish at 2026-10-10 20:18:57 + [2026-10-10 09:13:24] iteration 6412/ 6855 | consumed samples: 820736 | elapsed time per iteration (ms): 88938.9 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.47% | FLOPs estimator: workload-aware | logical tokens/s: 188636.8 | valid tokens/s: 188636.8 | physical tokens/s: 188976.5 | loss tokens/s: 142017.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.590154E-01 | lm-global-supervised-tokens: 1.263086E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.308534E-02 | lm-token-buffer-mib: 5.810239E+01 | global_load_balancing_loss: 9.997803E-01 | loss scale: 1.0 | grad norm: 0.064 | params norm: 11256.316 | params rms: 0.060949 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 10:56:39.934387 | finish at 2026-10-10 20:10:04 + [2026-10-10 09:14:50] iteration 6413/ 6855 | consumed samples: 820864 | elapsed time per iteration (ms): 86606.0 | throughput per GPU (TFLOP/s/GPU): 43.2 | MFU 13.86% | FLOPs estimator: workload-aware | logical tokens/s: 193718.6 | valid tokens/s: 193718.6 | physical tokens/s: 194285.7 | loss tokens/s: 145090.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.515298E-01 | lm-global-supervised-tokens: 1.256570E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.380481E-02 | lm-token-buffer-mib: 5.816144E+01 | global_load_balancing_loss: 1.000504E+00 | loss scale: 1.0 | grad norm: 0.074 | params norm: 11256.270 | params rms: 0.060949 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 10:37:59.872947 | finish at 2026-10-10 19:52:50 + [2026-10-10 09:16:18] iteration 6414/ 6855 | consumed samples: 820992 | elapsed time per iteration (ms): 87157.4 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.43% | FLOPs estimator: workload-aware | logical tokens/s: 192492.5 | valid tokens/s: 192492.5 | physical tokens/s: 192931.5 | loss tokens/s: 139229.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.692398E-01 | lm-global-supervised-tokens: 1.213492E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.340230E-02 | lm-token-buffer-mib: 5.811034E+01 | global_load_balancing_loss: 1.001319E+00 | loss scale: 1.0 | grad norm: 0.139 | params norm: 11256.223 | params rms: 0.060949 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 10:40:36.417664 | finish at 2026-10-10 19:56:54 + [2026-10-10 09:17:42] iteration 6415/ 6855 | consumed samples: 821120 | elapsed time per iteration (ms): 84751.9 | throughput per GPU (TFLOP/s/GPU): 43.6 | MFU 13.99% | FLOPs estimator: workload-aware | logical tokens/s: 197956.7 | valid tokens/s: 197956.7 | physical tokens/s: 198311.7 | loss tokens/s: 147924.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.784996E-01 | lm-global-supervised-tokens: 1.253686E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.414277E-02 | lm-token-buffer-mib: 5.806253E+01 | global_load_balancing_loss: 1.000641E+00 | loss scale: 1.0 | grad norm: 0.059 | params norm: 11256.175 | params rms: 0.060949 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 10:21:30.845003 | finish at 2026-10-10 19:39:13 + [2026-10-10 09:19:12] iteration 6416/ 6855 | consumed samples: 821248 | elapsed time per iteration (ms): 89729.1 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.59% | FLOPs estimator: workload-aware | logical tokens/s: 186976.2 | valid tokens/s: 186976.2 | physical tokens/s: 187324.4 | loss tokens/s: 139787.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.053613E-01 | lm-global-supervised-tokens: 1.254301E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.176760E-02 | lm-token-buffer-mib: 5.803600E+01 | global_load_balancing_loss: 1.000298E+00 | loss scale: 1.0 | grad norm: 0.263 | params norm: 11256.129 | params rms: 0.060948 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 10:56:31.080024 | finish at 2026-10-10 20:15:43 + [2026-10-10 09:20:36] iteration 6417/ 6855 | consumed samples: 821376 | elapsed time per iteration (ms): 84145.3 | throughput per GPU (TFLOP/s/GPU): 43.2 | MFU 13.84% | FLOPs estimator: workload-aware | logical tokens/s: 199383.5 | valid tokens/s: 199383.5 | physical tokens/s: 200009.6 | loss tokens/s: 145924.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.155937E-01 | lm-global-supervised-tokens: 1.227884E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.118768E-02 | lm-token-buffer-mib: 5.816657E+01 | global_load_balancing_loss: 9.993042E-01 | loss scale: 1.0 | grad norm: 0.082 | params norm: 11256.084 | params rms: 0.060948 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 10:14:15.656921 | finish at 2026-10-10 19:34:52 + [2026-10-10 09:22:03] iteration 6418/ 6855 | consumed samples: 821504 | elapsed time per iteration (ms): 87175.1 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.75% | FLOPs estimator: workload-aware | logical tokens/s: 192454.3 | valid tokens/s: 192454.3 | physical tokens/s: 192836.1 | loss tokens/s: 137425.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.786876E-01 | lm-global-supervised-tokens: 1.198006E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.167052E-02 | lm-token-buffer-mib: 5.803738E+01 | global_load_balancing_loss: 9.995300E-01 | loss scale: 1.0 | grad norm: 0.387 | params norm: 11256.039 | params rms: 0.060948 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 10:34:55.502693 | finish at 2026-10-10 19:56:59 + [2026-10-10 09:23:33] iteration 6419/ 6855 | consumed samples: 821632 | elapsed time per iteration (ms): 89270.6 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.49% | FLOPs estimator: workload-aware | logical tokens/s: 187936.2 | valid tokens/s: 187936.2 | physical tokens/s: 188418.1 | loss tokens/s: 140642.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.514947E-01 | lm-global-supervised-tokens: 1.255522E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.327380E-02 | lm-token-buffer-mib: 5.813841E+01 | global_load_balancing_loss: 9.999393E-01 | loss scale: 1.0 | grad norm: 0.101 | params norm: 11255.994 | params rms: 0.060948 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 10:48:41.962196 | finish at 2026-10-10 20:12:15 + [2026-10-10 09:25:24] iteration 6420/ 6855 | consumed samples: 821760 | elapsed time per iteration (ms): 111482.6 | throughput per GPU (TFLOP/s/GPU): 34.0 | MFU 10.89% | FLOPs estimator: workload-aware | logical tokens/s: 150491.1 | valid tokens/s: 150491.1 | physical tokens/s: 150835.1 | loss tokens/s: 112368.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.126077E-01 | lm-global-supervised-tokens: 1.252714E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.164633E-02 | lm-token-buffer-mib: 5.808383E+01 | global_load_balancing_loss: 9.998935E-01 | loss scale: 1.0 | grad norm: 0.412 | params norm: 11255.951 | params rms: 0.060947 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 13:28:14.913980 | finish at 2026-10-10 22:53:39 + [2026-10-10 09:26:48] iteration 6421/ 6855 | consumed samples: 821888 | elapsed time per iteration (ms): 83979.9 | throughput per GPU (TFLOP/s/GPU): 43.3 | MFU 13.87% | FLOPs estimator: workload-aware | logical tokens/s: 199776.2 | valid tokens/s: 199776.2 | physical tokens/s: 200287.4 | loss tokens/s: 149680.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.550966E-01 | lm-global-supervised-tokens: 1.257014E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.167967E-02 | lm-token-buffer-mib: 5.809541E+01 | global_load_balancing_loss: 9.999661E-01 | loss scale: 1.0 | grad norm: 0.070 | params norm: 11255.909 | params rms: 0.060947 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 10:07:27.266823 | finish at 2026-10-10 19:34:15 + [2026-10-10 09:28:15] iteration 6422/ 6855 | consumed samples: 822016 | elapsed time per iteration (ms): 87083.5 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.62% | FLOPs estimator: workload-aware | logical tokens/s: 192656.7 | valid tokens/s: 192656.7 | physical tokens/s: 193034.1 | loss tokens/s: 141609.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.979201E-01 | lm-global-supervised-tokens: 1.233184E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.446483E-02 | lm-token-buffer-mib: 5.808894E+01 | global_load_balancing_loss: 1.000112E+00 | loss scale: 1.0 | grad norm: 0.254 | params norm: 11255.867 | params rms: 0.060947 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 10:28:27.150402 | finish at 2026-10-10 19:56:42 + [2026-10-10 09:29:43] iteration 6423/ 6855 | consumed samples: 822144 | elapsed time per iteration (ms): 87645.5 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.49% | FLOPs estimator: workload-aware | logical tokens/s: 191421.0 | valid tokens/s: 191421.0 | physical tokens/s: 191889.9 | loss tokens/s: 146072.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.643556E-01 | lm-global-supervised-tokens: 1.280260E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.587803E-02 | lm-token-buffer-mib: 5.817919E+01 | global_load_balancing_loss: 1.000694E+00 | loss scale: 1.0 | grad norm: 0.272 | params norm: 11255.822 | params rms: 0.060947 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 10:31:02.854328 | finish at 2026-10-10 20:00:46 + [2026-10-10 09:31:09] iteration 6424/ 6855 | consumed samples: 822272 | elapsed time per iteration (ms): 85795.4 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.79% | FLOPs estimator: workload-aware | logical tokens/s: 195549.2 | valid tokens/s: 195549.2 | physical tokens/s: 196048.2 | loss tokens/s: 135566.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.253648E-01 | lm-global-supervised-tokens: 1.163102E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.124225E-02 | lm-token-buffer-mib: 5.802216E+01 | global_load_balancing_loss: 1.002340E+00 | loss scale: 1.0 | grad norm: 0.171 | params norm: 11255.776 | params rms: 0.060946 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 10:16:17.807083 | finish at 2026-10-10 19:47:26 + [2026-10-10 09:32:38] iteration 6425/ 6855 | consumed samples: 822400 | elapsed time per iteration (ms): 89185.1 | throughput per GPU (TFLOP/s/GPU): 41.7 | MFU 13.38% | FLOPs estimator: workload-aware | logical tokens/s: 188116.8 | valid tokens/s: 188116.8 | physical tokens/s: 188462.7 | loss tokens/s: 136280.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.326957E-01 | lm-global-supervised-tokens: 1.215416E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.186409E-02 | lm-token-buffer-mib: 5.819302E+01 | global_load_balancing_loss: 1.000939E+00 | loss scale: 1.0 | grad norm: 0.167 | params norm: 11255.731 | params rms: 0.060946 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 10:39:09.585037 | finish at 2026-10-10 20:11:47 + [2026-10-10 09:34:05] iteration 6426/ 6855 | consumed samples: 822528 | elapsed time per iteration (ms): 87525.3 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.49% | FLOPs estimator: workload-aware | logical tokens/s: 191683.6 | valid tokens/s: 191683.6 | physical tokens/s: 192219.6 | loss tokens/s: 140353.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.367314E-01 | lm-global-supervised-tokens: 1.228448E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.221615E-02 | lm-token-buffer-mib: 5.825208E+01 | global_load_balancing_loss: 1.000538E+00 | loss scale: 1.0 | grad norm: 0.363 | params norm: 11255.686 | params rms: 0.060946 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 10:25:48.360053 | finish at 2026-10-10 19:59:54 + [2026-10-10 09:35:37] iteration 6427/ 6855 | consumed samples: 822656 | elapsed time per iteration (ms): 92128.5 | throughput per GPU (TFLOP/s/GPU): 40.5 | MFU 12.99% | FLOPs estimator: workload-aware | logical tokens/s: 182106.1 | valid tokens/s: 182106.1 | physical tokens/s: 182475.5 | loss tokens/s: 135377.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.870816E-01 | lm-global-supervised-tokens: 1.247209E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.226426E-02 | lm-token-buffer-mib: 5.815302E+01 | global_load_balancing_loss: 9.987926E-01 | loss scale: 1.0 | grad norm: 0.266 | params norm: 11255.640 | params rms: 0.060946 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 10:57:11.017382 | finish at 2026-10-10 20:32:48 + [2026-10-10 09:37:05] iteration 6428/ 6855 | consumed samples: 822784 | elapsed time per iteration (ms): 87945.2 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.42% | FLOPs estimator: workload-aware | logical tokens/s: 190768.1 | valid tokens/s: 190768.1 | physical tokens/s: 191293.9 | loss tokens/s: 145239.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.693032E-01 | lm-global-supervised-tokens: 1.277309E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.141058E-02 | lm-token-buffer-mib: 5.815215E+01 | global_load_balancing_loss: 9.986823E-01 | loss scale: 1.0 | grad norm: 0.115 | params norm: 11255.594 | params rms: 0.060945 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 10:25:52.596619 | finish at 2026-10-10 20:02:58 + [2026-10-10 09:38:32] iteration 6429/ 6855 | consumed samples: 822912 | elapsed time per iteration (ms): 86150.4 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.61% | FLOPs estimator: workload-aware | logical tokens/s: 194743.2 | valid tokens/s: 194743.2 | physical tokens/s: 195293.5 | loss tokens/s: 149915.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.557689E-01 | lm-global-supervised-tokens: 1.291528E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.569481E-02 | lm-token-buffer-mib: 5.806953E+01 | global_load_balancing_loss: 9.995010E-01 | loss scale: 1.0 | grad norm: 0.167 | params norm: 11255.548 | params rms: 0.060945 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 10:11:40.086720 | finish at 2026-10-10 19:50:12 + [2026-10-10 09:40:19] iteration 6430/ 6855 | consumed samples: 823040 | elapsed time per iteration (ms): 107880.5 | throughput per GPU (TFLOP/s/GPU): 34.3 | MFU 10.99% | FLOPs estimator: workload-aware | logical tokens/s: 155516.4 | valid tokens/s: 155516.4 | physical tokens/s: 155868.1 | loss tokens/s: 118112.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.369211E-01 | lm-global-supervised-tokens: 1.274202E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.264128E-02 | lm-token-buffer-mib: 5.814082E+01 | global_load_balancing_loss: 1.000419E+00 | loss scale: 1.0 | grad norm: 0.101 | params norm: 11255.502 | params rms: 0.060945 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 12:44:09.192977 | finish at 2026-10-10 22:24:29 + [2026-10-10 09:41:47] iteration 6431/ 6855 | consumed samples: 823168 | elapsed time per iteration (ms): 88020.9 | throughput per GPU (TFLOP/s/GPU): 41.7 | MFU 13.38% | FLOPs estimator: workload-aware | logical tokens/s: 190604.8 | valid tokens/s: 190604.8 | physical tokens/s: 190934.2 | loss tokens/s: 142809.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.104986E-01 | lm-global-supervised-tokens: 1.257018E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.295533E-02 | lm-token-buffer-mib: 5.802679E+01 | global_load_balancing_loss: 1.000165E+00 | loss scale: 1.0 | grad norm: 0.058 | params norm: 11255.456 | params rms: 0.060945 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 10:22:00.850485 | finish at 2026-10-10 20:03:48 + [2026-10-10 09:43:15] iteration 6432/ 6855 | consumed samples: 823296 | elapsed time per iteration (ms): 87101.6 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.40% | FLOPs estimator: workload-aware | logical tokens/s: 192616.2 | valid tokens/s: 192616.2 | physical tokens/s: 193175.6 | loss tokens/s: 136674.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.073703E-01 | lm-global-supervised-tokens: 1.190460E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.221562E-01 | lm-token-buffer-mib: 5.818678E+01 | global_load_balancing_loss: 1.001104E+00 | loss scale: 1.0 | grad norm: 0.105 | params norm: 11255.409 | params rms: 0.060944 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 10:14:03.982923 | finish at 2026-10-10 19:57:19 + [2026-10-10 09:44:41] iteration 6433/ 6855 | consumed samples: 823424 | elapsed time per iteration (ms): 86722.7 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.69% | FLOPs estimator: workload-aware | logical tokens/s: 193457.5 | valid tokens/s: 193457.5 | physical tokens/s: 194081.1 | loss tokens/s: 141569.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.901815E-01 | lm-global-supervised-tokens: 1.227728E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.264018E-02 | lm-token-buffer-mib: 5.812920E+01 | global_load_balancing_loss: 1.001266E+00 | loss scale: 1.0 | grad norm: 0.094 | params norm: 11255.361 | params rms: 0.060944 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 10:09:56.973615 | finish at 2026-10-10 19:54:38 + [2026-10-10 09:46:06] iteration 6434/ 6855 | consumed samples: 823552 | elapsed time per iteration (ms): 84850.9 | throughput per GPU (TFLOP/s/GPU): 44.3 | MFU 14.19% | FLOPs estimator: workload-aware | logical tokens/s: 197725.4 | valid tokens/s: 197725.4 | physical tokens/s: 198091.2 | loss tokens/s: 141112.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.281744E-01 | lm-global-supervised-tokens: 1.197353E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.159715E-02 | lm-token-buffer-mib: 5.814292E+01 | global_load_balancing_loss: 1.000569E+00 | loss scale: 1.0 | grad norm: 0.064 | params norm: 11255.312 | params rms: 0.060944 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 9:55:22.243628 | finish at 2026-10-10 19:41:28 + [2026-10-10 09:47:30] iteration 6435/ 6855 | consumed samples: 823680 | elapsed time per iteration (ms): 84118.3 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.67% | FLOPs estimator: workload-aware | logical tokens/s: 199447.6 | valid tokens/s: 199447.6 | physical tokens/s: 200000.5 | loss tokens/s: 155304.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.581193E-01 | lm-global-supervised-tokens: 1.306391E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.531282E-02 | lm-token-buffer-mib: 5.808074E+01 | global_load_balancing_loss: 1.000404E+00 | loss scale: 1.0 | grad norm: 0.066 | params norm: 11255.264 | params rms: 0.060944 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 9:48:49.680376 | finish at 2026-10-10 19:36:20 + [2026-10-10 09:48:58] iteration 6436/ 6855 | consumed samples: 823808 | elapsed time per iteration (ms): 87832.1 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.55% | FLOPs estimator: workload-aware | logical tokens/s: 191014.4 | valid tokens/s: 191014.4 | physical tokens/s: 191502.7 | loss tokens/s: 144863.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.379884E-01 | lm-global-supervised-tokens: 1.272365E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.519333E-02 | lm-token-buffer-mib: 5.809931E+01 | global_load_balancing_loss: 9.993140E-01 | loss scale: 1.0 | grad norm: 0.059 | params norm: 11255.214 | params rms: 0.060943 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 10:13:21.641773 | finish at 2026-10-10 20:02:20 + [2026-10-10 09:50:23] iteration 6437/ 6855 | consumed samples: 823936 | elapsed time per iteration (ms): 84617.7 | throughput per GPU (TFLOP/s/GPU): 43.5 | MFU 13.93% | FLOPs estimator: workload-aware | logical tokens/s: 198269.5 | valid tokens/s: 198269.5 | physical tokens/s: 198703.8 | loss tokens/s: 146189.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.929821E-01 | lm-global-supervised-tokens: 1.237024E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.140879E-02 | lm-token-buffer-mib: 5.807281E+01 | global_load_balancing_loss: 1.000490E+00 | loss scale: 1.0 | grad norm: 0.045 | params norm: 11255.163 | params rms: 0.060943 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 9:49:30.219471 | finish at 2026-10-10 19:39:53 + [2026-10-10 09:51:50] iteration 6438/ 6855 | consumed samples: 824064 | elapsed time per iteration (ms): 87626.4 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.48% | FLOPs estimator: workload-aware | logical tokens/s: 191462.8 | valid tokens/s: 191462.8 | physical tokens/s: 192309.1 | loss tokens/s: 141588.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.874276E-01 | lm-global-supervised-tokens: 1.240690E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.711902E-02 | lm-token-buffer-mib: 5.809397E+01 | global_load_balancing_loss: 1.000244E+00 | loss scale: 1.0 | grad norm: 0.093 | params norm: 11255.111 | params rms: 0.060943 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 10:09:00.214166 | finish at 2026-10-10 20:00:51 + [2026-10-10 09:53:16] iteration 6439/ 6855 | consumed samples: 824192 | elapsed time per iteration (ms): 85357.2 | throughput per GPU (TFLOP/s/GPU): 41.5 | MFU 13.31% | FLOPs estimator: workload-aware | logical tokens/s: 196552.5 | valid tokens/s: 196552.4 | physical tokens/s: 197115.9 | loss tokens/s: 151461.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.814539E-01 | lm-global-supervised-tokens: 1.292831E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.432965E-02 | lm-token-buffer-mib: 5.814881E+01 | global_load_balancing_loss: 1.000049E+00 | loss scale: 1.0 | grad norm: 0.132 | params norm: 11255.061 | params rms: 0.060942 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 9:51:48.600616 | finish at 2026-10-10 19:45:04 + [2026-10-10 09:55:04] iteration 6440/ 6855 | consumed samples: 824320 | elapsed time per iteration (ms): 107918.5 | throughput per GPU (TFLOP/s/GPU): 33.8 | MFU 10.84% | FLOPs estimator: workload-aware | logical tokens/s: 155461.9 | valid tokens/s: 155461.9 | physical tokens/s: 155965.2 | loss tokens/s: 120351.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.617354E-01 | lm-global-supervised-tokens: 1.298812E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.159594E-02 | lm-token-buffer-mib: 5.812348E+01 | global_load_balancing_loss: 9.998760E-01 | loss scale: 1.0 | grad norm: 0.035 | params norm: 11255.009 | params rms: 0.060942 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 12:26:26.184206 | finish at 2026-10-10 22:21:30 +times across ranks (ms): + save-checkpoint: + rank 0: 24577.60 + rank 1: 24577.64 + rank 2: 24577.60 + rank 3: 24577.59 + rank 4: 24577.63 + rank 5: 24577.59 + rank 6: 24577.57 + rank 7: 24577.62 + rank 8: 24578.22 + rank 9: 24578.21 + rank 10: 24578.21 + rank 11: 24578.20 + rank 12: 24578.20 + rank 13: 24578.21 + rank 14: 24578.20 + rank 15: 24578.21 + rank 16: 24577.61 + rank 17: 24577.60 + rank 18: 24577.58 + rank 19: 24577.61 + rank 20: 24577.59 + rank 21: 24577.60 + rank 22: 24577.56 + rank 23: 24577.56 + rank 24: 24579.57 + rank 25: 24579.52 + rank 26: 24579.55 + rank 27: 24579.56 + rank 28: 24579.53 + rank 29: 24579.53 + rank 30: 24579.55 + rank 31: 24579.53 + rank 32: 24577.73 + rank 33: 24577.74 + rank 34: 24577.73 + rank 35: 24577.71 + rank 36: 24577.67 + rank 37: 24577.73 + rank 38: 24577.72 + rank 39: 24577.69 + rank 40: 24577.70 + rank 41: 24577.70 + rank 42: 24577.67 + rank 43: 24577.69 + rank 44: 24577.67 + rank 45: 24577.67 + rank 46: 24577.68 + rank 47: 24577.68 + rank 48: 24579.23 + rank 49: 24579.27 + rank 50: 24579.23 + rank 51: 24579.23 + rank 52: 24579.33 + rank 53: 24579.22 + rank 54: 24579.23 + rank 55: 24579.22 + rank 56: 24577.90 + rank 57: 24577.92 + rank 58: 24577.88 + rank 59: 24577.89 + rank 60: 24577.88 + rank 61: 24577.89 + rank 62: 24577.91 + rank 63: 24577.88 + [2026-10-10 09:56:58] iteration 6441/ 6855 | consumed samples: 824448 | elapsed time per iteration (ms): 90107.6 | throughput per GPU (TFLOP/s/GPU): 41.4 | MFU 13.27% | FLOPs estimator: workload-aware | logical tokens/s: 186189.8 | valid tokens/s: 186189.8 | physical tokens/s: 186566.4 | loss tokens/s: 134214.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.379972E-01 | lm-global-supervised-tokens: 1.209373E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 3.834421E-01 | lm-token-buffer-mib: 5.808276E+01 | global_load_balancing_loss: 1.000810E+00 | loss scale: 1.0 | grad norm: 0.086 | params norm: 11254.956 | params rms: 0.060942 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 10:21:44.560899 | finish at 2026-10-10 20:18:43 + [2026-10-10 09:58:29] iteration 6442/ 6855 | consumed samples: 824576 | elapsed time per iteration (ms): 90505.1 | throughput per GPU (TFLOP/s/GPU): 40.1 | MFU 12.84% | FLOPs estimator: workload-aware | logical tokens/s: 185372.7 | valid tokens/s: 185372.7 | physical tokens/s: 186093.6 | loss tokens/s: 137367.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.758147E-01 | lm-global-supervised-tokens: 1.243250E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.688932E-02 | lm-token-buffer-mib: 5.809153E+01 | global_load_balancing_loss: 1.000007E+00 | loss scale: 1.0 | grad norm: 0.075 | params norm: 11254.903 | params rms: 0.060942 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 10:22:58.599609 | finish at 2026-10-10 20:21:28 + [2026-10-10 09:59:56] iteration 6443/ 6855 | consumed samples: 824704 | elapsed time per iteration (ms): 86939.5 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.55% | FLOPs estimator: workload-aware | logical tokens/s: 192975.4 | valid tokens/s: 192975.4 | physical tokens/s: 193288.3 | loss tokens/s: 142802.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.336303E-01 | lm-global-supervised-tokens: 1.241514E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.341385E-02 | lm-token-buffer-mib: 5.807757E+01 | global_load_balancing_loss: 9.995537E-01 | loss scale: 1.0 | grad norm: 0.082 | params norm: 11254.852 | params rms: 0.060941 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 9:56:59.093488 | finish at 2026-10-10 19:56:55 + [2026-10-10 10:01:22] iteration 6444/ 6855 | consumed samples: 824832 | elapsed time per iteration (ms): 86372.1 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.63% | FLOPs estimator: workload-aware | logical tokens/s: 194243.4 | valid tokens/s: 194243.4 | physical tokens/s: 194650.8 | loss tokens/s: 140319.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.080305E-01 | lm-global-supervised-tokens: 1.211972E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.168795E-02 | lm-token-buffer-mib: 5.804727E+01 | global_load_balancing_loss: 1.000524E+00 | loss scale: 1.0 | grad norm: 0.110 | params norm: 11254.802 | params rms: 0.060941 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 9:51:38.927660 | finish at 2026-10-10 19:53:01 + [2026-10-10 10:02:44] iteration 6445/ 6855 | consumed samples: 824960 | elapsed time per iteration (ms): 82069.0 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.62% | FLOPs estimator: workload-aware | logical tokens/s: 204427.9 | valid tokens/s: 204427.9 | physical tokens/s: 205084.2 | loss tokens/s: 159752.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.134319E-01 | lm-global-supervised-tokens: 1.311069E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.372761E-02 | lm-token-buffer-mib: 5.812557E+01 | global_load_balancing_loss: 9.999907E-01 | loss scale: 1.0 | grad norm: 0.068 | params norm: 11254.752 | params rms: 0.060941 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 9:20:48.285799 | finish at 2026-10-10 19:23:33 + [2026-10-10 10:04:15] iteration 6446/ 6855 | consumed samples: 825088 | elapsed time per iteration (ms): 90395.2 | throughput per GPU (TFLOP/s/GPU): 41.1 | MFU 13.17% | FLOPs estimator: workload-aware | logical tokens/s: 185598.1 | valid tokens/s: 185598.1 | physical tokens/s: 186057.7 | loss tokens/s: 138081.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.600831E-01 | lm-global-supervised-tokens: 1.248192E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.258759E-02 | lm-token-buffer-mib: 5.823434E+01 | global_load_balancing_loss: 9.993346E-01 | loss scale: 1.0 | grad norm: 0.126 | params norm: 11254.702 | params rms: 0.060941 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 10:16:11.636416 | finish at 2026-10-10 20:20:26 + [2026-10-10 10:05:40] iteration 6447/ 6855 | consumed samples: 825216 | elapsed time per iteration (ms): 85279.1 | throughput per GPU (TFLOP/s/GPU): 43.7 | MFU 14.02% | FLOPs estimator: workload-aware | logical tokens/s: 196732.4 | valid tokens/s: 196732.4 | physical tokens/s: 197106.8 | loss tokens/s: 143473.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.806386E-01 | lm-global-supervised-tokens: 1.223532E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.328870E-02 | lm-token-buffer-mib: 5.819087E+01 | global_load_balancing_loss: 9.997827E-01 | loss scale: 1.0 | grad norm: 0.056 | params norm: 11254.652 | params rms: 0.060940 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 9:39:53.892134 | finish at 2026-10-10 19:45:34 + [2026-10-10 10:07:08] iteration 6448/ 6855 | consumed samples: 825344 | elapsed time per iteration (ms): 88077.7 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.50% | FLOPs estimator: workload-aware | logical tokens/s: 190482.0 | valid tokens/s: 190482.0 | physical tokens/s: 190838.8 | loss tokens/s: 138280.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.519002E-01 | lm-global-supervised-tokens: 1.217944E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.099765E-02 | lm-token-buffer-mib: 5.810780E+01 | global_load_balancing_loss: 1.001098E+00 | loss scale: 1.0 | grad norm: 0.072 | params norm: 11254.601 | params rms: 0.060940 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 9:57:27.620754 | finish at 2026-10-10 20:04:36 + [2026-10-10 10:08:35] iteration 6449/ 6855 | consumed samples: 825472 | elapsed time per iteration (ms): 86409.7 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.64% | FLOPs estimator: workload-aware | logical tokens/s: 194158.5 | valid tokens/s: 194158.5 | physical tokens/s: 194451.2 | loss tokens/s: 145404.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.838534E-01 | lm-global-supervised-tokens: 1.256432E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.169494E-02 | lm-token-buffer-mib: 5.813064E+01 | global_load_balancing_loss: 1.001172E+00 | loss scale: 1.0 | grad norm: 0.068 | params norm: 11254.551 | params rms: 0.060940 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 9:44:42.327615 | finish at 2026-10-10 19:53:17 + [2026-10-10 10:10:25] iteration 6450/ 6855 | consumed samples: 825600 | elapsed time per iteration (ms): 110404.6 | throughput per GPU (TFLOP/s/GPU): 34.3 | MFU 11.00% | FLOPs estimator: workload-aware | logical tokens/s: 151960.8 | valid tokens/s: 151960.8 | physical tokens/s: 152319.3 | loss tokens/s: 105532.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.198001E-01 | lm-global-supervised-tokens: 1.165130E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.160872E-02 | lm-token-buffer-mib: 5.818425E+01 | global_load_balancing_loss: 1.001275E+00 | loss scale: 1.0 | grad norm: 0.097 | params norm: 11254.501 | params rms: 0.060939 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 12:25:13.856782 | finish at 2026-10-10 22:35:39 + [2026-10-10 10:11:54] iteration 6451/ 6855 | consumed samples: 825728 | elapsed time per iteration (ms): 88749.7 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.49% | FLOPs estimator: workload-aware | logical tokens/s: 189039.2 | valid tokens/s: 189039.2 | physical tokens/s: 189691.7 | loss tokens/s: 136026.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.640325E-01 | lm-global-supervised-tokens: 1.207232E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.173522E-02 | lm-token-buffer-mib: 5.841942E+01 | global_load_balancing_loss: 1.000878E+00 | loss scale: 1.0 | grad norm: 0.290 | params norm: 11254.449 | params rms: 0.060939 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 9:57:34.859082 | finish at 2026-10-10 20:09:29 + [2026-10-10 10:13:18] iteration 6452/ 6855 | consumed samples: 825856 | elapsed time per iteration (ms): 84585.9 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.47% | FLOPs estimator: workload-aware | logical tokens/s: 198344.6 | valid tokens/s: 198344.6 | physical tokens/s: 199093.5 | loss tokens/s: 144565.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.357250E-01 | lm-global-supervised-tokens: 1.222819E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.311173E-02 | lm-token-buffer-mib: 5.848422E+01 | global_load_balancing_loss: 1.000604E+00 | loss scale: 1.0 | grad norm: 0.160 | params norm: 11254.397 | params rms: 0.060939 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 9:28:08.109945 | finish at 2026-10-10 19:41:26 + [2026-10-10 10:14:46] iteration 6453/ 6855 | consumed samples: 825984 | elapsed time per iteration (ms): 87275.6 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.68% | FLOPs estimator: workload-aware | logical tokens/s: 192231.8 | valid tokens/s: 192231.8 | physical tokens/s: 192558.8 | loss tokens/s: 141773.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.565042E-01 | lm-global-supervised-tokens: 1.237336E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.841891E-02 | lm-token-buffer-mib: 5.810431E+01 | global_load_balancing_loss: 9.991739E-01 | loss scale: 1.0 | grad norm: 0.123 | params norm: 11254.347 | params rms: 0.060939 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 9:44:44.805751 | finish at 2026-10-10 19:59:30 + [2026-10-10 10:16:10] iteration 6454/ 6855 | consumed samples: 826112 | elapsed time per iteration (ms): 84542.8 | throughput per GPU (TFLOP/s/GPU): 42.8 | MFU 13.73% | FLOPs estimator: workload-aware | logical tokens/s: 198446.1 | valid tokens/s: 198446.1 | physical tokens/s: 198862.9 | loss tokens/s: 149511.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.817168E-01 | lm-global-supervised-tokens: 1.264015E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.238059E-02 | lm-token-buffer-mib: 5.814498E+01 | global_load_balancing_loss: 9.991385E-01 | loss scale: 1.0 | grad norm: 0.077 | params norm: 11254.297 | params rms: 0.060938 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 9:25:01.671671 | finish at 2026-10-10 19:41:12 + [2026-10-10 10:17:37] iteration 6455/ 6855 | consumed samples: 826240 | elapsed time per iteration (ms): 86785.0 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.44% | FLOPs estimator: workload-aware | logical tokens/s: 193318.8 | valid tokens/s: 193318.8 | physical tokens/s: 193759.7 | loss tokens/s: 145419.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.411477E-01 | lm-global-supervised-tokens: 1.262018E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.707860E-02 | lm-token-buffer-mib: 5.814969E+01 | global_load_balancing_loss: 9.984634E-01 | loss scale: 1.0 | grad norm: 0.132 | params norm: 11254.247 | params rms: 0.060938 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 9:38:33.984489 | finish at 2026-10-10 19:56:11 + [2026-10-10 10:19:04] iteration 6456/ 6855 | consumed samples: 826368 | elapsed time per iteration (ms): 86758.5 | throughput per GPU (TFLOP/s/GPU): 41.5 | MFU 13.30% | FLOPs estimator: workload-aware | logical tokens/s: 193377.6 | valid tokens/s: 193377.6 | physical tokens/s: 193893.3 | loss tokens/s: 144398.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.395569E-01 | lm-global-supervised-tokens: 1.252778E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.219173E-02 | lm-token-buffer-mib: 5.819308E+01 | global_load_balancing_loss: 9.996762E-01 | loss scale: 1.0 | grad norm: 0.066 | params norm: 11254.197 | params rms: 0.060938 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 9:36:56.658663 | finish at 2026-10-10 19:56:00 + [2026-10-10 10:20:33] iteration 6457/ 6855 | consumed samples: 826496 | elapsed time per iteration (ms): 89264.7 | throughput per GPU (TFLOP/s/GPU): 41.6 | MFU 13.32% | FLOPs estimator: workload-aware | logical tokens/s: 187948.6 | valid tokens/s: 187948.6 | physical tokens/s: 188243.1 | loss tokens/s: 133609.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.651561E-01 | lm-global-supervised-tokens: 1.192658E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.156291E-02 | lm-token-buffer-mib: 5.813477E+01 | global_load_balancing_loss: 1.001064E+00 | loss scale: 1.0 | grad norm: 0.036 | params norm: 11254.149 | params rms: 0.060938 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 9:52:07.370312 | finish at 2026-10-10 20:12:40 + [2026-10-10 10:21:59] iteration 6458/ 6855 | consumed samples: 826624 | elapsed time per iteration (ms): 86146.5 | throughput per GPU (TFLOP/s/GPU): 43.1 | MFU 13.81% | FLOPs estimator: workload-aware | logical tokens/s: 194752.1 | valid tokens/s: 194752.1 | physical tokens/s: 195355.2 | loss tokens/s: 149570.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.178161E-01 | lm-global-supervised-tokens: 1.288496E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.145292E-02 | lm-token-buffer-mib: 5.805938E+01 | global_load_balancing_loss: 1.002121E+00 | loss scale: 1.0 | grad norm: 0.079 | params norm: 11254.101 | params rms: 0.060937 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 9:30:00.169631 | finish at 2026-10-10 19:51:59 + [2026-10-10 10:23:29] iteration 6459/ 6855 | consumed samples: 826752 | elapsed time per iteration (ms): 90389.3 | throughput per GPU (TFLOP/s/GPU): 41.3 | MFU 13.22% | FLOPs estimator: workload-aware | logical tokens/s: 185610.0 | valid tokens/s: 185610.0 | physical tokens/s: 186031.0 | loss tokens/s: 137193.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.233844E-01 | lm-global-supervised-tokens: 1.240081E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.367871E-02 | lm-token-buffer-mib: 5.815611E+01 | global_load_balancing_loss: 1.000982E+00 | loss scale: 1.0 | grad norm: 0.074 | params norm: 11254.052 | params rms: 0.060937 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 9:56:34.179365 | finish at 2026-10-10 20:20:04 + [2026-10-10 10:25:18] iteration 6460/ 6855 | consumed samples: 826880 | elapsed time per iteration (ms): 108998.0 | throughput per GPU (TFLOP/s/GPU): 33.6 | MFU 10.78% | FLOPs estimator: workload-aware | logical tokens/s: 153922.1 | valid tokens/s: 153922.1 | physical tokens/s: 154293.5 | loss tokens/s: 116397.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.497850E-01 | lm-global-supervised-tokens: 1.268708E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.536140E-02 | lm-token-buffer-mib: 5.812448E+01 | global_load_balancing_loss: 1.000700E+00 | loss scale: 1.0 | grad norm: 0.064 | params norm: 11254.005 | params rms: 0.060937 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 11:57:34.192917 | finish at 2026-10-10 22:22:53 + [2026-10-10 10:26:45] iteration 6461/ 6855 | consumed samples: 827008 | elapsed time per iteration (ms): 86643.9 | throughput per GPU (TFLOP/s/GPU): 43.5 | MFU 13.95% | FLOPs estimator: workload-aware | logical tokens/s: 193633.8 | valid tokens/s: 193633.8 | physical tokens/s: 194038.3 | loss tokens/s: 138651.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.895296E-01 | lm-global-supervised-tokens: 1.201328E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.393001E-02 | lm-token-buffer-mib: 5.811744E+01 | global_load_balancing_loss: 1.000360E+00 | loss scale: 1.0 | grad norm: 0.067 | params norm: 11253.959 | params rms: 0.060937 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 9:28:57.713382 | finish at 2026-10-10 19:55:43 + [2026-10-10 10:28:11] iteration 6462/ 6855 | consumed samples: 827136 | elapsed time per iteration (ms): 85993.5 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.58% | FLOPs estimator: workload-aware | logical tokens/s: 195097.7 | valid tokens/s: 195097.7 | physical tokens/s: 195768.3 | loss tokens/s: 139778.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.197247E-01 | lm-global-supervised-tokens: 1.202000E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.954575E-02 | lm-token-buffer-mib: 5.824953E+01 | global_load_balancing_loss: 9.995411E-01 | loss scale: 1.0 | grad norm: 0.102 | params norm: 11253.914 | params rms: 0.060936 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 9:23:15.448028 | finish at 2026-10-10 19:51:26 + [2026-10-10 10:29:42] iteration 6463/ 6855 | consumed samples: 827264 | elapsed time per iteration (ms): 90728.7 | throughput per GPU (TFLOP/s/GPU): 41.6 | MFU 13.34% | FLOPs estimator: workload-aware | logical tokens/s: 184915.5 | valid tokens/s: 184915.5 | physical tokens/s: 185350.8 | loss tokens/s: 137557.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.409015E-01 | lm-global-supervised-tokens: 1.248038E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.117027E-02 | lm-token-buffer-mib: 5.812049E+01 | global_load_balancing_loss: 9.999110E-01 | loss scale: 1.0 | grad norm: 0.068 | params norm: 11253.868 | params rms: 0.060936 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 9:52:45.660650 | finish at 2026-10-10 20:22:27 + [2026-10-10 10:31:09] iteration 6464/ 6855 | consumed samples: 827392 | elapsed time per iteration (ms): 87169.1 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.65% | FLOPs estimator: workload-aware | logical tokens/s: 192467.1 | valid tokens/s: 192467.1 | physical tokens/s: 192816.9 | loss tokens/s: 145450.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.037655E-01 | lm-global-supervised-tokens: 1.267876E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.184408E-02 | lm-token-buffer-mib: 5.806720E+01 | global_load_balancing_loss: 9.987713E-01 | loss scale: 1.0 | grad norm: 0.096 | params norm: 11253.822 | params rms: 0.060936 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 9:28:03.120821 | finish at 2026-10-10 19:59:12 + [2026-10-10 10:32:36] iteration 6465/ 6855 | consumed samples: 827520 | elapsed time per iteration (ms): 87117.1 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.66% | FLOPs estimator: workload-aware | logical tokens/s: 192582.1 | valid tokens/s: 192582.1 | physical tokens/s: 192930.6 | loss tokens/s: 143865.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.898168E-01 | lm-global-supervised-tokens: 1.253310E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.248627E-02 | lm-token-buffer-mib: 5.804584E+01 | global_load_balancing_loss: 9.998723E-01 | loss scale: 1.0 | grad norm: 0.063 | params norm: 11253.776 | params rms: 0.060936 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 9:26:15.672812 | finish at 2026-10-10 19:58:52 + [2026-10-10 10:34:06] iteration 6466/ 6855 | consumed samples: 827648 | elapsed time per iteration (ms): 90306.2 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.38% | FLOPs estimator: workload-aware | logical tokens/s: 185780.4 | valid tokens/s: 185780.4 | physical tokens/s: 186178.7 | loss tokens/s: 129380.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.413416E-01 | lm-global-supervised-tokens: 1.168388E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.330524E-02 | lm-token-buffer-mib: 5.821474E+01 | global_load_balancing_loss: 9.999200E-01 | loss scale: 1.0 | grad norm: 0.146 | params norm: 11253.729 | params rms: 0.060935 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 9:45:29.106431 | finish at 2026-10-10 20:19:35 + [2026-10-10 10:35:32] iteration 6467/ 6855 | consumed samples: 827776 | elapsed time per iteration (ms): 85917.1 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.67% | FLOPs estimator: workload-aware | logical tokens/s: 195271.4 | valid tokens/s: 195271.4 | physical tokens/s: 195726.4 | loss tokens/s: 150561.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.827965E-01 | lm-global-supervised-tokens: 1.293585E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.127038E-02 | lm-token-buffer-mib: 5.816316E+01 | global_load_balancing_loss: 9.998520E-01 | loss scale: 1.0 | grad norm: 0.114 | params norm: 11253.684 | params rms: 0.060935 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 9:15:35.853560 | finish at 2026-10-10 19:51:08 + [2026-10-10 10:37:01] iteration 6468/ 6855 | consumed samples: 827904 | elapsed time per iteration (ms): 89003.5 | throughput per GPU (TFLOP/s/GPU): 42.8 | MFU 13.73% | FLOPs estimator: workload-aware | logical tokens/s: 188500.2 | valid tokens/s: 188500.2 | physical tokens/s: 189154.8 | loss tokens/s: 139075.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.690338E-01 | lm-global-supervised-tokens: 1.237816E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.319872E-02 | lm-token-buffer-mib: 5.811547E+01 | global_load_balancing_loss: 9.999664E-01 | loss scale: 1.0 | grad norm: 0.105 | params norm: 11253.638 | params rms: 0.060935 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 9:34:04.363629 | finish at 2026-10-10 20:11:06 + [2026-10-10 10:38:29] iteration 6469/ 6855 | consumed samples: 828032 | elapsed time per iteration (ms): 87946.9 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.42% | FLOPs estimator: workload-aware | logical tokens/s: 190764.2 | valid tokens/s: 190764.1 | physical tokens/s: 191255.2 | loss tokens/s: 142790.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.691294E-01 | lm-global-supervised-tokens: 1.255801E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.765112E-02 | lm-token-buffer-mib: 5.810106E+01 | global_load_balancing_loss: 1.001525E+00 | loss scale: 1.0 | grad norm: 0.086 | params norm: 11253.593 | params rms: 0.060935 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 9:25:47.501333 | finish at 2026-10-10 20:04:17 + [2026-10-10 10:40:18] iteration 6470/ 6855 | consumed samples: 828160 | elapsed time per iteration (ms): 108423.1 | throughput per GPU (TFLOP/s/GPU): 33.5 | MFU 10.73% | FLOPs estimator: workload-aware | logical tokens/s: 154737.8 | valid tokens/s: 154737.8 | physical tokens/s: 155010.5 | loss tokens/s: 112973.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.006122E-01 | lm-global-supervised-tokens: 1.224894E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.176810E-02 | lm-token-buffer-mib: 5.813680E+01 | global_load_balancing_loss: 1.001931E+00 | loss scale: 1.0 | grad norm: 0.064 | params norm: 11253.548 | params rms: 0.060934 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 11:35:42.903870 | finish at 2026-10-10 22:16:01 + [2026-10-10 10:41:41] iteration 6471/ 6855 | consumed samples: 828288 | elapsed time per iteration (ms): 83827.8 | throughput per GPU (TFLOP/s/GPU): 43.3 | MFU 13.88% | FLOPs estimator: workload-aware | logical tokens/s: 200138.7 | valid tokens/s: 200138.7 | physical tokens/s: 200634.5 | loss tokens/s: 151432.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.474183E-01 | lm-global-supervised-tokens: 1.269426E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.413580E-02 | lm-token-buffer-mib: 5.811301E+01 | global_load_balancing_loss: 1.001203E+00 | loss scale: 1.0 | grad norm: 0.075 | params norm: 11253.505 | params rms: 0.060934 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 8:56:29.880249 | finish at 2026-10-10 19:38:11 + [2026-10-10 10:43:08] iteration 6472/ 6855 | consumed samples: 828416 | elapsed time per iteration (ms): 86438.6 | throughput per GPU (TFLOP/s/GPU): 43.7 | MFU 14.01% | FLOPs estimator: workload-aware | logical tokens/s: 194093.8 | valid tokens/s: 194093.8 | physical tokens/s: 194474.6 | loss tokens/s: 144378.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.638670E-01 | lm-global-supervised-tokens: 1.247985E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.324644E-02 | lm-token-buffer-mib: 5.803677E+01 | global_load_balancing_loss: 9.989695E-01 | loss scale: 1.0 | grad norm: 0.060 | params norm: 11253.462 | params rms: 0.060934 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 9:11:45.991586 | finish at 2026-10-10 19:54:54 + [2026-10-10 10:44:39] iteration 6473/ 6855 | consumed samples: 828544 | elapsed time per iteration (ms): 90761.2 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.53% | FLOPs estimator: workload-aware | logical tokens/s: 184849.6 | valid tokens/s: 184849.6 | physical tokens/s: 185280.0 | loss tokens/s: 133918.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.485872E-01 | lm-global-supervised-tokens: 1.215460E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.198586E-02 | lm-token-buffer-mib: 5.821606E+01 | global_load_balancing_loss: 9.994880E-01 | loss scale: 1.0 | grad norm: 0.090 | params norm: 11253.416 | params rms: 0.060934 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 9:37:50.795413 | finish at 2026-10-10 20:22:29 + [2026-10-10 10:46:06] iteration 6474/ 6855 | consumed samples: 828672 | elapsed time per iteration (ms): 86946.5 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.49% | FLOPs estimator: workload-aware | logical tokens/s: 192960.1 | valid tokens/s: 192960.1 | physical tokens/s: 193502.1 | loss tokens/s: 145443.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.181062E-01 | lm-global-supervised-tokens: 1.264580E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.420373E-02 | lm-token-buffer-mib: 5.819613E+01 | global_load_balancing_loss: 9.994228E-01 | loss scale: 1.0 | grad norm: 0.124 | params norm: 11253.372 | params rms: 0.060933 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 9:12:06.633420 | finish at 2026-10-10 19:58:12 + [2026-10-10 10:47:34] iteration 6475/ 6855 | consumed samples: 828800 | elapsed time per iteration (ms): 88255.9 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.69% | FLOPs estimator: workload-aware | logical tokens/s: 190097.4 | valid tokens/s: 190097.4 | physical tokens/s: 190684.1 | loss tokens/s: 135719.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.484125E-01 | lm-global-supervised-tokens: 1.197804E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.125914E-02 | lm-token-buffer-mib: 5.818768E+01 | global_load_balancing_loss: 1.000083E+00 | loss scale: 1.0 | grad norm: 0.049 | params norm: 11253.328 | params rms: 0.060933 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 9:18:57.227287 | finish at 2026-10-10 20:06:31 + [2026-10-10 10:49:01] iteration 6476/ 6855 | consumed samples: 828928 | elapsed time per iteration (ms): 87034.5 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.60% | FLOPs estimator: workload-aware | logical tokens/s: 192764.5 | valid tokens/s: 192764.5 | physical tokens/s: 193231.5 | loss tokens/s: 143166.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.009294E-01 | lm-global-supervised-tokens: 1.246044E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.531865E-02 | lm-token-buffer-mib: 5.820415E+01 | global_load_balancing_loss: 1.000389E+00 | loss scale: 1.0 | grad norm: 0.189 | params norm: 11253.284 | params rms: 0.060933 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 9:09:46.070576 | finish at 2026-10-10 19:58:47 + [2026-10-10 10:50:28] iteration 6477/ 6855 | consumed samples: 829056 | elapsed time per iteration (ms): 86716.7 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.79% | FLOPs estimator: workload-aware | logical tokens/s: 193471.5 | valid tokens/s: 193471.4 | physical tokens/s: 194015.9 | loss tokens/s: 139846.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.552706E-01 | lm-global-supervised-tokens: 1.212702E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.168270E-02 | lm-token-buffer-mib: 5.827593E+01 | global_load_balancing_loss: 1.000214E+00 | loss scale: 1.0 | grad norm: 0.149 | params norm: 11253.240 | params rms: 0.060933 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 9:06:18.902085 | finish at 2026-10-10 19:56:47 + [2026-10-10 10:51:50] iteration 6478/ 6855 | consumed samples: 829184 | elapsed time per iteration (ms): 82505.5 | throughput per GPU (TFLOP/s/GPU): 43.4 | MFU 13.92% | FLOPs estimator: workload-aware | logical tokens/s: 203346.6 | valid tokens/s: 203346.6 | physical tokens/s: 203882.1 | loss tokens/s: 157019.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.335131E-01 | lm-global-supervised-tokens: 1.295498E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.593769E-02 | lm-token-buffer-mib: 5.809452E+01 | global_load_balancing_loss: 1.000557E+00 | loss scale: 1.0 | grad norm: 0.074 | params norm: 11253.195 | params rms: 0.060932 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 8:38:24.567417 | finish at 2026-10-10 19:30:15 + [2026-10-10 10:53:17] iteration 6479/ 6855 | consumed samples: 829312 | elapsed time per iteration (ms): 86489.7 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.50% | FLOPs estimator: workload-aware | logical tokens/s: 193978.6 | valid tokens/s: 193978.6 | physical tokens/s: 194600.9 | loss tokens/s: 146900.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.609704E-01 | lm-global-supervised-tokens: 1.270536E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.205892E-02 | lm-token-buffer-mib: 5.809075E+01 | global_load_balancing_loss: 1.000437E+00 | loss scale: 1.0 | grad norm: 0.228 | params norm: 11253.153 | params rms: 0.060932 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 9:02:00.134043 | finish at 2026-10-10 19:55:17 + [2026-10-10 10:55:06] iteration 6480/ 6855 | consumed samples: 829440 | elapsed time per iteration (ms): 109154.1 | throughput per GPU (TFLOP/s/GPU): 34.0 | MFU 10.90% | FLOPs estimator: workload-aware | logical tokens/s: 153701.9 | valid tokens/s: 153701.9 | physical tokens/s: 153939.0 | loss tokens/s: 110465.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.904324E-01 | lm-global-supervised-tokens: 1.205771E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.783256E-02 | lm-token-buffer-mib: 5.802448E+01 | global_load_balancing_loss: 1.000547E+00 | loss scale: 1.0 | grad norm: 0.338 | params norm: 11253.110 | params rms: 0.060932 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 11:22:12.774603 | finish at 2026-10-10 22:17:19 +times across ranks (ms): + save-checkpoint: + rank 0: 25820.09 + rank 1: 25820.11 + rank 2: 25820.12 + rank 3: 25820.09 + rank 4: 25820.09 + rank 5: 25820.06 + rank 6: 25820.05 + rank 7: 25820.08 + rank 8: 25820.93 + rank 9: 25820.77 + rank 10: 25820.78 + rank 11: 25820.62 + rank 12: 25820.77 + rank 13: 25820.77 + rank 14: 25820.82 + rank 15: 25820.81 + rank 16: 25820.05 + rank 17: 25820.05 + rank 18: 25820.01 + rank 19: 25820.05 + rank 20: 25820.00 + rank 21: 25820.06 + rank 22: 25820.01 + rank 23: 25820.00 + rank 24: 25819.95 + rank 25: 25820.58 + rank 26: 25819.76 + rank 27: 25819.90 + rank 28: 25819.88 + rank 29: 25820.49 + rank 30: 25819.88 + rank 31: 25819.87 + rank 32: 25819.47 + rank 33: 25819.47 + rank 34: 25819.49 + rank 35: 25819.46 + rank 36: 25819.39 + rank 37: 25819.38 + rank 38: 25819.35 + rank 39: 25819.42 + rank 40: 25820.11 + rank 41: 25820.13 + rank 42: 25820.09 + rank 43: 25820.10 + rank 44: 25820.08 + rank 45: 25820.07 + rank 46: 25820.09 + rank 47: 25820.07 + rank 48: 25820.22 + rank 49: 25820.04 + rank 50: 25820.22 + rank 51: 25820.12 + rank 52: 25820.12 + rank 53: 25820.14 + rank 54: 25820.13 + rank 55: 25820.20 + rank 56: 25820.55 + rank 57: 25820.55 + rank 58: 25820.51 + rank 59: 25820.53 + rank 60: 25820.52 + rank 61: 25820.54 + rank 62: 25820.52 + rank 63: 25820.54 + [2026-10-10 10:57:15] iteration 6481/ 6855 | consumed samples: 829568 | elapsed time per iteration (ms): 102856.6 | throughput per GPU (TFLOP/s/GPU): 35.6 | MFU 11.43% | FLOPs estimator: workload-aware | logical tokens/s: 163111.9 | valid tokens/s: 163111.9 | physical tokens/s: 163717.7 | loss tokens/s: 121285.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.541963E-01 | lm-global-supervised-tokens: 1.247503E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 3.744332E+00 | lm-token-buffer-mib: 5.876237E+01 | global_load_balancing_loss: 9.995944E-01 | loss scale: 1.0 | grad norm: 0.157 | params norm: 11253.065 | params rms: 0.060932 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 10:41:08.386697 | finish at 2026-10-10 21:38:23 + [2026-10-10 10:58:46] iteration 6482/ 6855 | consumed samples: 829696 | elapsed time per iteration (ms): 90816.8 | throughput per GPU (TFLOP/s/GPU): 40.7 | MFU 13.05% | FLOPs estimator: workload-aware | logical tokens/s: 184736.7 | valid tokens/s: 184736.7 | physical tokens/s: 185095.4 | loss tokens/s: 138682.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.800460E-01 | lm-global-supervised-tokens: 1.259466E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.523763E-02 | lm-token-buffer-mib: 5.817667E+01 | global_load_balancing_loss: 9.998596E-01 | loss scale: 1.0 | grad norm: 0.178 | params norm: 11253.023 | params rms: 0.060931 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 9:24:34.675604 | finish at 2026-10-10 20:23:21 + [2026-10-10 11:00:13] iteration 6483/ 6855 | consumed samples: 829824 | elapsed time per iteration (ms): 87145.8 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.64% | FLOPs estimator: workload-aware | logical tokens/s: 192517.4 | valid tokens/s: 192517.4 | physical tokens/s: 192849.3 | loss tokens/s: 141306.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.779627E-01 | lm-global-supervised-tokens: 1.231424E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.269854E-02 | lm-token-buffer-mib: 5.810902E+01 | global_load_balancing_loss: 1.001168E+00 | loss scale: 1.0 | grad norm: 0.071 | params norm: 11252.980 | params rms: 0.060931 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 9:00:18.228507 | finish at 2026-10-10 20:00:31 + [2026-10-10 11:01:41] iteration 6484/ 6855 | consumed samples: 829952 | elapsed time per iteration (ms): 88230.1 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.67% | FLOPs estimator: workload-aware | logical tokens/s: 190152.6 | valid tokens/s: 190152.6 | physical tokens/s: 190492.1 | loss tokens/s: 139423.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.361141E-01 | lm-global-supervised-tokens: 1.230138E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.160168E-02 | lm-token-buffer-mib: 5.809043E+01 | global_load_balancing_loss: 1.000396E+00 | loss scale: 1.0 | grad norm: 0.095 | params norm: 11252.936 | params rms: 0.060931 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 9:05:33.370607 | finish at 2026-10-10 20:07:15 + [2026-10-10 11:03:05] iteration 6485/ 6855 | consumed samples: 830080 | elapsed time per iteration (ms): 84016.2 | throughput per GPU (TFLOP/s/GPU): 43.3 | MFU 13.87% | FLOPs estimator: workload-aware | logical tokens/s: 199690.1 | valid tokens/s: 199690.1 | physical tokens/s: 200076.3 | loss tokens/s: 151691.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.523364E-01 | lm-global-supervised-tokens: 1.274456E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.369420E-02 | lm-token-buffer-mib: 5.811586E+01 | global_load_balancing_loss: 1.000344E+00 | loss scale: 1.0 | grad norm: 0.125 | params norm: 11252.890 | params rms: 0.060931 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 8:38:05.996671 | finish at 2026-10-10 19:41:11 + [2026-10-10 11:04:33] iteration 6486/ 6855 | consumed samples: 830208 | elapsed time per iteration (ms): 87187.2 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.54% | FLOPs estimator: workload-aware | logical tokens/s: 192426.5 | valid tokens/s: 192426.5 | physical tokens/s: 192929.0 | loss tokens/s: 138095.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.445077E-01 | lm-global-supervised-tokens: 1.204019E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.151549E-02 | lm-token-buffer-mib: 5.816074E+01 | global_load_balancing_loss: 1.001014E+00 | loss scale: 1.0 | grad norm: 0.076 | params norm: 11252.846 | params rms: 0.060931 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 8:56:12.081576 | finish at 2026-10-10 20:00:45 + [2026-10-10 11:05:58] iteration 6487/ 6855 | consumed samples: 830336 | elapsed time per iteration (ms): 85052.1 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.60% | FLOPs estimator: workload-aware | logical tokens/s: 197257.3 | valid tokens/s: 197257.3 | physical tokens/s: 197893.9 | loss tokens/s: 151381.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.477190E-01 | lm-global-supervised-tokens: 1.287534E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.565693E-02 | lm-token-buffer-mib: 5.819833E+01 | global_load_balancing_loss: 9.998665E-01 | loss scale: 1.0 | grad norm: 0.096 | params norm: 11252.801 | params rms: 0.060930 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 8:41:39.185413 | finish at 2026-10-10 19:47:37 + [2026-10-10 11:07:23] iteration 6488/ 6855 | consumed samples: 830464 | elapsed time per iteration (ms): 85679.6 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.70% | FLOPs estimator: workload-aware | logical tokens/s: 195812.3 | valid tokens/s: 195812.3 | physical tokens/s: 196282.3 | loss tokens/s: 149959.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.658224E-01 | lm-global-supervised-tokens: 1.284844E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.806092E-02 | lm-token-buffer-mib: 5.812050E+01 | global_load_balancing_loss: 9.997498E-01 | loss scale: 1.0 | grad norm: 0.163 | params norm: 11252.756 | params rms: 0.060930 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 8:44:04.420725 | finish at 2026-10-10 19:51:28 + [2026-10-10 11:08:50] iteration 6489/ 6855 | consumed samples: 830592 | elapsed time per iteration (ms): 86589.4 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.53% | FLOPs estimator: workload-aware | logical tokens/s: 193755.6 | valid tokens/s: 193755.6 | physical tokens/s: 194281.0 | loss tokens/s: 145919.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.853113E-01 | lm-global-supervised-tokens: 1.263505E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.166386E-02 | lm-token-buffer-mib: 5.816873E+01 | global_load_balancing_loss: 9.996290E-01 | loss scale: 1.0 | grad norm: 0.050 | params norm: 11252.712 | params rms: 0.060930 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 8:48:11.720158 | finish at 2026-10-10 19:57:02 + [2026-10-10 11:10:43] iteration 6490/ 6855 | consumed samples: 830720 | elapsed time per iteration (ms): 113164.5 | throughput per GPU (TFLOP/s/GPU): 32.9 | MFU 10.55% | FLOPs estimator: workload-aware | logical tokens/s: 148254.9 | valid tokens/s: 148254.9 | physical tokens/s: 148526.3 | loss tokens/s: 108503.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.060542E-01 | lm-global-supervised-tokens: 1.227871E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.740647E-02 | lm-token-buffer-mib: 5.807727E+01 | global_load_balancing_loss: 9.993674E-01 | loss scale: 1.0 | grad norm: 0.108 | params norm: 11252.669 | params rms: 0.060930 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 11:28:25.038931 | finish at 2026-10-10 22:39:08 + [2026-10-10 11:12:12] iteration 6491/ 6855 | consumed samples: 830848 | elapsed time per iteration (ms): 89452.2 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.57% | FLOPs estimator: workload-aware | logical tokens/s: 187554.5 | valid tokens/s: 187554.5 | physical tokens/s: 187891.9 | loss tokens/s: 131667.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.879706E-01 | lm-global-supervised-tokens: 1.177791E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.977572E-02 | lm-token-buffer-mib: 5.818828E+01 | global_load_balancing_loss: 1.001817E+00 | loss scale: 1.0 | grad norm: 0.102 | params norm: 11252.625 | params rms: 0.060929 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 9:02:40.595483 | finish at 2026-10-10 20:14:53 + [2026-10-10 11:13:40] iteration 6492/ 6855 | consumed samples: 830976 | elapsed time per iteration (ms): 87496.0 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.43% | FLOPs estimator: workload-aware | logical tokens/s: 191747.6 | valid tokens/s: 191747.6 | physical tokens/s: 192282.5 | loss tokens/s: 145795.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.718656E-01 | lm-global-supervised-tokens: 1.275653E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.244693E-02 | lm-token-buffer-mib: 5.804904E+01 | global_load_balancing_loss: 9.996791E-01 | loss scale: 1.0 | grad norm: 0.064 | params norm: 11252.578 | params rms: 0.060929 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 8:49:21.060049 | finish at 2026-10-10 20:03:01 + [2026-10-10 11:15:08] iteration 6493/ 6855 | consumed samples: 831104 | elapsed time per iteration (ms): 88352.5 | throughput per GPU (TFLOP/s/GPU): 41.2 | MFU 13.22% | FLOPs estimator: workload-aware | logical tokens/s: 189888.5 | valid tokens/s: 189888.5 | physical tokens/s: 190424.9 | loss tokens/s: 145704.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.594233E-01 | lm-global-supervised-tokens: 1.287334E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.319109E-02 | lm-token-buffer-mib: 5.821213E+01 | global_load_balancing_loss: 9.991589E-01 | loss scale: 1.0 | grad norm: 0.089 | params norm: 11252.533 | params rms: 0.060929 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 8:53:03.605677 | finish at 2026-10-10 20:08:12 + [2026-10-10 11:16:37] iteration 6494/ 6855 | consumed samples: 831232 | elapsed time per iteration (ms): 88214.2 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.58% | FLOPs estimator: workload-aware | logical tokens/s: 190187.2 | valid tokens/s: 190187.2 | physical tokens/s: 190581.5 | loss tokens/s: 140247.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.793381E-01 | lm-global-supervised-tokens: 1.237185E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.147573E-02 | lm-token-buffer-mib: 5.807826E+01 | global_load_balancing_loss: 1.000763E+00 | loss scale: 1.0 | grad norm: 0.038 | params norm: 11252.490 | params rms: 0.060929 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 8:50:45.329220 | finish at 2026-10-10 20:07:22 + [2026-10-10 11:18:07] iteration 6495/ 6855 | consumed samples: 831360 | elapsed time per iteration (ms): 90547.3 | throughput per GPU (TFLOP/s/GPU): 41.1 | MFU 13.18% | FLOPs estimator: workload-aware | logical tokens/s: 185286.4 | valid tokens/s: 185286.4 | physical tokens/s: 185639.3 | loss tokens/s: 134300.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.885185E-01 | lm-global-supervised-tokens: 1.216056E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.233280E-02 | lm-token-buffer-mib: 5.809808E+01 | global_load_balancing_loss: 1.001138E+00 | loss scale: 1.0 | grad norm: 0.049 | params norm: 11252.448 | params rms: 0.060928 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 9:03:17.029152 | finish at 2026-10-10 20:21:24 + [2026-10-10 11:19:35] iteration 6496/ 6855 | consumed samples: 831488 | elapsed time per iteration (ms): 87782.0 | throughput per GPU (TFLOP/s/GPU): 42.8 | MFU 13.70% | FLOPs estimator: workload-aware | logical tokens/s: 191122.4 | valid tokens/s: 191122.4 | physical tokens/s: 191580.3 | loss tokens/s: 138833.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.141853E-01 | lm-global-supervised-tokens: 1.218712E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.649309E-02 | lm-token-buffer-mib: 5.803595E+01 | global_load_balancing_loss: 1.000630E+00 | loss scale: 1.0 | grad norm: 0.053 | params norm: 11252.403 | params rms: 0.060928 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 8:45:13.752317 | finish at 2026-10-10 20:04:49 + [2026-10-10 11:21:03] iteration 6497/ 6855 | consumed samples: 831616 | elapsed time per iteration (ms): 88287.3 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.52% | FLOPs estimator: workload-aware | logical tokens/s: 190029.1 | valid tokens/s: 190029.1 | physical tokens/s: 190430.8 | loss tokens/s: 136178.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.432172E-01 | lm-global-supervised-tokens: 1.202285E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.115191E-02 | lm-token-buffer-mib: 5.806789E+01 | global_load_balancing_loss: 1.001178E+00 | loss scale: 1.0 | grad norm: 0.066 | params norm: 11252.357 | params rms: 0.060928 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 8:46:46.861804 | finish at 2026-10-10 20:07:50 + [2026-10-10 11:22:29] iteration 6498/ 6855 | consumed samples: 831744 | elapsed time per iteration (ms): 86179.4 | throughput per GPU (TFLOP/s/GPU): 41.5 | MFU 13.31% | FLOPs estimator: workload-aware | logical tokens/s: 194677.5 | valid tokens/s: 194677.4 | physical tokens/s: 195182.8 | loss tokens/s: 151389.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.594002E-01 | lm-global-supervised-tokens: 1.304668E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.249354E-02 | lm-token-buffer-mib: 5.813633E+01 | global_load_balancing_loss: 1.000694E+00 | loss scale: 1.0 | grad norm: 0.049 | params norm: 11252.313 | params rms: 0.060928 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 8:32:46.043405 | finish at 2026-10-10 19:55:15 + [2026-10-10 11:23:58] iteration 6499/ 6855 | consumed samples: 831872 | elapsed time per iteration (ms): 88599.9 | throughput per GPU (TFLOP/s/GPU): 41.7 | MFU 13.37% | FLOPs estimator: workload-aware | logical tokens/s: 189358.9 | valid tokens/s: 189358.9 | physical tokens/s: 189731.6 | loss tokens/s: 140791.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.210425E-01 | lm-global-supervised-tokens: 1.247414E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.827744E-02 | lm-token-buffer-mib: 5.820357E+01 | global_load_balancing_loss: 9.995818E-01 | loss scale: 1.0 | grad norm: 0.055 | params norm: 11252.270 | params rms: 0.060927 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 8:45:41.576455 | finish at 2026-10-10 20:09:39 + [2026-10-10 11:25:49] iteration 6500/ 6855 | consumed samples: 832000 | elapsed time per iteration (ms): 110613.9 | throughput per GPU (TFLOP/s/GPU): 33.8 | MFU 10.84% | FLOPs estimator: workload-aware | logical tokens/s: 151673.4 | valid tokens/s: 151673.4 | physical tokens/s: 152063.8 | loss tokens/s: 112087.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.727228E-01 | lm-global-supervised-tokens: 1.239838E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.130209E-02 | lm-token-buffer-mib: 5.817162E+01 | global_load_balancing_loss: 9.992539E-01 | loss scale: 1.0 | grad norm: 0.044 | params norm: 11252.226 | params rms: 0.060927 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 10:54:27.935497 | finish at 2026-10-10 22:20:16 + [2026-10-10 11:27:14] iteration 6501/ 6855 | consumed samples: 832128 | elapsed time per iteration (ms): 85577.3 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.48% | FLOPs estimator: workload-aware | logical tokens/s: 196046.6 | valid tokens/s: 196046.6 | physical tokens/s: 196552.0 | loss tokens/s: 147601.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 5.888014E-01 | lm-global-supervised-tokens: 1.263130E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.275495E-02 | lm-token-buffer-mib: 5.813388E+01 | global_load_balancing_loss: 9.994922E-01 | loss scale: 1.0 | grad norm: 0.047 | params norm: 11252.181 | params rms: 0.060927 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 8:24:54.351903 | finish at 2026-10-10 19:52:08 + [2026-10-10 11:28:37] iteration 6502/ 6855 | consumed samples: 832256 | elapsed time per iteration (ms): 83234.0 | throughput per GPU (TFLOP/s/GPU): 43.1 | MFU 13.82% | FLOPs estimator: workload-aware | logical tokens/s: 201566.6 | valid tokens/s: 201566.6 | physical tokens/s: 201949.8 | loss tokens/s: 152331.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.416441E-01 | lm-global-supervised-tokens: 1.267915E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.250852E-02 | lm-token-buffer-mib: 5.810239E+01 | global_load_balancing_loss: 1.000364E+00 | loss scale: 1.0 | grad norm: 0.050 | params norm: 11252.135 | params rms: 0.060927 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 8:09:41.594246 | finish at 2026-10-10 19:38:19 + [2026-10-10 11:30:04] iteration 6503/ 6855 | consumed samples: 832384 | elapsed time per iteration (ms): 86402.2 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.77% | FLOPs estimator: workload-aware | logical tokens/s: 194175.5 | valid tokens/s: 194175.5 | physical tokens/s: 194589.4 | loss tokens/s: 139508.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.300411E-01 | lm-global-supervised-tokens: 1.205381E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.094395E-02 | lm-token-buffer-mib: 5.816717E+01 | global_load_balancing_loss: 1.000262E+00 | loss scale: 1.0 | grad norm: 0.045 | params norm: 11252.090 | params rms: 0.060926 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 8:26:53.566086 | finish at 2026-10-10 19:56:57 + [2026-10-10 11:31:32] iteration 6504/ 6855 | consumed samples: 832512 | elapsed time per iteration (ms): 88331.5 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.51% | FLOPs estimator: workload-aware | logical tokens/s: 189934.2 | valid tokens/s: 189934.2 | physical tokens/s: 190272.2 | loss tokens/s: 140763.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.174811E-01 | lm-global-supervised-tokens: 1.243385E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.155716E-02 | lm-token-buffer-mib: 5.807806E+01 | global_load_balancing_loss: 1.000417E+00 | loss scale: 1.0 | grad norm: 0.045 | params norm: 11252.045 | params rms: 0.060926 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 8:36:44.369155 | finish at 2026-10-10 20:08:16 + [2026-10-10 11:33:02] iteration 6505/ 6855 | consumed samples: 832640 | elapsed time per iteration (ms): 89645.7 | throughput per GPU (TFLOP/s/GPU): 40.9 | MFU 13.12% | FLOPs estimator: workload-aware | logical tokens/s: 187149.6 | valid tokens/s: 187149.6 | physical tokens/s: 187705.1 | loss tokens/s: 141214.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.748536E-01 | lm-global-supervised-tokens: 1.265924E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.136784E-02 | lm-token-buffer-mib: 5.821794E+01 | global_load_balancing_loss: 9.999616E-01 | loss scale: 1.0 | grad norm: 0.089 | params norm: 11251.998 | params rms: 0.060926 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 8:42:55.996578 | finish at 2026-10-10 20:15:58 + [2026-10-10 11:34:27] iteration 6506/ 6855 | consumed samples: 832768 | elapsed time per iteration (ms): 85497.0 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.70% | FLOPs estimator: workload-aware | logical tokens/s: 196231.0 | valid tokens/s: 196231.0 | physical tokens/s: 196598.5 | loss tokens/s: 149395.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.695596E-01 | lm-global-supervised-tokens: 1.277289E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.835158E-02 | lm-token-buffer-mib: 5.809251E+01 | global_load_balancing_loss: 1.000843E+00 | loss scale: 1.0 | grad norm: 0.060 | params norm: 11251.950 | params rms: 0.060926 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 8:17:18.448250 | finish at 2026-10-10 19:51:46 + [2026-10-10 11:35:56] iteration 6507/ 6855 | consumed samples: 832896 | elapsed time per iteration (ms): 88922.6 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.64% | FLOPs estimator: workload-aware | logical tokens/s: 188672.2 | valid tokens/s: 188672.2 | physical tokens/s: 189064.8 | loss tokens/s: 140000.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.703273E-01 | lm-global-supervised-tokens: 1.244922E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.463732E-02 | lm-token-buffer-mib: 5.805026E+01 | global_load_balancing_loss: 1.000813E+00 | loss scale: 1.0 | grad norm: 0.051 | params norm: 11251.905 | params rms: 0.060925 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 8:35:45.050540 | finish at 2026-10-10 20:11:41 + [2026-10-10 11:37:24] iteration 6508/ 6855 | consumed samples: 833024 | elapsed time per iteration (ms): 87700.3 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.55% | FLOPs estimator: workload-aware | logical tokens/s: 191301.5 | valid tokens/s: 191301.5 | physical tokens/s: 191780.0 | loss tokens/s: 141742.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.355605E-01 | lm-global-supervised-tokens: 1.243086E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.387207E-02 | lm-token-buffer-mib: 5.807448E+01 | global_load_balancing_loss: 1.000832E+00 | loss scale: 1.0 | grad norm: 0.075 | params norm: 11251.858 | params rms: 0.060925 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 8:27:12.005333 | finish at 2026-10-10 20:04:36 + [2026-10-10 11:38:52] iteration 6509/ 6855 | consumed samples: 833152 | elapsed time per iteration (ms): 88179.0 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.80% | FLOPs estimator: workload-aware | logical tokens/s: 190262.8 | valid tokens/s: 190262.8 | physical tokens/s: 190703.7 | loss tokens/s: 136494.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.476043E-01 | lm-global-supervised-tokens: 1.203592E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.636080E-02 | lm-token-buffer-mib: 5.810043E+01 | global_load_balancing_loss: 1.000118E+00 | loss scale: 1.0 | grad norm: 0.042 | params norm: 11251.809 | params rms: 0.060925 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 8:28:29.938585 | finish at 2026-10-10 20:07:22 + [2026-10-10 11:40:42] iteration 6510/ 6855 | consumed samples: 833280 | elapsed time per iteration (ms): 110055.2 | throughput per GPU (TFLOP/s/GPU): 33.1 | MFU 10.60% | FLOPs estimator: workload-aware | logical tokens/s: 152443.7 | valid tokens/s: 152443.7 | physical tokens/s: 152912.4 | loss tokens/s: 115825.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.757835E-01 | lm-global-supervised-tokens: 1.274721E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.220581E-02 | lm-token-buffer-mib: 5.828772E+01 | global_load_balancing_loss: 1.000342E+00 | loss scale: 1.0 | grad norm: 0.065 | params norm: 11251.760 | params rms: 0.060925 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 10:32:49.037619 | finish at 2026-10-10 22:13:31 + [2026-10-10 11:42:12] iteration 6511/ 6855 | consumed samples: 833408 | elapsed time per iteration (ms): 89686.0 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.45% | FLOPs estimator: workload-aware | logical tokens/s: 187065.6 | valid tokens/s: 187065.6 | physical tokens/s: 187532.2 | loss tokens/s: 138651.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.848781E-01 | lm-global-supervised-tokens: 1.243512E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.670226E-02 | lm-token-buffer-mib: 5.814425E+01 | global_load_balancing_loss: 9.996216E-01 | loss scale: 1.0 | grad norm: 0.101 | params norm: 11251.713 | params rms: 0.060924 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 8:34:11.983217 | finish at 2026-10-10 20:16:24 + [2026-10-10 11:43:38] iteration 6512/ 6855 | consumed samples: 833536 | elapsed time per iteration (ms): 86251.4 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.46% | FLOPs estimator: workload-aware | logical tokens/s: 194514.8 | valid tokens/s: 194514.8 | physical tokens/s: 195058.8 | loss tokens/s: 143831.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.527830E-01 | lm-global-supervised-tokens: 1.240565E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.128033E-02 | lm-token-buffer-mib: 5.832912E+01 | global_load_balancing_loss: 1.000928E+00 | loss scale: 1.0 | grad norm: 0.122 | params norm: 11251.667 | params rms: 0.060924 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 8:13:04.235268 | finish at 2026-10-10 19:56:42 + [2026-10-10 11:45:09] iteration 6513/ 6855 | consumed samples: 833664 | elapsed time per iteration (ms): 90895.0 | throughput per GPU (TFLOP/s/GPU): 41.5 | MFU 13.30% | FLOPs estimator: workload-aware | logical tokens/s: 184577.5 | valid tokens/s: 184577.5 | physical tokens/s: 185242.2 | loss tokens/s: 135643.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.868111E-01 | lm-global-supervised-tokens: 1.232929E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.890935E-02 | lm-token-buffer-mib: 5.834946E+01 | global_load_balancing_loss: 9.993260E-01 | loss scale: 1.0 | grad norm: 0.067 | params norm: 11251.619 | params rms: 0.060924 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 8:38:06.082818 | finish at 2026-10-10 20:23:15 + [2026-10-10 11:46:37] iteration 6514/ 6855 | consumed samples: 833792 | elapsed time per iteration (ms): 88007.9 | throughput per GPU (TFLOP/s/GPU): 41.4 | MFU 13.26% | FLOPs estimator: workload-aware | logical tokens/s: 190632.2 | valid tokens/s: 190632.2 | physical tokens/s: 191092.7 | loss tokens/s: 143992.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.722705E-01 | lm-global-supervised-tokens: 1.267246E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.503736E-02 | lm-token-buffer-mib: 5.825773E+01 | global_load_balancing_loss: 1.000170E+00 | loss scale: 1.0 | grad norm: 0.074 | params norm: 11251.572 | params rms: 0.060924 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 8:20:10.683168 | finish at 2026-10-10 20:06:48 + [2026-10-10 11:48:03] iteration 6515/ 6855 | consumed samples: 833920 | elapsed time per iteration (ms): 85816.1 | throughput per GPU (TFLOP/s/GPU): 43.2 | MFU 13.86% | FLOPs estimator: workload-aware | logical tokens/s: 195501.7 | valid tokens/s: 195501.7 | physical tokens/s: 195961.4 | loss tokens/s: 140177.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.234991E-01 | lm-global-supervised-tokens: 1.202944E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.382815E-02 | lm-token-buffer-mib: 5.803894E+01 | global_load_balancing_loss: 9.995643E-01 | loss scale: 1.0 | grad norm: 0.080 | params norm: 11251.526 | params rms: 0.060923 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 8:06:17.471690 | finish at 2026-10-10 19:54:20 + [2026-10-10 11:49:28] iteration 6516/ 6855 | consumed samples: 834048 | elapsed time per iteration (ms): 85618.3 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.39% | FLOPs estimator: workload-aware | logical tokens/s: 195953.1 | valid tokens/s: 195953.1 | physical tokens/s: 196552.9 | loss tokens/s: 144909.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.090522E-01 | lm-global-supervised-tokens: 1.240686E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.157423E-02 | lm-token-buffer-mib: 5.820882E+01 | global_load_balancing_loss: 9.998475E-01 | loss scale: 1.0 | grad norm: 0.073 | params norm: 11251.482 | params rms: 0.060923 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 8:03:44.606758 | finish at 2026-10-10 19:53:13 + [2026-10-10 11:50:57] iteration 6517/ 6855 | consumed samples: 834176 | elapsed time per iteration (ms): 88466.0 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.75% | FLOPs estimator: workload-aware | logical tokens/s: 189645.5 | valid tokens/s: 189645.5 | physical tokens/s: 190250.9 | loss tokens/s: 134780.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.651751E-01 | lm-global-supervised-tokens: 1.192346E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.414202E-02 | lm-token-buffer-mib: 5.807121E+01 | global_load_balancing_loss: 1.000406E+00 | loss scale: 1.0 | grad norm: 0.068 | params norm: 11251.434 | params rms: 0.060923 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 8:18:21.501822 | finish at 2026-10-10 20:09:18 + [2026-10-10 11:52:24] iteration 6518/ 6855 | consumed samples: 834304 | elapsed time per iteration (ms): 86951.5 | throughput per GPU (TFLOP/s/GPU): 41.6 | MFU 13.35% | FLOPs estimator: workload-aware | logical tokens/s: 192948.6 | valid tokens/s: 192948.6 | physical tokens/s: 193516.5 | loss tokens/s: 141292.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.145651E-01 | lm-global-supervised-tokens: 1.228556E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.691579E-02 | lm-token-buffer-mib: 5.823177E+01 | global_load_balancing_loss: 1.001233E+00 | loss scale: 1.0 | grad norm: 0.063 | params norm: 11251.385 | params rms: 0.060923 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 8:08:22.652587 | finish at 2026-10-10 20:00:46 + [2026-10-10 11:53:49] iteration 6519/ 6855 | consumed samples: 834432 | elapsed time per iteration (ms): 85221.3 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.53% | FLOPs estimator: workload-aware | logical tokens/s: 196866.0 | valid tokens/s: 196866.0 | physical tokens/s: 197435.5 | loss tokens/s: 145692.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.213194E-01 | lm-global-supervised-tokens: 1.241607E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.526125E-02 | lm-token-buffer-mib: 5.819407E+01 | global_load_balancing_loss: 1.000631E+00 | loss scale: 1.0 | grad norm: 0.032 | params norm: 11251.339 | params rms: 0.060922 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 7:57:14.356682 | finish at 2026-10-10 19:51:03 + [2026-10-10 11:55:36] iteration 6520/ 6855 | consumed samples: 834560 | elapsed time per iteration (ms): 107415.4 | throughput per GPU (TFLOP/s/GPU): 33.8 | MFU 10.85% | FLOPs estimator: workload-aware | logical tokens/s: 156189.6 | valid tokens/s: 156189.6 | physical tokens/s: 156596.4 | loss tokens/s: 118281.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.800041E-01 | lm-global-supervised-tokens: 1.270520E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.303645E-02 | lm-token-buffer-mib: 5.823392E+01 | global_load_balancing_loss: 1.000137E+00 | loss scale: 1.0 | grad norm: 0.048 | params norm: 11251.294 | params rms: 0.060922 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 9:59:44.167316 | finish at 2026-10-10 21:55:21 +times across ranks (ms): + save-checkpoint: + rank 0: 25173.01 + rank 1: 25173.00 + rank 2: 25173.00 + rank 3: 25172.97 + rank 4: 25172.98 + rank 5: 25172.96 + rank 6: 25172.94 + rank 7: 25173.01 + rank 8: 25172.59 + rank 9: 25172.56 + rank 10: 25172.55 + rank 11: 25172.57 + rank 12: 25172.57 + rank 13: 25172.55 + rank 14: 25172.59 + rank 15: 25172.55 + rank 16: 25173.54 + rank 17: 25173.55 + rank 18: 25173.48 + rank 19: 25173.52 + rank 20: 25173.48 + rank 21: 25173.52 + rank 22: 25173.49 + rank 23: 25173.50 + rank 24: 25173.31 + rank 25: 25173.24 + rank 26: 25173.33 + rank 27: 25173.32 + rank 28: 25173.28 + rank 29: 25173.33 + rank 30: 25173.29 + rank 31: 25173.23 + rank 32: 25172.78 + rank 33: 25172.92 + rank 34: 25172.82 + rank 35: 25172.78 + rank 36: 25172.77 + rank 37: 25172.77 + rank 38: 25172.75 + rank 39: 25172.77 + rank 40: 25173.19 + rank 41: 25173.15 + rank 42: 25173.24 + rank 43: 25173.19 + rank 44: 25173.05 + rank 45: 25173.16 + rank 46: 25173.18 + rank 47: 25173.12 + rank 48: 25173.24 + rank 49: 25173.08 + rank 50: 25173.28 + rank 51: 25173.06 + rank 52: 25173.24 + rank 53: 25173.17 + rank 54: 25173.21 + rank 55: 25173.24 + rank 56: 25173.58 + rank 57: 25173.58 + rank 58: 25173.54 + rank 59: 25173.54 + rank 60: 25173.55 + rank 61: 25173.56 + rank 62: 25173.60 + rank 63: 25173.57 + [2026-10-10 11:57:30] iteration 6521/ 6855 | consumed samples: 834688 | elapsed time per iteration (ms): 87908.8 | throughput per GPU (TFLOP/s/GPU): 41.3 | MFU 13.22% | FLOPs estimator: workload-aware | logical tokens/s: 190847.6 | valid tokens/s: 190847.6 | physical tokens/s: 191354.8 | loss tokens/s: 148621.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.461764E-01 | lm-global-supervised-tokens: 1.306518E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 7.261431E-01 | lm-token-buffer-mib: 5.821319E+01 | global_load_balancing_loss: 9.999371E-01 | loss scale: 1.0 | grad norm: 0.041 | params norm: 11251.249 | params rms: 0.060922 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 8:09:21.542825 | finish at 2026-10-10 20:06:51 + [2026-10-10 11:59:03] iteration 6522/ 6855 | consumed samples: 834816 | elapsed time per iteration (ms): 93741.5 | throughput per GPU (TFLOP/s/GPU): 38.7 | MFU 12.40% | FLOPs estimator: workload-aware | logical tokens/s: 178972.8 | valid tokens/s: 178972.8 | physical tokens/s: 179404.9 | loss tokens/s: 133762.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.517313E-01 | lm-global-supervised-tokens: 1.253905E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.408846E-02 | lm-token-buffer-mib: 5.813110E+01 | global_load_balancing_loss: 9.990721E-01 | loss scale: 1.0 | grad norm: 0.052 | params norm: 11251.205 | params rms: 0.060922 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 8:40:15.908828 | finish at 2026-10-10 20:39:19 + [2026-10-10 12:00:28] iteration 6523/ 6855 | consumed samples: 834944 | elapsed time per iteration (ms): 85047.1 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.50% | FLOPs estimator: workload-aware | logical tokens/s: 197269.5 | valid tokens/s: 197269.5 | physical tokens/s: 197839.2 | loss tokens/s: 151095.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.521161E-01 | lm-global-supervised-tokens: 1.285019E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.131812E-02 | lm-token-buffer-mib: 5.822684E+01 | global_load_balancing_loss: 1.000149E+00 | loss scale: 1.0 | grad norm: 0.068 | params norm: 11251.161 | params rms: 0.060921 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 7:50:35.637301 | finish at 2026-10-10 19:51:04 + [2026-10-10 12:01:53] iteration 6524/ 6855 | consumed samples: 835072 | elapsed time per iteration (ms): 85001.7 | throughput per GPU (TFLOP/s/GPU): 43.1 | MFU 13.82% | FLOPs estimator: workload-aware | logical tokens/s: 197374.2 | valid tokens/s: 197374.2 | physical tokens/s: 197878.1 | loss tokens/s: 149018.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.438156E-01 | lm-global-supervised-tokens: 1.266683E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.236067E-02 | lm-token-buffer-mib: 5.821172E+01 | global_load_balancing_loss: 9.999129E-01 | loss scale: 1.0 | grad norm: 0.047 | params norm: 11251.118 | params rms: 0.060921 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 7:48:55.574907 | finish at 2026-10-10 19:50:49 + [2026-10-10 12:03:20] iteration 6525/ 6855 | consumed samples: 835200 | elapsed time per iteration (ms): 86305.6 | throughput per GPU (TFLOP/s/GPU): 43.3 | MFU 13.88% | FLOPs estimator: workload-aware | logical tokens/s: 194392.8 | valid tokens/s: 194392.8 | physical tokens/s: 194710.8 | loss tokens/s: 143776.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.275389E-01 | lm-global-supervised-tokens: 1.240868E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.156295E-02 | lm-token-buffer-mib: 5.811078E+01 | global_load_balancing_loss: 9.993926E-01 | loss scale: 1.0 | grad norm: 0.067 | params norm: 11251.074 | params rms: 0.060921 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 7:54:40.840502 | finish at 2026-10-10 19:58:01 + [2026-10-10 12:04:43] iteration 6526/ 6855 | consumed samples: 835328 | elapsed time per iteration (ms): 83572.5 | throughput per GPU (TFLOP/s/GPU): 44.0 | MFU 14.11% | FLOPs estimator: workload-aware | logical tokens/s: 200750.0 | valid tokens/s: 200750.0 | physical tokens/s: 201338.7 | loss tokens/s: 152899.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.520484E-01 | lm-global-supervised-tokens: 1.277822E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.176523E-02 | lm-token-buffer-mib: 5.827830E+01 | global_load_balancing_loss: 1.000463E+00 | loss scale: 1.0 | grad norm: 0.041 | params norm: 11251.030 | params rms: 0.060921 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 7:38:15.358458 | finish at 2026-10-10 19:42:59 + [2026-10-10 12:06:11] iteration 6527/ 6855 | consumed samples: 835456 | elapsed time per iteration (ms): 87613.9 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.52% | FLOPs estimator: workload-aware | logical tokens/s: 191489.4 | valid tokens/s: 191489.4 | physical tokens/s: 192131.8 | loss tokens/s: 141701.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.353743E-01 | lm-global-supervised-tokens: 1.241504E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.481342E-02 | lm-token-buffer-mib: 5.864670E+01 | global_load_balancing_loss: 1.001904E+00 | loss scale: 1.0 | grad norm: 0.074 | params norm: 11250.985 | params rms: 0.060920 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 7:58:57.343386 | finish at 2026-10-10 20:05:08 + [2026-10-10 12:07:36] iteration 6528/ 6855 | consumed samples: 835584 | elapsed time per iteration (ms): 84672.6 | throughput per GPU (TFLOP/s/GPU): 43.2 | MFU 13.85% | FLOPs estimator: workload-aware | logical tokens/s: 198141.5 | valid tokens/s: 198141.5 | physical tokens/s: 198645.4 | loss tokens/s: 147802.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.756855E-01 | lm-global-supervised-tokens: 1.251482E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.357666E-02 | lm-token-buffer-mib: 5.808532E+01 | global_load_balancing_loss: 1.002100E+00 | loss scale: 1.0 | grad norm: 0.083 | params norm: 11250.938 | params rms: 0.060920 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 7:41:27.939197 | finish at 2026-10-10 19:49:04 + [2026-10-10 12:09:04] iteration 6529/ 6855 | consumed samples: 835712 | elapsed time per iteration (ms): 88578.2 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.46% | FLOPs estimator: workload-aware | logical tokens/s: 189405.1 | valid tokens/s: 189405.1 | physical tokens/s: 189994.1 | loss tokens/s: 147430.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.486966E-01 | lm-global-supervised-tokens: 1.305910E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.122789E-02 | lm-token-buffer-mib: 5.866150E+01 | global_load_balancing_loss: 9.997643E-01 | loss scale: 1.0 | grad norm: 0.084 | params norm: 11250.893 | params rms: 0.060920 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 8:01:16.482041 | finish at 2026-10-10 20:10:21 + [2026-10-10 12:10:53] iteration 6530/ 6855 | consumed samples: 835840 | elapsed time per iteration (ms): 109039.3 | throughput per GPU (TFLOP/s/GPU): 33.8 | MFU 10.83% | FLOPs estimator: workload-aware | logical tokens/s: 153863.1 | valid tokens/s: 153863.1 | physical tokens/s: 154218.2 | loss tokens/s: 114750.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.778517E-01 | lm-global-supervised-tokens: 1.251226E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.121286E-02 | lm-token-buffer-mib: 5.830751E+01 | global_load_balancing_loss: 9.995495E-01 | loss scale: 1.0 | grad norm: 0.055 | params norm: 11250.849 | params rms: 0.060920 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 9:50:37.768537 | finish at 2026-10-10 22:01:31 + [2026-10-10 12:12:23] iteration 6531/ 6855 | consumed samples: 835968 | elapsed time per iteration (ms): 89867.4 | throughput per GPU (TFLOP/s/GPU): 41.5 | MFU 13.32% | FLOPs estimator: workload-aware | logical tokens/s: 186687.4 | valid tokens/s: 186687.4 | physical tokens/s: 187343.9 | loss tokens/s: 135384.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.626762E-01 | lm-global-supervised-tokens: 1.216662E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.396745E-02 | lm-token-buffer-mib: 5.813649E+01 | global_load_balancing_loss: 9.997071E-01 | loss scale: 1.0 | grad norm: 0.289 | params norm: 11250.804 | params rms: 0.060919 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 8:05:17.048470 | finish at 2026-10-10 20:17:40 + [2026-10-10 12:13:52] iteration 6532/ 6855 | consumed samples: 836096 | elapsed time per iteration (ms): 88794.3 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.67% | FLOPs estimator: workload-aware | logical tokens/s: 188944.7 | valid tokens/s: 188944.7 | physical tokens/s: 189454.7 | loss tokens/s: 140148.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.827126E-01 | lm-global-supervised-tokens: 1.244439E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.848153E-02 | lm-token-buffer-mib: 5.814777E+01 | global_load_balancing_loss: 9.999562E-01 | loss scale: 1.0 | grad norm: 0.549 | params norm: 11250.757 | params rms: 0.060919 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 7:58:00.557616 | finish at 2026-10-10 20:11:52 + [2026-10-10 12:15:19] iteration 6533/ 6855 | consumed samples: 836224 | elapsed time per iteration (ms): 86902.2 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.49% | FLOPs estimator: workload-aware | logical tokens/s: 193058.4 | valid tokens/s: 193058.4 | physical tokens/s: 193510.4 | loss tokens/s: 141980.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.008700E-01 | lm-global-supervised-tokens: 1.233843E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.249613E-02 | lm-token-buffer-mib: 5.813705E+01 | global_load_balancing_loss: 9.997532E-01 | loss scale: 1.0 | grad norm: 0.189 | params norm: 11250.712 | params rms: 0.060919 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 7:46:22.509470 | finish at 2026-10-10 20:01:41 + [2026-10-10 12:16:48] iteration 6534/ 6855 | consumed samples: 836352 | elapsed time per iteration (ms): 88949.3 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.61% | FLOPs estimator: workload-aware | logical tokens/s: 188614.6 | valid tokens/s: 188614.6 | physical tokens/s: 189020.7 | loss tokens/s: 131216.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.764622E-01 | lm-global-supervised-tokens: 1.167162E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.402424E-02 | lm-token-buffer-mib: 5.818845E+01 | global_load_balancing_loss: 9.999622E-01 | loss scale: 1.0 | grad norm: 0.442 | params norm: 11250.669 | params rms: 0.060919 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 7:55:52.728301 | finish at 2026-10-10 20:12:40 + [2026-10-10 12:18:16] iteration 6535/ 6855 | consumed samples: 836480 | elapsed time per iteration (ms): 87913.4 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.43% | FLOPs estimator: workload-aware | logical tokens/s: 190836.8 | valid tokens/s: 190836.8 | physical tokens/s: 191409.7 | loss tokens/s: 145889.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.587922E-01 | lm-global-supervised-tokens: 1.282560E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.350345E-02 | lm-token-buffer-mib: 5.843053E+01 | global_load_balancing_loss: 9.991459E-01 | loss scale: 1.0 | grad norm: 0.078 | params norm: 11250.626 | params rms: 0.060918 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 7:48:52.277603 | finish at 2026-10-10 20:07:08 + [2026-10-10 12:19:43] iteration 6536/ 6855 | consumed samples: 836608 | elapsed time per iteration (ms): 87450.9 | throughput per GPU (TFLOP/s/GPU): 41.6 | MFU 13.33% | FLOPs estimator: workload-aware | logical tokens/s: 191846.9 | valid tokens/s: 191846.9 | physical tokens/s: 192564.7 | loss tokens/s: 139305.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.666949E-01 | lm-global-supervised-tokens: 1.218237E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.479460E-02 | lm-token-buffer-mib: 5.815914E+01 | global_load_balancing_loss: 1.000609E+00 | loss scale: 1.0 | grad norm: 0.449 | params norm: 11250.583 | params rms: 0.060918 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 7:44:56.833702 | finish at 2026-10-10 20:04:40 + [2026-10-10 12:21:09] iteration 6537/ 6855 | consumed samples: 836736 | elapsed time per iteration (ms): 86117.4 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.75% | FLOPs estimator: workload-aware | logical tokens/s: 194817.7 | valid tokens/s: 194817.7 | physical tokens/s: 195163.5 | loss tokens/s: 150282.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.383255E-01 | lm-global-supervised-tokens: 1.294192E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.665897E-02 | lm-token-buffer-mib: 5.807619E+01 | global_load_balancing_loss: 1.000557E+00 | loss scale: 1.0 | grad norm: 0.073 | params norm: 11250.539 | params rms: 0.060918 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 7:36:25.317863 | finish at 2026-10-10 19:57:35 + [2026-10-10 12:22:37] iteration 6538/ 6855 | consumed samples: 836864 | elapsed time per iteration (ms): 87338.9 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.53% | FLOPs estimator: workload-aware | logical tokens/s: 192092.9 | valid tokens/s: 192092.8 | physical tokens/s: 192525.3 | loss tokens/s: 134825.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.709780E-01 | lm-global-supervised-tokens: 1.177552E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.285887E-02 | lm-token-buffer-mib: 5.815270E+01 | global_load_balancing_loss: 1.000127E+00 | loss scale: 1.0 | grad norm: 0.189 | params norm: 11250.495 | params rms: 0.060918 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 7:41:26.422486 | finish at 2026-10-10 20:04:03 + [2026-10-10 12:24:02] iteration 6539/ 6855 | consumed samples: 836992 | elapsed time per iteration (ms): 85800.2 | throughput per GPU (TFLOP/s/GPU): 43.4 | MFU 13.91% | FLOPs estimator: workload-aware | logical tokens/s: 195537.9 | valid tokens/s: 195537.8 | physical tokens/s: 196134.3 | loss tokens/s: 147047.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.714949E-01 | lm-global-supervised-tokens: 1.261671E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.711859E-02 | lm-token-buffer-mib: 5.806667E+01 | global_load_balancing_loss: 1.001372E+00 | loss scale: 1.0 | grad norm: 0.184 | params norm: 11250.453 | params rms: 0.060918 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 7:31:52.851292 | finish at 2026-10-10 19:55:55 + [2026-10-10 12:25:51] iteration 6540/ 6855 | consumed samples: 837120 | elapsed time per iteration (ms): 108615.3 | throughput per GPU (TFLOP/s/GPU): 33.8 | MFU 10.82% | FLOPs estimator: workload-aware | logical tokens/s: 154464.5 | valid tokens/s: 154464.5 | physical tokens/s: 154844.6 | loss tokens/s: 110031.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.370704E-01 | lm-global-supervised-tokens: 1.195113E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.174165E-02 | lm-token-buffer-mib: 5.819381E+01 | global_load_balancing_loss: 1.000499E+00 | loss scale: 1.0 | grad norm: 0.384 | params norm: 11250.411 | params rms: 0.060917 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 9:30:13.823912 | finish at 2026-10-10 21:56:05 + [2026-10-10 12:27:15] iteration 6541/ 6855 | consumed samples: 837248 | elapsed time per iteration (ms): 84069.6 | throughput per GPU (TFLOP/s/GPU): 43.3 | MFU 13.87% | FLOPs estimator: workload-aware | logical tokens/s: 199562.9 | valid tokens/s: 199562.9 | physical tokens/s: 199990.0 | loss tokens/s: 141339.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.244140E-01 | lm-global-supervised-tokens: 1.188232E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.155155E-02 | lm-token-buffer-mib: 5.809190E+01 | global_load_balancing_loss: 1.000694E+00 | loss scale: 1.0 | grad norm: 0.150 | params norm: 11250.367 | params rms: 0.060917 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 7:19:57.869705 | finish at 2026-10-10 19:47:13 + [2026-10-10 12:28:42] iteration 6542/ 6855 | consumed samples: 837376 | elapsed time per iteration (ms): 87112.9 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.54% | FLOPs estimator: workload-aware | logical tokens/s: 192591.1 | valid tokens/s: 192591.1 | physical tokens/s: 193081.5 | loss tokens/s: 143695.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.143689E-01 | lm-global-supervised-tokens: 1.251772E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.156879E-02 | lm-token-buffer-mib: 5.801972E+01 | global_load_balancing_loss: 9.995980E-01 | loss scale: 1.0 | grad norm: 0.178 | params norm: 11250.322 | params rms: 0.060917 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 7:34:26.339348 | finish at 2026-10-10 20:03:08 + [2026-10-10 12:30:06] iteration 6543/ 6855 | consumed samples: 837504 | elapsed time per iteration (ms): 84214.7 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.69% | FLOPs estimator: workload-aware | logical tokens/s: 199219.6 | valid tokens/s: 199219.6 | physical tokens/s: 199837.1 | loss tokens/s: 150943.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.604545E-01 | lm-global-supervised-tokens: 1.271169E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.265448E-02 | lm-token-buffer-mib: 5.810726E+01 | global_load_balancing_loss: 1.000565E+00 | loss scale: 1.0 | grad norm: 0.065 | params norm: 11250.276 | params rms: 0.060917 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 7:17:54.978510 | finish at 2026-10-10 19:48:01 + [2026-10-10 12:31:38] iteration 6544/ 6855 | consumed samples: 837632 | elapsed time per iteration (ms): 91732.8 | throughput per GPU (TFLOP/s/GPU): 39.7 | MFU 12.73% | FLOPs estimator: workload-aware | logical tokens/s: 182891.6 | valid tokens/s: 182891.6 | physical tokens/s: 183456.7 | loss tokens/s: 133105.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.231787E-01 | lm-global-supervised-tokens: 1.221011E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.375790E-02 | lm-token-buffer-mib: 5.823454E+01 | global_load_balancing_loss: 9.994614E-01 | loss scale: 1.0 | grad norm: 0.113 | params norm: 11250.231 | params rms: 0.060916 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 7:55:28.899764 | finish at 2026-10-10 20:27:07 + [2026-10-10 12:33:06] iteration 6545/ 6855 | consumed samples: 837760 | elapsed time per iteration (ms): 88216.4 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.48% | FLOPs estimator: workload-aware | logical tokens/s: 190182.6 | valid tokens/s: 190182.6 | physical tokens/s: 190618.1 | loss tokens/s: 137738.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.923256E-01 | lm-global-supervised-tokens: 1.215078E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.448647E-02 | lm-token-buffer-mib: 5.815546E+01 | global_load_balancing_loss: 9.985478E-01 | loss scale: 1.0 | grad norm: 0.074 | params norm: 11250.187 | params rms: 0.060916 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 7:35:47.069190 | finish at 2026-10-10 20:08:53 + [2026-10-10 12:34:34] iteration 6546/ 6855 | consumed samples: 837888 | elapsed time per iteration (ms): 87210.8 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.49% | FLOPs estimator: workload-aware | logical tokens/s: 192374.3 | valid tokens/s: 192374.3 | physical tokens/s: 192846.6 | loss tokens/s: 146623.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.362008E-01 | lm-global-supervised-tokens: 1.278715E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.463979E-02 | lm-token-buffer-mib: 5.802533E+01 | global_load_balancing_loss: 9.998394E-01 | loss scale: 1.0 | grad norm: 0.052 | params norm: 11250.142 | params rms: 0.060916 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 7:29:08.134674 | finish at 2026-10-10 20:03:42 + [2026-10-10 12:36:01] iteration 6547/ 6855 | consumed samples: 838016 | elapsed time per iteration (ms): 87306.1 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.48% | FLOPs estimator: workload-aware | logical tokens/s: 192164.6 | valid tokens/s: 192164.6 | physical tokens/s: 192680.5 | loss tokens/s: 144376.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.347885E-01 | lm-global-supervised-tokens: 1.260497E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.129671E-02 | lm-token-buffer-mib: 5.814845E+01 | global_load_balancing_loss: 1.000269E+00 | loss scale: 1.0 | grad norm: 0.126 | params norm: 11250.098 | params rms: 0.060916 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 7:28:10.263419 | finish at 2026-10-10 20:04:11 + [2026-10-10 12:37:29] iteration 6548/ 6855 | consumed samples: 838144 | elapsed time per iteration (ms): 87774.9 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.44% | FLOPs estimator: workload-aware | logical tokens/s: 191138.6 | valid tokens/s: 191138.6 | physical tokens/s: 191584.7 | loss tokens/s: 140884.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.300682E-01 | lm-global-supervised-tokens: 1.236611E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.164408E-02 | lm-token-buffer-mib: 5.812550E+01 | global_load_balancing_loss: 1.002173E+00 | loss scale: 1.0 | grad norm: 0.038 | params norm: 11250.052 | params rms: 0.060915 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 7:29:06.885138 | finish at 2026-10-10 20:06:35 + [2026-10-10 12:38:54] iteration 6549/ 6855 | consumed samples: 838272 | elapsed time per iteration (ms): 85430.2 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.55% | FLOPs estimator: workload-aware | logical tokens/s: 196384.8 | valid tokens/s: 196384.8 | physical tokens/s: 196899.6 | loss tokens/s: 150928.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.314912E-01 | lm-global-supervised-tokens: 1.289382E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.150818E-02 | lm-token-buffer-mib: 5.804624E+01 | global_load_balancing_loss: 1.001852E+00 | loss scale: 1.0 | grad norm: 0.085 | params norm: 11250.009 | params rms: 0.060915 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 7:15:41.628317 | finish at 2026-10-10 19:54:36 + [2026-10-10 12:40:43] iteration 6550/ 6855 | consumed samples: 838400 | elapsed time per iteration (ms): 109119.7 | throughput per GPU (TFLOP/s/GPU): 33.4 | MFU 10.70% | FLOPs estimator: workload-aware | logical tokens/s: 153749.8 | valid tokens/s: 153749.8 | physical tokens/s: 154177.6 | loss tokens/s: 115381.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.729676E-01 | lm-global-supervised-tokens: 1.259039E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.175953E-02 | lm-token-buffer-mib: 5.833949E+01 | global_load_balancing_loss: 1.000980E+00 | loss scale: 1.0 | grad norm: 0.076 | params norm: 11249.964 | params rms: 0.060915 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 9:14:41.522084 | finish at 2026-10-10 21:55:25 + [2026-10-10 12:42:11] iteration 6551/ 6855 | consumed samples: 838528 | elapsed time per iteration (ms): 88283.6 | throughput per GPU (TFLOP/s/GPU): 41.5 | MFU 13.29% | FLOPs estimator: workload-aware | logical tokens/s: 190037.5 | valid tokens/s: 190037.5 | physical tokens/s: 190591.1 | loss tokens/s: 144881.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.111515E-01 | lm-global-supervised-tokens: 1.279062E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.560395E-02 | lm-token-buffer-mib: 5.816418E+01 | global_load_balancing_loss: 9.995109E-01 | loss scale: 1.0 | grad norm: 0.054 | params norm: 11249.920 | params rms: 0.060915 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 7:27:18.213051 | finish at 2026-10-10 20:09:30 + [2026-10-10 12:43:40] iteration 6552/ 6855 | consumed samples: 838656 | elapsed time per iteration (ms): 88737.9 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.62% | FLOPs estimator: workload-aware | logical tokens/s: 189064.8 | valid tokens/s: 189064.8 | physical tokens/s: 189354.3 | loss tokens/s: 134898.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.609419E-01 | lm-global-supervised-tokens: 1.197064E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.580699E-02 | lm-token-buffer-mib: 5.808733E+01 | global_load_balancing_loss: 9.994555E-01 | loss scale: 1.0 | grad norm: 0.069 | params norm: 11249.875 | params rms: 0.060914 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 7:28:07.578504 | finish at 2026-10-10 20:11:48 + [2026-10-10 12:45:10] iteration 6553/ 6855 | consumed samples: 838784 | elapsed time per iteration (ms): 89890.0 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.45% | FLOPs estimator: workload-aware | logical tokens/s: 186640.9 | valid tokens/s: 186640.9 | physical tokens/s: 187038.3 | loss tokens/s: 131289.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.295328E-01 | lm-global-supervised-tokens: 1.180161E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.455472E-02 | lm-token-buffer-mib: 5.828998E+01 | global_load_balancing_loss: 1.000628E+00 | loss scale: 1.0 | grad norm: 0.055 | params norm: 11249.828 | params rms: 0.060914 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 7:32:26.765415 | finish at 2026-10-10 20:17:37 + [2026-10-10 12:46:37] iteration 6554/ 6855 | consumed samples: 838912 | elapsed time per iteration (ms): 86851.0 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.60% | FLOPs estimator: workload-aware | logical tokens/s: 193172.3 | valid tokens/s: 193172.3 | physical tokens/s: 193642.3 | loss tokens/s: 134672.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.253588E-01 | lm-global-supervised-tokens: 1.169640E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.290419E-02 | lm-token-buffer-mib: 5.815015E+01 | global_load_balancing_loss: 9.995726E-01 | loss scale: 1.0 | grad norm: 0.051 | params norm: 11249.781 | params rms: 0.060914 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 7:15:42.152169 | finish at 2026-10-10 20:02:19 + [2026-10-10 12:48:06] iteration 6555/ 6855 | consumed samples: 839040 | elapsed time per iteration (ms): 89032.2 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.73% | FLOPs estimator: workload-aware | logical tokens/s: 188439.5 | valid tokens/s: 188439.5 | physical tokens/s: 188735.9 | loss tokens/s: 129516.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.345007E-01 | lm-global-supervised-tokens: 1.153112E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.139479E-02 | lm-token-buffer-mib: 5.807104E+01 | global_load_balancing_loss: 1.000548E+00 | loss scale: 1.0 | grad norm: 0.039 | params norm: 11249.734 | params rms: 0.060914 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 7:25:09.674549 | finish at 2026-10-10 20:13:16 + [2026-10-10 12:49:37] iteration 6556/ 6855 | consumed samples: 839168 | elapsed time per iteration (ms): 91527.0 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.47% | FLOPs estimator: workload-aware | logical tokens/s: 183303.0 | valid tokens/s: 183303.0 | physical tokens/s: 183616.7 | loss tokens/s: 137445.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.902430E-01 | lm-global-supervised-tokens: 1.257995E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.148841E-02 | lm-token-buffer-mib: 5.806035E+01 | global_load_balancing_loss: 9.984788E-01 | loss scale: 1.0 | grad norm: 0.053 | params norm: 11249.687 | params rms: 0.060913 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 7:36:06.572629 | finish at 2026-10-10 20:25:44 + [2026-10-10 12:51:02] iteration 6557/ 6855 | consumed samples: 839296 | elapsed time per iteration (ms): 84660.6 | throughput per GPU (TFLOP/s/GPU): 44.0 | MFU 14.09% | FLOPs estimator: workload-aware | logical tokens/s: 198169.7 | valid tokens/s: 198169.7 | physical tokens/s: 198797.2 | loss tokens/s: 145081.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.652279E-01 | lm-global-supervised-tokens: 1.228264E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.476236E-02 | lm-token-buffer-mib: 5.817709E+01 | global_load_balancing_loss: 9.995111E-01 | loss scale: 1.0 | grad norm: 0.045 | params norm: 11249.639 | params rms: 0.060913 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 7:00:28.850187 | finish at 2026-10-10 19:51:31 + [2026-10-10 12:52:32] iteration 6558/ 6855 | consumed samples: 839424 | elapsed time per iteration (ms): 90293.1 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.52% | FLOPs estimator: workload-aware | logical tokens/s: 185808.2 | valid tokens/s: 185808.1 | physical tokens/s: 186095.2 | loss tokens/s: 137771.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.193521E-01 | lm-global-supervised-tokens: 1.243979E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.861103E-02 | lm-token-buffer-mib: 5.816569E+01 | global_load_balancing_loss: 9.991638E-01 | loss scale: 1.0 | grad norm: 0.048 | params norm: 11249.590 | params rms: 0.060913 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 7:26:57.040326 | finish at 2026-10-10 20:19:29 + [2026-10-10 12:54:02] iteration 6559/ 6855 | consumed samples: 839552 | elapsed time per iteration (ms): 89937.5 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.39% | FLOPs estimator: workload-aware | logical tokens/s: 186542.3 | valid tokens/s: 186542.3 | physical tokens/s: 186960.0 | loss tokens/s: 129607.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.557484E-01 | lm-global-supervised-tokens: 1.165659E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.862974E-02 | lm-token-buffer-mib: 5.810829E+01 | global_load_balancing_loss: 1.001455E+00 | loss scale: 1.0 | grad norm: 0.056 | params norm: 11249.542 | params rms: 0.060913 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 7:23:41.507198 | finish at 2026-10-10 20:17:44 + [2026-10-10 12:55:52] iteration 6560/ 6855 | consumed samples: 839680 | elapsed time per iteration (ms): 109893.8 | throughput per GPU (TFLOP/s/GPU): 33.6 | MFU 10.76% | FLOPs estimator: workload-aware | logical tokens/s: 152667.2 | valid tokens/s: 152667.2 | physical tokens/s: 153135.9 | loss tokens/s: 108452.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.422461E-01 | lm-global-supervised-tokens: 1.191822E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.875019E-02 | lm-token-buffer-mib: 5.812241E+01 | global_load_balancing_loss: 1.001260E+00 | loss scale: 1.0 | grad norm: 0.044 | params norm: 11249.497 | params rms: 0.060912 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 9:00:18.676070 | finish at 2026-10-10 21:56:11 +times across ranks (ms): + save-checkpoint: + rank 0: 23899.99 + rank 1: 23900.02 + rank 2: 23900.02 + rank 3: 23900.02 + rank 4: 23900.03 + rank 5: 23900.04 + rank 6: 23899.99 + rank 7: 23900.02 + rank 8: 23899.63 + rank 9: 23899.60 + rank 10: 23899.59 + rank 11: 23899.61 + rank 12: 23899.61 + rank 13: 23899.59 + rank 14: 23899.61 + rank 15: 23899.59 + rank 16: 23899.12 + rank 17: 23899.12 + rank 18: 23899.12 + rank 19: 23899.13 + rank 20: 23899.10 + rank 21: 23899.13 + rank 22: 23899.11 + rank 23: 23899.10 + rank 24: 23899.73 + rank 25: 23899.71 + rank 26: 23899.77 + rank 27: 23899.76 + rank 28: 23899.72 + rank 29: 23899.75 + rank 30: 23899.75 + rank 31: 23899.74 + rank 32: 23899.33 + rank 33: 23899.34 + rank 34: 23899.34 + rank 35: 23899.31 + rank 36: 23899.31 + rank 37: 23899.30 + rank 38: 23899.32 + rank 39: 23899.31 + rank 40: 23899.31 + rank 41: 23899.31 + rank 42: 23899.28 + rank 43: 23899.32 + rank 44: 23899.27 + rank 45: 23899.28 + rank 46: 23899.28 + rank 47: 23899.28 + rank 48: 23898.90 + rank 49: 23898.94 + rank 50: 23898.92 + rank 51: 23898.89 + rank 52: 23898.93 + rank 53: 23898.89 + rank 54: 23898.94 + rank 55: 23898.98 + rank 56: 23899.92 + rank 57: 23899.86 + rank 58: 23899.86 + rank 59: 23899.85 + rank 60: 23899.85 + rank 61: 23899.90 + rank 62: 23899.85 + rank 63: 23899.89 + [2026-10-10 12:57:41] iteration 6561/ 6855 | consumed samples: 839808 | elapsed time per iteration (ms): 84849.4 | throughput per GPU (TFLOP/s/GPU): 43.7 | MFU 14.01% | FLOPs estimator: workload-aware | logical tokens/s: 197729.3 | valid tokens/s: 197729.3 | physical tokens/s: 198120.0 | loss tokens/s: 149872.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.509919E-01 | lm-global-supervised-tokens: 1.271660E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 4.945239E-01 | lm-token-buffer-mib: 5.817345E+01 | global_load_balancing_loss: 1.000036E+00 | loss scale: 1.0 | grad norm: 0.050 | params norm: 11249.453 | params rms: 0.060912 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6:55:45.729501 | finish at 2026-10-10 19:53:27 + [2026-10-10 12:59:16] iteration 6562/ 6855 | consumed samples: 839936 | elapsed time per iteration (ms): 95125.4 | throughput per GPU (TFLOP/s/GPU): 38.9 | MFU 12.47% | FLOPs estimator: workload-aware | logical tokens/s: 176368.1 | valid tokens/s: 176368.1 | physical tokens/s: 176903.3 | loss tokens/s: 128740.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.690476E-01 | lm-global-supervised-tokens: 1.224645E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.534416E-02 | lm-token-buffer-mib: 5.815459E+01 | global_load_balancing_loss: 1.000135E+00 | loss scale: 1.0 | grad norm: 0.053 | params norm: 11249.409 | params rms: 0.060912 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 7:44:31.743686 | finish at 2026-10-10 20:43:48 + [2026-10-10 13:00:48] iteration 6563/ 6855 | consumed samples: 840064 | elapsed time per iteration (ms): 91370.0 | throughput per GPU (TFLOP/s/GPU): 41.3 | MFU 13.24% | FLOPs estimator: workload-aware | logical tokens/s: 183617.8 | valid tokens/s: 183617.8 | physical tokens/s: 184040.4 | loss tokens/s: 136227.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.330062E-01 | lm-global-supervised-tokens: 1.244708E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.167385E-02 | lm-token-buffer-mib: 5.815290E+01 | global_load_balancing_loss: 1.001402E+00 | loss scale: 1.0 | grad norm: 0.055 | params norm: 11249.365 | params rms: 0.060912 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 7:24:40.025904 | finish at 2026-10-10 20:25:28 + [2026-10-10 13:02:15] iteration 6564/ 6855 | consumed samples: 840192 | elapsed time per iteration (ms): 86740.5 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.65% | FLOPs estimator: workload-aware | logical tokens/s: 193418.0 | valid tokens/s: 193418.0 | physical tokens/s: 193959.6 | loss tokens/s: 147430.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.128015E-01 | lm-global-supervised-tokens: 1.278818E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.178918E-02 | lm-token-buffer-mib: 5.821951E+01 | global_load_balancing_loss: 1.000346E+00 | loss scale: 1.0 | grad norm: 0.101 | params norm: 11249.319 | params rms: 0.060911 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 7:00:41.497148 | finish at 2026-10-10 20:02:56 + [2026-10-10 13:03:44] iteration 6565/ 6855 | consumed samples: 840320 | elapsed time per iteration (ms): 89565.2 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.67% | FLOPs estimator: workload-aware | logical tokens/s: 187317.4 | valid tokens/s: 187317.4 | physical tokens/s: 187718.8 | loss tokens/s: 133455.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.558923E-01 | lm-global-supervised-tokens: 1.195295E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.598155E-02 | lm-token-buffer-mib: 5.802702E+01 | global_load_balancing_loss: 1.000350E+00 | loss scale: 1.0 | grad norm: 0.107 | params norm: 11249.274 | params rms: 0.060911 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 7:12:53.910377 | finish at 2026-10-10 20:16:38 + [2026-10-10 13:05:13] iteration 6566/ 6855 | consumed samples: 840448 | elapsed time per iteration (ms): 88370.2 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.57% | FLOPs estimator: workload-aware | logical tokens/s: 189851.3 | valid tokens/s: 189851.3 | physical tokens/s: 190304.7 | loss tokens/s: 134181.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.194220E-01 | lm-global-supervised-tokens: 1.185767E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.151554E-02 | lm-token-buffer-mib: 5.807428E+01 | global_load_balancing_loss: 1.001583E+00 | loss scale: 1.0 | grad norm: 0.033 | params norm: 11249.230 | params rms: 0.060911 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 7:05:38.980886 | finish at 2026-10-10 20:10:52 + [2026-10-10 13:06:38] iteration 6567/ 6855 | consumed samples: 840576 | elapsed time per iteration (ms): 84975.5 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.77% | FLOPs estimator: workload-aware | logical tokens/s: 197435.9 | valid tokens/s: 197435.9 | physical tokens/s: 197994.6 | loss tokens/s: 149767.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.752432E-01 | lm-global-supervised-tokens: 1.272660E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.530840E-02 | lm-token-buffer-mib: 5.805013E+01 | global_load_balancing_loss: 9.993928E-01 | loss scale: 1.0 | grad norm: 0.104 | params norm: 11249.188 | params rms: 0.060911 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6:47:52.940872 | finish at 2026-10-10 19:54:30 + [2026-10-10 13:08:05] iteration 6568/ 6855 | consumed samples: 840704 | elapsed time per iteration (ms): 87620.8 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.67% | FLOPs estimator: workload-aware | logical tokens/s: 191475.3 | valid tokens/s: 191475.3 | physical tokens/s: 191865.0 | loss tokens/s: 143671.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.563281E-01 | lm-global-supervised-tokens: 1.258864E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.407989E-02 | lm-token-buffer-mib: 5.806749E+01 | global_load_balancing_loss: 9.996710E-01 | loss scale: 1.0 | grad norm: 0.117 | params norm: 11249.144 | params rms: 0.060910 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6:59:07.163378 | finish at 2026-10-10 20:07:12 + [2026-10-10 13:09:33] iteration 6569/ 6855 | consumed samples: 840832 | elapsed time per iteration (ms): 87776.0 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.50% | FLOPs estimator: workload-aware | logical tokens/s: 191136.3 | valid tokens/s: 191136.3 | physical tokens/s: 191429.5 | loss tokens/s: 143075.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.239754E-01 | lm-global-supervised-tokens: 1.255859E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.319695E-02 | lm-token-buffer-mib: 5.808987E+01 | global_load_balancing_loss: 9.992432E-01 | loss scale: 1.0 | grad norm: 0.040 | params norm: 11249.102 | params rms: 0.060910 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6:58:23.926801 | finish at 2026-10-10 20:07:57 + [2026-10-10 13:11:20] iteration 6570/ 6855 | consumed samples: 840960 | elapsed time per iteration (ms): 107253.0 | throughput per GPU (TFLOP/s/GPU): 33.8 | MFU 10.85% | FLOPs estimator: workload-aware | logical tokens/s: 156426.4 | valid tokens/s: 156426.4 | physical tokens/s: 156749.9 | loss tokens/s: 113200.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.387084E-01 | lm-global-supervised-tokens: 1.214112E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.150856E-02 | lm-token-buffer-mib: 5.806548E+01 | global_load_balancing_loss: 9.992065E-01 | loss scale: 1.0 | grad norm: 0.064 | params norm: 11249.060 | params rms: 0.060910 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 8:29:27.109423 | finish at 2026-10-10 21:40:47 + [2026-10-10 13:12:45] iteration 6571/ 6855 | consumed samples: 841088 | elapsed time per iteration (ms): 84865.2 | throughput per GPU (TFLOP/s/GPU): 43.1 | MFU 13.82% | FLOPs estimator: workload-aware | logical tokens/s: 197692.5 | valid tokens/s: 197692.5 | physical tokens/s: 198253.2 | loss tokens/s: 147073.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.818098E-01 | lm-global-supervised-tokens: 1.248140E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.143171E-02 | lm-token-buffer-mib: 5.806825E+01 | global_load_balancing_loss: 9.990601E-01 | loss scale: 1.0 | grad norm: 0.109 | params norm: 11249.017 | params rms: 0.060910 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6:41:41.703270 | finish at 2026-10-10 19:54:27 + [2026-10-10 13:14:15] iteration 6572/ 6855 | consumed samples: 841216 | elapsed time per iteration (ms): 89956.7 | throughput per GPU (TFLOP/s/GPU): 40.7 | MFU 13.03% | FLOPs estimator: workload-aware | logical tokens/s: 186502.4 | valid tokens/s: 186502.4 | physical tokens/s: 186893.7 | loss tokens/s: 137050.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.276481E-01 | lm-global-supervised-tokens: 1.232862E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.578978E-02 | lm-token-buffer-mib: 5.805311E+01 | global_load_balancing_loss: 1.001729E+00 | loss scale: 1.0 | grad norm: 0.131 | params norm: 11248.975 | params rms: 0.060910 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 7:04:17.732900 | finish at 2026-10-10 20:18:33 + [2026-10-10 13:15:38] iteration 6573/ 6855 | consumed samples: 841344 | elapsed time per iteration (ms): 82922.2 | throughput per GPU (TFLOP/s/GPU): 43.5 | MFU 13.93% | FLOPs estimator: workload-aware | logical tokens/s: 202324.3 | valid tokens/s: 202324.3 | physical tokens/s: 203001.0 | loss tokens/s: 151910.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.489053E-01 | lm-global-supervised-tokens: 1.259673E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.634526E-02 | lm-token-buffer-mib: 5.829582E+01 | global_load_balancing_loss: 1.000579E+00 | loss scale: 1.0 | grad norm: 0.051 | params norm: 11248.933 | params rms: 0.060909 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6:29:44.064559 | finish at 2026-10-10 19:45:22 + [2026-10-10 13:17:10] iteration 6574/ 6855 | consumed samples: 841472 | elapsed time per iteration (ms): 91693.4 | throughput per GPU (TFLOP/s/GPU): 41.1 | MFU 13.16% | FLOPs estimator: workload-aware | logical tokens/s: 182970.7 | valid tokens/s: 182970.7 | physical tokens/s: 183439.1 | loss tokens/s: 129778.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.799809E-01 | lm-global-supervised-tokens: 1.189986E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.402081E-02 | lm-token-buffer-mib: 5.814481E+01 | global_load_balancing_loss: 1.000769E+00 | loss scale: 1.0 | grad norm: 0.085 | params norm: 11248.889 | params rms: 0.060909 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 7:09:25.855356 | finish at 2026-10-10 20:26:35 + [2026-10-10 13:18:39] iteration 6575/ 6855 | consumed samples: 841600 | elapsed time per iteration (ms): 89268.5 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.42% | FLOPs estimator: workload-aware | logical tokens/s: 187940.9 | valid tokens/s: 187940.9 | physical tokens/s: 188252.3 | loss tokens/s: 138062.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.879912E-01 | lm-global-supervised-tokens: 1.232461E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.193169E-02 | lm-token-buffer-mib: 5.810818E+01 | global_load_balancing_loss: 1.001712E+00 | loss scale: 1.0 | grad norm: 0.038 | params norm: 11248.844 | params rms: 0.060909 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6:56:35.193777 | finish at 2026-10-10 20:15:14 + [2026-10-10 13:20:05] iteration 6576/ 6855 | consumed samples: 841728 | elapsed time per iteration (ms): 86562.0 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.64% | FLOPs estimator: workload-aware | logical tokens/s: 193816.7 | valid tokens/s: 193816.7 | physical tokens/s: 194230.8 | loss tokens/s: 144056.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.847056E-01 | lm-global-supervised-tokens: 1.246980E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.132420E-02 | lm-token-buffer-mib: 5.810298E+01 | global_load_balancing_loss: 1.000163E+00 | loss scale: 1.0 | grad norm: 0.067 | params norm: 11248.799 | params rms: 0.060909 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6:42:30.798077 | finish at 2026-10-10 20:02:36 + [2026-10-10 13:21:34] iteration 6577/ 6855 | consumed samples: 841856 | elapsed time per iteration (ms): 88972.1 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.43% | FLOPs estimator: workload-aware | logical tokens/s: 188566.4 | valid tokens/s: 188566.4 | physical tokens/s: 188994.6 | loss tokens/s: 135392.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.875305E-01 | lm-global-supervised-tokens: 1.204615E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.265601E-02 | lm-token-buffer-mib: 5.810857E+01 | global_load_balancing_loss: 1.000779E+00 | loss scale: 1.0 | grad norm: 0.050 | params norm: 11248.758 | params rms: 0.060908 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6:52:14.251825 | finish at 2026-10-10 20:13:49 + [2026-10-10 13:23:00] iteration 6578/ 6855 | consumed samples: 841984 | elapsed time per iteration (ms): 85947.0 | throughput per GPU (TFLOP/s/GPU): 43.5 | MFU 13.93% | FLOPs estimator: workload-aware | logical tokens/s: 195202.5 | valid tokens/s: 195202.5 | physical tokens/s: 195689.5 | loss tokens/s: 137942.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.392286E-01 | lm-global-supervised-tokens: 1.185573E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.483395E-02 | lm-token-buffer-mib: 5.812443E+01 | global_load_balancing_loss: 9.991210E-01 | loss scale: 1.0 | grad norm: 0.042 | params norm: 11248.715 | params rms: 0.060908 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6:36:47.327395 | finish at 2026-10-10 19:59:48 + [2026-10-10 13:24:32] iteration 6579/ 6855 | consumed samples: 842112 | elapsed time per iteration (ms): 91189.8 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.41% | FLOPs estimator: workload-aware | logical tokens/s: 183980.7 | valid tokens/s: 183980.7 | physical tokens/s: 184310.4 | loss tokens/s: 132420.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.931472E-01 | lm-global-supervised-tokens: 1.207538E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.210965E-02 | lm-token-buffer-mib: 5.808508E+01 | global_load_balancing_loss: 9.985422E-01 | loss scale: 1.0 | grad norm: 0.044 | params norm: 11248.672 | params rms: 0.060908 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6:59:28.392571 | finish at 2026-10-10 20:24:00 + [2026-10-10 13:26:19] iteration 6580/ 6855 | consumed samples: 842240 | elapsed time per iteration (ms): 107480.0 | throughput per GPU (TFLOP/s/GPU): 34.5 | MFU 11.07% | FLOPs estimator: workload-aware | logical tokens/s: 156095.2 | valid tokens/s: 156095.2 | physical tokens/s: 156415.6 | loss tokens/s: 117290.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 5.871520E-01 | lm-global-supervised-tokens: 1.260641E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.525723E-02 | lm-token-buffer-mib: 5.811385E+01 | global_load_balancing_loss: 9.995483E-01 | loss scale: 1.0 | grad norm: 0.049 | params norm: 11248.629 | params rms: 0.060908 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 8:12:37.004791 | finish at 2026-10-10 21:38:56 + [2026-10-10 13:27:43] iteration 6581/ 6855 | consumed samples: 842368 | elapsed time per iteration (ms): 84045.1 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.42% | FLOPs estimator: workload-aware | logical tokens/s: 199620.0 | valid tokens/s: 199620.0 | physical tokens/s: 200141.5 | loss tokens/s: 149275.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.059152E-01 | lm-global-supervised-tokens: 1.254583E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.475490E-02 | lm-token-buffer-mib: 5.808608E+01 | global_load_balancing_loss: 1.001137E+00 | loss scale: 1.0 | grad norm: 0.077 | params norm: 11248.587 | params rms: 0.060907 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6:23:48.360659 | finish at 2026-10-10 19:51:31 + [2026-10-10 13:29:08] iteration 6582/ 6855 | consumed samples: 842496 | elapsed time per iteration (ms): 84826.6 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.78% | FLOPs estimator: workload-aware | logical tokens/s: 197781.9 | valid tokens/s: 197781.9 | physical tokens/s: 198526.7 | loss tokens/s: 145657.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.389835E-01 | lm-global-supervised-tokens: 1.235558E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.753336E-02 | lm-token-buffer-mib: 5.809410E+01 | global_load_balancing_loss: 1.000530E+00 | loss scale: 1.0 | grad norm: 0.050 | params norm: 11248.546 | params rms: 0.060907 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6:25:57.648152 | finish at 2026-10-10 19:55:06 + [2026-10-10 13:30:36] iteration 6583/ 6855 | consumed samples: 842624 | elapsed time per iteration (ms): 88400.3 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.40% | FLOPs estimator: workload-aware | logical tokens/s: 189786.9 | valid tokens/s: 189786.9 | physical tokens/s: 190128.1 | loss tokens/s: 137292.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.629506E-01 | lm-global-supervised-tokens: 1.213671E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.581703E-02 | lm-token-buffer-mib: 5.821684E+01 | global_load_balancing_loss: 1.001338E+00 | loss scale: 1.0 | grad norm: 0.055 | params norm: 11248.500 | params rms: 0.060907 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6:40:44.877586 | finish at 2026-10-10 20:11:21 + [2026-10-10 13:32:01] iteration 6584/ 6855 | consumed samples: 842752 | elapsed time per iteration (ms): 84272.9 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.75% | FLOPs estimator: workload-aware | logical tokens/s: 199081.2 | valid tokens/s: 199081.2 | physical tokens/s: 199728.5 | loss tokens/s: 156043.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.456330E-01 | lm-global-supervised-tokens: 1.315026E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.189980E-02 | lm-token-buffer-mib: 5.826094E+01 | global_load_balancing_loss: 1.000269E+00 | loss scale: 1.0 | grad norm: 0.110 | params norm: 11248.453 | params rms: 0.060907 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6:20:37.955864 | finish at 2026-10-10 19:52:39 + [2026-10-10 13:33:26] iteration 6585/ 6855 | consumed samples: 842880 | elapsed time per iteration (ms): 85390.8 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.44% | FLOPs estimator: workload-aware | logical tokens/s: 196475.6 | valid tokens/s: 196475.6 | physical tokens/s: 197002.8 | loss tokens/s: 148538.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.188558E-01 | lm-global-supervised-tokens: 1.268382E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.151608E-02 | lm-token-buffer-mib: 5.810401E+01 | global_load_balancing_loss: 9.999828E-01 | loss scale: 1.0 | grad norm: 0.081 | params norm: 11248.407 | params rms: 0.060906 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6:24:15.509305 | finish at 2026-10-10 19:57:41 + [2026-10-10 13:34:52] iteration 6586/ 6855 | consumed samples: 843008 | elapsed time per iteration (ms): 86355.8 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.64% | FLOPs estimator: workload-aware | logical tokens/s: 194279.5 | valid tokens/s: 194279.5 | physical tokens/s: 194921.4 | loss tokens/s: 141269.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.421754E-01 | lm-global-supervised-tokens: 1.219944E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.131962E-02 | lm-token-buffer-mib: 5.806885E+01 | global_load_balancing_loss: 9.995661E-01 | loss scale: 1.0 | grad norm: 0.082 | params norm: 11248.360 | params rms: 0.060906 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6:27:09.710946 | finish at 2026-10-10 20:02:02 + [2026-10-10 13:36:20] iteration 6587/ 6855 | consumed samples: 843136 | elapsed time per iteration (ms): 87567.0 | throughput per GPU (TFLOP/s/GPU): 41.6 | MFU 13.32% | FLOPs estimator: workload-aware | logical tokens/s: 191592.7 | valid tokens/s: 191592.7 | physical tokens/s: 192043.8 | loss tokens/s: 145215.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.799981E-01 | lm-global-supervised-tokens: 1.271612E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.597182E-02 | lm-token-buffer-mib: 5.818900E+01 | global_load_balancing_loss: 1.000137E+00 | loss scale: 1.0 | grad norm: 0.127 | params norm: 11248.312 | params rms: 0.060906 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6:31:07.964986 | finish at 2026-10-10 20:07:28 + [2026-10-10 13:37:44] iteration 6588/ 6855 | consumed samples: 843264 | elapsed time per iteration (ms): 84605.4 | throughput per GPU (TFLOP/s/GPU): 43.7 | MFU 14.00% | FLOPs estimator: workload-aware | logical tokens/s: 198299.4 | valid tokens/s: 198299.4 | physical tokens/s: 198779.7 | loss tokens/s: 153010.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.721112E-01 | lm-global-supervised-tokens: 1.294550E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.122223E-02 | lm-token-buffer-mib: 5.815834E+01 | global_load_balancing_loss: 1.000237E+00 | loss scale: 1.0 | grad norm: 0.046 | params norm: 11248.265 | params rms: 0.060906 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6:16:29.643542 | finish at 2026-10-10 19:54:14 + [2026-10-10 13:39:14] iteration 6589/ 6855 | consumed samples: 843392 | elapsed time per iteration (ms): 89488.5 | throughput per GPU (TFLOP/s/GPU): 41.5 | MFU 13.30% | FLOPs estimator: workload-aware | logical tokens/s: 187477.9 | valid tokens/s: 187477.9 | physical tokens/s: 187928.6 | loss tokens/s: 140669.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.481902E-01 | lm-global-supervised-tokens: 1.258834E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.149405E-02 | lm-token-buffer-mib: 5.820851E+01 | global_load_balancing_loss: 1.000229E+00 | loss scale: 1.0 | grad norm: 0.135 | params norm: 11248.219 | params rms: 0.060905 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6:36:43.950418 | finish at 2026-10-10 20:15:58 + [2026-10-10 13:41:02] iteration 6590/ 6855 | consumed samples: 843520 | elapsed time per iteration (ms): 107868.5 | throughput per GPU (TFLOP/s/GPU): 34.0 | MFU 10.90% | FLOPs estimator: workload-aware | logical tokens/s: 155532.9 | valid tokens/s: 155532.9 | physical tokens/s: 155897.7 | loss tokens/s: 117052.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.338445E-01 | lm-global-supervised-tokens: 1.262627E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.426089E-02 | lm-token-buffer-mib: 5.806837E+01 | global_load_balancing_loss: 1.000891E+00 | loss scale: 1.0 | grad norm: 0.054 | params norm: 11248.174 | params rms: 0.060905 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 7:56:25.144475 | finish at 2026-10-10 21:37:27 + [2026-10-10 13:42:31] iteration 6591/ 6855 | consumed samples: 843648 | elapsed time per iteration (ms): 89159.8 | throughput per GPU (TFLOP/s/GPU): 41.4 | MFU 13.28% | FLOPs estimator: workload-aware | logical tokens/s: 188169.7 | valid tokens/s: 188169.7 | physical tokens/s: 188538.1 | loss tokens/s: 135997.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.210659E-01 | lm-global-supervised-tokens: 1.212552E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.140220E-02 | lm-token-buffer-mib: 5.814176E+01 | global_load_balancing_loss: 1.000766E+00 | loss scale: 1.0 | grad norm: 0.098 | params norm: 11248.127 | params rms: 0.060905 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6:32:18.181864 | finish at 2026-10-10 20:14:49 + [2026-10-10 13:43:56] iteration 6592/ 6855 | consumed samples: 843776 | elapsed time per iteration (ms): 85391.1 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.75% | FLOPs estimator: workload-aware | logical tokens/s: 196474.5 | valid tokens/s: 196474.5 | physical tokens/s: 196910.4 | loss tokens/s: 149037.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.553690E-01 | lm-global-supervised-tokens: 1.272646E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.467788E-02 | lm-token-buffer-mib: 5.815475E+01 | global_load_balancing_loss: 9.989668E-01 | loss scale: 1.0 | grad norm: 0.033 | params norm: 11248.079 | params rms: 0.060905 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6:14:17.847430 | finish at 2026-10-10 19:58:14 + [2026-10-10 13:45:21] iteration 6593/ 6855 | consumed samples: 843904 | elapsed time per iteration (ms): 84914.7 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.65% | FLOPs estimator: workload-aware | logical tokens/s: 197577.0 | valid tokens/s: 197577.0 | physical tokens/s: 198294.3 | loss tokens/s: 148273.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.154686E-01 | lm-global-supervised-tokens: 1.259062E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.139055E-02 | lm-token-buffer-mib: 5.818548E+01 | global_load_balancing_loss: 1.000105E+00 | loss scale: 1.0 | grad norm: 0.143 | params norm: 11248.032 | params rms: 0.060904 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6:10:47.645287 | finish at 2026-10-10 19:56:09 + [2026-10-10 13:46:52] iteration 6594/ 6855 | consumed samples: 844032 | elapsed time per iteration (ms): 90996.9 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.39% | FLOPs estimator: workload-aware | logical tokens/s: 184371.1 | valid tokens/s: 184371.1 | physical tokens/s: 184873.7 | loss tokens/s: 136004.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.594110E-01 | lm-global-supervised-tokens: 1.237596E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.894367E-02 | lm-token-buffer-mib: 5.842035E+01 | global_load_balancing_loss: 1.001537E+00 | loss scale: 1.0 | grad norm: 0.070 | params norm: 11247.986 | params rms: 0.060904 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6:35:50.189303 | finish at 2026-10-10 20:22:43 + [2026-10-10 13:48:16] iteration 6595/ 6855 | consumed samples: 844160 | elapsed time per iteration (ms): 83456.2 | throughput per GPU (TFLOP/s/GPU): 43.3 | MFU 13.89% | FLOPs estimator: workload-aware | logical tokens/s: 201030.3 | valid tokens/s: 201030.3 | physical tokens/s: 201518.4 | loss tokens/s: 151665.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.595065E-01 | lm-global-supervised-tokens: 1.265738E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.165883E-02 | lm-token-buffer-mib: 5.823873E+01 | global_load_balancing_loss: 9.991215E-01 | loss scale: 1.0 | grad norm: 0.046 | params norm: 11247.940 | params rms: 0.060904 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6:01:38.604779 | finish at 2026-10-10 19:49:54 + [2026-10-10 13:49:41] iteration 6596/ 6855 | consumed samples: 844288 | elapsed time per iteration (ms): 85672.8 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.77% | FLOPs estimator: workload-aware | logical tokens/s: 195828.7 | valid tokens/s: 195828.7 | physical tokens/s: 196190.2 | loss tokens/s: 147764.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.778025E-01 | lm-global-supervised-tokens: 1.265937E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.128106E-02 | lm-token-buffer-mib: 5.809620E+01 | global_load_balancing_loss: 1.000155E+00 | loss scale: 1.0 | grad norm: 0.054 | params norm: 11247.892 | params rms: 0.060904 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6:09:49.249412 | finish at 2026-10-10 19:59:31 + [2026-10-10 13:51:09] iteration 6597/ 6855 | consumed samples: 844416 | elapsed time per iteration (ms): 87401.2 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.69% | FLOPs estimator: workload-aware | logical tokens/s: 191955.4 | valid tokens/s: 191955.4 | physical tokens/s: 192487.4 | loss tokens/s: 138989.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.609892E-01 | lm-global-supervised-tokens: 1.214787E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.140994E-02 | lm-token-buffer-mib: 5.820033E+01 | global_load_balancing_loss: 9.999665E-01 | loss scale: 1.0 | grad norm: 0.037 | params norm: 11247.844 | params rms: 0.060903 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6:15:49.516258 | finish at 2026-10-10 20:06:58 + [2026-10-10 13:52:41] iteration 6598/ 6855 | consumed samples: 844544 | elapsed time per iteration (ms): 92513.2 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.43% | FLOPs estimator: workload-aware | logical tokens/s: 181348.9 | valid tokens/s: 181348.9 | physical tokens/s: 181752.4 | loss tokens/s: 129358.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.946696E-01 | lm-global-supervised-tokens: 1.196732E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.156367E-02 | lm-token-buffer-mib: 5.803569E+01 | global_load_balancing_loss: 1.000755E+00 | loss scale: 1.0 | grad norm: 0.042 | params norm: 11247.796 | params rms: 0.060903 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6:36:15.890328 | finish at 2026-10-10 20:28:57 + [2026-10-10 13:54:09] iteration 6599/ 6855 | consumed samples: 844672 | elapsed time per iteration (ms): 87182.7 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.46% | FLOPs estimator: workload-aware | logical tokens/s: 192436.0 | valid tokens/s: 192436.0 | physical tokens/s: 193001.5 | loss tokens/s: 146860.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.483119E-01 | lm-global-supervised-tokens: 1.280368E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.426796E-02 | lm-token-buffer-mib: 5.847442E+01 | global_load_balancing_loss: 9.996780E-01 | loss scale: 1.0 | grad norm: 0.043 | params norm: 11247.748 | params rms: 0.060903 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6:11:58.762695 | finish at 2026-10-10 20:06:07 + [2026-10-10 13:56:02] iteration 6600/ 6855 | consumed samples: 844800 | elapsed time per iteration (ms): 113259.1 | throughput per GPU (TFLOP/s/GPU): 33.0 | MFU 10.58% | FLOPs estimator: workload-aware | logical tokens/s: 148130.9 | valid tokens/s: 148130.9 | physical tokens/s: 148501.7 | loss tokens/s: 108319.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.160946E-01 | lm-global-supervised-tokens: 1.226818E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.159994E-02 | lm-token-buffer-mib: 5.805326E+01 | global_load_balancing_loss: 1.000221E+00 | loss scale: 1.0 | grad norm: 0.050 | params norm: 11247.702 | params rms: 0.060903 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 8:01:21.070368 | finish at 2026-10-10 21:57:23 +times across ranks (ms): + save-checkpoint: + rank 0: 24781.89 + rank 1: 24781.91 + rank 2: 24781.90 + rank 3: 24781.88 + rank 4: 24781.88 + rank 5: 24781.88 + rank 6: 24781.87 + rank 7: 24781.87 + rank 8: 24782.42 + rank 9: 24782.37 + rank 10: 24782.36 + rank 11: 24782.37 + rank 12: 24782.38 + rank 13: 24782.37 + rank 14: 24782.41 + rank 15: 24782.36 + rank 16: 24783.03 + rank 17: 24782.97 + rank 18: 24782.98 + rank 19: 24782.97 + rank 20: 24782.95 + rank 21: 24783.04 + rank 22: 24782.95 + rank 23: 24782.95 + rank 24: 24782.30 + rank 25: 24782.19 + rank 26: 24782.40 + rank 27: 24782.33 + rank 28: 24782.28 + rank 29: 24782.42 + rank 30: 24782.26 + rank 31: 24782.20 + rank 32: 24782.85 + rank 33: 24782.81 + rank 34: 24782.81 + rank 35: 24782.84 + rank 36: 24782.69 + rank 37: 24782.74 + rank 38: 24782.75 + rank 39: 24782.72 + rank 40: 24782.17 + rank 41: 24782.13 + rank 42: 24782.19 + rank 43: 24782.18 + rank 44: 24782.12 + rank 45: 24782.17 + rank 46: 24782.17 + rank 47: 24781.97 + rank 48: 24781.24 + rank 49: 24781.16 + rank 50: 24781.14 + rank 51: 24781.12 + rank 52: 24781.18 + rank 53: 24781.07 + rank 54: 24781.17 + rank 55: 24781.18 + rank 56: 24782.21 + rank 57: 24782.19 + rank 58: 24782.16 + rank 59: 24782.16 + rank 60: 24782.16 + rank 61: 24782.19 + rank 62: 24782.16 + rank 63: 24781.44 + [2026-10-10 13:58:21] iteration 6601/ 6855 | consumed samples: 844928 | elapsed time per iteration (ms): 114151.1 | throughput per GPU (TFLOP/s/GPU): 31.9 | MFU 10.23% | FLOPs estimator: workload-aware | logical tokens/s: 146973.3 | valid tokens/s: 146973.3 | physical tokens/s: 147529.8 | loss tokens/s: 110418.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.466807E-01 | lm-global-supervised-tokens: 1.260442E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.252488E-02 | lm-token-buffer-mib: 5.826735E+01 | global_load_balancing_loss: 9.997835E-01 | loss scale: 1.0 | grad norm: 0.066 | params norm: 11247.655 | params rms: 0.060902 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 8:03:14.385859 | finish at 2026-10-10 22:01:35 + [2026-10-10 13:59:57] iteration 6602/ 6855 | consumed samples: 845056 | elapsed time per iteration (ms): 96264.7 | throughput per GPU (TFLOP/s/GPU): 38.9 | MFU 12.47% | FLOPs estimator: workload-aware | logical tokens/s: 174282.1 | valid tokens/s: 174282.1 | physical tokens/s: 174649.9 | loss tokens/s: 126055.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.305166E-01 | lm-global-supervised-tokens: 1.213470E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.318926E-02 | lm-token-buffer-mib: 5.812333E+01 | global_load_balancing_loss: 9.999831E-01 | loss scale: 1.0 | grad norm: 0.068 | params norm: 11247.608 | params rms: 0.060902 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6:45:54.967938 | finish at 2026-10-10 20:45:52 + [2026-10-10 14:01:25] iteration 6603/ 6855 | consumed samples: 845184 | elapsed time per iteration (ms): 88240.5 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.52% | FLOPs estimator: workload-aware | logical tokens/s: 190130.1 | valid tokens/s: 190130.1 | physical tokens/s: 190616.2 | loss tokens/s: 139007.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.112533E-01 | lm-global-supervised-tokens: 1.226612E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.460049E-02 | lm-token-buffer-mib: 5.802289E+01 | global_load_balancing_loss: 1.000732E+00 | loss scale: 1.0 | grad norm: 0.072 | params norm: 11247.563 | params rms: 0.060902 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6:10:36.598063 | finish at 2026-10-10 20:12:02 + [2026-10-10 14:02:52] iteration 6604/ 6855 | consumed samples: 845312 | elapsed time per iteration (ms): 86412.2 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.74% | FLOPs estimator: workload-aware | logical tokens/s: 194152.9 | valid tokens/s: 194152.9 | physical tokens/s: 194657.0 | loss tokens/s: 140151.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.359860E-01 | lm-global-supervised-tokens: 1.211077E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.291041E-02 | lm-token-buffer-mib: 5.824142E+01 | global_load_balancing_loss: 1.000860E+00 | loss scale: 1.0 | grad norm: 0.065 | params norm: 11247.519 | params rms: 0.060902 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6:01:29.472846 | finish at 2026-10-10 20:04:21 + [2026-10-10 14:04:20] iteration 6605/ 6855 | consumed samples: 845440 | elapsed time per iteration (ms): 87736.4 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.67% | FLOPs estimator: workload-aware | logical tokens/s: 191222.3 | valid tokens/s: 191222.3 | physical tokens/s: 191630.5 | loss tokens/s: 144702.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.813102E-01 | lm-global-supervised-tokens: 1.269569E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.521308E-02 | lm-token-buffer-mib: 5.810078E+01 | global_load_balancing_loss: 9.998292E-01 | loss scale: 1.0 | grad norm: 0.227 | params norm: 11247.475 | params rms: 0.060901 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6:05:34.103847 | finish at 2026-10-10 20:09:54 + [2026-10-10 14:05:43] iteration 6606/ 6855 | consumed samples: 845568 | elapsed time per iteration (ms): 83523.6 | throughput per GPU (TFLOP/s/GPU): 43.4 | MFU 13.90% | FLOPs estimator: workload-aware | logical tokens/s: 200867.7 | valid tokens/s: 200867.7 | physical tokens/s: 201401.0 | loss tokens/s: 148568.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.682879E-01 | lm-global-supervised-tokens: 1.240899E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.534410E-02 | lm-token-buffer-mib: 5.816584E+01 | global_load_balancing_loss: 9.992149E-01 | loss scale: 1.0 | grad norm: 0.270 | params norm: 11247.427 | params rms: 0.060901 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5:46:37.368826 | finish at 2026-10-10 19:52:21 + [2026-10-10 14:07:07] iteration 6607/ 6855 | consumed samples: 845696 | elapsed time per iteration (ms): 84001.3 | throughput per GPU (TFLOP/s/GPU): 43.8 | MFU 14.05% | FLOPs estimator: workload-aware | logical tokens/s: 199724.9 | valid tokens/s: 199724.9 | physical tokens/s: 200146.5 | loss tokens/s: 144877.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.404414E-01 | lm-global-supervised-tokens: 1.216988E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.107691E-02 | lm-token-buffer-mib: 5.815025E+01 | global_load_balancing_loss: 1.001100E+00 | loss scale: 1.0 | grad norm: 0.062 | params norm: 11247.381 | params rms: 0.060901 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5:47:12.316334 | finish at 2026-10-10 19:54:19 + [2026-10-10 14:08:34] iteration 6608/ 6855 | consumed samples: 845824 | elapsed time per iteration (ms): 86792.7 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.73% | FLOPs estimator: workload-aware | logical tokens/s: 193301.8 | valid tokens/s: 193301.8 | physical tokens/s: 193999.6 | loss tokens/s: 139585.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.911758E-01 | lm-global-supervised-tokens: 1.211502E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.552416E-02 | lm-token-buffer-mib: 5.812082E+01 | global_load_balancing_loss: 1.000581E+00 | loss scale: 1.0 | grad norm: 0.157 | params norm: 11247.335 | params rms: 0.060901 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5:57:17.807807 | finish at 2026-10-10 20:05:52 + [2026-10-10 14:10:00] iteration 6609/ 6855 | consumed samples: 845952 | elapsed time per iteration (ms): 85593.3 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.55% | FLOPs estimator: workload-aware | logical tokens/s: 196010.3 | valid tokens/s: 196010.1 | physical tokens/s: 196404.7 | loss tokens/s: 145764.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.191586E-01 | lm-global-supervised-tokens: 1.247647E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.184757E-02 | lm-token-buffer-mib: 5.805779E+01 | global_load_balancing_loss: 1.000271E+00 | loss scale: 1.0 | grad norm: 0.071 | params norm: 11247.287 | params rms: 0.060900 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5:50:55.954817 | finish at 2026-10-10 20:00:55 + [2026-10-10 14:11:46] iteration 6610/ 6855 | consumed samples: 846080 | elapsed time per iteration (ms): 106148.3 | throughput per GPU (TFLOP/s/GPU): 34.4 | MFU 11.02% | FLOPs estimator: workload-aware | logical tokens/s: 158054.3 | valid tokens/s: 158054.3 | physical tokens/s: 158327.3 | loss tokens/s: 119202.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.934615E-01 | lm-global-supervised-tokens: 1.265310E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.132660E-02 | lm-token-buffer-mib: 5.819028E+01 | global_load_balancing_loss: 1.000129E+00 | loss scale: 1.0 | grad norm: 0.072 | params norm: 11247.239 | params rms: 0.060900 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 7:13:26.343822 | finish at 2026-10-10 21:25:12 + [2026-10-10 14:13:11] iteration 6611/ 6855 | consumed samples: 846208 | elapsed time per iteration (ms): 85795.9 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.78% | FLOPs estimator: workload-aware | logical tokens/s: 195547.2 | valid tokens/s: 195547.2 | physical tokens/s: 195930.0 | loss tokens/s: 143629.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.550720E-01 | lm-global-supervised-tokens: 1.232279E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.212076E-02 | lm-token-buffer-mib: 5.805531E+01 | global_load_balancing_loss: 1.000696E+00 | loss scale: 1.0 | grad norm: 0.061 | params norm: 11247.192 | params rms: 0.060900 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5:48:54.193576 | finish at 2026-10-10 20:02:06 + [2026-10-10 14:14:37] iteration 6612/ 6855 | consumed samples: 846336 | elapsed time per iteration (ms): 85618.0 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.55% | FLOPs estimator: workload-aware | logical tokens/s: 195953.7 | valid tokens/s: 195953.7 | physical tokens/s: 196489.8 | loss tokens/s: 150711.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.201825E-01 | lm-global-supervised-tokens: 1.290363E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.471683E-02 | lm-token-buffer-mib: 5.831477E+01 | global_load_balancing_loss: 1.000620E+00 | loss scale: 1.0 | grad norm: 0.098 | params norm: 11247.145 | params rms: 0.060900 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5:46:45.167461 | finish at 2026-10-10 20:01:22 + [2026-10-10 14:16:05] iteration 6613/ 6855 | consumed samples: 846464 | elapsed time per iteration (ms): 87823.3 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.60% | FLOPs estimator: workload-aware | logical tokens/s: 191033.4 | valid tokens/s: 191033.4 | physical tokens/s: 191398.7 | loss tokens/s: 138264.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.681215E-01 | lm-global-supervised-tokens: 1.214283E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.455869E-02 | lm-token-buffer-mib: 5.812129E+01 | global_load_balancing_loss: 1.000116E+00 | loss scale: 1.0 | grad norm: 0.036 | params norm: 11247.098 | params rms: 0.060899 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5:54:13.234764 | finish at 2026-10-10 20:10:18 + [2026-10-10 14:17:31] iteration 6614/ 6855 | consumed samples: 846592 | elapsed time per iteration (ms): 86114.0 | throughput per GPU (TFLOP/s/GPU): 43.4 | MFU 13.92% | FLOPs estimator: workload-aware | logical tokens/s: 194824.8 | valid tokens/s: 194824.8 | physical tokens/s: 195177.1 | loss tokens/s: 133220.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.077059E-01 | lm-global-supervised-tokens: 1.147219E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.302448E-02 | lm-token-buffer-mib: 5.806091E+01 | global_load_balancing_loss: 9.998287E-01 | loss scale: 1.0 | grad norm: 0.045 | params norm: 11247.049 | params rms: 0.060899 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5:45:53.485167 | finish at 2026-10-10 20:03:25 + [2026-10-10 14:19:00] iteration 6615/ 6855 | consumed samples: 846720 | elapsed time per iteration (ms): 89090.4 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.56% | FLOPs estimator: workload-aware | logical tokens/s: 188316.8 | valid tokens/s: 188316.7 | physical tokens/s: 188688.9 | loss tokens/s: 133883.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.586870E-01 | lm-global-supervised-tokens: 1.192777E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.841869E-02 | lm-token-buffer-mib: 5.810722E+01 | global_load_balancing_loss: 9.994355E-01 | loss scale: 1.0 | grad norm: 0.065 | params norm: 11247.000 | params rms: 0.060899 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5:56:21.689701 | finish at 2026-10-10 20:15:22 + [2026-10-10 14:20:27] iteration 6616/ 6855 | consumed samples: 846848 | elapsed time per iteration (ms): 86870.9 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.43% | FLOPs estimator: workload-aware | logical tokens/s: 193128.1 | valid tokens/s: 193128.1 | physical tokens/s: 193638.6 | loss tokens/s: 148288.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.642621E-01 | lm-global-supervised-tokens: 1.288192E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.146031E-02 | lm-token-buffer-mib: 5.820514E+01 | global_load_balancing_loss: 9.997565E-01 | loss scale: 1.0 | grad norm: 0.039 | params norm: 11246.952 | params rms: 0.060899 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5:46:02.133285 | finish at 2026-10-10 20:06:29 + [2026-10-10 14:21:54] iteration 6617/ 6855 | consumed samples: 846976 | elapsed time per iteration (ms): 87002.3 | throughput per GPU (TFLOP/s/GPU): 41.6 | MFU 13.34% | FLOPs estimator: workload-aware | logical tokens/s: 192835.4 | valid tokens/s: 192835.4 | physical tokens/s: 193421.3 | loss tokens/s: 144804.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.384030E-01 | lm-global-supervised-tokens: 1.259832E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.399060E-02 | lm-token-buffer-mib: 5.834008E+01 | global_load_balancing_loss: 1.000426E+00 | loss scale: 1.0 | grad norm: 0.084 | params norm: 11246.903 | params rms: 0.060898 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5:45:06.546498 | finish at 2026-10-10 20:07:01 + [2026-10-10 14:23:23] iteration 6618/ 6855 | consumed samples: 847104 | elapsed time per iteration (ms): 89067.0 | throughput per GPU (TFLOP/s/GPU): 43.1 | MFU 13.82% | FLOPs estimator: workload-aware | logical tokens/s: 188366.2 | valid tokens/s: 188366.2 | physical tokens/s: 188626.7 | loss tokens/s: 132978.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.957690E-01 | lm-global-supervised-tokens: 1.184404E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.192264E-02 | lm-token-buffer-mib: 5.803377E+01 | global_load_balancing_loss: 1.000434E+00 | loss scale: 1.0 | grad norm: 0.041 | params norm: 11246.854 | params rms: 0.060898 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5:51:48.882257 | finish at 2026-10-10 20:15:12 + [2026-10-10 14:24:54] iteration 6619/ 6855 | consumed samples: 847232 | elapsed time per iteration (ms): 90927.2 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.50% | FLOPs estimator: workload-aware | logical tokens/s: 184510.6 | valid tokens/s: 184510.6 | physical tokens/s: 184936.5 | loss tokens/s: 128511.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.039044E-01 | lm-global-supervised-tokens: 1.168514E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.162900E-02 | lm-token-buffer-mib: 5.812123E+01 | global_load_balancing_loss: 1.001470E+00 | loss scale: 1.0 | grad norm: 0.063 | params norm: 11246.804 | params rms: 0.060898 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5:57:38.821638 | finish at 2026-10-10 20:22:33 + [2026-10-10 14:26:44] iteration 6620/ 6855 | consumed samples: 847360 | elapsed time per iteration (ms): 110040.9 | throughput per GPU (TFLOP/s/GPU): 33.3 | MFU 10.66% | FLOPs estimator: workload-aware | logical tokens/s: 152463.5 | valid tokens/s: 152463.5 | physical tokens/s: 152756.0 | loss tokens/s: 115071.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.737518E-01 | lm-global-supervised-tokens: 1.266253E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.613065E-02 | lm-token-buffer-mib: 5.814882E+01 | global_load_balancing_loss: 1.001055E+00 | loss scale: 1.0 | grad norm: 0.066 | params norm: 11246.756 | params rms: 0.060898 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 7:10:59.609145 | finish at 2026-10-10 21:37:44 + [2026-10-10 14:28:10] iteration 6621/ 6855 | consumed samples: 847488 | elapsed time per iteration (ms): 85662.6 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.60% | FLOPs estimator: workload-aware | logical tokens/s: 195851.9 | valid tokens/s: 195851.8 | physical tokens/s: 196286.8 | loss tokens/s: 148231.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.415493E-01 | lm-global-supervised-tokens: 1.269786E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.921627E-02 | lm-token-buffer-mib: 5.818823E+01 | global_load_balancing_loss: 9.999134E-01 | loss scale: 1.0 | grad norm: 0.091 | params norm: 11246.712 | params rms: 0.060897 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5:34:05.046178 | finish at 2026-10-10 20:02:15 + [2026-10-10 14:29:36] iteration 6622/ 6855 | consumed samples: 847616 | elapsed time per iteration (ms): 85957.9 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.49% | FLOPs estimator: workload-aware | logical tokens/s: 195178.7 | valid tokens/s: 195178.7 | physical tokens/s: 195582.0 | loss tokens/s: 146792.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.515314E-01 | lm-global-supervised-tokens: 1.261794E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.351078E-02 | lm-token-buffer-mib: 5.815516E+01 | global_load_balancing_loss: 9.999089E-01 | loss scale: 1.0 | grad norm: 0.094 | params norm: 11246.669 | params rms: 0.060897 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5:33:48.198433 | finish at 2026-10-10 20:03:24 + [2026-10-10 14:31:02] iteration 6623/ 6855 | consumed samples: 847744 | elapsed time per iteration (ms): 86024.8 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.67% | FLOPs estimator: workload-aware | logical tokens/s: 195026.6 | valid tokens/s: 195026.6 | physical tokens/s: 195493.9 | loss tokens/s: 145965.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.362332E-01 | lm-global-supervised-tokens: 1.255665E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.128902E-02 | lm-token-buffer-mib: 5.809263E+01 | global_load_balancing_loss: 9.995013E-01 | loss scale: 1.0 | grad norm: 0.091 | params norm: 11246.626 | params rms: 0.060897 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5:32:37.762243 | finish at 2026-10-10 20:03:39 + [2026-10-10 14:32:26] iteration 6624/ 6855 | consumed samples: 847872 | elapsed time per iteration (ms): 84712.1 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.40% | FLOPs estimator: workload-aware | logical tokens/s: 198048.5 | valid tokens/s: 198048.5 | physical tokens/s: 198574.2 | loss tokens/s: 154226.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.469642E-01 | lm-global-supervised-tokens: 1.306486E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.520515E-02 | lm-token-buffer-mib: 5.808025E+01 | global_load_balancing_loss: 9.999336E-01 | loss scale: 1.0 | grad norm: 0.126 | params norm: 11246.584 | params rms: 0.060897 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5:26:08.504469 | finish at 2026-10-10 19:58:35 + [2026-10-10 14:33:51] iteration 6625/ 6855 | consumed samples: 848000 | elapsed time per iteration (ms): 84332.5 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.64% | FLOPs estimator: workload-aware | logical tokens/s: 198940.4 | valid tokens/s: 198940.4 | physical tokens/s: 199503.8 | loss tokens/s: 146879.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 5.918573E-01 | lm-global-supervised-tokens: 1.238673E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.437141E-02 | lm-token-buffer-mib: 5.811309E+01 | global_load_balancing_loss: 1.000809E+00 | loss scale: 1.0 | grad norm: 0.170 | params norm: 11246.542 | params rms: 0.060896 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5:23:16.475434 | finish at 2026-10-10 19:57:07 + [2026-10-10 14:35:17] iteration 6626/ 6855 | consumed samples: 848128 | elapsed time per iteration (ms): 86314.2 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.74% | FLOPs estimator: workload-aware | logical tokens/s: 194373.6 | valid tokens/s: 194373.6 | physical tokens/s: 194730.2 | loss tokens/s: 142478.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.591859E-01 | lm-global-supervised-tokens: 1.229787E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.185554E-02 | lm-token-buffer-mib: 5.808165E+01 | global_load_balancing_loss: 1.000197E+00 | loss scale: 1.0 | grad norm: 0.103 | params norm: 11246.499 | params rms: 0.060896 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5:29:25.940481 | finish at 2026-10-10 20:04:43 + [2026-10-10 14:36:46] iteration 6627/ 6855 | consumed samples: 848256 | elapsed time per iteration (ms): 88472.4 | throughput per GPU (TFLOP/s/GPU): 41.6 | MFU 13.33% | FLOPs estimator: workload-aware | logical tokens/s: 189631.7 | valid tokens/s: 189631.7 | physical tokens/s: 190050.8 | loss tokens/s: 135673.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.341648E-01 | lm-global-supervised-tokens: 1.200331E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.281166E-02 | lm-token-buffer-mib: 5.825814E+01 | global_load_balancing_loss: 1.000986E+00 | loss scale: 1.0 | grad norm: 0.087 | params norm: 11246.456 | params rms: 0.060896 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5:36:11.697839 | finish at 2026-10-10 20:12:57 + [2026-10-10 14:38:15] iteration 6628/ 6855 | consumed samples: 848384 | elapsed time per iteration (ms): 89356.1 | throughput per GPU (TFLOP/s/GPU): 41.7 | MFU 13.37% | FLOPs estimator: workload-aware | logical tokens/s: 187756.7 | valid tokens/s: 187756.7 | physical tokens/s: 188084.7 | loss tokens/s: 135053.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.759685E-01 | lm-global-supervised-tokens: 1.206783E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.565334E-02 | lm-token-buffer-mib: 5.821572E+01 | global_load_balancing_loss: 9.993172E-01 | loss scale: 1.0 | grad norm: 0.159 | params norm: 11246.412 | params rms: 0.060896 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5:38:03.832879 | finish at 2026-10-10 20:16:19 + [2026-10-10 14:39:43] iteration 6629/ 6855 | consumed samples: 848512 | elapsed time per iteration (ms): 88215.3 | throughput per GPU (TFLOP/s/GPU): 41.2 | MFU 13.20% | FLOPs estimator: workload-aware | logical tokens/s: 190185.0 | valid tokens/s: 190185.0 | physical tokens/s: 190619.7 | loss tokens/s: 137149.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.459672E-01 | lm-global-supervised-tokens: 1.209864E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.178124E-02 | lm-token-buffer-mib: 5.811243E+01 | global_load_balancing_loss: 1.000633E+00 | loss scale: 1.0 | grad norm: 0.042 | params norm: 11246.367 | params rms: 0.060895 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5:32:16.648497 | finish at 2026-10-10 20:12:00 + [2026-10-10 14:41:31] iteration 6630/ 6855 | consumed samples: 848640 | elapsed time per iteration (ms): 108047.0 | throughput per GPU (TFLOP/s/GPU): 34.0 | MFU 10.89% | FLOPs estimator: workload-aware | logical tokens/s: 155276.7 | valid tokens/s: 155276.7 | physical tokens/s: 155686.2 | loss tokens/s: 112844.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.619087E-01 | lm-global-supervised-tokens: 1.219246E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.105132E-02 | lm-token-buffer-mib: 5.806385E+01 | global_load_balancing_loss: 9.999609E-01 | loss scale: 1.0 | grad norm: 0.224 | params norm: 11246.322 | params rms: 0.060895 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6:45:10.578042 | finish at 2026-10-10 21:26:42 + [2026-10-10 14:43:04] iteration 6631/ 6855 | consumed samples: 848768 | elapsed time per iteration (ms): 92525.5 | throughput per GPU (TFLOP/s/GPU): 40.8 | MFU 13.07% | FLOPs estimator: workload-aware | logical tokens/s: 181324.9 | valid tokens/s: 181324.9 | physical tokens/s: 181720.1 | loss tokens/s: 131160.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.738876E-01 | lm-global-supervised-tokens: 1.213564E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.298551E-02 | lm-token-buffer-mib: 5.817190E+01 | global_load_balancing_loss: 9.998122E-01 | loss scale: 1.0 | grad norm: 0.098 | params norm: 11246.275 | params rms: 0.060895 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5:45:25.713882 | finish at 2026-10-10 20:28:29 + [2026-10-10 14:44:29] iteration 6632/ 6855 | consumed samples: 848896 | elapsed time per iteration (ms): 85766.7 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.66% | FLOPs estimator: workload-aware | logical tokens/s: 195614.6 | valid tokens/s: 195614.6 | physical tokens/s: 196202.5 | loss tokens/s: 148549.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.256113E-01 | lm-global-supervised-tokens: 1.274061E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.207869E-02 | lm-token-buffer-mib: 5.811968E+01 | global_load_balancing_loss: 1.000079E+00 | loss scale: 1.0 | grad norm: 0.187 | params norm: 11246.229 | params rms: 0.060895 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5:18:45.965174 | finish at 2026-10-10 20:03:15 + [2026-10-10 14:45:58] iteration 6633/ 6855 | consumed samples: 849024 | elapsed time per iteration (ms): 88150.3 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.41% | FLOPs estimator: workload-aware | logical tokens/s: 190324.7 | valid tokens/s: 190324.7 | physical tokens/s: 190696.3 | loss tokens/s: 130724.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.192864E-01 | lm-global-supervised-tokens: 1.152338E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.147584E-02 | lm-token-buffer-mib: 5.810232E+01 | global_load_balancing_loss: 1.000408E+00 | loss scale: 1.0 | grad norm: 0.064 | params norm: 11246.182 | params rms: 0.060894 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5:26:09.370999 | finish at 2026-10-10 20:12:07 + [2026-10-10 14:47:23] iteration 6634/ 6855 | consumed samples: 849152 | elapsed time per iteration (ms): 85049.5 | throughput per GPU (TFLOP/s/GPU): 43.1 | MFU 13.82% | FLOPs estimator: workload-aware | logical tokens/s: 197263.3 | valid tokens/s: 197263.3 | physical tokens/s: 197809.1 | loss tokens/s: 148528.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.498130E-01 | lm-global-supervised-tokens: 1.263231E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.305297E-02 | lm-token-buffer-mib: 5.807687E+01 | global_load_balancing_loss: 1.000068E+00 | loss scale: 1.0 | grad norm: 0.246 | params norm: 11246.135 | params rms: 0.060894 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5:13:15.947454 | finish at 2026-10-10 20:00:39 + [2026-10-10 14:48:46] iteration 6635/ 6855 | consumed samples: 849280 | elapsed time per iteration (ms): 83326.9 | throughput per GPU (TFLOP/s/GPU): 43.8 | MFU 14.04% | FLOPs estimator: workload-aware | logical tokens/s: 201341.8 | valid tokens/s: 201341.7 | physical tokens/s: 201745.6 | loss tokens/s: 147620.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.405320E-01 | lm-global-supervised-tokens: 1.230077E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.160133E-02 | lm-token-buffer-mib: 5.804433E+01 | global_load_balancing_loss: 1.001653E+00 | loss scale: 1.0 | grad norm: 0.083 | params norm: 11246.088 | params rms: 0.060894 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5:05:31.913595 | finish at 2026-10-10 19:54:18 + [2026-10-10 14:50:16] iteration 6636/ 6855 | consumed samples: 849408 | elapsed time per iteration (ms): 90443.3 | throughput per GPU (TFLOP/s/GPU): 41.3 | MFU 13.25% | FLOPs estimator: workload-aware | logical tokens/s: 185499.8 | valid tokens/s: 185499.8 | physical tokens/s: 185814.7 | loss tokens/s: 128771.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 5.967845E-01 | lm-global-supervised-tokens: 1.164654E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.214712E-02 | lm-token-buffer-mib: 5.811508E+01 | global_load_balancing_loss: 1.000403E+00 | loss scale: 1.0 | grad norm: 0.140 | params norm: 11246.041 | params rms: 0.060894 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5:30:07.086879 | finish at 2026-10-10 20:20:23 + [2026-10-10 14:51:47] iteration 6637/ 6855 | consumed samples: 849536 | elapsed time per iteration (ms): 90599.1 | throughput per GPU (TFLOP/s/GPU): 40.8 | MFU 13.08% | FLOPs estimator: workload-aware | logical tokens/s: 185180.7 | valid tokens/s: 185180.7 | physical tokens/s: 185660.4 | loss tokens/s: 137944.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.337649E-01 | lm-global-supervised-tokens: 1.249761E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.315260E-02 | lm-token-buffer-mib: 5.830238E+01 | global_load_balancing_loss: 9.989235E-01 | loss scale: 1.0 | grad norm: 0.162 | params norm: 11245.993 | params rms: 0.060893 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5:29:10.593378 | finish at 2026-10-10 20:20:58 + [2026-10-10 14:53:16] iteration 6638/ 6855 | consumed samples: 849664 | elapsed time per iteration (ms): 89086.5 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.61% | FLOPs estimator: workload-aware | logical tokens/s: 188325.0 | valid tokens/s: 188325.0 | physical tokens/s: 188745.3 | loss tokens/s: 139905.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.950484E-01 | lm-global-supervised-tokens: 1.246365E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.367728E-02 | lm-token-buffer-mib: 5.825220E+01 | global_load_balancing_loss: 9.984729E-01 | loss scale: 1.0 | grad norm: 0.103 | params norm: 11245.944 | params rms: 0.060893 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5:22:11.767380 | finish at 2026-10-10 20:15:28 + [2026-10-10 14:54:42] iteration 6639/ 6855 | consumed samples: 849792 | elapsed time per iteration (ms): 85413.4 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.66% | FLOPs estimator: workload-aware | logical tokens/s: 196423.1 | valid tokens/s: 196423.1 | physical tokens/s: 197149.3 | loss tokens/s: 143882.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.487052E-01 | lm-global-supervised-tokens: 1.228947E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.189143E-02 | lm-token-buffer-mib: 5.840710E+01 | global_load_balancing_loss: 9.995891E-01 | loss scale: 1.0 | grad norm: 0.222 | params norm: 11245.896 | params rms: 0.060893 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5:07:29.285889 | finish at 2026-10-10 20:02:11 + [2026-10-10 14:56:30] iteration 6640/ 6855 | consumed samples: 849920 | elapsed time per iteration (ms): 108599.3 | throughput per GPU (TFLOP/s/GPU): 34.0 | MFU 10.89% | FLOPs estimator: workload-aware | logical tokens/s: 154486.6 | valid tokens/s: 154486.6 | physical tokens/s: 154875.1 | loss tokens/s: 113383.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.008285E-01 | lm-global-supervised-tokens: 1.231342E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.533478E-02 | lm-token-buffer-mib: 5.818029E+01 | global_load_balancing_loss: 1.001816E+00 | loss scale: 1.0 | grad norm: 0.144 | params norm: 11245.849 | params rms: 0.060893 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6:29:08.854555 | finish at 2026-10-10 21:25:39 +times across ranks (ms): + save-checkpoint: + rank 0: 25586.40 + rank 1: 25586.55 + rank 2: 25586.44 + rank 3: 25586.40 + rank 4: 25586.42 + rank 5: 25586.39 + rank 6: 25586.39 + rank 7: 25586.45 + rank 8: 25585.46 + rank 9: 25585.41 + rank 10: 25585.42 + rank 11: 25585.41 + rank 12: 25585.42 + rank 13: 25585.43 + rank 14: 25585.47 + rank 15: 25585.42 + rank 16: 25585.97 + rank 17: 25585.96 + rank 18: 25585.95 + rank 19: 25585.97 + rank 20: 25585.95 + rank 21: 25585.97 + rank 22: 25585.95 + rank 23: 25585.95 + rank 24: 25585.71 + rank 25: 25585.68 + rank 26: 25585.75 + rank 27: 25585.77 + rank 28: 25585.61 + rank 29: 25585.75 + rank 30: 25585.65 + rank 31: 25585.70 + rank 32: 25586.26 + rank 33: 25586.17 + rank 34: 25586.11 + rank 35: 25594.91 + rank 36: 25586.13 + rank 37: 25585.96 + rank 38: 25586.14 + rank 39: 25586.11 + rank 40: 25585.55 + rank 41: 25585.52 + rank 42: 25585.54 + rank 43: 25585.54 + rank 44: 25585.55 + rank 45: 25585.52 + rank 46: 25585.53 + rank 47: 25585.51 + rank 48: 25585.66 + rank 49: 25585.30 + rank 50: 25585.56 + rank 51: 25585.41 + rank 52: 25585.64 + rank 53: 25585.52 + rank 54: 25585.42 + rank 55: 25585.58 + rank 56: 25586.22 + rank 57: 25586.22 + rank 58: 25586.17 + rank 59: 25586.17 + rank 60: 25586.16 + rank 61: 25586.21 + rank 62: 25586.16 + rank 63: 25586.21 + [2026-10-10 14:58:23] iteration 6641/ 6855 | consumed samples: 850048 | elapsed time per iteration (ms): 87359.9 | throughput per GPU (TFLOP/s/GPU): 40.4 | MFU 12.96% | FLOPs estimator: workload-aware | logical tokens/s: 192047.1 | valid tokens/s: 192047.1 | physical tokens/s: 192499.2 | loss tokens/s: 150761.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.160656E-01 | lm-global-supervised-tokens: 1.317052E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 3.303303E-01 | lm-token-buffer-mib: 5.808759E+01 | global_load_balancing_loss: 1.001171E+00 | loss scale: 1.0 | grad norm: 0.131 | params norm: 11245.802 | params rms: 0.060892 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5:11:35.009467 | finish at 2026-10-10 20:09:58 + [2026-10-10 14:59:54] iteration 6642/ 6855 | consumed samples: 850176 | elapsed time per iteration (ms): 91079.1 | throughput per GPU (TFLOP/s/GPU): 40.5 | MFU 12.97% | FLOPs estimator: workload-aware | logical tokens/s: 184203.8 | valid tokens/s: 184203.8 | physical tokens/s: 184646.5 | loss tokens/s: 139343.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.296033E-01 | lm-global-supervised-tokens: 1.269128E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.689349E-02 | lm-token-buffer-mib: 5.810744E+01 | global_load_balancing_loss: 1.000329E+00 | loss scale: 1.0 | grad norm: 0.135 | params norm: 11245.754 | params rms: 0.060892 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5:23:19.854371 | finish at 2026-10-10 20:23:14 + [2026-10-10 15:01:21] iteration 6643/ 6855 | consumed samples: 850304 | elapsed time per iteration (ms): 86969.6 | throughput per GPU (TFLOP/s/GPU): 41.6 | MFU 13.34% | FLOPs estimator: workload-aware | logical tokens/s: 192907.9 | valid tokens/s: 192907.9 | physical tokens/s: 193423.6 | loss tokens/s: 140385.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.192876E-01 | lm-global-supervised-tokens: 1.220928E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.134268E-02 | lm-token-buffer-mib: 5.820712E+01 | global_load_balancing_loss: 1.000913E+00 | loss scale: 1.0 | grad norm: 0.145 | params norm: 11245.707 | params rms: 0.060892 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5:07:17.553979 | finish at 2026-10-10 20:08:39 + [2026-10-10 15:02:48] iteration 6644/ 6855 | consumed samples: 850432 | elapsed time per iteration (ms): 86682.1 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.65% | FLOPs estimator: workload-aware | logical tokens/s: 193548.3 | valid tokens/s: 193548.3 | physical tokens/s: 194184.9 | loss tokens/s: 148195.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.864035E-01 | lm-global-supervised-tokens: 1.284592E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.319198E-02 | lm-token-buffer-mib: 5.832667E+01 | global_load_balancing_loss: 1.000063E+00 | loss scale: 1.0 | grad norm: 0.060 | params norm: 11245.659 | params rms: 0.060892 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5:04:49.921704 | finish at 2026-10-10 20:07:38 + [2026-10-10 15:04:14] iteration 6645/ 6855 | consumed samples: 850560 | elapsed time per iteration (ms): 86002.7 | throughput per GPU (TFLOP/s/GPU): 43.3 | MFU 13.88% | FLOPs estimator: workload-aware | logical tokens/s: 195077.5 | valid tokens/s: 195077.5 | physical tokens/s: 195708.5 | loss tokens/s: 145176.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.888290E-01 | lm-global-supervised-tokens: 1.248558E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.140021E-02 | lm-token-buffer-mib: 5.822980E+01 | global_load_balancing_loss: 9.995421E-01 | loss scale: 1.0 | grad norm: 0.125 | params norm: 11245.611 | params rms: 0.060891 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5:01:00.562463 | finish at 2026-10-10 20:05:15 + [2026-10-10 15:05:42] iteration 6646/ 6855 | consumed samples: 850688 | elapsed time per iteration (ms): 87710.6 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.40% | FLOPs estimator: workload-aware | logical tokens/s: 191279.1 | valid tokens/s: 191279.1 | physical tokens/s: 191886.8 | loss tokens/s: 137902.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.898166E-01 | lm-global-supervised-tokens: 1.209554E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.240635E-02 | lm-token-buffer-mib: 5.818404E+01 | global_load_balancing_loss: 9.995542E-01 | loss scale: 1.0 | grad norm: 0.055 | params norm: 11245.563 | params rms: 0.060891 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5:05:31.520362 | finish at 2026-10-10 20:11:13 + [2026-10-10 15:07:06] iteration 6647/ 6855 | consumed samples: 850816 | elapsed time per iteration (ms): 84336.1 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.76% | FLOPs estimator: workload-aware | logical tokens/s: 198932.3 | valid tokens/s: 198932.3 | physical tokens/s: 199437.5 | loss tokens/s: 155165.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.420100E-01 | lm-global-supervised-tokens: 1.308606E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.343178E-02 | lm-token-buffer-mib: 5.820839E+01 | global_load_balancing_loss: 1.000372E+00 | loss scale: 1.0 | grad norm: 0.101 | params norm: 11245.515 | params rms: 0.060891 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4:52:21.917400 | finish at 2026-10-10 19:59:28 + [2026-10-10 15:08:33] iteration 6648/ 6855 | consumed samples: 850944 | elapsed time per iteration (ms): 86977.7 | throughput per GPU (TFLOP/s/GPU): 41.7 | MFU 13.37% | FLOPs estimator: workload-aware | logical tokens/s: 192890.6 | valid tokens/s: 192890.6 | physical tokens/s: 193234.4 | loss tokens/s: 145958.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.521162E-01 | lm-global-supervised-tokens: 1.269509E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.394900E-02 | lm-token-buffer-mib: 5.812047E+01 | global_load_balancing_loss: 1.000286E+00 | loss scale: 1.0 | grad norm: 0.090 | params norm: 11245.465 | params rms: 0.060891 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5:00:04.386416 | finish at 2026-10-10 20:08:37 + [2026-10-10 15:09:59] iteration 6649/ 6855 | consumed samples: 851072 | elapsed time per iteration (ms): 86223.5 | throughput per GPU (TFLOP/s/GPU): 43.1 | MFU 13.81% | FLOPs estimator: workload-aware | logical tokens/s: 194578.2 | valid tokens/s: 194578.2 | physical tokens/s: 194971.8 | loss tokens/s: 141835.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.698888E-01 | lm-global-supervised-tokens: 1.222952E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.497044E-02 | lm-token-buffer-mib: 5.815098E+01 | global_load_balancing_loss: 1.000497E+00 | loss scale: 1.0 | grad norm: 0.042 | params norm: 11245.416 | params rms: 0.060890 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4:56:02.044588 | finish at 2026-10-10 20:06:01 + [2026-10-10 15:11:47] iteration 6650/ 6855 | consumed samples: 851200 | elapsed time per iteration (ms): 107982.8 | throughput per GPU (TFLOP/s/GPU): 34.2 | MFU 10.98% | FLOPs estimator: workload-aware | logical tokens/s: 155368.5 | valid tokens/s: 155368.5 | physical tokens/s: 155836.2 | loss tokens/s: 112978.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.778762E-01 | lm-global-supervised-tokens: 1.219974E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.120518E-02 | lm-token-buffer-mib: 5.813498E+01 | global_load_balancing_loss: 1.000527E+00 | loss scale: 1.0 | grad norm: 0.135 | params norm: 11245.370 | params rms: 0.060890 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 6:08:56.474246 | finish at 2026-10-10 21:20:44 + [2026-10-10 15:13:14] iteration 6651/ 6855 | consumed samples: 851328 | elapsed time per iteration (ms): 86616.7 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.59% | FLOPs estimator: workload-aware | logical tokens/s: 193694.4 | valid tokens/s: 193694.4 | physical tokens/s: 194375.0 | loss tokens/s: 146578.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.960387E-01 | lm-global-supervised-tokens: 1.269615E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.168780E-02 | lm-token-buffer-mib: 5.811518E+01 | global_load_balancing_loss: 9.997735E-01 | loss scale: 1.0 | grad norm: 0.123 | params norm: 11245.322 | params rms: 0.060890 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4:54:29.800610 | finish at 2026-10-10 20:07:44 + [2026-10-10 15:14:40] iteration 6652/ 6855 | consumed samples: 851456 | elapsed time per iteration (ms): 86210.1 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.45% | FLOPs estimator: workload-aware | logical tokens/s: 194607.8 | valid tokens/s: 194607.7 | physical tokens/s: 195126.5 | loss tokens/s: 153295.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.647689E-01 | lm-global-supervised-tokens: 1.321561E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.165480E-02 | lm-token-buffer-mib: 5.810056E+01 | global_load_balancing_loss: 1.000002E+00 | loss scale: 1.0 | grad norm: 0.102 | params norm: 11245.274 | params rms: 0.060889 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4:51:40.645011 | finish at 2026-10-10 20:06:21 + [2026-10-10 15:16:10] iteration 6653/ 6855 | consumed samples: 851584 | elapsed time per iteration (ms): 89509.6 | throughput per GPU (TFLOP/s/GPU): 41.3 | MFU 13.22% | FLOPs estimator: workload-aware | logical tokens/s: 187434.0 | valid tokens/s: 187433.9 | physical tokens/s: 187846.8 | loss tokens/s: 137990.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.294448E-01 | lm-global-supervised-tokens: 1.235144E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.308643E-02 | lm-token-buffer-mib: 5.827976E+01 | global_load_balancing_loss: 1.000564E+00 | loss scale: 1.0 | grad norm: 0.221 | params norm: 11245.227 | params rms: 0.060889 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5:01:20.943712 | finish at 2026-10-10 20:17:31 + [2026-10-10 15:17:36] iteration 6654/ 6855 | consumed samples: 851712 | elapsed time per iteration (ms): 86429.3 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.77% | FLOPs estimator: workload-aware | logical tokens/s: 194114.4 | valid tokens/s: 194114.4 | physical tokens/s: 194655.1 | loss tokens/s: 149762.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.855363E-01 | lm-global-supervised-tokens: 1.294383E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.455172E-02 | lm-token-buffer-mib: 5.820200E+01 | global_load_balancing_loss: 9.994529E-01 | loss scale: 1.0 | grad norm: 0.050 | params norm: 11245.181 | params rms: 0.060889 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4:49:32.287781 | finish at 2026-10-10 20:07:08 + [2026-10-10 15:19:02] iteration 6655/ 6855 | consumed samples: 851840 | elapsed time per iteration (ms): 86081.8 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.44% | FLOPs estimator: workload-aware | logical tokens/s: 194898.3 | valid tokens/s: 194898.3 | physical tokens/s: 195491.3 | loss tokens/s: 152372.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.535220E-01 | lm-global-supervised-tokens: 1.311653E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.155085E-02 | lm-token-buffer-mib: 5.815489E+01 | global_load_balancing_loss: 1.001025E+00 | loss scale: 1.0 | grad norm: 0.175 | params norm: 11245.135 | params rms: 0.060889 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4:46:56.359901 | finish at 2026-10-10 20:05:58 + [2026-10-10 15:20:33] iteration 6656/ 6855 | consumed samples: 851968 | elapsed time per iteration (ms): 90874.4 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.41% | FLOPs estimator: workload-aware | logical tokens/s: 184619.2 | valid tokens/s: 184619.2 | physical tokens/s: 185048.8 | loss tokens/s: 135372.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.875404E-01 | lm-global-supervised-tokens: 1.230189E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.570506E-02 | lm-token-buffer-mib: 5.824409E+01 | global_load_balancing_loss: 1.000944E+00 | loss scale: 1.0 | grad norm: 0.043 | params norm: 11245.088 | params rms: 0.060888 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5:01:24.001120 | finish at 2026-10-10 20:21:57 + [2026-10-10 15:22:03] iteration 6657/ 6855 | consumed samples: 852096 | elapsed time per iteration (ms): 89657.6 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.40% | FLOPs estimator: workload-aware | logical tokens/s: 187125.1 | valid tokens/s: 187125.1 | physical tokens/s: 187381.7 | loss tokens/s: 137115.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.336997E-01 | lm-global-supervised-tokens: 1.229347E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.345076E-02 | lm-token-buffer-mib: 5.807125E+01 | global_load_balancing_loss: 9.994726E-01 | loss scale: 1.0 | grad norm: 0.113 | params norm: 11245.041 | params rms: 0.060888 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4:55:52.203464 | finish at 2026-10-10 20:17:55 + [2026-10-10 15:23:32] iteration 6658/ 6855 | consumed samples: 852224 | elapsed time per iteration (ms): 89599.0 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.62% | FLOPs estimator: workload-aware | logical tokens/s: 187247.5 | valid tokens/s: 187247.5 | physical tokens/s: 187721.5 | loss tokens/s: 138475.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.546655E-01 | lm-global-supervised-tokens: 1.240725E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.257625E-02 | lm-token-buffer-mib: 5.805945E+01 | global_load_balancing_loss: 1.000400E+00 | loss scale: 1.0 | grad norm: 0.104 | params norm: 11244.995 | params rms: 0.060888 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4:54:11.006189 | finish at 2026-10-10 20:17:43 + [2026-10-10 15:25:01] iteration 6659/ 6855 | consumed samples: 852352 | elapsed time per iteration (ms): 88414.1 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.64% | FLOPs estimator: workload-aware | logical tokens/s: 189756.4 | valid tokens/s: 189756.4 | physical tokens/s: 190041.2 | loss tokens/s: 139070.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.036183E-01 | lm-global-supervised-tokens: 1.229583E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.154013E-02 | lm-token-buffer-mib: 5.805762E+01 | global_load_balancing_loss: 1.000265E+00 | loss scale: 1.0 | grad norm: 0.047 | params norm: 11244.949 | params rms: 0.060888 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4:48:49.164334 | finish at 2026-10-10 20:13:50 + [2026-10-10 15:26:50] iteration 6660/ 6855 | consumed samples: 852480 | elapsed time per iteration (ms): 108970.1 | throughput per GPU (TFLOP/s/GPU): 33.7 | MFU 10.80% | FLOPs estimator: workload-aware | logical tokens/s: 153961.3 | valid tokens/s: 153961.3 | physical tokens/s: 154258.6 | loss tokens/s: 114819.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.911226E-01 | lm-global-supervised-tokens: 1.251194E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.483754E-02 | lm-token-buffer-mib: 5.807495E+01 | global_load_balancing_loss: 1.000974E+00 | loss scale: 1.0 | grad norm: 0.127 | params norm: 11244.903 | params rms: 0.060887 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5:54:09.177761 | finish at 2026-10-10 21:20:59 + [2026-10-10 15:28:18] iteration 6661/ 6855 | consumed samples: 852608 | elapsed time per iteration (ms): 88852.6 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.74% | FLOPs estimator: workload-aware | logical tokens/s: 188820.2 | valid tokens/s: 188820.2 | physical tokens/s: 189334.8 | loss tokens/s: 141932.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.317493E-01 | lm-global-supervised-tokens: 1.261106E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.487347E-02 | lm-token-buffer-mib: 5.806402E+01 | global_load_balancing_loss: 9.998573E-01 | loss scale: 1.0 | grad norm: 0.097 | params norm: 11244.858 | params rms: 0.060887 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4:47:17.395353 | finish at 2026-10-10 20:15:36 + [2026-10-10 15:29:47] iteration 6662/ 6855 | consumed samples: 852736 | elapsed time per iteration (ms): 88802.5 | throughput per GPU (TFLOP/s/GPU): 41.4 | MFU 13.28% | FLOPs estimator: workload-aware | logical tokens/s: 188927.1 | valid tokens/s: 188927.1 | physical tokens/s: 189709.0 | loss tokens/s: 141120.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.367982E-01 | lm-global-supervised-tokens: 1.253184E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.128400E-02 | lm-token-buffer-mib: 5.839465E+01 | global_load_balancing_loss: 9.989798E-01 | loss scale: 1.0 | grad norm: 0.255 | params norm: 11244.814 | params rms: 0.060887 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4:45:38.889956 | finish at 2026-10-10 20:15:26 + [2026-10-10 15:31:15] iteration 6663/ 6855 | consumed samples: 852864 | elapsed time per iteration (ms): 87855.7 | throughput per GPU (TFLOP/s/GPU): 41.1 | MFU 13.18% | FLOPs estimator: workload-aware | logical tokens/s: 190962.8 | valid tokens/s: 190962.8 | physical tokens/s: 191424.8 | loss tokens/s: 138448.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.245511E-01 | lm-global-supervised-tokens: 1.216346E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.549983E-02 | lm-token-buffer-mib: 5.818829E+01 | global_load_balancing_loss: 9.998522E-01 | loss scale: 1.0 | grad norm: 0.094 | params norm: 11244.770 | params rms: 0.060887 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4:41:08.296097 | finish at 2026-10-10 20:12:23 + [2026-10-10 15:32:42] iteration 6664/ 6855 | consumed samples: 852992 | elapsed time per iteration (ms): 86689.7 | throughput per GPU (TFLOP/s/GPU): 42.8 | MFU 13.71% | FLOPs estimator: workload-aware | logical tokens/s: 193531.6 | valid tokens/s: 193531.6 | physical tokens/s: 193946.3 | loss tokens/s: 141263.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.538084E-01 | lm-global-supervised-tokens: 1.224609E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.078674E-02 | lm-token-buffer-mib: 5.814075E+01 | global_load_balancing_loss: 1.000759E+00 | loss scale: 1.0 | grad norm: 0.201 | params norm: 11244.727 | params rms: 0.060887 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4:35:57.735775 | finish at 2026-10-10 20:08:40 + [2026-10-10 15:34:14] iteration 6665/ 6855 | consumed samples: 853120 | elapsed time per iteration (ms): 92045.3 | throughput per GPU (TFLOP/s/GPU): 40.8 | MFU 13.08% | FLOPs estimator: workload-aware | logical tokens/s: 182270.6 | valid tokens/s: 182270.6 | physical tokens/s: 182622.0 | loss tokens/s: 130160.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.349064E-01 | lm-global-supervised-tokens: 1.198067E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.170384E-02 | lm-token-buffer-mib: 5.807773E+01 | global_load_balancing_loss: 1.000590E+00 | loss scale: 1.0 | grad norm: 0.112 | params norm: 11244.685 | params rms: 0.060886 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4:51:28.599210 | finish at 2026-10-10 20:25:42 + [2026-10-10 15:35:39] iteration 6666/ 6855 | consumed samples: 853248 | elapsed time per iteration (ms): 85516.6 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.68% | FLOPs estimator: workload-aware | logical tokens/s: 196186.3 | valid tokens/s: 196186.3 | physical tokens/s: 196569.1 | loss tokens/s: 143968.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.181838E-01 | lm-global-supervised-tokens: 1.231172E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.359657E-02 | lm-token-buffer-mib: 5.837446E+01 | global_load_balancing_loss: 1.000676E+00 | loss scale: 1.0 | grad norm: 0.126 | params norm: 11244.641 | params rms: 0.060886 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4:29:22.639768 | finish at 2026-10-10 20:05:02 + [2026-10-10 15:37:09] iteration 6667/ 6855 | consumed samples: 853376 | elapsed time per iteration (ms): 89679.6 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.39% | FLOPs estimator: workload-aware | logical tokens/s: 187078.7 | valid tokens/s: 187078.7 | physical tokens/s: 187570.5 | loss tokens/s: 132528.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.663490E-01 | lm-global-supervised-tokens: 1.188513E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.145779E-02 | lm-token-buffer-mib: 5.809647E+01 | global_load_balancing_loss: 1.000921E+00 | loss scale: 1.0 | grad norm: 0.200 | params norm: 11244.598 | params rms: 0.060886 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4:40:59.761662 | finish at 2026-10-10 20:18:09 + [2026-10-10 15:38:36] iteration 6668/ 6855 | consumed samples: 853504 | elapsed time per iteration (ms): 87357.8 | throughput per GPU (TFLOP/s/GPU): 41.4 | MFU 13.27% | FLOPs estimator: workload-aware | logical tokens/s: 192051.3 | valid tokens/s: 192051.3 | physical tokens/s: 192637.4 | loss tokens/s: 147087.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.121451E-01 | lm-global-supervised-tokens: 1.284927E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.525704E-02 | lm-token-buffer-mib: 5.820055E+01 | global_load_balancing_loss: 1.000698E+00 | loss scale: 1.0 | grad norm: 0.194 | params norm: 11244.555 | params rms: 0.060886 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4:32:15.910652 | finish at 2026-10-10 20:10:52 + [2026-10-10 15:40:03] iteration 6669/ 6855 | consumed samples: 853632 | elapsed time per iteration (ms): 86102.6 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.67% | FLOPs estimator: workload-aware | logical tokens/s: 194850.3 | valid tokens/s: 194850.3 | physical tokens/s: 195228.7 | loss tokens/s: 149422.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.823282E-01 | lm-global-supervised-tokens: 1.286563E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.569418E-02 | lm-token-buffer-mib: 5.818859E+01 | global_load_balancing_loss: 9.996160E-01 | loss scale: 1.0 | grad norm: 0.109 | params norm: 11244.512 | params rms: 0.060885 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4:26:55.082110 | finish at 2026-10-10 20:06:58 + [2026-10-10 15:41:51] iteration 6670/ 6855 | consumed samples: 853760 | elapsed time per iteration (ms): 108474.3 | throughput per GPU (TFLOP/s/GPU): 33.8 | MFU 10.83% | FLOPs estimator: workload-aware | logical tokens/s: 154665.3 | valid tokens/s: 154665.3 | physical tokens/s: 154951.6 | loss tokens/s: 117373.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.383225E-01 | lm-global-supervised-tokens: 1.273197E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.518994E-02 | lm-token-buffer-mib: 5.811646E+01 | global_load_balancing_loss: 9.998890E-01 | loss scale: 1.0 | grad norm: 0.073 | params norm: 11244.467 | params rms: 0.060885 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5:34:27.749276 | finish at 2026-10-10 21:16:19 + [2026-10-10 15:43:21] iteration 6671/ 6855 | consumed samples: 853888 | elapsed time per iteration (ms): 90182.8 | throughput per GPU (TFLOP/s/GPU): 40.7 | MFU 13.06% | FLOPs estimator: workload-aware | logical tokens/s: 186035.3 | valid tokens/s: 186035.3 | physical tokens/s: 186615.4 | loss tokens/s: 143254.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.672445E-01 | lm-global-supervised-tokens: 1.291905E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.146874E-02 | lm-token-buffer-mib: 5.849381E+01 | global_load_balancing_loss: 1.000544E+00 | loss scale: 1.0 | grad norm: 0.105 | params norm: 11244.423 | params rms: 0.060885 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4:36:33.639641 | finish at 2026-10-10 20:19:55 + [2026-10-10 15:44:49] iteration 6672/ 6855 | consumed samples: 854016 | elapsed time per iteration (ms): 87786.6 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.54% | FLOPs estimator: workload-aware | logical tokens/s: 191113.5 | valid tokens/s: 191113.5 | physical tokens/s: 191681.6 | loss tokens/s: 144242.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.775030E-01 | lm-global-supervised-tokens: 1.266252E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.180980E-02 | lm-token-buffer-mib: 5.811953E+01 | global_load_balancing_loss: 1.000911E+00 | loss scale: 1.0 | grad norm: 0.118 | params norm: 11244.381 | params rms: 0.060885 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4:27:44.941625 | finish at 2026-10-10 20:12:34 + [2026-10-10 15:46:19] iteration 6673/ 6855 | consumed samples: 854144 | elapsed time per iteration (ms): 89591.4 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.49% | FLOPs estimator: workload-aware | logical tokens/s: 187263.4 | valid tokens/s: 187263.4 | physical tokens/s: 187765.2 | loss tokens/s: 138306.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.847626E-01 | lm-global-supervised-tokens: 1.239105E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.148398E-02 | lm-token-buffer-mib: 5.823671E+01 | global_load_balancing_loss: 9.991722E-01 | loss scale: 1.0 | grad norm: 0.109 | params norm: 11244.339 | params rms: 0.060884 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4:31:45.631833 | finish at 2026-10-10 20:18:04 + [2026-10-10 15:47:41] iteration 6674/ 6855 | consumed samples: 854272 | elapsed time per iteration (ms): 82252.4 | throughput per GPU (TFLOP/s/GPU): 43.3 | MFU 13.87% | FLOPs estimator: workload-aware | logical tokens/s: 203972.4 | valid tokens/s: 203972.4 | physical tokens/s: 204650.6 | loss tokens/s: 155687.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.566758E-01 | lm-global-supervised-tokens: 1.280570E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.366286E-02 | lm-token-buffer-mib: 5.814255E+01 | global_load_balancing_loss: 9.990231E-01 | loss scale: 1.0 | grad norm: 0.083 | params norm: 11244.293 | params rms: 0.060884 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4:08:07.678085 | finish at 2026-10-10 19:55:48 + [2026-10-10 15:49:11] iteration 6675/ 6855 | consumed samples: 854400 | elapsed time per iteration (ms): 90178.9 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.70% | FLOPs estimator: workload-aware | logical tokens/s: 186043.0 | valid tokens/s: 186043.0 | physical tokens/s: 186335.4 | loss tokens/s: 127282.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.913263E-01 | lm-global-supervised-tokens: 1.147823E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.454065E-02 | lm-token-buffer-mib: 5.809359E+01 | global_load_balancing_loss: 9.996930E-01 | loss scale: 1.0 | grad norm: 0.046 | params norm: 11244.247 | params rms: 0.060884 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4:30:32.201357 | finish at 2026-10-10 20:19:43 + [2026-10-10 15:50:39] iteration 6676/ 6855 | consumed samples: 854528 | elapsed time per iteration (ms): 88365.3 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.65% | FLOPs estimator: workload-aware | logical tokens/s: 189862.0 | valid tokens/s: 189862.0 | physical tokens/s: 190292.7 | loss tokens/s: 143309.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.718842E-01 | lm-global-supervised-tokens: 1.266360E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.173663E-02 | lm-token-buffer-mib: 5.816776E+01 | global_load_balancing_loss: 1.001350E+00 | loss scale: 1.0 | grad norm: 0.073 | params norm: 11244.202 | params rms: 0.060884 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4:23:37.383483 | finish at 2026-10-10 20:14:17 + [2026-10-10 15:52:09] iteration 6677/ 6855 | consumed samples: 854656 | elapsed time per iteration (ms): 89682.5 | throughput per GPU (TFLOP/s/GPU): 41.7 | MFU 13.36% | FLOPs estimator: workload-aware | logical tokens/s: 187073.2 | valid tokens/s: 187073.2 | physical tokens/s: 187527.2 | loss tokens/s: 132368.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.781680E-01 | lm-global-supervised-tokens: 1.187111E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.386280E-02 | lm-token-buffer-mib: 5.829825E+01 | global_load_balancing_loss: 1.000847E+00 | loss scale: 1.0 | grad norm: 0.070 | params norm: 11244.157 | params rms: 0.060883 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4:26:03.493213 | finish at 2026-10-10 20:18:13 + [2026-10-10 15:53:34] iteration 6678/ 6855 | consumed samples: 854784 | elapsed time per iteration (ms): 84664.4 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.62% | FLOPs estimator: workload-aware | logical tokens/s: 198160.5 | valid tokens/s: 198160.5 | physical tokens/s: 198615.4 | loss tokens/s: 151056.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.369249E-01 | lm-global-supervised-tokens: 1.278914E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.154812E-02 | lm-token-buffer-mib: 5.814267E+01 | global_load_balancing_loss: 1.000429E+00 | loss scale: 1.0 | grad norm: 0.072 | params norm: 11244.110 | params rms: 0.060883 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4:09:45.590251 | finish at 2026-10-10 20:03:19 + [2026-10-10 15:55:02] iteration 6679/ 6855 | consumed samples: 854912 | elapsed time per iteration (ms): 87875.3 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.56% | FLOPs estimator: workload-aware | logical tokens/s: 190920.3 | valid tokens/s: 190920.3 | physical tokens/s: 191303.3 | loss tokens/s: 138552.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.698172E-01 | lm-global-supervised-tokens: 1.217533E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.437279E-02 | lm-token-buffer-mib: 5.808112E+01 | global_load_balancing_loss: 1.000485E+00 | loss scale: 1.0 | grad norm: 0.057 | params norm: 11244.064 | params rms: 0.060883 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4:17:46.052872 | finish at 2026-10-10 20:12:48 + [2026-10-10 15:56:49] iteration 6680/ 6855 | consumed samples: 855040 | elapsed time per iteration (ms): 107621.7 | throughput per GPU (TFLOP/s/GPU): 33.7 | MFU 10.81% | FLOPs estimator: workload-aware | logical tokens/s: 155890.2 | valid tokens/s: 155890.2 | physical tokens/s: 156235.5 | loss tokens/s: 115993.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.771488E-01 | lm-global-supervised-tokens: 1.248341E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.303945E-02 | lm-token-buffer-mib: 5.816601E+01 | global_load_balancing_loss: 1.000615E+00 | loss scale: 1.0 | grad norm: 0.044 | params norm: 11244.019 | params rms: 0.060883 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 5:13:53.798426 | finish at 2026-10-10 21:10:43 +times across ranks (ms): + save-checkpoint: + rank 0: 26116.36 + rank 1: 26116.34 + rank 2: 26116.35 + rank 3: 26116.33 + rank 4: 26116.33 + rank 5: 26116.29 + rank 6: 26116.29 + rank 7: 26116.32 + rank 8: 26116.04 + rank 9: 26115.95 + rank 10: 26115.97 + rank 11: 26116.00 + rank 12: 26116.13 + rank 13: 26115.98 + rank 14: 26116.01 + rank 15: 26115.98 + rank 16: 26115.90 + rank 17: 26115.98 + rank 18: 26115.95 + rank 19: 26115.99 + rank 20: 26115.94 + rank 21: 26115.96 + rank 22: 26115.97 + rank 23: 26116.01 + rank 24: 26116.37 + rank 25: 26116.31 + rank 26: 26116.36 + rank 27: 26116.41 + rank 28: 26116.78 + rank 29: 26116.28 + rank 30: 26116.33 + rank 31: 26116.34 + rank 32: 26116.96 + rank 33: 26117.26 + rank 34: 26117.20 + rank 35: 26117.22 + rank 36: 26117.17 + rank 37: 26117.19 + rank 38: 26117.16 + rank 39: 26117.11 + rank 40: 26116.56 + rank 41: 26116.53 + rank 42: 26116.54 + rank 43: 26116.56 + rank 44: 26116.52 + rank 45: 26116.52 + rank 46: 26116.54 + rank 47: 26116.53 + rank 48: 26116.41 + rank 49: 26116.38 + rank 50: 26116.38 + rank 51: 26116.40 + rank 52: 26116.47 + rank 53: 26116.36 + rank 54: 26116.41 + rank 55: 26116.41 + rank 56: 26115.95 + rank 57: 26115.92 + rank 58: 26115.90 + rank 59: 26115.88 + rank 60: 26115.92 + rank 61: 26115.91 + rank 62: 26115.94 + rank 63: 26115.91 + [2026-10-10 15:58:46] iteration 6681/ 6855 | consumed samples: 855168 | elapsed time per iteration (ms): 90894.1 | throughput per GPU (TFLOP/s/GPU): 39.8 | MFU 12.76% | FLOPs estimator: workload-aware | logical tokens/s: 184579.5 | valid tokens/s: 184579.5 | physical tokens/s: 185014.6 | loss tokens/s: 140970.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.499755E-01 | lm-global-supervised-tokens: 1.281336E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 5.632985E-01 | lm-token-buffer-mib: 5.805054E+01 | global_load_balancing_loss: 9.999126E-01 | loss scale: 1.0 | grad norm: 0.134 | params norm: 11243.974 | params rms: 0.060882 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4:23:35.573143 | finish at 2026-10-10 20:22:22 + [2026-10-10 16:00:19] iteration 6682/ 6855 | consumed samples: 855296 | elapsed time per iteration (ms): 92730.6 | throughput per GPU (TFLOP/s/GPU): 40.3 | MFU 12.93% | FLOPs estimator: workload-aware | logical tokens/s: 180924.1 | valid tokens/s: 180924.1 | physical tokens/s: 181331.8 | loss tokens/s: 131644.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.977196E-01 | lm-global-supervised-tokens: 1.220747E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.473540E-02 | lm-token-buffer-mib: 5.820217E+01 | global_load_balancing_loss: 1.000524E+00 | loss scale: 1.0 | grad norm: 0.048 | params norm: 11243.932 | params rms: 0.060882 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4:27:22.392212 | finish at 2026-10-10 20:27:41 + [2026-10-10 16:01:48] iteration 6683/ 6855 | consumed samples: 855424 | elapsed time per iteration (ms): 89017.2 | throughput per GPU (TFLOP/s/GPU): 41.6 | MFU 13.35% | FLOPs estimator: workload-aware | logical tokens/s: 188471.1 | valid tokens/s: 188471.1 | physical tokens/s: 188923.2 | loss tokens/s: 138764.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.553121E-01 | lm-global-supervised-tokens: 1.235240E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.525167E-02 | lm-token-buffer-mib: 5.810254E+01 | global_load_balancing_loss: 1.001043E+00 | loss scale: 1.0 | grad norm: 0.098 | params norm: 11243.889 | params rms: 0.060882 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4:15:10.963935 | finish at 2026-10-10 20:16:59 + [2026-10-10 16:03:16] iteration 6684/ 6855 | consumed samples: 855552 | elapsed time per iteration (ms): 87733.5 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.76% | FLOPs estimator: workload-aware | logical tokens/s: 191229.0 | valid tokens/s: 191229.0 | physical tokens/s: 191827.3 | loss tokens/s: 138948.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.801948E-01 | lm-global-supervised-tokens: 1.219048E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.274902E-02 | lm-token-buffer-mib: 5.805996E+01 | global_load_balancing_loss: 1.000061E+00 | loss scale: 1.0 | grad norm: 0.043 | params norm: 11243.845 | params rms: 0.060882 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4:10:02.430213 | finish at 2026-10-10 20:13:18 + [2026-10-10 16:04:44] iteration 6685/ 6855 | consumed samples: 855680 | elapsed time per iteration (ms): 88502.8 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.65% | FLOPs estimator: workload-aware | logical tokens/s: 189566.9 | valid tokens/s: 189566.9 | physical tokens/s: 189891.2 | loss tokens/s: 137358.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.992601E-01 | lm-global-supervised-tokens: 1.215664E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.330603E-02 | lm-token-buffer-mib: 5.802462E+01 | global_load_balancing_loss: 1.000042E+00 | loss scale: 1.0 | grad norm: 0.080 | params norm: 11243.801 | params rms: 0.060882 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4:10:45.480659 | finish at 2026-10-10 20:15:30 + [2026-10-10 16:06:09] iteration 6686/ 6855 | consumed samples: 855808 | elapsed time per iteration (ms): 84416.7 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.67% | FLOPs estimator: workload-aware | logical tokens/s: 198742.2 | valid tokens/s: 198742.2 | physical tokens/s: 199147.2 | loss tokens/s: 147395.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.345551E-01 | lm-global-supervised-tokens: 1.244261E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.161338E-02 | lm-token-buffer-mib: 5.809240E+01 | global_load_balancing_loss: 1.000373E+00 | loss scale: 1.0 | grad norm: 0.039 | params norm: 11243.757 | params rms: 0.060881 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3:57:46.414383 | finish at 2026-10-10 20:03:55 + [2026-10-10 16:07:33] iteration 6687/ 6855 | consumed samples: 855936 | elapsed time per iteration (ms): 84431.7 | throughput per GPU (TFLOP/s/GPU): 43.4 | MFU 13.90% | FLOPs estimator: workload-aware | logical tokens/s: 198706.9 | valid tokens/s: 198706.9 | physical tokens/s: 199156.7 | loss tokens/s: 149807.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.237371E-01 | lm-global-supervised-tokens: 1.264846E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.146743E-02 | lm-token-buffer-mib: 5.821282E+01 | global_load_balancing_loss: 9.990807E-01 | loss scale: 1.0 | grad norm: 0.086 | params norm: 11243.712 | params rms: 0.060881 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3:56:24.521513 | finish at 2026-10-10 20:03:58 + [2026-10-10 16:09:00] iteration 6688/ 6855 | consumed samples: 856064 | elapsed time per iteration (ms): 87072.1 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.59% | FLOPs estimator: workload-aware | logical tokens/s: 192681.3 | valid tokens/s: 192681.2 | physical tokens/s: 193380.6 | loss tokens/s: 145565.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.824537E-01 | lm-global-supervised-tokens: 1.267471E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.178398E-02 | lm-token-buffer-mib: 5.804855E+01 | global_load_balancing_loss: 9.986210E-01 | loss scale: 1.0 | grad norm: 0.045 | params norm: 11243.669 | params rms: 0.060881 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4:02:21.046341 | finish at 2026-10-10 20:11:21 + [2026-10-10 16:10:30] iteration 6689/ 6855 | consumed samples: 856192 | elapsed time per iteration (ms): 89317.4 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.43% | FLOPs estimator: workload-aware | logical tokens/s: 187837.1 | valid tokens/s: 187837.1 | physical tokens/s: 188264.3 | loss tokens/s: 144034.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.856240E-01 | lm-global-supervised-tokens: 1.286480E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.724341E-02 | lm-token-buffer-mib: 5.811127E+01 | global_load_balancing_loss: 1.000438E+00 | loss scale: 1.0 | grad norm: 0.086 | params norm: 11243.626 | params rms: 0.060881 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4:07:06.687961 | finish at 2026-10-10 20:17:36 + [2026-10-10 16:12:18] iteration 6690/ 6855 | consumed samples: 856320 | elapsed time per iteration (ms): 108334.0 | throughput per GPU (TFLOP/s/GPU): 33.6 | MFU 10.78% | FLOPs estimator: workload-aware | logical tokens/s: 154865.3 | valid tokens/s: 154865.3 | physical tokens/s: 155156.3 | loss tokens/s: 117125.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.499577E-01 | lm-global-supervised-tokens: 1.268869E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.505157E-02 | lm-token-buffer-mib: 5.812167E+01 | global_load_balancing_loss: 1.000256E+00 | loss scale: 1.0 | grad norm: 0.062 | params norm: 11243.582 | params rms: 0.060880 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4:57:55.113755 | finish at 2026-10-10 21:10:13 + [2026-10-10 16:13:46] iteration 6691/ 6855 | consumed samples: 856448 | elapsed time per iteration (ms): 88021.5 | throughput per GPU (TFLOP/s/GPU): 41.2 | MFU 13.20% | FLOPs estimator: workload-aware | logical tokens/s: 190603.4 | valid tokens/s: 190603.4 | physical tokens/s: 191104.2 | loss tokens/s: 142537.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.513008E-01 | lm-global-supervised-tokens: 1.254639E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.141533E-02 | lm-token-buffer-mib: 5.822258E+01 | global_load_balancing_loss: 1.002011E+00 | loss scale: 1.0 | grad norm: 0.079 | params norm: 11243.538 | params rms: 0.060880 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4:00:35.524532 | finish at 2026-10-10 20:14:21 + [2026-10-10 16:15:16] iteration 6692/ 6855 | consumed samples: 856576 | elapsed time per iteration (ms): 90291.3 | throughput per GPU (TFLOP/s/GPU): 40.7 | MFU 13.03% | FLOPs estimator: workload-aware | logical tokens/s: 185811.7 | valid tokens/s: 185811.7 | physical tokens/s: 186339.5 | loss tokens/s: 141560.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.514957E-01 | lm-global-supervised-tokens: 1.278169E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.272183E-02 | lm-token-buffer-mib: 5.808102E+01 | global_load_balancing_loss: 1.002435E+00 | loss scale: 1.0 | grad norm: 0.165 | params norm: 11243.496 | params rms: 0.060880 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4:05:17.474875 | finish at 2026-10-10 20:20:34 + [2026-10-10 16:16:41] iteration 6693/ 6855 | consumed samples: 856704 | elapsed time per iteration (ms): 85008.3 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.66% | FLOPs estimator: workload-aware | logical tokens/s: 197359.7 | valid tokens/s: 197359.7 | physical tokens/s: 197860.7 | loss tokens/s: 151266.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.395640E-01 | lm-global-supervised-tokens: 1.285891E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.213396E-02 | lm-token-buffer-mib: 5.815167E+01 | global_load_balancing_loss: 1.000677E+00 | loss scale: 1.0 | grad norm: 0.109 | params norm: 11243.454 | params rms: 0.060880 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3:49:31.338701 | finish at 2026-10-10 20:06:13 + [2026-10-10 16:18:09] iteration 6694/ 6855 | consumed samples: 856832 | elapsed time per iteration (ms): 87874.7 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.65% | FLOPs estimator: workload-aware | logical tokens/s: 190921.4 | valid tokens/s: 190921.4 | physical tokens/s: 191324.2 | loss tokens/s: 131926.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.251329E-01 | lm-global-supervised-tokens: 1.159297E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.574834E-02 | lm-token-buffer-mib: 5.807993E+01 | global_load_balancing_loss: 1.000141E+00 | loss scale: 1.0 | grad norm: 0.091 | params norm: 11243.412 | params rms: 0.060879 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3:55:47.819533 | finish at 2026-10-10 20:13:57 + [2026-10-10 16:19:35] iteration 6695/ 6855 | consumed samples: 856960 | elapsed time per iteration (ms): 85521.6 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.60% | FLOPs estimator: workload-aware | logical tokens/s: 196174.9 | valid tokens/s: 196174.9 | physical tokens/s: 196569.4 | loss tokens/s: 144506.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.171117E-01 | lm-global-supervised-tokens: 1.235843E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.158996E-02 | lm-token-buffer-mib: 5.816137E+01 | global_load_balancing_loss: 9.985576E-01 | loss scale: 1.0 | grad norm: 0.086 | params norm: 11243.369 | params rms: 0.060879 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3:48:03.454781 | finish at 2026-10-10 20:07:38 + [2026-10-10 16:21:06] iteration 6696/ 6855 | consumed samples: 857088 | elapsed time per iteration (ms): 91567.7 | throughput per GPU (TFLOP/s/GPU): 41.5 | MFU 13.30% | FLOPs estimator: workload-aware | logical tokens/s: 183221.6 | valid tokens/s: 183221.6 | physical tokens/s: 183775.9 | loss tokens/s: 134051.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.378740E-01 | lm-global-supervised-tokens: 1.227475E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.480569E-02 | lm-token-buffer-mib: 5.832396E+01 | global_load_balancing_loss: 9.979700E-01 | loss scale: 1.0 | grad norm: 0.108 | params norm: 11243.326 | params rms: 0.060879 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4:02:39.263035 | finish at 2026-10-10 20:23:45 + [2026-10-10 16:22:31] iteration 6697/ 6855 | consumed samples: 857216 | elapsed time per iteration (ms): 85206.2 | throughput per GPU (TFLOP/s/GPU): 42.8 | MFU 13.71% | FLOPs estimator: workload-aware | logical tokens/s: 196901.2 | valid tokens/s: 196901.2 | physical tokens/s: 197444.9 | loss tokens/s: 148551.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.419280E-01 | lm-global-supervised-tokens: 1.265748E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.521486E-02 | lm-token-buffer-mib: 5.827705E+01 | global_load_balancing_loss: 9.998860E-01 | loss scale: 1.0 | grad norm: 0.084 | params norm: 11243.282 | params rms: 0.060879 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3:44:22.574647 | finish at 2026-10-10 20:06:54 + [2026-10-10 16:23:58] iteration 6698/ 6855 | consumed samples: 857344 | elapsed time per iteration (ms): 86713.4 | throughput per GPU (TFLOP/s/GPU): 43.1 | MFU 13.82% | FLOPs estimator: workload-aware | logical tokens/s: 193478.2 | valid tokens/s: 193478.2 | physical tokens/s: 193894.9 | loss tokens/s: 140385.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.953317E-01 | lm-global-supervised-tokens: 1.217335E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.080708E-02 | lm-token-buffer-mib: 5.815180E+01 | global_load_balancing_loss: 1.000295E+00 | loss scale: 1.0 | grad norm: 0.167 | params norm: 11243.237 | params rms: 0.060878 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3:46:54.009397 | finish at 2026-10-10 20:10:52 + [2026-10-10 16:25:23] iteration 6699/ 6855 | consumed samples: 857472 | elapsed time per iteration (ms): 84673.0 | throughput per GPU (TFLOP/s/GPU): 43.5 | MFU 13.96% | FLOPs estimator: workload-aware | logical tokens/s: 198141.0 | valid tokens/s: 198141.0 | physical tokens/s: 198538.9 | loss tokens/s: 145877.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.299670E-01 | lm-global-supervised-tokens: 1.235186E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.310081E-02 | lm-token-buffer-mib: 5.806558E+01 | global_load_balancing_loss: 1.001671E+00 | loss scale: 1.0 | grad norm: 0.155 | params norm: 11243.190 | params rms: 0.060878 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3:40:08.994933 | finish at 2026-10-10 20:05:32 + [2026-10-10 16:27:10] iteration 6700/ 6855 | consumed samples: 857600 | elapsed time per iteration (ms): 107154.1 | throughput per GPU (TFLOP/s/GPU): 34.2 | MFU 10.96% | FLOPs estimator: workload-aware | logical tokens/s: 156570.6 | valid tokens/s: 156570.6 | physical tokens/s: 156850.2 | loss tokens/s: 117751.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.622605E-01 | lm-global-supervised-tokens: 1.261754E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.237166E-02 | lm-token-buffer-mib: 5.809615E+01 | global_load_balancing_loss: 1.001881E+00 | loss scale: 1.0 | grad norm: 0.115 | params norm: 11243.143 | params rms: 0.060878 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4:36:48.879615 | finish at 2026-10-10 21:03:59 + [2026-10-10 16:28:38] iteration 6701/ 6855 | consumed samples: 857728 | elapsed time per iteration (ms): 87734.7 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.40% | FLOPs estimator: workload-aware | logical tokens/s: 191226.3 | valid tokens/s: 191226.3 | physical tokens/s: 191730.2 | loss tokens/s: 145295.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.325459E-01 | lm-global-supervised-tokens: 1.274741E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.122713E-02 | lm-token-buffer-mib: 5.814977E+01 | global_load_balancing_loss: 1.000259E+00 | loss scale: 1.0 | grad norm: 0.098 | params norm: 11243.095 | params rms: 0.060878 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3:45:11.147136 | finish at 2026-10-10 20:13:49 + [2026-10-10 16:30:04] iteration 6702/ 6855 | consumed samples: 857856 | elapsed time per iteration (ms): 86445.1 | throughput per GPU (TFLOP/s/GPU): 43.5 | MFU 13.93% | FLOPs estimator: workload-aware | logical tokens/s: 194079.3 | valid tokens/s: 194079.3 | physical tokens/s: 194471.3 | loss tokens/s: 139595.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.661531E-01 | lm-global-supervised-tokens: 1.206731E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.096079E-02 | lm-token-buffer-mib: 5.808329E+01 | global_load_balancing_loss: 1.000066E+00 | loss scale: 1.0 | grad norm: 0.150 | params norm: 11243.049 | params rms: 0.060877 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3:40:26.101150 | finish at 2026-10-10 20:10:30 + [2026-10-10 16:31:27] iteration 6703/ 6855 | consumed samples: 857984 | elapsed time per iteration (ms): 83106.5 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.78% | FLOPs estimator: workload-aware | logical tokens/s: 201875.4 | valid tokens/s: 201875.4 | physical tokens/s: 202310.0 | loss tokens/s: 155203.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.297361E-01 | lm-global-supervised-tokens: 1.289842E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.459188E-02 | lm-token-buffer-mib: 5.811478E+01 | global_load_balancing_loss: 9.998314E-01 | loss scale: 1.0 | grad norm: 0.140 | params norm: 11243.003 | params rms: 0.060877 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3:30:32.193785 | finish at 2026-10-10 20:01:59 + [2026-10-10 16:32:56] iteration 6704/ 6855 | consumed samples: 858112 | elapsed time per iteration (ms): 88309.2 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.68% | FLOPs estimator: workload-aware | logical tokens/s: 189982.2 | valid tokens/s: 189982.2 | physical tokens/s: 190423.8 | loss tokens/s: 138041.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.110713E-01 | lm-global-supervised-tokens: 1.219034E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.386397E-02 | lm-token-buffer-mib: 5.823929E+01 | global_load_balancing_loss: 1.000346E+00 | loss scale: 1.0 | grad norm: 0.141 | params norm: 11242.956 | params rms: 0.060877 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3:42:14.693347 | finish at 2026-10-10 20:15:10 + [2026-10-10 16:34:26] iteration 6705/ 6855 | consumed samples: 858240 | elapsed time per iteration (ms): 90680.9 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.39% | FLOPs estimator: workload-aware | logical tokens/s: 185013.0 | valid tokens/s: 185013.0 | physical tokens/s: 185471.9 | loss tokens/s: 132913.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.729014E-01 | lm-global-supervised-tokens: 1.205272E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.151146E-02 | lm-token-buffer-mib: 5.811507E+01 | global_load_balancing_loss: 9.985597E-01 | loss scale: 1.0 | grad norm: 0.088 | params norm: 11242.910 | params rms: 0.060877 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3:46:42.140522 | finish at 2026-10-10 20:21:08 + [2026-10-10 16:35:53] iteration 6706/ 6855 | consumed samples: 858368 | elapsed time per iteration (ms): 87203.2 | throughput per GPU (TFLOP/s/GPU): 41.4 | MFU 13.26% | FLOPs estimator: workload-aware | logical tokens/s: 192392.1 | valid tokens/s: 192392.0 | physical tokens/s: 192838.7 | loss tokens/s: 142128.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.197924E-01 | lm-global-supervised-tokens: 1.239410E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.143046E-02 | lm-token-buffer-mib: 5.813140E+01 | global_load_balancing_loss: 9.994202E-01 | loss scale: 1.0 | grad norm: 0.169 | params norm: 11242.865 | params rms: 0.060876 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3:36:33.282606 | finish at 2026-10-10 20:12:27 + [2026-10-10 16:37:24] iteration 6707/ 6855 | consumed samples: 858496 | elapsed time per iteration (ms): 90418.2 | throughput per GPU (TFLOP/s/GPU): 41.1 | MFU 13.19% | FLOPs estimator: workload-aware | logical tokens/s: 185550.6 | valid tokens/s: 185550.6 | physical tokens/s: 185918.8 | loss tokens/s: 135910.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.305920E-01 | lm-global-supervised-tokens: 1.228874E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.051141E-02 | lm-token-buffer-mib: 5.826294E+01 | global_load_balancing_loss: 1.000242E+00 | loss scale: 1.0 | grad norm: 0.074 | params norm: 11242.821 | params rms: 0.060876 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3:43:01.888062 | finish at 2026-10-10 20:20:26 + [2026-10-10 16:38:52] iteration 6708/ 6855 | consumed samples: 858624 | elapsed time per iteration (ms): 88409.7 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.58% | FLOPs estimator: workload-aware | logical tokens/s: 189766.3 | valid tokens/s: 189766.3 | physical tokens/s: 190270.0 | loss tokens/s: 139747.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.314086E-01 | lm-global-supervised-tokens: 1.235503E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.175417E-02 | lm-token-buffer-mib: 5.831107E+01 | global_load_balancing_loss: 1.000010E+00 | loss scale: 1.0 | grad norm: 0.143 | params norm: 11242.775 | params rms: 0.060876 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3:36:36.222839 | finish at 2026-10-10 20:15:28 + [2026-10-10 16:40:19] iteration 6709/ 6855 | consumed samples: 858752 | elapsed time per iteration (ms): 86819.2 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.78% | FLOPs estimator: workload-aware | logical tokens/s: 193243.1 | valid tokens/s: 193243.1 | physical tokens/s: 193845.3 | loss tokens/s: 149412.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.965888E-01 | lm-global-supervised-tokens: 1.297191E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.139603E-02 | lm-token-buffer-mib: 5.849844E+01 | global_load_balancing_loss: 9.999467E-01 | loss scale: 1.0 | grad norm: 0.098 | params norm: 11242.729 | params rms: 0.060876 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3:31:15.608288 | finish at 2026-10-10 20:11:35 + [2026-10-10 16:42:11] iteration 6710/ 6855 | consumed samples: 858880 | elapsed time per iteration (ms): 111611.7 | throughput per GPU (TFLOP/s/GPU): 33.5 | MFU 10.74% | FLOPs estimator: workload-aware | logical tokens/s: 150317.4 | valid tokens/s: 150317.4 | physical tokens/s: 150706.6 | loss tokens/s: 109169.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.752995E-01 | lm-global-supervised-tokens: 1.218459E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.488969E-02 | lm-token-buffer-mib: 5.822939E+01 | global_load_balancing_loss: 1.001324E+00 | loss scale: 1.0 | grad norm: 0.143 | params norm: 11242.683 | params rms: 0.060875 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 4:29:43.696578 | finish at 2026-10-10 21:11:54 + [2026-10-10 16:43:37] iteration 6711/ 6855 | consumed samples: 859008 | elapsed time per iteration (ms): 86729.5 | throughput per GPU (TFLOP/s/GPU): 42.8 | MFU 13.73% | FLOPs estimator: workload-aware | logical tokens/s: 193442.9 | valid tokens/s: 193442.9 | physical tokens/s: 193882.5 | loss tokens/s: 140523.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.521066E-01 | lm-global-supervised-tokens: 1.218757E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.367653E-02 | lm-token-buffer-mib: 5.803771E+01 | global_load_balancing_loss: 1.000611E+00 | loss scale: 1.0 | grad norm: 0.056 | params norm: 11242.640 | params rms: 0.060875 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3:28:09.053810 | finish at 2026-10-10 20:11:46 + [2026-10-10 16:45:04] iteration 6712/ 6855 | consumed samples: 859136 | elapsed time per iteration (ms): 86188.6 | throughput per GPU (TFLOP/s/GPU): 42.8 | MFU 13.72% | FLOPs estimator: workload-aware | logical tokens/s: 194656.8 | valid tokens/s: 194656.8 | physical tokens/s: 195171.5 | loss tokens/s: 147308.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.484431E-01 | lm-global-supervised-tokens: 1.269631E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.165152E-02 | lm-token-buffer-mib: 5.804917E+01 | global_load_balancing_loss: 9.999353E-01 | loss scale: 1.0 | grad norm: 0.147 | params norm: 11242.597 | params rms: 0.060875 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3:25:24.967593 | finish at 2026-10-10 20:10:29 + [2026-10-10 16:46:28] iteration 6713/ 6855 | consumed samples: 859264 | elapsed time per iteration (ms): 84656.7 | throughput per GPU (TFLOP/s/GPU): 43.2 | MFU 13.85% | FLOPs estimator: workload-aware | logical tokens/s: 198179.1 | valid tokens/s: 198179.1 | physical tokens/s: 198646.5 | loss tokens/s: 151465.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.136453E-01 | lm-global-supervised-tokens: 1.282260E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.118852E-02 | lm-token-buffer-mib: 5.812587E+01 | global_load_balancing_loss: 1.000951E+00 | loss scale: 1.0 | grad norm: 0.052 | params norm: 11242.554 | params rms: 0.060875 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3:20:21.249388 | finish at 2026-10-10 20:06:50 + [2026-10-10 16:47:53] iteration 6714/ 6855 | consumed samples: 859392 | elapsed time per iteration (ms): 84755.9 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.76% | FLOPs estimator: workload-aware | logical tokens/s: 197946.0 | valid tokens/s: 197946.0 | physical tokens/s: 198463.0 | loss tokens/s: 153072.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.880172E-01 | lm-global-supervised-tokens: 1.297378E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.332457E-02 | lm-token-buffer-mib: 5.827707E+01 | global_load_balancing_loss: 1.000723E+00 | loss scale: 1.0 | grad norm: 0.203 | params norm: 11242.512 | params rms: 0.060875 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3:19:10.581248 | finish at 2026-10-10 20:07:04 + [2026-10-10 16:49:25] iteration 6715/ 6855 | consumed samples: 859520 | elapsed time per iteration (ms): 92278.2 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.43% | FLOPs estimator: workload-aware | logical tokens/s: 181810.9 | valid tokens/s: 181810.9 | physical tokens/s: 182223.4 | loss tokens/s: 128574.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.461700E-01 | lm-global-supervised-tokens: 1.186459E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.336614E-02 | lm-token-buffer-mib: 5.807818E+01 | global_load_balancing_loss: 1.001158E+00 | loss scale: 1.0 | grad norm: 0.054 | params norm: 11242.471 | params rms: 0.060874 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3:35:18.950391 | finish at 2026-10-10 20:24:44 + [2026-10-10 16:50:52] iteration 6716/ 6855 | consumed samples: 859648 | elapsed time per iteration (ms): 86361.8 | throughput per GPU (TFLOP/s/GPU): 43.4 | MFU 13.92% | FLOPs estimator: workload-aware | logical tokens/s: 194266.4 | valid tokens/s: 194266.4 | physical tokens/s: 194783.5 | loss tokens/s: 141812.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.791115E-01 | lm-global-supervised-tokens: 1.224721E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.095596E-02 | lm-token-buffer-mib: 5.809287E+01 | global_load_balancing_loss: 1.000697E+00 | loss scale: 1.0 | grad norm: 0.202 | params norm: 11242.431 | params rms: 0.060874 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3:20:04.289630 | finish at 2026-10-10 20:10:56 + [2026-10-10 16:52:18] iteration 6717/ 6855 | consumed samples: 859776 | elapsed time per iteration (ms): 85994.3 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.53% | FLOPs estimator: workload-aware | logical tokens/s: 195096.3 | valid tokens/s: 195096.3 | physical tokens/s: 195734.3 | loss tokens/s: 149419.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.679187E-01 | lm-global-supervised-tokens: 1.284924E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.002780E-02 | lm-token-buffer-mib: 5.836126E+01 | global_load_balancing_loss: 9.986953E-01 | loss scale: 1.0 | grad norm: 0.083 | params norm: 11242.389 | params rms: 0.060874 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3:17:47.220195 | finish at 2026-10-10 20:10:05 + [2026-10-10 16:53:43] iteration 6718/ 6855 | consumed samples: 859904 | elapsed time per iteration (ms): 84932.4 | throughput per GPU (TFLOP/s/GPU): 43.5 | MFU 13.95% | FLOPs estimator: workload-aware | logical tokens/s: 197535.2 | valid tokens/s: 197535.2 | physical tokens/s: 198177.1 | loss tokens/s: 144964.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.570616E-01 | lm-global-supervised-tokens: 1.231222E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.519877E-02 | lm-token-buffer-mib: 5.836592E+01 | global_load_balancing_loss: 9.992868E-01 | loss scale: 1.0 | grad norm: 0.159 | params norm: 11242.349 | params rms: 0.060874 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3:13:55.734454 | finish at 2026-10-10 20:07:38 + [2026-10-10 16:55:10] iteration 6719/ 6855 | consumed samples: 860032 | elapsed time per iteration (ms): 87056.8 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.66% | FLOPs estimator: workload-aware | logical tokens/s: 192714.2 | valid tokens/s: 192714.2 | physical tokens/s: 193133.0 | loss tokens/s: 142351.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.510274E-01 | lm-global-supervised-tokens: 1.239270E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.137839E-02 | lm-token-buffer-mib: 5.803868E+01 | global_load_balancing_loss: 9.994728E-01 | loss scale: 1.0 | grad norm: 0.037 | params norm: 11242.306 | params rms: 0.060873 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3:17:19.730686 | finish at 2026-10-10 20:12:29 + [2026-10-10 16:56:56] iteration 6720/ 6855 | consumed samples: 860160 | elapsed time per iteration (ms): 105935.6 | throughput per GPU (TFLOP/s/GPU): 33.9 | MFU 10.87% | FLOPs estimator: workload-aware | logical tokens/s: 158371.7 | valid tokens/s: 158371.7 | physical tokens/s: 158765.9 | loss tokens/s: 118504.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.235249E-01 | lm-global-supervised-tokens: 1.255387E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.169450E-02 | lm-token-buffer-mib: 5.813449E+01 | global_load_balancing_loss: 1.001326E+00 | loss scale: 1.0 | grad norm: 0.119 | params norm: 11242.262 | params rms: 0.060873 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3:58:21.308742 | finish at 2026-10-10 20:55:17 +times across ranks (ms): + save-checkpoint: + rank 0: 24499.94 + rank 1: 24499.96 + rank 2: 24500.01 + rank 3: 24499.95 + rank 4: 24499.93 + rank 5: 24499.94 + rank 6: 24499.91 + rank 7: 24499.93 + rank 8: 24501.48 + rank 9: 24501.46 + rank 10: 24501.31 + rank 11: 24501.46 + rank 12: 24501.47 + rank 13: 24501.45 + rank 14: 24501.47 + rank 15: 24501.46 + rank 16: 24500.09 + rank 17: 24500.09 + rank 18: 24500.09 + rank 19: 24500.09 + rank 20: 24500.07 + rank 21: 24500.07 + rank 22: 24500.08 + rank 23: 24500.08 + rank 24: 24500.61 + rank 25: 24500.53 + rank 26: 24500.64 + rank 27: 24500.64 + rank 28: 24500.58 + rank 29: 24500.73 + rank 30: 24501.08 + rank 31: 24500.60 + rank 32: 24500.44 + rank 33: 24500.46 + rank 34: 24500.44 + rank 35: 24500.32 + rank 36: 24500.36 + rank 37: 24500.35 + rank 38: 24500.40 + rank 39: 24500.34 + rank 40: 24500.66 + rank 41: 24500.65 + rank 42: 24500.64 + rank 43: 24500.65 + rank 44: 24500.63 + rank 45: 24500.62 + rank 46: 24500.62 + rank 47: 24500.63 + rank 48: 24500.70 + rank 49: 24500.67 + rank 50: 24500.63 + rank 51: 24500.56 + rank 52: 24500.73 + rank 53: 24500.69 + rank 54: 24500.55 + rank 55: 24500.64 + rank 56: 24500.37 + rank 57: 24500.34 + rank 58: 24500.29 + rank 59: 24500.15 + rank 60: 24500.30 + rank 61: 24500.33 + rank 62: 24500.33 + rank 63: 24500.32 + [2026-10-10 16:59:04] iteration 6721/ 6855 | consumed samples: 860288 | elapsed time per iteration (ms): 103541.4 | throughput per GPU (TFLOP/s/GPU): 35.8 | MFU 11.46% | FLOPs estimator: workload-aware | logical tokens/s: 162032.8 | valid tokens/s: 162032.7 | physical tokens/s: 162606.2 | loss tokens/s: 122483.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.443220E-01 | lm-global-supervised-tokens: 1.268214E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 5.249560E-01 | lm-token-buffer-mib: 5.841247E+01 | global_load_balancing_loss: 1.000163E+00 | loss scale: 1.0 | grad norm: 0.092 | params norm: 11242.216 | params rms: 0.060873 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3:51:14.552993 | finish at 2026-10-10 20:50:18 + [2026-10-10 17:00:37] iteration 6722/ 6855 | consumed samples: 860416 | elapsed time per iteration (ms): 93591.9 | throughput per GPU (TFLOP/s/GPU): 40.3 | MFU 12.93% | FLOPs estimator: workload-aware | logical tokens/s: 179258.8 | valid tokens/s: 179258.8 | physical tokens/s: 179643.7 | loss tokens/s: 130247.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.658647E-01 | lm-global-supervised-tokens: 1.219013E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.561347E-02 | lm-token-buffer-mib: 5.814454E+01 | global_load_balancing_loss: 1.000153E+00 | loss scale: 1.0 | grad norm: 0.104 | params norm: 11242.172 | params rms: 0.060873 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3:27:27.726203 | finish at 2026-10-10 20:28:05 + [2026-10-10 17:02:03] iteration 6723/ 6855 | consumed samples: 860544 | elapsed time per iteration (ms): 85741.8 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.67% | FLOPs estimator: workload-aware | logical tokens/s: 195670.4 | valid tokens/s: 195670.4 | physical tokens/s: 196021.6 | loss tokens/s: 147099.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.700341E-01 | lm-global-supervised-tokens: 1.261259E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.432292E-02 | lm-token-buffer-mib: 5.809149E+01 | global_load_balancing_loss: 1.000430E+00 | loss scale: 1.0 | grad norm: 0.094 | params norm: 11242.126 | params rms: 0.060872 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3:08:37.919592 | finish at 2026-10-10 20:10:41 + [2026-10-10 17:03:28] iteration 6724/ 6855 | consumed samples: 860672 | elapsed time per iteration (ms): 84481.2 | throughput per GPU (TFLOP/s/GPU): 42.8 | MFU 13.71% | FLOPs estimator: workload-aware | logical tokens/s: 198590.7 | valid tokens/s: 198590.6 | physical tokens/s: 199172.4 | loss tokens/s: 149129.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.739916E-01 | lm-global-supervised-tokens: 1.259862E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.011841E-02 | lm-token-buffer-mib: 5.833379E+01 | global_load_balancing_loss: 9.985281E-01 | loss scale: 1.0 | grad norm: 0.040 | params norm: 11242.079 | params rms: 0.060872 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3:04:27.033949 | finish at 2026-10-10 20:07:55 + [2026-10-10 17:04:54] iteration 6725/ 6855 | consumed samples: 860800 | elapsed time per iteration (ms): 86257.0 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.62% | FLOPs estimator: workload-aware | logical tokens/s: 194501.8 | valid tokens/s: 194501.8 | physical tokens/s: 195052.4 | loss tokens/s: 146511.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.375715E-01 | lm-global-supervised-tokens: 1.263763E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.290883E-02 | lm-token-buffer-mib: 5.821675E+01 | global_load_balancing_loss: 9.998378E-01 | loss scale: 1.0 | grad norm: 0.081 | params norm: 11242.031 | params rms: 0.060872 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3:06:53.411949 | finish at 2026-10-10 20:11:47 + [2026-10-10 17:06:24] iteration 6726/ 6855 | consumed samples: 860928 | elapsed time per iteration (ms): 90328.5 | throughput per GPU (TFLOP/s/GPU): 40.5 | MFU 12.97% | FLOPs estimator: workload-aware | logical tokens/s: 185735.2 | valid tokens/s: 185735.1 | physical tokens/s: 186216.5 | loss tokens/s: 143161.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.584805E-01 | lm-global-supervised-tokens: 1.293158E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 4.771465E+00 | lm-token-buffer-mib: 5.815952E+01 | global_load_balancing_loss: 1.001560E+00 | loss scale: 1.0 | grad norm: 0.093 | params norm: 11241.984 | params rms: 0.060872 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3:14:12.376842 | finish at 2026-10-10 20:20:37 + [2026-10-10 17:07:51] iteration 6727/ 6855 | consumed samples: 861056 | elapsed time per iteration (ms): 86339.8 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.60% | FLOPs estimator: workload-aware | logical tokens/s: 194314.9 | valid tokens/s: 194314.9 | physical tokens/s: 194907.4 | loss tokens/s: 144912.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.702341E-01 | lm-global-supervised-tokens: 1.251173E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.707980E-02 | lm-token-buffer-mib: 5.825739E+01 | global_load_balancing_loss: 1.000991E+00 | loss scale: 1.0 | grad norm: 0.049 | params norm: 11241.937 | params rms: 0.060871 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3:04:11.496582 | finish at 2026-10-10 20:12:02 + [2026-10-10 17:09:16] iteration 6728/ 6855 | consumed samples: 861184 | elapsed time per iteration (ms): 85397.0 | throughput per GPU (TFLOP/s/GPU): 43.1 | MFU 13.80% | FLOPs estimator: workload-aware | logical tokens/s: 196460.2 | valid tokens/s: 196460.2 | physical tokens/s: 196943.3 | loss tokens/s: 142731.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.650201E-01 | lm-global-supervised-tokens: 1.218885E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.434206E-02 | lm-token-buffer-mib: 5.802820E+01 | global_load_balancing_loss: 1.000818E+00 | loss scale: 1.0 | grad norm: 0.089 | params norm: 11241.889 | params rms: 0.060871 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3:00:45.417617 | finish at 2026-10-10 20:10:01 + [2026-10-10 17:10:48] iteration 6729/ 6855 | consumed samples: 861312 | elapsed time per iteration (ms): 92157.2 | throughput per GPU (TFLOP/s/GPU): 40.1 | MFU 12.86% | FLOPs estimator: workload-aware | logical tokens/s: 182049.4 | valid tokens/s: 182049.4 | physical tokens/s: 182401.6 | loss tokens/s: 138742.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.415796E-01 | lm-global-supervised-tokens: 1.278613E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.143258E-02 | lm-token-buffer-mib: 5.817605E+01 | global_load_balancing_loss: 9.999054E-01 | loss scale: 1.0 | grad norm: 0.062 | params norm: 11241.840 | params rms: 0.060871 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3:13:31.806882 | finish at 2026-10-10 20:24:20 + [2026-10-10 17:12:41] iteration 6730/ 6855 | consumed samples: 861440 | elapsed time per iteration (ms): 112411.9 | throughput per GPU (TFLOP/s/GPU): 32.8 | MFU 10.52% | FLOPs estimator: workload-aware | logical tokens/s: 149247.5 | valid tokens/s: 149247.5 | physical tokens/s: 149678.7 | loss tokens/s: 112604.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.417063E-01 | lm-global-supervised-tokens: 1.265811E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.539640E-02 | lm-token-buffer-mib: 5.807583E+01 | global_load_balancing_loss: 1.001123E+00 | loss scale: 1.0 | grad norm: 0.047 | params norm: 11241.792 | params rms: 0.060871 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3:54:11.486224 | finish at 2026-10-10 21:06:52 + [2026-10-10 17:14:09] iteration 6731/ 6855 | consumed samples: 861568 | elapsed time per iteration (ms): 88075.6 | throughput per GPU (TFLOP/s/GPU): 41.6 | MFU 13.34% | FLOPs estimator: workload-aware | logical tokens/s: 190485.6 | valid tokens/s: 190485.6 | physical tokens/s: 191015.8 | loss tokens/s: 136696.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.342541E-01 | lm-global-supervised-tokens: 1.203963E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.815801E-02 | lm-token-buffer-mib: 5.810770E+01 | global_load_balancing_loss: 9.996254E-01 | loss scale: 1.0 | grad norm: 0.087 | params norm: 11241.744 | params rms: 0.060870 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3:02:01.380390 | finish at 2026-10-10 20:16:10 + [2026-10-10 17:15:35] iteration 6732/ 6855 | consumed samples: 861696 | elapsed time per iteration (ms): 86612.4 | throughput per GPU (TFLOP/s/GPU): 43.3 | MFU 13.87% | FLOPs estimator: workload-aware | logical tokens/s: 193704.4 | valid tokens/s: 193704.4 | physical tokens/s: 194098.5 | loss tokens/s: 146683.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.244165E-01 | lm-global-supervised-tokens: 1.270460E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.160328E-02 | lm-token-buffer-mib: 5.824104E+01 | global_load_balancing_loss: 1.000291E+00 | loss scale: 1.0 | grad norm: 0.038 | params norm: 11241.697 | params rms: 0.060870 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2:57:33.319430 | finish at 2026-10-10 20:13:09 + [2026-10-10 17:17:05] iteration 6733/ 6855 | consumed samples: 861824 | elapsed time per iteration (ms): 89930.7 | throughput per GPU (TFLOP/s/GPU): 41.1 | MFU 13.17% | FLOPs estimator: workload-aware | logical tokens/s: 186556.9 | valid tokens/s: 186556.9 | physical tokens/s: 187095.4 | loss tokens/s: 139354.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.614979E-01 | lm-global-supervised-tokens: 1.253220E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.160278E-02 | lm-token-buffer-mib: 5.811150E+01 | global_load_balancing_loss: 9.998211E-01 | loss scale: 1.0 | grad norm: 0.106 | params norm: 11241.649 | params rms: 0.060870 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3:02:51.543866 | finish at 2026-10-10 20:19:57 + [2026-10-10 17:18:35] iteration 6734/ 6855 | consumed samples: 861952 | elapsed time per iteration (ms): 90278.3 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.43% | FLOPs estimator: workload-aware | logical tokens/s: 185838.5 | valid tokens/s: 185838.5 | physical tokens/s: 186141.1 | loss tokens/s: 137546.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.826009E-01 | lm-global-supervised-tokens: 1.241748E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.611259E-02 | lm-token-buffer-mib: 5.808499E+01 | global_load_balancing_loss: 9.997792E-01 | loss scale: 1.0 | grad norm: 0.044 | params norm: 11241.600 | params rms: 0.060870 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3:02:03.679383 | finish at 2026-10-10 20:20:39 + [2026-10-10 17:20:06] iteration 6735/ 6855 | consumed samples: 862080 | elapsed time per iteration (ms): 90320.2 | throughput per GPU (TFLOP/s/GPU): 41.5 | MFU 13.32% | FLOPs estimator: workload-aware | logical tokens/s: 185752.4 | valid tokens/s: 185752.4 | physical tokens/s: 186242.1 | loss tokens/s: 130793.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.390612E-01 | lm-global-supervised-tokens: 1.181329E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.343098E-02 | lm-token-buffer-mib: 5.825117E+01 | global_load_balancing_loss: 9.997470E-01 | loss scale: 1.0 | grad norm: 0.118 | params norm: 11241.551 | params rms: 0.060869 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3:00:38.425541 | finish at 2026-10-10 20:20:44 + [2026-10-10 17:21:31] iteration 6736/ 6855 | consumed samples: 862208 | elapsed time per iteration (ms): 85286.5 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.58% | FLOPs estimator: workload-aware | logical tokens/s: 196715.2 | valid tokens/s: 196715.2 | physical tokens/s: 197232.9 | loss tokens/s: 147235.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.312329E-01 | lm-global-supervised-tokens: 1.255723E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.152354E-02 | lm-token-buffer-mib: 5.816779E+01 | global_load_balancing_loss: 1.000087E+00 | loss scale: 1.0 | grad norm: 0.090 | params norm: 11241.504 | params rms: 0.060869 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2:49:09.090064 | finish at 2026-10-10 20:10:40 + [2026-10-10 17:22:59] iteration 6737/ 6855 | consumed samples: 862336 | elapsed time per iteration (ms): 87801.2 | throughput per GPU (TFLOP/s/GPU): 41.7 | MFU 13.36% | FLOPs estimator: workload-aware | logical tokens/s: 191081.6 | valid tokens/s: 191081.6 | physical tokens/s: 191591.7 | loss tokens/s: 142482.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.122873E-01 | lm-global-supervised-tokens: 1.251015E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.288733E-02 | lm-token-buffer-mib: 5.830953E+01 | global_load_balancing_loss: 1.000152E+00 | loss scale: 1.0 | grad norm: 0.040 | params norm: 11241.458 | params rms: 0.060869 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2:52:40.540915 | finish at 2026-10-10 20:15:39 + [2026-10-10 17:24:27] iteration 6738/ 6855 | consumed samples: 862464 | elapsed time per iteration (ms): 88164.6 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.67% | FLOPs estimator: workload-aware | logical tokens/s: 190294.0 | valid tokens/s: 190294.0 | physical tokens/s: 190738.8 | loss tokens/s: 139522.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.688225E-01 | lm-global-supervised-tokens: 1.230094E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.301071E-02 | lm-token-buffer-mib: 5.811607E+01 | global_load_balancing_loss: 9.999599E-01 | loss scale: 1.0 | grad norm: 0.061 | params norm: 11241.409 | params rms: 0.060869 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2:51:55.252692 | finish at 2026-10-10 20:16:22 + [2026-10-10 17:25:55] iteration 6739/ 6855 | consumed samples: 862592 | elapsed time per iteration (ms): 88131.5 | throughput per GPU (TFLOP/s/GPU): 41.7 | MFU 13.35% | FLOPs estimator: workload-aware | logical tokens/s: 190365.7 | valid tokens/s: 190365.7 | physical tokens/s: 190733.6 | loss tokens/s: 143600.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.569056E-01 | lm-global-supervised-tokens: 1.265569E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.593838E-02 | lm-token-buffer-mib: 5.815907E+01 | global_load_balancing_loss: 1.000334E+00 | loss scale: 1.0 | grad norm: 0.042 | params norm: 11241.361 | params rms: 0.060868 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2:50:23.250571 | finish at 2026-10-10 20:16:18 + [2026-10-10 17:27:44] iteration 6740/ 6855 | consumed samples: 862720 | elapsed time per iteration (ms): 108405.6 | throughput per GPU (TFLOP/s/GPU): 34.0 | MFU 10.90% | FLOPs estimator: workload-aware | logical tokens/s: 154763.0 | valid tokens/s: 154763.0 | physical tokens/s: 155271.7 | loss tokens/s: 112674.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.088731E-01 | lm-global-supervised-tokens: 1.221450E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.264277E-02 | lm-token-buffer-mib: 5.801785E+01 | global_load_balancing_loss: 1.001314E+00 | loss scale: 1.0 | grad norm: 0.091 | params norm: 11241.316 | params rms: 0.060868 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3:27:46.639073 | finish at 2026-10-10 20:55:30 + [2026-10-10 17:29:12] iteration 6741/ 6855 | consumed samples: 862848 | elapsed time per iteration (ms): 88076.2 | throughput per GPU (TFLOP/s/GPU): 41.6 | MFU 13.34% | FLOPs estimator: workload-aware | logical tokens/s: 190484.9 | valid tokens/s: 190484.9 | physical tokens/s: 191113.6 | loss tokens/s: 141838.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.811460E-01 | lm-global-supervised-tokens: 1.249256E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.091891E-02 | lm-token-buffer-mib: 5.826760E+01 | global_load_balancing_loss: 1.000445E+00 | loss scale: 1.0 | grad norm: 0.040 | params norm: 11241.269 | params rms: 0.060868 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2:47:20.687943 | finish at 2026-10-10 20:16:32 + [2026-10-10 17:30:39] iteration 6742/ 6855 | consumed samples: 862976 | elapsed time per iteration (ms): 87085.7 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.56% | FLOPs estimator: workload-aware | logical tokens/s: 192651.9 | valid tokens/s: 192651.9 | physical tokens/s: 193057.2 | loss tokens/s: 150924.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.760051E-01 | lm-global-supervised-tokens: 1.314333E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.273419E-02 | lm-token-buffer-mib: 5.821490E+01 | global_load_balancing_loss: 1.000166E+00 | loss scale: 1.0 | grad norm: 0.081 | params norm: 11241.220 | params rms: 0.060868 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2:44:00.679389 | finish at 2026-10-10 20:14:39 + [2026-10-10 17:32:05] iteration 6743/ 6855 | consumed samples: 863104 | elapsed time per iteration (ms): 86520.8 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.67% | FLOPs estimator: workload-aware | logical tokens/s: 193909.4 | valid tokens/s: 193909.4 | physical tokens/s: 194346.3 | loss tokens/s: 145340.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.990734E-01 | lm-global-supervised-tokens: 1.257500E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.238880E-02 | lm-token-buffer-mib: 5.819976E+01 | global_load_balancing_loss: 1.001243E+00 | loss scale: 1.0 | grad norm: 0.068 | params norm: 11241.172 | params rms: 0.060867 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2:41:30.325527 | finish at 2026-10-10 20:13:36 + [2026-10-10 17:33:36] iteration 6744/ 6855 | consumed samples: 863232 | elapsed time per iteration (ms): 91092.1 | throughput per GPU (TFLOP/s/GPU): 40.9 | MFU 13.10% | FLOPs estimator: workload-aware | logical tokens/s: 184178.3 | valid tokens/s: 184178.3 | physical tokens/s: 184486.6 | loss tokens/s: 127972.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.557022E-01 | lm-global-supervised-tokens: 1.165728E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.346371E-02 | lm-token-buffer-mib: 5.816575E+01 | global_load_balancing_loss: 1.000576E+00 | loss scale: 1.0 | grad norm: 0.038 | params norm: 11241.125 | params rms: 0.060867 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2:48:31.227985 | finish at 2026-10-10 20:22:08 + [2026-10-10 17:35:05] iteration 6745/ 6855 | consumed samples: 863360 | elapsed time per iteration (ms): 88326.9 | throughput per GPU (TFLOP/s/GPU): 40.8 | MFU 13.09% | FLOPs estimator: workload-aware | logical tokens/s: 189944.3 | valid tokens/s: 189944.3 | physical tokens/s: 190514.2 | loss tokens/s: 146652.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.520048E-01 | lm-global-supervised-tokens: 1.295340E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.266301E-02 | lm-token-buffer-mib: 5.808682E+01 | global_load_balancing_loss: 1.000011E+00 | loss scale: 1.0 | grad norm: 0.144 | params norm: 11241.078 | params rms: 0.060867 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2:41:55.956063 | finish at 2026-10-10 20:17:01 + [2026-10-10 17:36:31] iteration 6746/ 6855 | consumed samples: 863488 | elapsed time per iteration (ms): 86760.5 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.63% | FLOPs estimator: workload-aware | logical tokens/s: 193373.5 | valid tokens/s: 193373.5 | physical tokens/s: 193824.2 | loss tokens/s: 143949.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.664134E-01 | lm-global-supervised-tokens: 1.248913E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.397291E-02 | lm-token-buffer-mib: 5.819225E+01 | global_load_balancing_loss: 9.996581E-01 | loss scale: 1.0 | grad norm: 0.085 | params norm: 11241.031 | params rms: 0.060867 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2:37:36.898289 | finish at 2026-10-10 20:14:08 + [2026-10-10 17:38:00] iteration 6747/ 6855 | consumed samples: 863616 | elapsed time per iteration (ms): 88142.8 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.47% | FLOPs estimator: workload-aware | logical tokens/s: 190340.6 | valid tokens/s: 190340.6 | physical tokens/s: 190915.1 | loss tokens/s: 143566.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.759822E-01 | lm-global-supervised-tokens: 1.265439E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.296138E-02 | lm-token-buffer-mib: 5.827448E+01 | global_load_balancing_loss: 9.987163E-01 | loss scale: 1.0 | grad norm: 0.089 | params norm: 11240.986 | params rms: 0.060866 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2:38:39.421277 | finish at 2026-10-10 20:16:39 + [2026-10-10 17:39:29] iteration 6748/ 6855 | consumed samples: 863744 | elapsed time per iteration (ms): 89096.4 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.50% | FLOPs estimator: workload-aware | logical tokens/s: 188303.4 | valid tokens/s: 188303.4 | physical tokens/s: 188764.3 | loss tokens/s: 135262.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.437879E-01 | lm-global-supervised-tokens: 1.205141E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.149337E-02 | lm-token-buffer-mib: 5.815716E+01 | global_load_balancing_loss: 9.993715E-01 | loss scale: 1.0 | grad norm: 0.115 | params norm: 11240.940 | params rms: 0.060866 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2:38:53.314802 | finish at 2026-10-10 20:18:22 + [2026-10-10 17:40:59] iteration 6749/ 6855 | consumed samples: 863872 | elapsed time per iteration (ms): 89832.6 | throughput per GPU (TFLOP/s/GPU): 41.2 | MFU 13.20% | FLOPs estimator: workload-aware | logical tokens/s: 186760.0 | valid tokens/s: 186760.0 | physical tokens/s: 187283.7 | loss tokens/s: 136770.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.571267E-01 | lm-global-supervised-tokens: 1.228647E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.162467E-02 | lm-token-buffer-mib: 5.803592E+01 | global_load_balancing_loss: 9.994658E-01 | loss scale: 1.0 | grad norm: 0.050 | params norm: 11240.892 | params rms: 0.060866 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2:38:42.252580 | finish at 2026-10-10 20:19:41 + [2026-10-10 17:42:48] iteration 6750/ 6855 | consumed samples: 864000 | elapsed time per iteration (ms): 109929.7 | throughput per GPU (TFLOP/s/GPU): 34.2 | MFU 10.95% | FLOPs estimator: workload-aware | logical tokens/s: 152617.2 | valid tokens/s: 152617.2 | physical tokens/s: 152964.4 | loss tokens/s: 113865.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.510136E-01 | lm-global-supervised-tokens: 1.251721E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.306747E-02 | lm-token-buffer-mib: 5.816138E+01 | global_load_balancing_loss: 1.001133E+00 | loss scale: 1.0 | grad norm: 0.041 | params norm: 11240.844 | params rms: 0.060866 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 3:12:22.613432 | finish at 2026-10-10 20:55:11 + [2026-10-10 17:44:14] iteration 6751/ 6855 | consumed samples: 864128 | elapsed time per iteration (ms): 85532.1 | throughput per GPU (TFLOP/s/GPU): 43.3 | MFU 13.87% | FLOPs estimator: workload-aware | logical tokens/s: 196151.0 | valid tokens/s: 196151.0 | physical tokens/s: 196577.3 | loss tokens/s: 144045.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.178255E-01 | lm-global-supervised-tokens: 1.232056E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.233783E-02 | lm-token-buffer-mib: 5.810078E+01 | global_load_balancing_loss: 1.000725E+00 | loss scale: 1.0 | grad norm: 0.053 | params norm: 11240.800 | params rms: 0.060865 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2:28:15.342165 | finish at 2026-10-10 20:12:29 + [2026-10-10 17:45:43] iteration 6752/ 6855 | consumed samples: 864256 | elapsed time per iteration (ms): 89037.0 | throughput per GPU (TFLOP/s/GPU): 41.7 | MFU 13.36% | FLOPs estimator: workload-aware | logical tokens/s: 188429.4 | valid tokens/s: 188429.4 | physical tokens/s: 188918.6 | loss tokens/s: 138163.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.777875E-01 | lm-global-supervised-tokens: 1.230163E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.577399E-02 | lm-token-buffer-mib: 5.811935E+01 | global_load_balancing_loss: 1.002270E+00 | loss scale: 1.0 | grad norm: 0.055 | params norm: 11240.756 | params rms: 0.060865 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2:32:50.816028 | finish at 2026-10-10 20:18:34 + [2026-10-10 17:47:10] iteration 6753/ 6855 | consumed samples: 864384 | elapsed time per iteration (ms): 87417.0 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.70% | FLOPs estimator: workload-aware | logical tokens/s: 191921.3 | valid tokens/s: 191921.3 | physical tokens/s: 192542.9 | loss tokens/s: 143102.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.393687E-01 | lm-global-supervised-tokens: 1.250957E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.156099E-02 | lm-token-buffer-mib: 5.810419E+01 | global_load_balancing_loss: 1.000771E+00 | loss scale: 1.0 | grad norm: 0.064 | params norm: 11240.712 | params rms: 0.060865 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2:28:36.532328 | finish at 2026-10-10 20:15:47 + [2026-10-10 17:48:38] iteration 6754/ 6855 | consumed samples: 864512 | elapsed time per iteration (ms): 87739.1 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.45% | FLOPs estimator: workload-aware | logical tokens/s: 191216.9 | valid tokens/s: 191216.9 | physical tokens/s: 191697.3 | loss tokens/s: 142089.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.694884E-01 | lm-global-supervised-tokens: 1.246680E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.334399E-02 | lm-token-buffer-mib: 5.814024E+01 | global_load_balancing_loss: 9.996528E-01 | loss scale: 1.0 | grad norm: 0.081 | params norm: 11240.669 | params rms: 0.060865 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2:27:41.653529 | finish at 2026-10-10 20:16:20 + [2026-10-10 17:50:01] iteration 6755/ 6855 | consumed samples: 864640 | elapsed time per iteration (ms): 82992.0 | throughput per GPU (TFLOP/s/GPU): 44.0 | MFU 14.11% | FLOPs estimator: workload-aware | logical tokens/s: 202154.0 | valid tokens/s: 202154.0 | physical tokens/s: 202724.0 | loss tokens/s: 150569.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.293403E-01 | lm-global-supervised-tokens: 1.249604E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.235696E-02 | lm-token-buffer-mib: 5.821970E+01 | global_load_balancing_loss: 1.001286E+00 | loss scale: 1.0 | grad norm: 0.107 | params norm: 11240.624 | params rms: 0.060864 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2:18:19.195051 | finish at 2026-10-10 20:08:20 + [2026-10-10 17:51:28] iteration 6756/ 6855 | consumed samples: 864768 | elapsed time per iteration (ms): 86428.4 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.44% | FLOPs estimator: workload-aware | logical tokens/s: 194116.1 | valid tokens/s: 194116.1 | physical tokens/s: 194659.0 | loss tokens/s: 143970.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.250864E-01 | lm-global-supervised-tokens: 1.244312E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.153724E-02 | lm-token-buffer-mib: 5.810481E+01 | global_load_balancing_loss: 9.997470E-01 | loss scale: 1.0 | grad norm: 0.097 | params norm: 11240.580 | params rms: 0.060864 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2:22:36.409527 | finish at 2026-10-10 20:14:04 + [2026-10-10 17:52:55] iteration 6757/ 6855 | consumed samples: 864896 | elapsed time per iteration (ms): 87590.5 | throughput per GPU (TFLOP/s/GPU): 41.3 | MFU 13.23% | FLOPs estimator: workload-aware | logical tokens/s: 191540.7 | valid tokens/s: 191540.7 | physical tokens/s: 192263.3 | loss tokens/s: 143924.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.418197E-01 | lm-global-supervised-tokens: 1.260645E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.158138E-02 | lm-token-buffer-mib: 5.818985E+01 | global_load_balancing_loss: 9.993163E-01 | loss scale: 1.0 | grad norm: 0.119 | params norm: 11240.535 | params rms: 0.060864 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2:23:03.872166 | finish at 2026-10-10 20:15:59 + [2026-10-10 17:54:25] iteration 6758/ 6855 | consumed samples: 865024 | elapsed time per iteration (ms): 89531.6 | throughput per GPU (TFLOP/s/GPU): 41.6 | MFU 13.32% | FLOPs estimator: workload-aware | logical tokens/s: 187388.6 | valid tokens/s: 187388.6 | physical tokens/s: 187910.1 | loss tokens/s: 143740.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.095221E-01 | lm-global-supervised-tokens: 1.286936E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.168491E-02 | lm-token-buffer-mib: 5.806948E+01 | global_load_balancing_loss: 9.994425E-01 | loss scale: 1.0 | grad norm: 0.162 | params norm: 11240.493 | params rms: 0.060864 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2:24:44.569548 | finish at 2026-10-10 20:19:09 + [2026-10-10 17:55:51] iteration 6759/ 6855 | consumed samples: 865152 | elapsed time per iteration (ms): 86131.7 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.51% | FLOPs estimator: workload-aware | logical tokens/s: 194785.2 | valid tokens/s: 194785.2 | physical tokens/s: 195267.3 | loss tokens/s: 140503.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.485758E-01 | lm-global-supervised-tokens: 1.210177E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.234483E-02 | lm-token-buffer-mib: 5.813380E+01 | global_load_balancing_loss: 1.000202E+00 | loss scale: 1.0 | grad norm: 0.073 | params norm: 11240.451 | params rms: 0.060863 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2:17:48.643204 | finish at 2026-10-10 20:13:39 + [2026-10-10 17:57:42] iteration 6760/ 6855 | consumed samples: 865280 | elapsed time per iteration (ms): 110866.1 | throughput per GPU (TFLOP/s/GPU): 33.0 | MFU 10.57% | FLOPs estimator: workload-aware | logical tokens/s: 151328.3 | valid tokens/s: 151328.3 | physical tokens/s: 151677.6 | loss tokens/s: 112336.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.030383E-01 | lm-global-supervised-tokens: 1.245428E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.158742E-02 | lm-token-buffer-mib: 5.822459E+01 | global_load_balancing_loss: 1.001183E+00 | loss scale: 1.0 | grad norm: 0.381 | params norm: 11240.408 | params rms: 0.060863 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2:55:32.280254 | finish at 2026-10-10 20:53:14 +times across ranks (ms): + save-checkpoint: + rank 0: 24039.05 + rank 1: 24039.06 + rank 2: 24039.10 + rank 3: 24039.04 + rank 4: 24039.09 + rank 5: 24039.02 + rank 6: 24039.03 + rank 7: 24039.04 + rank 8: 24039.64 + rank 9: 24039.65 + rank 10: 24039.63 + rank 11: 24039.63 + rank 12: 24039.63 + rank 13: 24039.63 + rank 14: 24039.63 + rank 15: 24039.64 + rank 16: 24039.84 + rank 17: 24039.82 + rank 18: 24039.85 + rank 19: 24039.83 + rank 20: 24039.82 + rank 21: 24039.82 + rank 22: 24039.84 + rank 23: 24039.82 + rank 24: 24039.19 + rank 25: 24039.16 + rank 26: 24039.17 + rank 27: 24039.18 + rank 28: 24039.15 + rank 29: 24039.17 + rank 30: 24039.15 + rank 31: 24039.16 + rank 32: 24039.53 + rank 33: 24039.53 + rank 34: 24039.51 + rank 35: 24039.61 + rank 36: 24039.46 + rank 37: 24039.46 + rank 38: 24039.49 + rank 39: 24039.47 + rank 40: 24037.77 + rank 41: 24037.74 + rank 42: 24037.76 + rank 43: 24037.81 + rank 44: 24037.72 + rank 45: 24037.74 + rank 46: 24037.72 + rank 47: 24037.73 + rank 48: 24039.44 + rank 49: 24039.44 + rank 50: 24039.52 + rank 51: 24039.45 + rank 52: 24039.47 + rank 53: 24039.41 + rank 54: 24039.42 + rank 55: 24039.41 + rank 56: 24039.75 + rank 57: 24039.74 + rank 58: 24039.73 + rank 59: 24039.75 + rank 60: 24039.76 + rank 61: 24039.77 + rank 62: 24039.73 + rank 63: 24039.76 + [2026-10-10 17:59:52] iteration 6761/ 6855 | consumed samples: 865408 | elapsed time per iteration (ms): 103410.9 | throughput per GPU (TFLOP/s/GPU): 35.3 | MFU 11.32% | FLOPs estimator: workload-aware | logical tokens/s: 162237.8 | valid tokens/s: 162237.8 | physical tokens/s: 162632.5 | loss tokens/s: 120078.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.705101E-01 | lm-global-supervised-tokens: 1.241747E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.317131E+00 | lm-token-buffer-mib: 5.813785E+01 | global_load_balancing_loss: 9.996999E-01 | loss scale: 1.0 | grad norm: 0.298 | params norm: 11240.365 | params rms: 0.060863 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2:42:00.628421 | finish at 2026-10-10 20:41:52 + [2026-10-10 18:01:22] iteration 6762/ 6855 | consumed samples: 865536 | elapsed time per iteration (ms): 90325.7 | throughput per GPU (TFLOP/s/GPU): 40.3 | MFU 12.92% | FLOPs estimator: workload-aware | logical tokens/s: 185740.6 | valid tokens/s: 185740.5 | physical tokens/s: 186183.7 | loss tokens/s: 147259.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.294458E-01 | lm-global-supervised-tokens: 1.330135E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.849206E-02 | lm-token-buffer-mib: 5.806757E+01 | global_load_balancing_loss: 1.000257E+00 | loss scale: 1.0 | grad norm: 0.158 | params norm: 11240.322 | params rms: 0.060863 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2:20:00.286335 | finish at 2026-10-10 20:21:22 + [2026-10-10 18:02:51] iteration 6763/ 6855 | consumed samples: 865664 | elapsed time per iteration (ms): 89318.7 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.43% | FLOPs estimator: workload-aware | logical tokens/s: 187835.3 | valid tokens/s: 187835.3 | physical tokens/s: 188258.1 | loss tokens/s: 136020.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.024161E-01 | lm-global-supervised-tokens: 1.214922E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.285311E-02 | lm-token-buffer-mib: 5.805499E+01 | global_load_balancing_loss: 1.000524E+00 | loss scale: 1.0 | grad norm: 0.111 | params norm: 11240.277 | params rms: 0.060862 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2:16:57.324750 | finish at 2026-10-10 20:19:49 + [2026-10-10 18:04:18] iteration 6764/ 6855 | consumed samples: 865792 | elapsed time per iteration (ms): 86287.8 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.44% | FLOPs estimator: workload-aware | logical tokens/s: 194433.1 | valid tokens/s: 194433.1 | physical tokens/s: 194865.7 | loss tokens/s: 149133.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.551248E-01 | lm-global-supervised-tokens: 1.286843E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 8.436432E-02 | lm-token-buffer-mib: 5.822277E+01 | global_load_balancing_loss: 1.000079E+00 | loss scale: 1.0 | grad norm: 0.206 | params norm: 11240.233 | params rms: 0.060862 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2:10:52.187051 | finish at 2026-10-10 20:15:10 + [2026-10-10 18:05:43] iteration 6765/ 6855 | consumed samples: 865920 | elapsed time per iteration (ms): 85630.6 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.68% | FLOPs estimator: workload-aware | logical tokens/s: 195925.3 | valid tokens/s: 195925.2 | physical tokens/s: 196406.3 | loss tokens/s: 148945.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.928330E-01 | lm-global-supervised-tokens: 1.275428E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.485620E-02 | lm-token-buffer-mib: 5.805772E+01 | global_load_balancing_loss: 1.000744E+00 | loss scale: 1.0 | grad norm: 0.055 | params norm: 11240.187 | params rms: 0.060862 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2:08:26.757839 | finish at 2026-10-10 20:14:10 + [2026-10-10 18:07:08] iteration 6766/ 6855 | consumed samples: 866048 | elapsed time per iteration (ms): 84780.5 | throughput per GPU (TFLOP/s/GPU): 43.2 | MFU 13.83% | FLOPs estimator: workload-aware | logical tokens/s: 197890.0 | valid tokens/s: 197890.0 | physical tokens/s: 198389.9 | loss tokens/s: 146679.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.792029E-01 | lm-global-supervised-tokens: 1.243555E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.374916E-02 | lm-token-buffer-mib: 5.828083E+01 | global_load_balancing_loss: 1.000366E+00 | loss scale: 1.0 | grad norm: 0.214 | params norm: 11240.141 | params rms: 0.060862 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2:05:45.465279 | finish at 2026-10-10 20:12:54 + [2026-10-10 18:08:35] iteration 6767/ 6855 | consumed samples: 866176 | elapsed time per iteration (ms): 87158.7 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.76% | FLOPs estimator: workload-aware | logical tokens/s: 192490.5 | valid tokens/s: 192490.4 | physical tokens/s: 192893.8 | loss tokens/s: 143239.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.647423E-01 | lm-global-supervised-tokens: 1.248457E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.147604E-02 | lm-token-buffer-mib: 5.815207E+01 | global_load_balancing_loss: 9.996044E-01 | loss scale: 1.0 | grad norm: 0.107 | params norm: 11240.095 | params rms: 0.060861 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2:07:49.964235 | finish at 2026-10-10 20:16:25 + [2026-10-10 18:10:02] iteration 6768/ 6855 | consumed samples: 866304 | elapsed time per iteration (ms): 86366.8 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.61% | FLOPs estimator: workload-aware | logical tokens/s: 194255.1 | valid tokens/s: 194255.0 | physical tokens/s: 194523.8 | loss tokens/s: 148527.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.507686E-01 | lm-global-supervised-tokens: 1.282786E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.669566E-02 | lm-token-buffer-mib: 5.806738E+01 | global_load_balancing_loss: 1.000572E+00 | loss scale: 1.0 | grad norm: 0.056 | params norm: 11240.049 | params rms: 0.060861 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2:05:13.913991 | finish at 2026-10-10 20:15:16 + [2026-10-10 18:11:32] iteration 6769/ 6855 | consumed samples: 866432 | elapsed time per iteration (ms): 90003.9 | throughput per GPU (TFLOP/s/GPU): 40.9 | MFU 13.10% | FLOPs estimator: workload-aware | logical tokens/s: 186404.9 | valid tokens/s: 186404.9 | physical tokens/s: 186882.3 | loss tokens/s: 138127.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.548570E-01 | lm-global-supervised-tokens: 1.243197E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.200226E-02 | lm-token-buffer-mib: 5.810673E+01 | global_load_balancing_loss: 9.998361E-01 | loss scale: 1.0 | grad norm: 0.207 | params norm: 11240.002 | params rms: 0.060861 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2:09:00.331304 | finish at 2026-10-10 20:20:32 + [2026-10-10 18:13:20] iteration 6770/ 6855 | consumed samples: 866560 | elapsed time per iteration (ms): 108072.4 | throughput per GPU (TFLOP/s/GPU): 33.6 | MFU 10.76% | FLOPs estimator: workload-aware | logical tokens/s: 155240.2 | valid tokens/s: 155240.2 | physical tokens/s: 155631.2 | loss tokens/s: 116407.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.195269E-01 | lm-global-supervised-tokens: 1.258042E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.228869E-02 | lm-token-buffer-mib: 5.807581E+01 | global_load_balancing_loss: 9.997544E-01 | loss scale: 1.0 | grad norm: 0.199 | params norm: 11239.957 | params rms: 0.060861 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2:33:06.156703 | finish at 2026-10-10 20:46:26 + [2026-10-10 18:14:51] iteration 6771/ 6855 | consumed samples: 866688 | elapsed time per iteration (ms): 90791.1 | throughput per GPU (TFLOP/s/GPU): 41.1 | MFU 13.18% | FLOPs estimator: workload-aware | logical tokens/s: 184788.8 | valid tokens/s: 184788.8 | physical tokens/s: 185346.7 | loss tokens/s: 131177.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.612149E-01 | lm-global-supervised-tokens: 1.190974E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.795792E-02 | lm-token-buffer-mib: 5.806041E+01 | global_load_balancing_loss: 1.000998E+00 | loss scale: 1.0 | grad norm: 0.166 | params norm: 11239.912 | params rms: 0.060860 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2:07:06.450960 | finish at 2026-10-10 20:21:57 + [2026-10-10 18:16:17] iteration 6772/ 6855 | consumed samples: 866816 | elapsed time per iteration (ms): 86908.9 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.78% | FLOPs estimator: workload-aware | logical tokens/s: 193043.5 | valid tokens/s: 193043.5 | physical tokens/s: 193505.5 | loss tokens/s: 142104.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.323946E-01 | lm-global-supervised-tokens: 1.235014E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.151175E-02 | lm-token-buffer-mib: 5.833143E+01 | global_load_balancing_loss: 1.000897E+00 | loss scale: 1.0 | grad norm: 0.159 | params norm: 11239.867 | params rms: 0.060860 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2:00:13.441136 | finish at 2026-10-10 20:16:31 + [2026-10-10 18:17:45] iteration 6773/ 6855 | consumed samples: 866944 | elapsed time per iteration (ms): 87953.1 | throughput per GPU (TFLOP/s/GPU): 41.6 | MFU 13.34% | FLOPs estimator: workload-aware | logical tokens/s: 190750.6 | valid tokens/s: 190750.6 | physical tokens/s: 191167.9 | loss tokens/s: 143882.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.394522E-01 | lm-global-supervised-tokens: 1.265486E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.226583E-02 | lm-token-buffer-mib: 5.809248E+01 | global_load_balancing_loss: 9.998515E-01 | loss scale: 1.0 | grad norm: 0.055 | params norm: 11239.825 | params rms: 0.060860 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2:00:12.152692 | finish at 2026-10-10 20:17:58 + [2026-10-10 18:19:11] iteration 6774/ 6855 | consumed samples: 867072 | elapsed time per iteration (ms): 85221.6 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.44% | FLOPs estimator: workload-aware | logical tokens/s: 196865.6 | valid tokens/s: 196865.6 | physical tokens/s: 197281.0 | loss tokens/s: 148848.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.591480E-01 | lm-global-supervised-tokens: 1.268510E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.298527E-02 | lm-token-buffer-mib: 5.806713E+01 | global_load_balancing_loss: 9.993849E-01 | loss scale: 1.0 | grad norm: 0.139 | params norm: 11239.781 | params rms: 0.060860 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1:55:02.945916 | finish at 2026-10-10 20:14:14 + [2026-10-10 18:20:38] iteration 6775/ 6855 | consumed samples: 867200 | elapsed time per iteration (ms): 87511.3 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.65% | FLOPs estimator: workload-aware | logical tokens/s: 191714.5 | valid tokens/s: 191714.5 | physical tokens/s: 192132.5 | loss tokens/s: 136664.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.666244E-01 | lm-global-supervised-tokens: 1.195968E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.763524E-02 | lm-token-buffer-mib: 5.809073E+01 | global_load_balancing_loss: 9.992416E-01 | loss scale: 1.0 | grad norm: 0.079 | params norm: 11239.738 | params rms: 0.060860 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1:56:40.901318 | finish at 2026-10-10 20:17:19 + [2026-10-10 18:22:02] iteration 6776/ 6855 | consumed samples: 867328 | elapsed time per iteration (ms): 84092.6 | throughput per GPU (TFLOP/s/GPU): 43.7 | MFU 14.01% | FLOPs estimator: workload-aware | logical tokens/s: 199508.4 | valid tokens/s: 199508.4 | physical tokens/s: 200022.0 | loss tokens/s: 142654.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.107364E-01 | lm-global-supervised-tokens: 1.199621E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.147547E-02 | lm-token-buffer-mib: 5.813767E+01 | global_load_balancing_loss: 9.997386E-01 | loss scale: 1.0 | grad norm: 0.105 | params norm: 11239.694 | params rms: 0.060859 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1:50:43.315465 | finish at 2026-10-10 20:12:46 + [2026-10-10 18:23:33] iteration 6777/ 6855 | consumed samples: 867456 | elapsed time per iteration (ms): 91137.5 | throughput per GPU (TFLOP/s/GPU): 40.7 | MFU 13.05% | FLOPs estimator: workload-aware | logical tokens/s: 184085.8 | valid tokens/s: 184085.8 | physical tokens/s: 184477.1 | loss tokens/s: 134643.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.448527E-01 | lm-global-supervised-tokens: 1.227103E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.806817E-02 | lm-token-buffer-mib: 5.820445E+01 | global_load_balancing_loss: 9.996094E-01 | loss scale: 1.0 | grad norm: 0.045 | params norm: 11239.649 | params rms: 0.060859 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1:58:28.724092 | finish at 2026-10-10 20:22:02 + [2026-10-10 18:25:01] iteration 6778/ 6855 | consumed samples: 867584 | elapsed time per iteration (ms): 87437.9 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.51% | FLOPs estimator: workload-aware | logical tokens/s: 191874.6 | valid tokens/s: 191874.6 | physical tokens/s: 192331.2 | loss tokens/s: 143176.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.282210E-01 | lm-global-supervised-tokens: 1.251907E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.766457E-02 | lm-token-buffer-mib: 5.813326E+01 | global_load_balancing_loss: 1.001137E+00 | loss scale: 1.0 | grad norm: 0.109 | params norm: 11239.605 | params rms: 0.060859 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1:52:12.719021 | finish at 2026-10-10 20:17:13 + [2026-10-10 18:26:30] iteration 6779/ 6855 | consumed samples: 867712 | elapsed time per iteration (ms): 89475.7 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.46% | FLOPs estimator: workload-aware | logical tokens/s: 187505.2 | valid tokens/s: 187505.2 | physical tokens/s: 187859.1 | loss tokens/s: 138799.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.429428E-01 | lm-global-supervised-tokens: 1.241917E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.209883E-02 | lm-token-buffer-mib: 5.816521E+01 | global_load_balancing_loss: 1.001608E+00 | loss scale: 1.0 | grad norm: 0.050 | params norm: 11239.563 | params rms: 0.060859 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1:53:20.154823 | finish at 2026-10-10 20:19:50 + [2026-10-10 18:28:21] iteration 6780/ 6855 | consumed samples: 867840 | elapsed time per iteration (ms): 111215.5 | throughput per GPU (TFLOP/s/GPU): 33.3 | MFU 10.67% | FLOPs estimator: workload-aware | logical tokens/s: 150853.1 | valid tokens/s: 150853.1 | physical tokens/s: 151410.1 | loss tokens/s: 107018.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.227376E-01 | lm-global-supervised-tokens: 1.190207E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.268541E-02 | lm-token-buffer-mib: 5.849200E+01 | global_load_balancing_loss: 1.001704E+00 | loss scale: 1.0 | grad norm: 0.246 | params norm: 11239.519 | params rms: 0.060858 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 2:19:01.162688 | finish at 2026-10-10 20:47:23 + [2026-10-10 18:29:48] iteration 6781/ 6855 | consumed samples: 867968 | elapsed time per iteration (ms): 86465.1 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.69% | FLOPs estimator: workload-aware | logical tokens/s: 194034.1 | valid tokens/s: 194034.1 | physical tokens/s: 194542.5 | loss tokens/s: 146853.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.663249E-01 | lm-global-supervised-tokens: 1.269771E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.150881E-02 | lm-token-buffer-mib: 5.811191E+01 | global_load_balancing_loss: 1.000987E+00 | loss scale: 1.0 | grad norm: 0.222 | params norm: 11239.477 | params rms: 0.060858 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1:46:38.417174 | finish at 2026-10-10 20:16:26 + [2026-10-10 18:31:14] iteration 6782/ 6855 | consumed samples: 868096 | elapsed time per iteration (ms): 86306.5 | throughput per GPU (TFLOP/s/GPU): 42.7 | MFU 13.69% | FLOPs estimator: workload-aware | logical tokens/s: 194390.7 | valid tokens/s: 194390.7 | physical tokens/s: 194801.3 | loss tokens/s: 141728.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.323640E-01 | lm-global-supervised-tokens: 1.223205E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.136572E-02 | lm-token-buffer-mib: 5.806509E+01 | global_load_balancing_loss: 9.992831E-01 | loss scale: 1.0 | grad norm: 0.054 | params norm: 11239.435 | params rms: 0.060858 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1:45:00.370999 | finish at 2026-10-10 20:16:15 + [2026-10-10 18:32:37] iteration 6783/ 6855 | consumed samples: 868224 | elapsed time per iteration (ms): 82986.5 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.78% | FLOPs estimator: workload-aware | logical tokens/s: 202167.5 | valid tokens/s: 202167.5 | physical tokens/s: 202661.5 | loss tokens/s: 154833.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.637257E-01 | lm-global-supervised-tokens: 1.284904E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.201511E-02 | lm-token-buffer-mib: 5.821329E+01 | global_load_balancing_loss: 9.991573E-01 | loss scale: 1.0 | grad norm: 0.125 | params norm: 11239.391 | params rms: 0.060858 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1:39:35.026028 | finish at 2026-10-10 20:12:12 + [2026-10-10 18:34:00] iteration 6784/ 6855 | consumed samples: 868352 | elapsed time per iteration (ms): 83047.7 | throughput per GPU (TFLOP/s/GPU): 43.4 | MFU 13.92% | FLOPs estimator: workload-aware | logical tokens/s: 202018.6 | valid tokens/s: 202018.6 | physical tokens/s: 202576.2 | loss tokens/s: 149680.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.753749E-01 | lm-global-supervised-tokens: 1.243062E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.670701E-02 | lm-token-buffer-mib: 5.815842E+01 | global_load_balancing_loss: 9.988189E-01 | loss scale: 1.0 | grad norm: 0.037 | params norm: 11239.347 | params rms: 0.060857 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1:38:16.385764 | finish at 2026-10-10 20:12:17 + [2026-10-10 18:35:30] iteration 6785/ 6855 | consumed samples: 868480 | elapsed time per iteration (ms): 89525.9 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.44% | FLOPs estimator: workload-aware | logical tokens/s: 187399.5 | valid tokens/s: 187399.5 | physical tokens/s: 187866.4 | loss tokens/s: 137139.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.792988E-01 | lm-global-supervised-tokens: 1.227756E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.874846E-02 | lm-token-buffer-mib: 5.824316E+01 | global_load_balancing_loss: 1.000373E+00 | loss scale: 1.0 | grad norm: 0.060 | params norm: 11239.301 | params rms: 0.060857 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1:44:26.812642 | finish at 2026-10-10 20:19:57 + [2026-10-10 18:36:58] iteration 6786/ 6855 | consumed samples: 868608 | elapsed time per iteration (ms): 88588.9 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.48% | FLOPs estimator: workload-aware | logical tokens/s: 189382.6 | valid tokens/s: 189382.6 | physical tokens/s: 189780.3 | loss tokens/s: 133528.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.306353E-01 | lm-global-supervised-tokens: 1.182916E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.133729E-02 | lm-token-buffer-mib: 5.810722E+01 | global_load_balancing_loss: 1.000770E+00 | loss scale: 1.0 | grad norm: 0.050 | params norm: 11239.256 | params rms: 0.060857 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1:41:52.634468 | finish at 2026-10-10 20:18:51 + [2026-10-10 18:38:25] iteration 6787/ 6855 | consumed samples: 868736 | elapsed time per iteration (ms): 86507.8 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.55% | FLOPs estimator: workload-aware | logical tokens/s: 193937.6 | valid tokens/s: 193937.6 | physical tokens/s: 194492.6 | loss tokens/s: 147243.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.582662E-01 | lm-global-supervised-tokens: 1.273774E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.127340E-02 | lm-token-buffer-mib: 5.830090E+01 | global_load_balancing_loss: 9.994895E-01 | loss scale: 1.0 | grad norm: 0.040 | params norm: 11239.210 | params rms: 0.060857 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1:38:02.528137 | finish at 2026-10-10 20:16:27 + [2026-10-10 18:39:52] iteration 6788/ 6855 | consumed samples: 868864 | elapsed time per iteration (ms): 86758.2 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.44% | FLOPs estimator: workload-aware | logical tokens/s: 193378.8 | valid tokens/s: 193378.8 | physical tokens/s: 193767.0 | loss tokens/s: 148369.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.518506E-01 | lm-global-supervised-tokens: 1.287228E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.247611E-02 | lm-token-buffer-mib: 5.818658E+01 | global_load_balancing_loss: 9.992372E-01 | loss scale: 1.0 | grad norm: 0.039 | params norm: 11239.166 | params rms: 0.060856 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1:36:52.801360 | finish at 2026-10-10 20:16:44 + [2026-10-10 18:41:18] iteration 6789/ 6855 | consumed samples: 868992 | elapsed time per iteration (ms): 86538.2 | throughput per GPU (TFLOP/s/GPU): 42.8 | MFU 13.73% | FLOPs estimator: workload-aware | logical tokens/s: 193869.2 | valid tokens/s: 193869.2 | physical tokens/s: 194306.9 | loss tokens/s: 143028.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.670015E-01 | lm-global-supervised-tokens: 1.237743E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.122171E-02 | lm-token-buffer-mib: 5.806206E+01 | global_load_balancing_loss: 1.000711E+00 | loss scale: 1.0 | grad norm: 0.042 | params norm: 11239.123 | params rms: 0.060856 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1:35:11.523507 | finish at 2026-10-10 20:16:30 + [2026-10-10 18:43:06] iteration 6790/ 6855 | consumed samples: 869120 | elapsed time per iteration (ms): 108137.1 | throughput per GPU (TFLOP/s/GPU): 34.1 | MFU 10.93% | FLOPs estimator: workload-aware | logical tokens/s: 155147.2 | valid tokens/s: 155147.1 | physical tokens/s: 155498.0 | loss tokens/s: 117476.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.718059E-01 | lm-global-supervised-tokens: 1.270354E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.502704E-02 | lm-token-buffer-mib: 5.813995E+01 | global_load_balancing_loss: 1.001506E+00 | loss scale: 1.0 | grad norm: 0.037 | params norm: 11239.079 | params rms: 0.060856 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1:57:08.914164 | finish at 2026-10-10 20:40:15 + [2026-10-10 18:44:29] iteration 6791/ 6855 | consumed samples: 869248 | elapsed time per iteration (ms): 82241.0 | throughput per GPU (TFLOP/s/GPU): 43.9 | MFU 14.07% | FLOPs estimator: workload-aware | logical tokens/s: 203999.9 | valid tokens/s: 203999.9 | physical tokens/s: 204390.7 | loss tokens/s: 156565.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.619306E-01 | lm-global-supervised-tokens: 1.287606E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.479947E-02 | lm-token-buffer-mib: 5.805463E+01 | global_load_balancing_loss: 9.994839E-01 | loss scale: 1.0 | grad norm: 0.035 | params norm: 11239.037 | params rms: 0.060856 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1:27:43.424393 | finish at 2026-10-10 20:12:12 + [2026-10-10 18:45:57] iteration 6792/ 6855 | consumed samples: 869376 | elapsed time per iteration (ms): 88458.2 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.40% | FLOPs estimator: workload-aware | logical tokens/s: 189662.4 | valid tokens/s: 189662.4 | physical tokens/s: 190185.9 | loss tokens/s: 141849.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.493956E-01 | lm-global-supervised-tokens: 1.254775E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.671816E-02 | lm-token-buffer-mib: 5.829765E+01 | global_load_balancing_loss: 1.001155E+00 | loss scale: 1.0 | grad norm: 0.039 | params norm: 11238.994 | params rms: 0.060855 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1:32:52.866208 | finish at 2026-10-10 20:18:50 + [2026-10-10 18:47:26] iteration 6793/ 6855 | consumed samples: 869504 | elapsed time per iteration (ms): 88598.3 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.60% | FLOPs estimator: workload-aware | logical tokens/s: 189362.7 | valid tokens/s: 189362.7 | physical tokens/s: 189781.0 | loss tokens/s: 140124.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.876794E-01 | lm-global-supervised-tokens: 1.241474E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.101034E-02 | lm-token-buffer-mib: 5.830304E+01 | global_load_balancing_loss: 1.000752E+00 | loss scale: 1.0 | grad norm: 0.039 | params norm: 11238.951 | params rms: 0.060855 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1:31:33.094096 | finish at 2026-10-10 20:18:59 + [2026-10-10 18:48:54] iteration 6794/ 6855 | consumed samples: 869632 | elapsed time per iteration (ms): 88261.7 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.55% | FLOPs estimator: workload-aware | logical tokens/s: 190084.6 | valid tokens/s: 190084.6 | physical tokens/s: 190498.8 | loss tokens/s: 140538.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.449100E-01 | lm-global-supervised-tokens: 1.240415E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.216590E-02 | lm-token-buffer-mib: 5.810908E+01 | global_load_balancing_loss: 1.000603E+00 | loss scale: 1.0 | grad norm: 0.038 | params norm: 11238.908 | params rms: 0.060855 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1:29:43.963142 | finish at 2026-10-10 20:18:38 + [2026-10-10 18:50:20] iteration 6795/ 6855 | consumed samples: 869760 | elapsed time per iteration (ms): 85816.4 | throughput per GPU (TFLOP/s/GPU): 42.9 | MFU 13.74% | FLOPs estimator: workload-aware | logical tokens/s: 195500.6 | valid tokens/s: 195500.6 | physical tokens/s: 196158.7 | loss tokens/s: 142665.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.949614E-01 | lm-global-supervised-tokens: 1.224306E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.178359E-02 | lm-token-buffer-mib: 5.814534E+01 | global_load_balancing_loss: 9.994034E-01 | loss scale: 1.0 | grad norm: 0.044 | params norm: 11238.866 | params rms: 0.060855 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1:25:48.982587 | finish at 2026-10-10 20:16:09 + [2026-10-10 18:51:45] iteration 6796/ 6855 | consumed samples: 869888 | elapsed time per iteration (ms): 85567.4 | throughput per GPU (TFLOP/s/GPU): 43.3 | MFU 13.88% | FLOPs estimator: workload-aware | logical tokens/s: 196069.9 | valid tokens/s: 196069.8 | physical tokens/s: 196544.9 | loss tokens/s: 138813.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.744533E-01 | lm-global-supervised-tokens: 1.187791E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.157063E-02 | lm-token-buffer-mib: 5.840984E+01 | global_load_balancing_loss: 9.985476E-01 | loss scale: 1.0 | grad norm: 0.042 | params norm: 11238.822 | params rms: 0.060855 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1:24:08.476753 | finish at 2026-10-10 20:15:54 + [2026-10-10 18:53:12] iteration 6797/ 6855 | consumed samples: 870016 | elapsed time per iteration (ms): 86414.8 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.62% | FLOPs estimator: workload-aware | logical tokens/s: 194146.8 | valid tokens/s: 194146.8 | physical tokens/s: 194556.9 | loss tokens/s: 140587.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.653933E-01 | lm-global-supervised-tokens: 1.214886E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.123910E-02 | lm-token-buffer-mib: 5.810012E+01 | global_load_balancing_loss: 1.000082E+00 | loss scale: 1.0 | grad norm: 0.039 | params norm: 11238.777 | params rms: 0.060854 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1:23:32.057566 | finish at 2026-10-10 20:16:44 + [2026-10-10 18:54:39] iteration 6798/ 6855 | consumed samples: 870144 | elapsed time per iteration (ms): 86918.6 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.44% | FLOPs estimator: workload-aware | logical tokens/s: 193022.1 | valid tokens/s: 193022.0 | physical tokens/s: 193430.7 | loss tokens/s: 147193.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.117749E-01 | lm-global-supervised-tokens: 1.279385E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.549582E-02 | lm-token-buffer-mib: 5.808852E+01 | global_load_balancing_loss: 9.994065E-01 | loss scale: 1.0 | grad norm: 0.038 | params norm: 11238.733 | params rms: 0.060854 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1:22:34.361985 | finish at 2026-10-10 20:17:13 + [2026-10-10 18:56:12] iteration 6799/ 6855 | consumed samples: 870272 | elapsed time per iteration (ms): 93236.8 | throughput per GPU (TFLOP/s/GPU): 40.1 | MFU 12.86% | FLOPs estimator: workload-aware | logical tokens/s: 179941.7 | valid tokens/s: 179941.7 | physical tokens/s: 180475.6 | loss tokens/s: 131865.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.575649E-01 | lm-global-supervised-tokens: 1.229473E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.537336E-02 | lm-token-buffer-mib: 5.820051E+01 | global_load_balancing_loss: 1.002108E+00 | loss scale: 1.0 | grad norm: 0.040 | params norm: 11238.689 | params rms: 0.060854 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1:27:01.263014 | finish at 2026-10-10 20:23:13 + [2026-10-10 18:58:00] iteration 6800/ 6855 | consumed samples: 870400 | elapsed time per iteration (ms): 107705.2 | throughput per GPU (TFLOP/s/GPU): 34.0 | MFU 10.90% | FLOPs estimator: workload-aware | logical tokens/s: 155769.5 | valid tokens/s: 155769.5 | physical tokens/s: 156145.1 | loss tokens/s: 114961.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.238847E-01 | lm-global-supervised-tokens: 1.238198E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.189530E-02 | lm-token-buffer-mib: 5.807439E+01 | global_load_balancing_loss: 1.001363E+00 | loss scale: 1.0 | grad norm: 0.045 | params norm: 11238.644 | params rms: 0.060854 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1:38:43.783257 | finish at 2026-10-10 20:36:43 +times across ranks (ms): + save-checkpoint: + rank 0: 25819.70 + rank 1: 25819.66 + rank 2: 25819.68 + rank 3: 25819.64 + rank 4: 25819.67 + rank 5: 25819.63 + rank 6: 25819.62 + rank 7: 25819.67 + rank 8: 25820.55 + rank 9: 25820.52 + rank 10: 25820.56 + rank 11: 25820.53 + rank 12: 25820.54 + rank 13: 25820.52 + rank 14: 25820.54 + rank 15: 25820.51 + rank 16: 25819.57 + rank 17: 25819.56 + rank 18: 25819.54 + rank 19: 25819.60 + rank 20: 25819.57 + rank 21: 25819.59 + rank 22: 25819.54 + rank 23: 25819.54 + rank 24: 25820.38 + rank 25: 25820.35 + rank 26: 25820.38 + rank 27: 25820.33 + rank 28: 25820.28 + rank 29: 25820.33 + rank 30: 25820.94 + rank 31: 25820.33 + rank 32: 25819.56 + rank 33: 25819.54 + rank 34: 25819.56 + rank 35: 25819.61 + rank 36: 25819.55 + rank 37: 25819.55 + rank 38: 25819.65 + rank 39: 25819.54 + rank 40: 25822.82 + rank 41: 25822.76 + rank 42: 25822.80 + rank 43: 25822.80 + rank 44: 25822.76 + rank 45: 25822.78 + rank 46: 25822.81 + rank 47: 25822.78 + rank 48: 25822.54 + rank 49: 25822.42 + rank 50: 25822.59 + rank 51: 25822.55 + rank 52: 25822.56 + rank 53: 25822.50 + rank 54: 25822.41 + rank 55: 25822.52 + rank 56: 25820.57 + rank 57: 25820.51 + rank 58: 25820.51 + rank 59: 25820.50 + rank 60: 25820.54 + rank 61: 25820.50 + rank 62: 25820.54 + rank 63: 25820.36 + [2026-10-10 18:59:51] iteration 6801/ 6855 | consumed samples: 870528 | elapsed time per iteration (ms): 85798.7 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.62% | FLOPs estimator: workload-aware | logical tokens/s: 195541.6 | valid tokens/s: 195541.6 | physical tokens/s: 195830.1 | loss tokens/s: 138998.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.270385E-01 | lm-global-supervised-tokens: 1.192592E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 4.894089E-01 | lm-token-buffer-mib: 5.815502E+01 | global_load_balancing_loss: 1.000174E+00 | loss scale: 1.0 | grad norm: 0.042 | params norm: 11238.601 | params rms: 0.060853 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1:17:13.129316 | finish at 2026-10-10 20:17:04 + [2026-10-10 19:01:25] iteration 6802/ 6855 | consumed samples: 870656 | elapsed time per iteration (ms): 93435.6 | throughput per GPU (TFLOP/s/GPU): 40.0 | MFU 12.81% | FLOPs estimator: workload-aware | logical tokens/s: 179558.7 | valid tokens/s: 179558.7 | physical tokens/s: 179902.1 | loss tokens/s: 127022.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.674693E-01 | lm-global-supervised-tokens: 1.186846E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.753188E-02 | lm-token-buffer-mib: 5.802325E+01 | global_load_balancing_loss: 1.000433E+00 | loss scale: 1.0 | grad norm: 0.046 | params norm: 11238.558 | params rms: 0.060853 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1:22:32.088862 | finish at 2026-10-10 20:23:57 + [2026-10-10 19:02:46] iteration 6803/ 6855 | consumed samples: 870784 | elapsed time per iteration (ms): 81625.9 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.79% | FLOPs estimator: workload-aware | logical tokens/s: 205537.8 | valid tokens/s: 205537.8 | physical tokens/s: 206368.5 | loss tokens/s: 158741.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.467447E-01 | lm-global-supervised-tokens: 1.295744E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.440311E-02 | lm-token-buffer-mib: 5.810371E+01 | global_load_balancing_loss: 1.000091E+00 | loss scale: 1.0 | grad norm: 0.077 | params norm: 11238.514 | params rms: 0.060853 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1:10:44.545549 | finish at 2026-10-10 20:13:31 + [2026-10-10 19:04:13] iteration 6804/ 6855 | consumed samples: 870912 | elapsed time per iteration (ms): 87005.7 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.49% | FLOPs estimator: workload-aware | logical tokens/s: 192828.2 | valid tokens/s: 192828.2 | physical tokens/s: 193194.5 | loss tokens/s: 145184.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.425394E-01 | lm-global-supervised-tokens: 1.263192E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.247441E-02 | lm-token-buffer-mib: 5.826636E+01 | global_load_balancing_loss: 9.995235E-01 | loss scale: 1.0 | grad norm: 0.100 | params norm: 11238.470 | params rms: 0.060853 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1:13:57.290523 | finish at 2026-10-10 20:18:11 + [2026-10-10 19:05:43] iteration 6805/ 6855 | consumed samples: 871040 | elapsed time per iteration (ms): 89239.8 | throughput per GPU (TFLOP/s/GPU): 41.9 | MFU 13.42% | FLOPs estimator: workload-aware | logical tokens/s: 188001.0 | valid tokens/s: 188001.0 | physical tokens/s: 188446.7 | loss tokens/s: 134363.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.532978E-01 | lm-global-supervised-tokens: 1.199054E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.578742E-02 | lm-token-buffer-mib: 5.806375E+01 | global_load_balancing_loss: 9.997291E-01 | loss scale: 1.0 | grad norm: 0.125 | params norm: 11238.425 | params rms: 0.060852 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1:14:21.988759 | finish at 2026-10-10 20:20:05 + [2026-10-10 19:07:10] iteration 6806/ 6855 | consumed samples: 871168 | elapsed time per iteration (ms): 87813.4 | throughput per GPU (TFLOP/s/GPU): 41.5 | MFU 13.31% | FLOPs estimator: workload-aware | logical tokens/s: 191054.5 | valid tokens/s: 191054.5 | physical tokens/s: 191545.8 | loss tokens/s: 143191.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.770739E-01 | lm-global-supervised-tokens: 1.257409E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.160278E-02 | lm-token-buffer-mib: 5.818530E+01 | global_load_balancing_loss: 1.000772E+00 | loss scale: 1.0 | grad norm: 0.099 | params norm: 11238.380 | params rms: 0.060852 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1:11:42.856637 | finish at 2026-10-10 20:18:53 + [2026-10-10 19:08:38] iteration 6807/ 6855 | consumed samples: 871296 | elapsed time per iteration (ms): 87826.3 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.40% | FLOPs estimator: workload-aware | logical tokens/s: 191026.4 | valid tokens/s: 191026.4 | physical tokens/s: 191588.7 | loss tokens/s: 138965.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.512821E-01 | lm-global-supervised-tokens: 1.220480E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 2.128462E-02 | lm-token-buffer-mib: 5.822140E+01 | global_load_balancing_loss: 9.994897E-01 | loss scale: 1.0 | grad norm: 0.050 | params norm: 11238.336 | params rms: 0.060852 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1:10:15.663460 | finish at 2026-10-10 20:18:54 + [2026-10-10 19:10:06] iteration 6808/ 6855 | consumed samples: 871424 | elapsed time per iteration (ms): 88246.6 | throughput per GPU (TFLOP/s/GPU): 42.1 | MFU 13.49% | FLOPs estimator: workload-aware | logical tokens/s: 190116.5 | valid tokens/s: 190116.5 | physical tokens/s: 190630.1 | loss tokens/s: 140968.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.642734E-01 | lm-global-supervised-tokens: 1.243999E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.126697E-02 | lm-token-buffer-mib: 5.817393E+01 | global_load_balancing_loss: 1.000174E+00 | loss scale: 1.0 | grad norm: 0.208 | params norm: 11238.291 | params rms: 0.060852 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1:09:07.591541 | finish at 2026-10-10 20:19:14 + [2026-10-10 19:11:36] iteration 6809/ 6855 | consumed samples: 871552 | elapsed time per iteration (ms): 89248.8 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.55% | FLOPs estimator: workload-aware | logical tokens/s: 187982.3 | valid tokens/s: 187982.3 | physical tokens/s: 188335.2 | loss tokens/s: 136852.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.012886E-01 | lm-global-supervised-tokens: 1.221394E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.844979E-02 | lm-token-buffer-mib: 5.820435E+01 | global_load_balancing_loss: 9.996593E-01 | loss scale: 1.0 | grad norm: 0.220 | params norm: 11238.249 | params rms: 0.060851 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1:08:25.445240 | finish at 2026-10-10 20:20:01 + [2026-10-10 19:13:25] iteration 6810/ 6855 | consumed samples: 871680 | elapsed time per iteration (ms): 109439.6 | throughput per GPU (TFLOP/s/GPU): 33.4 | MFU 10.71% | FLOPs estimator: workload-aware | logical tokens/s: 153301.2 | valid tokens/s: 153301.2 | physical tokens/s: 153765.1 | loss tokens/s: 117711.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.473513E-01 | lm-global-supervised-tokens: 1.288232E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.164580E-02 | lm-token-buffer-mib: 5.806001E+01 | global_load_balancing_loss: 9.999439E-01 | loss scale: 1.0 | grad norm: 0.053 | params norm: 11238.204 | params rms: 0.060851 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1:22:04.780755 | finish at 2026-10-10 20:35:30 + [2026-10-10 19:14:53] iteration 6811/ 6855 | consumed samples: 871808 | elapsed time per iteration (ms): 87618.3 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.52% | FLOPs estimator: workload-aware | logical tokens/s: 191480.1 | valid tokens/s: 191480.1 | physical tokens/s: 191824.5 | loss tokens/s: 146268.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.880934E-01 | lm-global-supervised-tokens: 1.281575E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.468188E-02 | lm-token-buffer-mib: 5.807098E+01 | global_load_balancing_loss: 1.000848E+00 | loss scale: 1.0 | grad norm: 0.140 | params norm: 11238.160 | params rms: 0.060851 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1:04:15.203352 | finish at 2026-10-10 20:19:08 + [2026-10-10 19:16:20] iteration 6812/ 6855 | consumed samples: 871936 | elapsed time per iteration (ms): 87108.9 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.38% | FLOPs estimator: workload-aware | logical tokens/s: 192600.2 | valid tokens/s: 192600.2 | physical tokens/s: 193054.3 | loss tokens/s: 144957.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.009496E-01 | lm-global-supervised-tokens: 1.262710E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.303862E-02 | lm-token-buffer-mib: 5.816726E+01 | global_load_balancing_loss: 1.001549E+00 | loss scale: 1.0 | grad norm: 0.076 | params norm: 11238.115 | params rms: 0.060851 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1:02:25.683277 | finish at 2026-10-10 20:18:46 + [2026-10-10 19:17:47] iteration 6813/ 6855 | consumed samples: 872064 | elapsed time per iteration (ms): 86808.3 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.60% | FLOPs estimator: workload-aware | logical tokens/s: 193267.1 | valid tokens/s: 193267.1 | physical tokens/s: 193748.3 | loss tokens/s: 142058.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.227601E-01 | lm-global-supervised-tokens: 1.233188E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.358710E-02 | lm-token-buffer-mib: 5.807548E+01 | global_load_balancing_loss: 1.001579E+00 | loss scale: 1.0 | grad norm: 0.109 | params norm: 11238.071 | params rms: 0.060851 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1:00:45.948120 | finish at 2026-10-10 20:18:33 + [2026-10-10 19:19:15] iteration 6814/ 6855 | consumed samples: 872192 | elapsed time per iteration (ms): 88071.3 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.52% | FLOPs estimator: workload-aware | logical tokens/s: 190495.3 | valid tokens/s: 190495.3 | physical tokens/s: 190891.1 | loss tokens/s: 140042.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.242721E-01 | lm-global-supervised-tokens: 1.233369E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.127213E-02 | lm-token-buffer-mib: 5.811695E+01 | global_load_balancing_loss: 1.001345E+00 | loss scale: 1.0 | grad norm: 0.054 | params norm: 11238.029 | params rms: 0.060850 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1:00:10.923917 | finish at 2026-10-10 20:19:26 + [2026-10-10 19:20:40] iteration 6815/ 6855 | consumed samples: 872320 | elapsed time per iteration (ms): 85042.6 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.38% | FLOPs estimator: workload-aware | logical tokens/s: 197279.8 | valid tokens/s: 197279.8 | physical tokens/s: 197896.6 | loss tokens/s: 156331.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.456861E-01 | lm-global-supervised-tokens: 1.329486E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.628611E-02 | lm-token-buffer-mib: 5.823618E+01 | global_load_balancing_loss: 9.987087E-01 | loss scale: 1.0 | grad norm: 0.110 | params norm: 11237.985 | params rms: 0.060850 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 0:56:41.702118 | finish at 2026-10-10 20:17:22 + [2026-10-10 19:22:08] iteration 6816/ 6855 | consumed samples: 872448 | elapsed time per iteration (ms): 88423.4 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.45% | FLOPs estimator: workload-aware | logical tokens/s: 189737.1 | valid tokens/s: 189737.1 | physical tokens/s: 190089.2 | loss tokens/s: 134254.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 5.975763E-01 | lm-global-supervised-tokens: 1.187127E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.632565E-02 | lm-token-buffer-mib: 5.813323E+01 | global_load_balancing_loss: 9.994114E-01 | loss scale: 1.0 | grad norm: 0.062 | params norm: 11237.941 | params rms: 0.060850 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 0:57:28.512848 | finish at 2026-10-10 20:19:37 + [2026-10-10 19:23:34] iteration 6817/ 6855 | consumed samples: 872576 | elapsed time per iteration (ms): 85693.0 | throughput per GPU (TFLOP/s/GPU): 41.6 | MFU 13.33% | FLOPs estimator: workload-aware | logical tokens/s: 195781.9 | valid tokens/s: 195781.9 | physical tokens/s: 196560.9 | loss tokens/s: 150549.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.645821E-01 | lm-global-supervised-tokens: 1.290106E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.307541E-02 | lm-token-buffer-mib: 5.814384E+01 | global_load_balancing_loss: 9.989662E-01 | loss scale: 1.0 | grad norm: 0.098 | params norm: 11237.896 | params rms: 0.060850 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 0:54:16.333387 | finish at 2026-10-10 20:17:50 + [2026-10-10 19:24:58] iteration 6818/ 6855 | consumed samples: 872704 | elapsed time per iteration (ms): 83854.4 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.77% | FLOPs estimator: workload-aware | logical tokens/s: 200074.9 | valid tokens/s: 200074.9 | physical tokens/s: 200516.7 | loss tokens/s: 150287.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.614697E-01 | lm-global-supervised-tokens: 1.260229E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.554886E-02 | lm-token-buffer-mib: 5.820231E+01 | global_load_balancing_loss: 1.000005E+00 | loss scale: 1.0 | grad norm: 0.035 | params norm: 11237.850 | params rms: 0.060849 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 0:51:42.612515 | finish at 2026-10-10 20:16:40 + [2026-10-10 19:26:28] iteration 6819/ 6855 | consumed samples: 872832 | elapsed time per iteration (ms): 89908.2 | throughput per GPU (TFLOP/s/GPU): 41.6 | MFU 13.34% | FLOPs estimator: workload-aware | logical tokens/s: 186603.7 | valid tokens/s: 186603.6 | physical tokens/s: 187075.9 | loss tokens/s: 135489.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.963133E-01 | lm-global-supervised-tokens: 1.218160E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.511132E-02 | lm-token-buffer-mib: 5.818662E+01 | global_load_balancing_loss: 9.992821E-01 | loss scale: 1.0 | grad norm: 0.094 | params norm: 11237.805 | params rms: 0.060849 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 0:53:56.694077 | finish at 2026-10-10 20:20:24 + [2026-10-10 19:28:17] iteration 6820/ 6855 | consumed samples: 872960 | elapsed time per iteration (ms): 109451.2 | throughput per GPU (TFLOP/s/GPU): 33.9 | MFU 10.86% | FLOPs estimator: workload-aware | logical tokens/s: 153284.4 | valid tokens/s: 153284.4 | physical tokens/s: 153753.3 | loss tokens/s: 112696.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.809206E-01 | lm-global-supervised-tokens: 1.233477E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.663160E-02 | lm-token-buffer-mib: 5.812921E+01 | global_load_balancing_loss: 1.001612E+00 | loss scale: 1.0 | grad norm: 0.041 | params norm: 11237.760 | params rms: 0.060849 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 1:03:50.792476 | finish at 2026-10-10 20:32:08 + [2026-10-10 19:29:46] iteration 6821/ 6855 | consumed samples: 873088 | elapsed time per iteration (ms): 88702.3 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.64% | FLOPs estimator: workload-aware | logical tokens/s: 189140.8 | valid tokens/s: 189140.8 | physical tokens/s: 189406.7 | loss tokens/s: 136654.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.566918E-01 | lm-global-supervised-tokens: 1.212155E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.409065E-02 | lm-token-buffer-mib: 5.804307E+01 | global_load_balancing_loss: 1.000985E+00 | loss scale: 1.0 | grad norm: 0.039 | params norm: 11237.716 | params rms: 0.060849 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 0:50:15.876808 | finish at 2026-10-10 20:20:02 + [2026-10-10 19:31:14] iteration 6822/ 6855 | consumed samples: 873216 | elapsed time per iteration (ms): 88246.1 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.53% | FLOPs estimator: workload-aware | logical tokens/s: 190118.3 | valid tokens/s: 190118.3 | physical tokens/s: 190516.2 | loss tokens/s: 138147.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.892533E-01 | lm-global-supervised-tokens: 1.219094E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.453841E-02 | lm-token-buffer-mib: 5.804646E+01 | global_load_balancing_loss: 1.000525E+00 | loss scale: 1.0 | grad norm: 0.102 | params norm: 11237.671 | params rms: 0.060848 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 0:48:32.120724 | finish at 2026-10-10 20:19:46 + [2026-10-10 19:32:41] iteration 6823/ 6855 | consumed samples: 873344 | elapsed time per iteration (ms): 86575.0 | throughput per GPU (TFLOP/s/GPU): 42.8 | MFU 13.72% | FLOPs estimator: workload-aware | logical tokens/s: 193787.8 | valid tokens/s: 193787.8 | physical tokens/s: 194177.7 | loss tokens/s: 143209.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 5.932363E-01 | lm-global-supervised-tokens: 1.239835E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.494573E-02 | lm-token-buffer-mib: 5.809438E+01 | global_load_balancing_loss: 1.000211E+00 | loss scale: 1.0 | grad norm: 0.122 | params norm: 11237.626 | params rms: 0.060848 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 0:46:10.401138 | finish at 2026-10-10 20:18:51 + [2026-10-10 19:34:08] iteration 6824/ 6855 | consumed samples: 873472 | elapsed time per iteration (ms): 87647.0 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.66% | FLOPs estimator: workload-aware | logical tokens/s: 191417.5 | valid tokens/s: 191417.5 | physical tokens/s: 191986.0 | loss tokens/s: 139434.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.315418E-01 | lm-global-supervised-tokens: 1.222098E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.507679E-02 | lm-token-buffer-mib: 5.817294E+01 | global_load_balancing_loss: 9.994662E-01 | loss scale: 1.0 | grad norm: 0.114 | params norm: 11237.583 | params rms: 0.060848 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 0:45:17.056677 | finish at 2026-10-10 20:19:25 + [2026-10-10 19:35:36] iteration 6825/ 6855 | consumed samples: 873600 | elapsed time per iteration (ms): 87958.7 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.60% | FLOPs estimator: workload-aware | logical tokens/s: 190738.7 | valid tokens/s: 190738.7 | physical tokens/s: 191198.6 | loss tokens/s: 134141.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.733478E-01 | lm-global-supervised-tokens: 1.179889E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.147927E-02 | lm-token-buffer-mib: 5.810836E+01 | global_load_balancing_loss: 1.000280E+00 | loss scale: 1.0 | grad norm: 0.097 | params norm: 11237.543 | params rms: 0.060848 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 0:43:58.762114 | finish at 2026-10-10 20:19:35 + [2026-10-10 19:37:06] iteration 6826/ 6855 | consumed samples: 873728 | elapsed time per iteration (ms): 90184.5 | throughput per GPU (TFLOP/s/GPU): 41.3 | MFU 13.24% | FLOPs estimator: workload-aware | logical tokens/s: 186032.0 | valid tokens/s: 186032.0 | physical tokens/s: 186478.2 | loss tokens/s: 135783.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.043152E-01 | lm-global-supervised-tokens: 1.224557E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.493564E-02 | lm-token-buffer-mib: 5.812247E+01 | global_load_balancing_loss: 1.000103E+00 | loss scale: 1.0 | grad norm: 0.271 | params norm: 11237.501 | params rms: 0.060847 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 0:43:35.350064 | finish at 2026-10-10 20:20:42 + [2026-10-10 19:38:36] iteration 6827/ 6855 | consumed samples: 873856 | elapsed time per iteration (ms): 89641.6 | throughput per GPU (TFLOP/s/GPU): 43.2 | MFU 13.85% | FLOPs estimator: workload-aware | logical tokens/s: 187158.7 | valid tokens/s: 187158.7 | physical tokens/s: 187503.5 | loss tokens/s: 137165.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.014815E-01 | lm-global-supervised-tokens: 1.229573E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.347985E-02 | lm-token-buffer-mib: 5.817153E+01 | global_load_balancing_loss: 1.000705E+00 | loss scale: 1.0 | grad norm: 0.190 | params norm: 11237.457 | params rms: 0.060847 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 0:41:49.963983 | finish at 2026-10-10 20:20:26 + [2026-10-10 19:40:01] iteration 6828/ 6855 | consumed samples: 873984 | elapsed time per iteration (ms): 84579.6 | throughput per GPU (TFLOP/s/GPU): 43.7 | MFU 14.02% | FLOPs estimator: workload-aware | logical tokens/s: 198359.6 | valid tokens/s: 198359.6 | physical tokens/s: 198829.9 | loss tokens/s: 149788.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.850324E-01 | lm-global-supervised-tokens: 1.266902E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.136633E-02 | lm-token-buffer-mib: 5.820233E+01 | global_load_balancing_loss: 9.999699E-01 | loss scale: 1.0 | grad norm: 0.233 | params norm: 11237.416 | params rms: 0.060847 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 0:38:03.650413 | finish at 2026-10-10 20:18:04 + [2026-10-10 19:41:26] iteration 6829/ 6855 | consumed samples: 874112 | elapsed time per iteration (ms): 85191.6 | throughput per GPU (TFLOP/s/GPU): 43.2 | MFU 13.83% | FLOPs estimator: workload-aware | logical tokens/s: 196934.3 | valid tokens/s: 196934.3 | physical tokens/s: 197319.0 | loss tokens/s: 148196.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.628441E-01 | lm-global-supervised-tokens: 1.262509E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.416436E-02 | lm-token-buffer-mib: 5.807211E+01 | global_load_balancing_loss: 9.990795E-01 | loss scale: 1.0 | grad norm: 0.156 | params norm: 11237.374 | params rms: 0.060847 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 0:36:54.981825 | finish at 2026-10-10 20:18:21 + [2026-10-10 19:43:19] iteration 6830/ 6855 | consumed samples: 874240 | elapsed time per iteration (ms): 112882.0 | throughput per GPU (TFLOP/s/GPU): 33.7 | MFU 10.81% | FLOPs estimator: workload-aware | logical tokens/s: 148625.4 | valid tokens/s: 148625.4 | physical tokens/s: 149115.9 | loss tokens/s: 103837.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.081891E-01 | lm-global-supervised-tokens: 1.172136E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.150673E-02 | lm-token-buffer-mib: 5.833035E+01 | global_load_balancing_loss: 1.000038E+00 | loss scale: 1.0 | grad norm: 0.223 | params norm: 11237.333 | params rms: 0.060847 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 0:47:02.049940 | finish at 2026-10-10 20:30:21 + [2026-10-10 19:44:45] iteration 6831/ 6855 | consumed samples: 874368 | elapsed time per iteration (ms): 86085.7 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.39% | FLOPs estimator: workload-aware | logical tokens/s: 194889.1 | valid tokens/s: 194889.0 | physical tokens/s: 195341.2 | loss tokens/s: 148108.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.299393E-01 | lm-global-supervised-tokens: 1.275001E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.387469E-02 | lm-token-buffer-mib: 5.819402E+01 | global_load_balancing_loss: 1.001619E+00 | loss scale: 1.0 | grad norm: 0.099 | params norm: 11237.290 | params rms: 0.060846 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 0:34:26.056841 | finish at 2026-10-10 20:19:11 + [2026-10-10 19:46:14] iteration 6832/ 6855 | consumed samples: 874496 | elapsed time per iteration (ms): 88812.9 | throughput per GPU (TFLOP/s/GPU): 41.8 | MFU 13.38% | FLOPs estimator: workload-aware | logical tokens/s: 188904.8 | valid tokens/s: 188904.8 | physical tokens/s: 189451.0 | loss tokens/s: 139686.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.491995E-01 | lm-global-supervised-tokens: 1.240598E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.144650E-02 | lm-token-buffer-mib: 5.812814E+01 | global_load_balancing_loss: 1.001280E+00 | loss scale: 1.0 | grad norm: 0.161 | params norm: 11237.248 | params rms: 0.060846 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 0:34:02.696861 | finish at 2026-10-10 20:20:16 + [2026-10-10 19:47:43] iteration 6833/ 6855 | consumed samples: 874624 | elapsed time per iteration (ms): 89665.0 | throughput per GPU (TFLOP/s/GPU): 41.3 | MFU 13.23% | FLOPs estimator: workload-aware | logical tokens/s: 187109.9 | valid tokens/s: 187109.9 | physical tokens/s: 187614.7 | loss tokens/s: 137361.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.432319E-01 | lm-global-supervised-tokens: 1.231651E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.549726E-02 | lm-token-buffer-mib: 5.818382E+01 | global_load_balancing_loss: 1.000082E+00 | loss scale: 1.0 | grad norm: 0.100 | params norm: 11237.207 | params rms: 0.060846 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 0:32:52.629176 | finish at 2026-10-10 20:20:36 + [2026-10-10 19:49:09] iteration 6834/ 6855 | consumed samples: 874752 | elapsed time per iteration (ms): 86012.2 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.46% | FLOPs estimator: workload-aware | logical tokens/s: 195055.6 | valid tokens/s: 195055.6 | physical tokens/s: 195579.6 | loss tokens/s: 145967.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.291714E-01 | lm-global-supervised-tokens: 1.255502E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.413060E-02 | lm-token-buffer-mib: 5.809551E+01 | global_load_balancing_loss: 1.000517E+00 | loss scale: 1.0 | grad norm: 0.069 | params norm: 11237.166 | params rms: 0.060846 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 0:30:06.256979 | finish at 2026-10-10 20:19:16 + [2026-10-10 19:50:39] iteration 6835/ 6855 | consumed samples: 874880 | elapsed time per iteration (ms): 89428.0 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.59% | FLOPs estimator: workload-aware | logical tokens/s: 187605.3 | valid tokens/s: 187605.3 | physical tokens/s: 187984.6 | loss tokens/s: 145365.1 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.317691E-01 | lm-global-supervised-tokens: 1.299972E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.143336E-02 | lm-token-buffer-mib: 5.831721E+01 | global_load_balancing_loss: 9.981682E-01 | loss scale: 1.0 | grad norm: 0.115 | params norm: 11237.124 | params rms: 0.060845 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 0:29:48.560519 | finish at 2026-10-10 20:20:27 + [2026-10-10 19:52:11] iteration 6836/ 6855 | consumed samples: 875008 | elapsed time per iteration (ms): 91979.8 | throughput per GPU (TFLOP/s/GPU): 41.4 | MFU 13.28% | FLOPs estimator: workload-aware | logical tokens/s: 182400.3 | valid tokens/s: 182400.3 | physical tokens/s: 182756.3 | loss tokens/s: 136249.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.728318E-01 | lm-global-supervised-tokens: 1.253222E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.147696E-02 | lm-token-buffer-mib: 5.818880E+01 | global_load_balancing_loss: 9.980045E-01 | loss scale: 1.0 | grad norm: 0.157 | params norm: 11237.081 | params rms: 0.060845 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 0:29:07.616281 | finish at 2026-10-10 20:21:18 + [2026-10-10 19:53:37] iteration 6837/ 6855 | consumed samples: 875136 | elapsed time per iteration (ms): 86268.5 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.58% | FLOPs estimator: workload-aware | logical tokens/s: 194476.3 | valid tokens/s: 194476.3 | physical tokens/s: 195175.4 | loss tokens/s: 140735.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.634676E-01 | lm-global-supervised-tokens: 1.214103E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.524724E-02 | lm-token-buffer-mib: 5.833624E+01 | global_load_balancing_loss: 1.001052E+00 | loss scale: 1.0 | grad norm: 0.144 | params norm: 11237.039 | params rms: 0.060845 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 0:25:52.832736 | finish at 2026-10-10 20:19:30 + [2026-10-10 19:54:59] iteration 6838/ 6855 | consumed samples: 875264 | elapsed time per iteration (ms): 82412.3 | throughput per GPU (TFLOP/s/GPU): 42.8 | MFU 13.73% | FLOPs estimator: workload-aware | logical tokens/s: 203576.6 | valid tokens/s: 203576.6 | physical tokens/s: 203985.3 | loss tokens/s: 151516.8 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.081218E-01 | lm-global-supervised-tokens: 1.248685E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.540972E-02 | lm-token-buffer-mib: 5.811208E+01 | global_load_balancing_loss: 1.001223E+00 | loss scale: 1.0 | grad norm: 0.095 | params norm: 11236.998 | params rms: 0.060845 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 0:23:21.008960 | finish at 2026-10-10 20:18:20 + [2026-10-10 19:56:28] iteration 6839/ 6855 | consumed samples: 875392 | elapsed time per iteration (ms): 88476.5 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.62% | FLOPs estimator: workload-aware | logical tokens/s: 189622.3 | valid tokens/s: 189622.3 | physical tokens/s: 190048.0 | loss tokens/s: 137732.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.213981E-01 | lm-global-supervised-tokens: 1.218607E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.140779E-02 | lm-token-buffer-mib: 5.810979E+01 | global_load_balancing_loss: 1.001112E+00 | loss scale: 1.0 | grad norm: 0.242 | params norm: 11236.955 | params rms: 0.060844 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 0:23:35.624233 | finish at 2026-10-10 20:20:04 + [2026-10-10 19:58:17] iteration 6840/ 6855 | consumed samples: 875520 | elapsed time per iteration (ms): 109167.9 | throughput per GPU (TFLOP/s/GPU): 33.9 | MFU 10.86% | FLOPs estimator: workload-aware | logical tokens/s: 153682.6 | valid tokens/s: 153682.6 | physical tokens/s: 154040.3 | loss tokens/s: 112814.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.520537E-01 | lm-global-supervised-tokens: 1.231568E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.203351E-02 | lm-token-buffer-mib: 5.801294E+01 | global_load_balancing_loss: 1.000795E+00 | loss scale: 1.0 | grad norm: 0.074 | params norm: 11236.913 | params rms: 0.060844 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 0:27:17.518369 | finish at 2026-10-10 20:25:35 +times across ranks (ms): + save-checkpoint: + rank 0: 24082.40 + rank 1: 24082.42 + rank 2: 24082.43 + rank 3: 24082.39 + rank 4: 24082.45 + rank 5: 24082.42 + rank 6: 24082.40 + rank 7: 24082.48 + rank 8: 24082.51 + rank 9: 24082.48 + rank 10: 24082.50 + rank 11: 24082.48 + rank 12: 24082.49 + rank 13: 24082.49 + rank 14: 24082.52 + rank 15: 24082.49 + rank 16: 24082.18 + rank 17: 24082.28 + rank 18: 24082.19 + rank 19: 24082.17 + rank 20: 24082.17 + rank 21: 24082.18 + rank 22: 24082.17 + rank 23: 24082.17 + rank 24: 24082.58 + rank 25: 24082.61 + rank 26: 24082.60 + rank 27: 24082.63 + rank 28: 24082.63 + rank 29: 24082.61 + rank 30: 24082.59 + rank 31: 24082.60 + rank 32: 24083.42 + rank 33: 24083.43 + rank 34: 24083.42 + rank 35: 24083.42 + rank 36: 24083.43 + rank 37: 24083.41 + rank 38: 24083.43 + rank 39: 24083.43 + rank 40: 24082.35 + rank 41: 24082.33 + rank 42: 24082.34 + rank 43: 24082.35 + rank 44: 24082.33 + rank 45: 24082.33 + rank 46: 24082.35 + rank 47: 24082.38 + rank 48: 24082.09 + rank 49: 24082.12 + rank 50: 24082.10 + rank 51: 24082.08 + rank 52: 24082.10 + rank 53: 24082.10 + rank 54: 24082.17 + rank 55: 24082.09 + rank 56: 24082.43 + rank 57: 24082.42 + rank 58: 24082.38 + rank 59: 24082.39 + rank 60: 24082.41 + rank 61: 24082.41 + rank 62: 24082.38 + rank 63: 24082.45 + [2026-10-10 20:00:13] iteration 6841/ 6855 | consumed samples: 875648 | elapsed time per iteration (ms): 91255.6 | throughput per GPU (TFLOP/s/GPU): 40.3 | MFU 12.91% | FLOPs estimator: workload-aware | logical tokens/s: 183848.4 | valid tokens/s: 183848.3 | physical tokens/s: 184368.0 | loss tokens/s: 138685.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.037164E-01 | lm-global-supervised-tokens: 1.265585E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.057980E+00 | lm-token-buffer-mib: 5.844946E+01 | global_load_balancing_loss: 1.000585E+00 | loss scale: 1.0 | grad norm: 0.228 | params norm: 11236.871 | params rms: 0.060844 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 0:21:17.577966 | finish at 2026-10-10 20:21:30 + [2026-10-10 20:01:47] iteration 6842/ 6855 | consumed samples: 875776 | elapsed time per iteration (ms): 94887.3 | throughput per GPU (TFLOP/s/GPU): 39.1 | MFU 12.54% | FLOPs estimator: workload-aware | logical tokens/s: 176810.9 | valid tokens/s: 176810.9 | physical tokens/s: 177208.7 | loss tokens/s: 131393.3 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.408519E-01 | lm-global-supervised-tokens: 1.246756E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.344377E-02 | lm-token-buffer-mib: 5.817964E+01 | global_load_balancing_loss: 1.000450E+00 | loss scale: 1.0 | grad norm: 0.065 | params norm: 11236.826 | params rms: 0.060844 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 0:20:33.535511 | finish at 2026-10-10 20:22:21 + [2026-10-10 20:03:13] iteration 6843/ 6855 | consumed samples: 875904 | elapsed time per iteration (ms): 85678.1 | throughput per GPU (TFLOP/s/GPU): 42.5 | MFU 13.62% | FLOPs estimator: workload-aware | logical tokens/s: 195816.2 | valid tokens/s: 195816.2 | physical tokens/s: 196246.5 | loss tokens/s: 153244.5 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 7.030673E-01 | lm-global-supervised-tokens: 1.312970E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.212442E-02 | lm-token-buffer-mib: 5.808212E+01 | global_load_balancing_loss: 9.995152E-01 | loss scale: 1.0 | grad norm: 0.074 | params norm: 11236.779 | params rms: 0.060844 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 0:17:08.137390 | finish at 2026-10-10 20:20:21 + [2026-10-10 20:04:42] iteration 6844/ 6855 | consumed samples: 876032 | elapsed time per iteration (ms): 89327.9 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.57% | FLOPs estimator: workload-aware | logical tokens/s: 187815.5 | valid tokens/s: 187815.5 | physical tokens/s: 188182.3 | loss tokens/s: 138099.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.474896E-01 | lm-global-supervised-tokens: 1.233612E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.138555E-02 | lm-token-buffer-mib: 5.813380E+01 | global_load_balancing_loss: 9.992276E-01 | loss scale: 1.0 | grad norm: 0.128 | params norm: 11236.734 | params rms: 0.060843 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 0:16:22.607324 | finish at 2026-10-10 20:21:05 + [2026-10-10 20:06:11] iteration 6845/ 6855 | consumed samples: 876160 | elapsed time per iteration (ms): 88692.6 | throughput per GPU (TFLOP/s/GPU): 41.1 | MFU 13.19% | FLOPs estimator: workload-aware | logical tokens/s: 189161.3 | valid tokens/s: 189161.3 | physical tokens/s: 189771.5 | loss tokens/s: 138839.9 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.655381E-01 | lm-global-supervised-tokens: 1.231407E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.119278E-02 | lm-token-buffer-mib: 5.810937E+01 | global_load_balancing_loss: 9.996653E-01 | loss scale: 1.0 | grad norm: 0.068 | params norm: 11236.691 | params rms: 0.060843 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 0:14:46.925938 | finish at 2026-10-10 20:20:58 + [2026-10-10 20:07:39] iteration 6846/ 6855 | consumed samples: 876288 | elapsed time per iteration (ms): 87737.8 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.59% | FLOPs estimator: workload-aware | logical tokens/s: 191219.8 | valid tokens/s: 191219.8 | physical tokens/s: 191676.2 | loss tokens/s: 136434.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.494727E-01 | lm-global-supervised-tokens: 1.197042E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.358887E-02 | lm-token-buffer-mib: 5.822880E+01 | global_load_balancing_loss: 9.996169E-01 | loss scale: 1.0 | grad norm: 0.083 | params norm: 11236.645 | params rms: 0.060843 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 0:13:09.640280 | finish at 2026-10-10 20:20:48 + [2026-10-10 20:09:06] iteration 6847/ 6855 | consumed samples: 876416 | elapsed time per iteration (ms): 87373.6 | throughput per GPU (TFLOP/s/GPU): 43.0 | MFU 13.77% | FLOPs estimator: workload-aware | logical tokens/s: 192016.6 | valid tokens/s: 192016.6 | physical tokens/s: 192736.7 | loss tokens/s: 144286.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.789681E-01 | lm-global-supervised-tokens: 1.260679E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.493679E-02 | lm-token-buffer-mib: 5.833685E+01 | global_load_balancing_loss: 1.000610E+00 | loss scale: 1.0 | grad norm: 0.080 | params norm: 11236.600 | params rms: 0.060843 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 0:11:38.988703 | finish at 2026-10-10 20:20:45 + [2026-10-10 20:10:37] iteration 6848/ 6855 | consumed samples: 876544 | elapsed time per iteration (ms): 90583.7 | throughput per GPU (TFLOP/s/GPU): 41.7 | MFU 13.38% | FLOPs estimator: workload-aware | logical tokens/s: 185211.7 | valid tokens/s: 185211.7 | physical tokens/s: 185554.7 | loss tokens/s: 133802.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.545710E-01 | lm-global-supervised-tokens: 1.212028E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 8.773562E-02 | lm-token-buffer-mib: 5.803394E+01 | global_load_balancing_loss: 1.001577E+00 | loss scale: 1.0 | grad norm: 0.080 | params norm: 11236.556 | params rms: 0.060842 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 0:10:34.085556 | finish at 2026-10-10 20:21:11 + [2026-10-10 20:12:05] iteration 6849/ 6855 | consumed samples: 876672 | elapsed time per iteration (ms): 88630.4 | throughput per GPU (TFLOP/s/GPU): 42.3 | MFU 13.57% | FLOPs estimator: workload-aware | logical tokens/s: 189293.9 | valid tokens/s: 189293.9 | physical tokens/s: 189682.4 | loss tokens/s: 136343.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.720818E-01 | lm-global-supervised-tokens: 1.208416E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.543009E-02 | lm-token-buffer-mib: 5.808747E+01 | global_load_balancing_loss: 1.001060E+00 | loss scale: 1.0 | grad norm: 0.116 | params norm: 11236.511 | params rms: 0.060842 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 0:08:51.782640 | finish at 2026-10-10 20:20:57 + [2026-10-10 20:13:54] iteration 6850/ 6855 | consumed samples: 876800 | elapsed time per iteration (ms): 108379.3 | throughput per GPU (TFLOP/s/GPU): 33.9 | MFU 10.85% | FLOPs estimator: workload-aware | logical tokens/s: 154800.3 | valid tokens/s: 154800.3 | physical tokens/s: 155203.3 | loss tokens/s: 115412.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 5.862445E-01 | lm-global-supervised-tokens: 1.250832E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.137683E-02 | lm-token-buffer-mib: 5.817834E+01 | global_load_balancing_loss: 1.000918E+00 | loss scale: 1.0 | grad norm: 0.068 | params norm: 11236.465 | params rms: 0.060842 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 0:09:01.896703 | finish at 2026-10-10 20:22:56 + [2026-10-10 20:15:19] iteration 6851/ 6855 | consumed samples: 876928 | elapsed time per iteration (ms): 85573.7 | throughput per GPU (TFLOP/s/GPU): 42.4 | MFU 13.60% | FLOPs estimator: workload-aware | logical tokens/s: 196055.0 | valid tokens/s: 196055.0 | physical tokens/s: 196540.2 | loss tokens/s: 148562.7 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.577148E-01 | lm-global-supervised-tokens: 1.271306E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.172315E-02 | lm-token-buffer-mib: 5.807663E+01 | global_load_balancing_loss: 9.992023E-01 | loss scale: 1.0 | grad norm: 0.102 | params norm: 11236.421 | params rms: 0.060842 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 0:05:42.294717 | finish at 2026-10-10 20:21:02 + [2026-10-10 20:16:45] iteration 6852/ 6855 | consumed samples: 877056 | elapsed time per iteration (ms): 85521.4 | throughput per GPU (TFLOP/s/GPU): 42.6 | MFU 13.64% | FLOPs estimator: workload-aware | logical tokens/s: 196175.6 | valid tokens/s: 196175.6 | physical tokens/s: 196799.6 | loss tokens/s: 148252.0 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.265352E-01 | lm-global-supervised-tokens: 1.267872E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.676994E-02 | lm-token-buffer-mib: 5.838431E+01 | global_load_balancing_loss: 9.988999E-01 | loss scale: 1.0 | grad norm: 0.082 | params norm: 11236.377 | params rms: 0.060841 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 0:04:16.564229 | finish at 2026-10-10 20:21:01 + [2026-10-10 20:18:10] iteration 6853/ 6855 | consumed samples: 877184 | elapsed time per iteration (ms): 84691.1 | throughput per GPU (TFLOP/s/GPU): 43.5 | MFU 13.94% | FLOPs estimator: workload-aware | logical tokens/s: 198098.7 | valid tokens/s: 198098.7 | physical tokens/s: 198363.7 | loss tokens/s: 144917.4 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.452237E-01 | lm-global-supervised-tokens: 1.227321E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.477332E-02 | lm-token-buffer-mib: 5.806231E+01 | global_load_balancing_loss: 9.991233E-01 | loss scale: 1.0 | grad norm: 0.056 | params norm: 11236.333 | params rms: 0.060841 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 0:02:49.382158 | finish at 2026-10-10 20:20:59 + [2026-10-10 20:19:38] iteration 6854/ 6855 | consumed samples: 877312 | elapsed time per iteration (ms): 88371.4 | throughput per GPU (TFLOP/s/GPU): 42.0 | MFU 13.47% | FLOPs estimator: workload-aware | logical tokens/s: 189848.8 | valid tokens/s: 189848.8 | physical tokens/s: 190228.1 | loss tokens/s: 145569.2 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.668813E-01 | lm-global-supervised-tokens: 1.286415E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.215139E-02 | lm-token-buffer-mib: 5.825079E+01 | global_load_balancing_loss: 1.000218E+00 | loss scale: 1.0 | grad norm: 0.251 | params norm: 11236.288 | params rms: 0.060841 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 0:01:28.371398 | finish at 2026-10-10 20:21:06 + [2026-10-10 20:21:05] iteration 6855/ 6855 | consumed samples: 877440 | elapsed time per iteration (ms): 87391.2 | throughput per GPU (TFLOP/s/GPU): 42.2 | MFU 13.54% | FLOPs estimator: workload-aware | logical tokens/s: 191978.0 | valid tokens/s: 191978.0 | physical tokens/s: 192433.4 | loss tokens/s: 142213.6 | learning rate: 5.000000E-04 | global batch size: 128 | lm loss: 6.734186E-01 | lm-global-supervised-tokens: 1.242822E+07 | lm-global-samples: 1.280000E+02 | lm-token-prepare-seconds: 1.081588E-02 | lm-token-buffer-mib: 5.808039E+01 | global_load_balancing_loss: 1.000700E+00 | loss scale: 1.0 | grad norm: 0.183 | params norm: 11236.244 | params rms: 0.060841 | number of skipped iterations: 0 | number of nan iterations: 0 | remaining time: 0:00:00 | finish at 2026-10-10 20:21:05 +times across ranks (ms): + evaluate: + rank 0: 255474.30 + rank 1: 255474.28 + rank 2: 255474.27 + rank 3: 255474.34 + rank 4: 255474.30 + rank 5: 255474.33 + rank 6: 255474.33 + rank 7: 255474.31 + rank 8: 255473.95 + rank 9: 255473.98 + rank 10: 255473.98 + rank 11: 255474.00 + rank 12: 255473.98 + rank 13: 255474.00 + rank 14: 255474.00 + rank 15: 255473.98 + rank 16: 255477.50 + rank 17: 255477.52 + rank 18: 255477.53 + rank 19: 255477.50 + rank 20: 255477.53 + rank 21: 255477.48 + rank 22: 255477.53 + rank 23: 255477.53 + rank 24: 255475.45 + rank 25: 255475.50 + rank 26: 255475.47 + rank 27: 255475.44 + rank 28: 255475.50 + rank 29: 255475.47 + rank 30: 255475.47 + rank 31: 255475.48 + rank 32: 255483.19 + rank 33: 255483.39 + rank 34: 255483.02 + rank 35: 255483.33 + rank 36: 255483.20 + rank 37: 255483.81 + rank 38: 255483.88 + rank 39: 255482.52 + rank 40: 255474.53 + rank 41: 255474.58 + rank 42: 255474.53 + rank 43: 255474.53 + rank 44: 255474.55 + rank 45: 255474.56 + rank 46: 255474.53 + rank 47: 255474.56 + rank 48: 255473.50 + rank 49: 255473.52 + rank 50: 255473.56 + rank 51: 255473.56 + rank 52: 255473.45 + rank 53: 255473.55 + rank 54: 255473.55 + rank 55: 255473.55 + rank 56: 255474.55 + rank 57: 255474.52 + rank 58: 255474.53 + rank 59: 255474.52 + rank 60: 255474.53 + rank 61: 255474.52 + rank 62: 255474.52 + rank 63: 255474.52 +------------------------------------------------------------------------------------------------------------------ + validation loss at iteration 6855 on validation set | lm loss value: 3.189261E-01 | lm loss PPL: 1.375650E+00 | +------------------------------------------------------------------------------------------------------------------ diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/wandb/wandb/run-20261003_153655-neb76hvj/files/requirements.txt b/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/wandb/wandb/run-20261003_153655-neb76hvj/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..f41a3ace63feeb2be75122f04b45a4bcb2ede2ad --- /dev/null +++ b/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/wandb/wandb/run-20261003_153655-neb76hvj/files/requirements.txt @@ -0,0 +1,202 @@ +jupyter_core==5.9.1 +triton==3.4.0 +decorator==5.2.1 +pybind11==3.0.1 +importlib_metadata==8.7.0 +joblib==1.5.2 +pillow==12.0.0 +multiprocess==0.70.18 +asttokens==3.0.1 +prettytable==3.17.0 +networkx==3.4.2 +aniso8601==10.0.1 +setuptools==69.0.0 +pandas==2.3.3 +kiwisolver==1.4.9 +aiohttp==3.13.2 +Flask-RESTful==0.3.10 +Jinja2==3.1.6 +urllib3==2.5.0 +hf-xet==1.5.1 +nltk==3.9.2 +fonttools==4.61.0 +triton_dist==3.4.0 +requests==2.32.5 +ipykernel==7.1.0 +multidict==6.7.0 +typer==0.25.1 +linghe==0.4.5 +pyparsing==3.2.5 +python-dateutil==2.9.0.post0 +pytest-random-order==1.2.0 +pydantic==2.12.5 +click==8.4.1 +safetensors==0.8.0 +openai==2.9.0 +Flask==3.1.2 +ipython==8.37.0 +pyzmq==27.1.0 +fla-core==0.5.0 +jmespath==1.1.0 +onnxscript==0.5.6 +nvidia-cudnn-cu12==9.10.2.21 +virtualenv-clone==0.5.7 +huggingface_hub==1.19.0 +nvidia-cuda-runtime-cu12==12.8.90 +nvitop==1.6.0 +transformer_engine==2.11.0.dev0+3ff0b8d4 +wrapt==2.0.1 +nvidia-ml-py==13.580.82 +coverage==7.12.0 +dill==0.4.0 +ml_dtypes==0.5.4 +s3transfer==0.19.2 +idna==3.18 +filelock==3.29.4 +pyarrow==22.0.0 +PyNaCl==1.6.2 +zarr==2.18.3 +gitdb==4.0.12 +sentencepiece==0.2.1 +annotated-doc==0.0.4 +tokenizers==0.22.2 +pexpect==4.9.0 +pytest-mock==3.15.1 +torchvision==0.23.0 +boto3==1.43.99 +tzdata==2025.2 +modelscope==1.32.0 +tomli==2.3.0 +jsonschema==4.26.0 +transformers==5.13.0 +markdown-it-py==4.2.0 +nvidia-cuda-nvrtc-cu12==12.8.93 +Werkzeug==3.1.4 +pip==25.3 +flash-linear-attention==0.5.0 +numpy==2.2.6 +wcwidth==0.2.14 +wheel==0.45.1 +iniconfig==2.3.0 +debugpy==1.8.17 +packaging==26.2 +zipp==3.23.0 +pluggy==1.6.0 +frozenlist==1.8.0 +jsonschema-specifications==2025.9.1 +exceptiongroup==1.3.1 +cffi==2.0.0 +nvidia-nvtx-cu12==12.8.90 +charset-normalizer==3.4.4 +accelerate==1.12.0 +nvidia-cusparse-cu12==12.5.8.93 +protobuf==6.33.1 +PyYAML==6.0.3 +mdurl==0.1.2 +torch==2.8.0 +six==1.17.0 +nvidia-nvshmem-cu12==3.3.9 +pytest==9.0.1 +mpmath==1.3.0 +xxhash==3.6.0 +nvidia-cufft-cu12==11.3.3.83 +typing_extensions==4.15.0 +attrs==25.4.0 +executing==2.2.1 +nvidia-cusparselt-cu12==0.7.1 +tiktoken==0.12.0 +nvidia-cuda-cupti-cu12==12.8.90 +sympy==1.14.0 +matplotlib==3.10.8 +rich==15.0.0 +nvidia-cusolver-cu12==11.7.3.90 +causal_conv1d==1.5.4 +simplejson==3.20.2 +prompt_toolkit==3.0.52 +typing-inspection==0.4.2 +MarkupSafe==3.0.3 +rpds-py==0.30.0 +nvidia-modelopt==0.39.0 +PyGithub==2.9.1 +contourpy==1.3.2 +jiter==0.12.0 +tqdm==4.68.2 +cycler==0.12.1 +nest-asyncio==1.6.0 +propcache==0.4.1 +pytest-cov==7.0.0 +cuda-core==0.2.0 +parso==0.8.5 +itsdangerous==2.2.0 +bracex==3.0.1 +nvidia-cufile-cu12==1.13.1.3 +jedi==0.19.2 +Pygments==2.20.0 +jupyter_client==8.6.3 +swanlab==0.7.5 +shellingham==1.5.4 +datasets==4.4.1 +nvidia-curand-cu12==10.3.9.90 +async-timeout==5.0.1 +smmap==5.0.2 +stack-data==0.6.3 +flash_attn==2.8.3 +tensorstore==0.1.78 +referencing==0.37.0 +matplotlib-inline==0.2.1 +einops==0.8.2 +certifi==2026.5.20 +mamba_ssm==2.2.6.post3 +pure_eval==0.2.3 +aiosignal==1.4.0 +sniffio==1.3.1 +Cython==0.29.24 +pydantic_core==2.41.5 +onnx==1.19.1 +onnx-ir==0.1.12 +annotated-types==0.7.0 +xattr==1.3.0 +wcmatch==10.2.1 +torchprofile==0.0.4 +psutil==7.1.3 +platformdirs==4.5.0 +sentry-sdk==2.46.0 +PyJWT==2.13.0 +ptyprocess==0.7.0 +nvidia-cublas-cu12==12.8.4.1 +PuLP==3.3.0 +apex==0.1 +scipy==1.15.3 +comm==0.2.3 +tornado==6.5.2 +traitlets==5.14.3 +distro==1.9.0 +yarl==1.22.0 +GitPython==3.1.45 +ninja==1.13.0 +blinker==1.9.0 +httpcore==1.0.9 +nvidia-nccl-cu12==2.27.3 +multi-storage-client==0.51.0 +nvidia-nvjitlink-cu12==12.8.93 +httpx==0.28.1 +fasteners==0.20 +h11==0.16.0 +wandb==0.23.0 +torchaudio==2.8.0 +aiohappyeyeballs==2.6.1 +numcodecs==0.13.1 +nvshmem4py-cu12==0.1.2 +opentelemetry-api==1.44.0 +asciitree==0.3.3 +anyio==4.14.0 +botocore==1.43.99 +fsspec==2026.6.0 +absl-py==2.3.1 +pytz==2025.2 +cryptography==49.0.0 +lark==1.3.1 +regex==2026.5.9 +pycparser==3.0 +pyecharts==2.0.9 +cuda-python==12.4.0 diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/wandb/wandb/run-20261003_153655-neb76hvj/files/wandb-metadata.json b/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/wandb/wandb/run-20261003_153655-neb76hvj/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..c055e59b0254ca5a67f34dd2a441732a0c606dc1 --- /dev/null +++ b/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/wandb/wandb/run-20261003_153655-neb76hvj/files/wandb-metadata.json @@ -0,0 +1,384 @@ +{ + "os": "Linux-5.15.0-94-generic-x86_64-with-glibc2.39", + "python": "CPython 3.10.19", + "startedAt": "2026-10-03T07:36:55.078128Z", + "args": [ + "--use-mcore-models", + "--hybrid-attention-ratio", + "0.0625", + "--hybrid-mlp-ratio", + "0.5", + "--is-hybrid-model", + "--linear-attention-type", + "gated_delta_net", + "--linear-conv-kernel-dim", + "4", + "--linear-key-head-dim", + "64", + "--linear-value-head-dim", + "64", + "--linear-num-key-heads", + "8", + "--linear-num-value-heads", + "32", + "--disable-bias-linear", + "--seq-length", + "131072", + "--max-position-embeddings", + "131072", + "--num-layers", + "56", + "--hidden-size", + "2048", + "--ffn-hidden-size", + "4800", + "--num-attention-heads", + "32", + "--init-method-std", + "0.02", + "--attention-dropout", + "0.0", + "--hidden-dropout", + "0.0", + "--normalization", + "RMSNorm", + "--swiglu", + "--group-query-attention", + "--num-query-groups", + "8", + "--no-masked-softmax-fusion", + "--use-flash-attn", + "--add-qkv-bias", + "--mtp-num-layers", + "0", + "--mtp-loss-scaling-factor", + "0.1", + "--norm-epsilon", + "1e-06", + "--mtp-linear-attention-type", + "gated_delta_net", + "--position-embedding-type", + "none", + "--linear-attention-freq", + "[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1]", + "--optimizer", + "adam", + "--adam-beta1", + "0.9", + "--adam-beta2", + "0.95", + "--adam-eps", + "1e-15", + "--weight-decay", + "0.1", + "--clip-grad", + "1.0", + "--data-path", + "/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_{0..78}_input_ids_tmp", + "--split", + "999,1,0", + "--data-cache-path", + "/mnt/hdd/cpt/binidx_cache_stage15_final_20261001_v2", + "--no-create-attention-mask-in-dataloader", + "--num-experts", + "192", + "--moe-grouped-gemm", + "--moe-router-topk", + "4", + "--moe-router-score-function", + "softmax", + "--overlap-param-gather", + "--overlap-grad-reduce", + "--moe-expert-capacity-factor", + "0", + "--moe-shared-expert-intermediate-size", + "512", + "--moe-router-dtype", + "fp32", + "--moe-permute-fusion", + "--moe-ffn-hidden-size", + "512", + "--moe-shared-expert-gate", + "--moe-shared-expert-overlap", + "--moe-layer-freq", + "([0]*2+[1]*54)", + "--moe-aux-loss-coeff", + "0.05", + "--moe-router-load-balancing-type", + "global_aux_loss", + "--micro-batch-size", + "1", + "--global-batch-size", + "128", + "--lr", + "5e-4", + "--train-samples", + "877440", + "--lr-warmup-samples", + "0", + "--lr-decay-samples", + "877440", + "--lr-decay-style", + "constant", + "--min-lr", + "0.0005", + "--bf16", + "--async-save", + "--ckpt-fully-parallel-load", + "--auto-detect-ckpt-format", + "--load-complemental-dataset", + "loss_mask", + "--tensor-model-parallel-size", + "1", + "--pipeline-model-parallel-size", + "1", + "--context-parallel-size", + "8", + "--expert-model-parallel-size", + "8", + "--sequence-parallel", + "--use-distributed-optimizer", + "--log-interval", + "1", + "--log-throughput", + "--save-interval", + "40", + "--eval-interval", + "99999999", + "--eval-iters", + "10", + "--increase-log-level-iters", + "5", + "--timing-log-level", + "0", + "--wandb-project", + "qwen2.5_3b_mom_cpt", + "--tokenizer-type", + "HuggingFaceTokenizer", + "--tokenizer-model", + "/mnt/ssd/yulan/pretrain-linear-moe/preprocess/modify_tokenizer/YuLan-Mini-Qwen-Template_new_sp", + "--no-adam-bias-correction", + "--corrected-weight-decay", + "--auto-generate-cu-seqlens", + "--cp-comm-type", + "a2a", + "--log-moe-activations", + "--final-logits-clamp-value", + "30", + "--attention-output-gate", + "--gated-norm", + "--gated-norm-lora-rank", + "16", + "--dataloader-type", + "cyclic", + "--lm-loss-normalization", + "global-token", + "--prefetch-factor", + "8", + "--activation-func-clamp-value", + "7", + "--activation-func-clamp-mode", + "soft", + "--gdn-split-in-proj", + "--gdn-fused-split-forward", + "--gdn-alpha-weight-scale", + "1.0", + "--gdn-alpha-weight-scale-mode", + "matrix", + "--gdn-lr-mult-alpha", + "0.5", + "--start-samples", + "0", + "--ckpt-format", + "torch_dist", + "--rerun-mode", + "validate_results", + "--distributed-timeout-minutes", + "120", + "--dist-ckpt-strictness", + "assume_ok_unexpected", + "--gdn-backend", + "fla", + "--gdn-gated-norm-backend", + "linghe", + "--moe-router-gemm-backend", + "te", + "--log-params-norm", + "--timing-log-option", + "all", + "--no-barrier-with-level-1-timing", + "--log-metrics-interval", + "10", + "--cross-entropy-loss-fusion", + "--cross-entropy-fusion-impl", + "linear_triton", + "--linear-cross-entropy-num-chunks", + "8", + "--num-workers", + "4", + "--moe-router-sqrt-gate", + "--moe-router-fusion", + "--moe-token-dispatcher-type", + "alltoall", + "--log-hidden-states", + "embeddings", + "residual_attn", + "input_layernorm", + "attention::q_layernorm", + "attention::k_layernorm", + "attention::core_attention", + "attention::o_proj", + "residual_mlp", + "pre_mlp_layernorm", + "mlp", + "moe::router", + "moe::shared_experts", + "moe::routed_experts", + "gated_delta::in_proj", + "gated_delta::conv1d", + "gated_delta::query", + "gated_delta::key", + "gated_delta::value", + "gated_delta::gate", + "gated_delta::beta", + "gated_delta::alpha", + "gated_delta::g", + "gated_delta::core_attention", + "gated_delta::out_norm", + "gated_delta::out_proj", + "final_layernorm", + "--log-params", + "attention::linear_qkv", + "attention::o_proj", + "mlp::linear_fc1", + "mlp::linear_fc2", + "input_layernorm", + "pre_mlp_layernorm", + "embedding", + "lm_head", + "gated_delta::in_proj", + "gated_delta::in_proj_bias", + "gated_delta::conv1d", + "gated_delta::conv1d_bias", + "gated_delta::dt_bias", + "gated_delta::A_log", + "gated_delta::out_norm", + "gated_delta::out_norm_bias", + "gated_delta::out_proj", + "gated_delta::out_proj_bias", + "--log-update-rms", + "--log-per-module-grad-rms", + "--log-per-layer-grad-rms", + "--document-packing-algorithm", + "obfd", + "--exit-on-missing-checkpoint", + "--no-initialization", + "--untie-embeddings-and-output-weights", + "--recompute-granularity", + "full", + "--recompute-method", + "block", + "--recompute-num-layers", + "56", + "--no-save-step-one", + "--save-retain-interval", + "1000000000", + "--exit-signal-handler", + "--fast-thd-a2a", + "--hybrid-adamh", + "--reuse-training-communicators", + "--ddp-bucket-size", + "40000000", + "--load", + "/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage14_120B_8node_gdnlralpha_off_20260925a/checkpoints", + "--continual-pretrain", + "--save", + "/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints", + "--tensorboard-dir", + "/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/tensorboard", + "--wandb-save-dir", + "/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/wandb", + "--wandb-exp-name", + "stage14_120B_8node_gdnlralpha_off_20260925a", + "--runtime-audit-dir", + "/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/runtime_audit" + ], + "program": "/mnt/ssd/yulan/huyiwen-cpt/YuLan-Pretrain-triton-dist-dispatcher/pretrain_gpt.py", + "email": "huyiwen0301@hotmail.com", + "root": "/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/wandb", + "host": "cs-3554a-53569-server", + "executable": "/mnt/ssd/yulan/pretrain-linear-moe-dev-worktree/YuLan-Pretrain/.venv_800b_ebcloud/bin/python", + "cpu_count": 64, + "cpu_count_logical": 128, + "gpu": "NVIDIA A800-SXM4-80GB", + "gpu_count": 8, + "disk": { + "/": { + "total": "31526391808", + "used": "1319878656" + } + }, + "memory": { + "total": "1081804955648" + }, + "gpu_nvidia": [ + { + "name": "NVIDIA A800-SXM4-80GB", + "memoryTotal": "85899345920", + "cudaCores": 6912, + "architecture": "Ampere", + "uuid": "GPU-40999d1b-2b80-14f4-48d7-334e7dc3e650" + }, + { + "name": "NVIDIA A800-SXM4-80GB", + "memoryTotal": "85899345920", + "cudaCores": 6912, + "architecture": "Ampere", + "uuid": "GPU-c854c8e4-7352-b317-f9e8-7ab1d7a12c08" + }, + { + "name": "NVIDIA A800-SXM4-80GB", + "memoryTotal": "85899345920", + "cudaCores": 6912, + "architecture": "Ampere", + "uuid": "GPU-f7d2842f-48e9-649c-55a4-4efe7a6e20f7" + }, + { + "name": "NVIDIA A800-SXM4-80GB", + "memoryTotal": "85899345920", + "cudaCores": 6912, + "architecture": "Ampere", + "uuid": "GPU-03925119-14d4-a11c-d740-0ba854285052" + }, + { + "name": "NVIDIA A800-SXM4-80GB", + "memoryTotal": "85899345920", + "cudaCores": 6912, + "architecture": "Ampere", + "uuid": "GPU-7cbabdfb-9388-3fbd-babc-d0b03ac9f1bb" + }, + { + "name": "NVIDIA A800-SXM4-80GB", + "memoryTotal": "85899345920", + "cudaCores": 6912, + "architecture": "Ampere", + "uuid": "GPU-4999b9f4-9851-ce6a-4296-c230e2813b5f" + }, + { + "name": "NVIDIA A800-SXM4-80GB", + "memoryTotal": "85899345920", + "cudaCores": 6912, + "architecture": "Ampere", + "uuid": "GPU-9e23b382-2f8e-e424-f025-c2b311d96c54" + }, + { + "name": "NVIDIA A800-SXM4-80GB", + "memoryTotal": "85899345920", + "cudaCores": 6912, + "architecture": "Ampere", + "uuid": "GPU-4241d9ca-7546-c1c3-dba8-bdc5786a3777" + } + ], + "cudaVersion": "13.3", + "writerId": "i4bo5dz9n6n9rgehz11zfcuyno7peo6e" +} \ No newline at end of file diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/wandb/wandb/run-20261003_153655-neb76hvj/logs/debug-internal.log b/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/wandb/wandb/run-20261003_153655-neb76hvj/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..27efabf33b4846f8f5d627c9acb2f5991cfdaf14 --- /dev/null +++ b/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/wandb/wandb/run-20261003_153655-neb76hvj/logs/debug-internal.log @@ -0,0 +1,116 @@ +{"time":"2026-10-03T15:36:55.391747798+08:00","level":"INFO","msg":"stream: starting","core version":"0.23.0"} +{"time":"2026-10-03T15:36:56.343971627+08:00","level":"INFO","msg":"stream: created new stream","id":"neb76hvj"} +{"time":"2026-10-03T15:36:56.344046883+08:00","level":"INFO","msg":"handler: started","stream_id":"neb76hvj"} +{"time":"2026-10-03T15:36:56.344730971+08:00","level":"INFO","msg":"stream: started","id":"neb76hvj"} +{"time":"2026-10-03T15:36:56.344743257+08:00","level":"INFO","msg":"writer: started","stream_id":"neb76hvj"} +{"time":"2026-10-03T15:36:56.344739432+08:00","level":"INFO","msg":"sender: started","stream_id":"neb76hvj"} +{"time":"2026-10-03T20:46:23.552125568+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/huyiwen/qwen2.5_3b_mom_cpt/neb76hvj/file_stream\": EOF"} +{"time":"2026-10-03T20:46:36.077615671+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/huyiwen/qwen2.5_3b_mom_cpt/neb76hvj/file_stream\": EOF"} +{"time":"2026-10-03T20:48:50.064249591+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/huyiwen/qwen2.5_3b_mom_cpt/neb76hvj/file_stream\": dial tcp 34.8.250.101:443: connect: connection timed out"} +{"time":"2026-10-03T20:49:08.79949489+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/huyiwen/qwen2.5_3b_mom_cpt/neb76hvj/file_stream\": EOF"} +{"time":"2026-10-03T20:54:11.796774742+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/huyiwen/qwen2.5_3b_mom_cpt/neb76hvj/file_stream\": EOF"} +{"time":"2026-10-03T20:54:25.486179586+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/huyiwen/qwen2.5_3b_mom_cpt/neb76hvj/file_stream\": EOF"} +{"time":"2026-10-03T20:54:40.545937717+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/huyiwen/qwen2.5_3b_mom_cpt/neb76hvj/file_stream\": EOF"} +{"time":"2026-10-03T20:55:00.580305387+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/huyiwen/qwen2.5_3b_mom_cpt/neb76hvj/file_stream\": EOF"} +{"time":"2026-10-03T20:55:28.634028057+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/huyiwen/qwen2.5_3b_mom_cpt/neb76hvj/file_stream\": EOF"} +{"time":"2026-10-03T20:56:17.718812384+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/huyiwen/qwen2.5_3b_mom_cpt/neb76hvj/file_stream\": EOF"} +{"time":"2026-10-03T20:56:30.19182662+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/huyiwen/qwen2.5_3b_mom_cpt/neb76hvj/file_stream\": EOF"} +{"time":"2026-10-03T20:56:45.115099873+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/huyiwen/qwen2.5_3b_mom_cpt/neb76hvj/file_stream\": EOF"} +{"time":"2026-10-03T20:57:04.950041949+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/huyiwen/qwen2.5_3b_mom_cpt/neb76hvj/file_stream\": EOF"} +{"time":"2026-10-03T20:57:33.290989029+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/huyiwen/qwen2.5_3b_mom_cpt/neb76hvj/file_stream\": EOF"} +{"time":"2026-10-03T20:58:17.249014078+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/huyiwen/qwen2.5_3b_mom_cpt/neb76hvj/file_stream\": EOF"} +{"time":"2026-10-03T21:18:07.6748799+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/huyiwen/qwen2.5_3b_mom_cpt/neb76hvj/file_stream\": EOF"} +{"time":"2026-10-03T21:18:20.477958703+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/huyiwen/qwen2.5_3b_mom_cpt/neb76hvj/file_stream\": EOF"} +{"time":"2026-10-03T21:26:22.679695426+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/huyiwen/qwen2.5_3b_mom_cpt/neb76hvj/file_stream\": EOF"} +{"time":"2026-10-03T21:26:52.680964647+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/huyiwen/qwen2.5_3b_mom_cpt/neb76hvj/file_stream\": EOF"} +{"time":"2026-10-03T21:27:05.58031186+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/huyiwen/qwen2.5_3b_mom_cpt/neb76hvj/file_stream\": EOF"} +{"time":"2026-10-03T21:27:20.891877642+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/huyiwen/qwen2.5_3b_mom_cpt/neb76hvj/file_stream\": EOF"} +{"time":"2026-10-03T21:27:39.777644747+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/huyiwen/qwen2.5_3b_mom_cpt/neb76hvj/file_stream\": EOF"} +{"time":"2026-10-03T21:28:07.705963999+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/huyiwen/qwen2.5_3b_mom_cpt/neb76hvj/file_stream\": EOF"} +{"time":"2026-10-03T21:28:55.980689331+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/huyiwen/qwen2.5_3b_mom_cpt/neb76hvj/file_stream\": EOF"} +{"time":"2026-10-03T21:46:22.751047592+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/huyiwen/qwen2.5_3b_mom_cpt/neb76hvj/file_stream\": EOF"} +{"time":"2026-10-03T21:46:35.471208755+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/huyiwen/qwen2.5_3b_mom_cpt/neb76hvj/file_stream\": EOF"} +{"time":"2026-10-03T21:46:50.725116709+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/huyiwen/qwen2.5_3b_mom_cpt/neb76hvj/file_stream\": EOF"} +{"time":"2026-10-03T21:48:13.853271803+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/huyiwen/qwen2.5_3b_mom_cpt/neb76hvj/file_stream\": EOF"} +{"time":"2026-10-03T21:48:26.620263443+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/huyiwen/qwen2.5_3b_mom_cpt/neb76hvj/file_stream\": EOF"} +{"time":"2026-10-03T21:48:41.252419412+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/huyiwen/qwen2.5_3b_mom_cpt/neb76hvj/file_stream\": EOF"} +{"time":"2026-10-03T21:48:59.941891806+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/huyiwen/qwen2.5_3b_mom_cpt/neb76hvj/file_stream\": EOF"} +{"time":"2026-10-03T22:23:17.148564204+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/huyiwen/qwen2.5_3b_mom_cpt/neb76hvj/file_stream\": EOF"} +{"time":"2026-10-03T22:24:34.961977946+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/huyiwen/qwen2.5_3b_mom_cpt/neb76hvj/file_stream\": EOF"} +{"time":"2026-10-03T22:24:49.868897624+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/huyiwen/qwen2.5_3b_mom_cpt/neb76hvj/file_stream\": EOF"} +{"time":"2026-10-03T22:25:09.837019477+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/huyiwen/qwen2.5_3b_mom_cpt/neb76hvj/file_stream\": EOF"} +{"time":"2026-10-03T22:25:22.740851635+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/huyiwen/qwen2.5_3b_mom_cpt/neb76hvj/file_stream\": EOF"} +{"time":"2026-10-03T22:25:41.180815099+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/huyiwen/qwen2.5_3b_mom_cpt/neb76hvj/file_stream\": EOF"} +{"time":"2026-10-03T22:26:02.099461715+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/huyiwen/qwen2.5_3b_mom_cpt/neb76hvj/file_stream\": EOF"} +{"time":"2026-10-03T23:16:37.70080957+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/huyiwen/qwen2.5_3b_mom_cpt/neb76hvj/file_stream\": EOF"} +{"time":"2026-10-03T23:16:50.079410395+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/huyiwen/qwen2.5_3b_mom_cpt/neb76hvj/file_stream\": EOF"} +{"time":"2026-10-03T23:17:06.416669273+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/huyiwen/qwen2.5_3b_mom_cpt/neb76hvj/file_stream\": EOF"} +{"time":"2026-10-03T23:17:19.195204077+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/huyiwen/qwen2.5_3b_mom_cpt/neb76hvj/file_stream\": EOF"} +{"time":"2026-10-03T23:17:33.957777145+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/huyiwen/qwen2.5_3b_mom_cpt/neb76hvj/file_stream\": EOF"} +{"time":"2026-10-03T23:17:56.940595699+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/huyiwen/qwen2.5_3b_mom_cpt/neb76hvj/file_stream\": EOF"} +{"time":"2026-10-03T23:19:15.017046246+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/huyiwen/qwen2.5_3b_mom_cpt/neb76hvj/file_stream\": EOF"} +{"time":"2026-10-03T23:19:28.467688489+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/huyiwen/qwen2.5_3b_mom_cpt/neb76hvj/file_stream\": EOF"} +{"time":"2026-10-04T00:16:56.18210566+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/huyiwen/qwen2.5_3b_mom_cpt/neb76hvj/file_stream\": EOF"} +{"time":"2026-10-04T00:17:09.315476895+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/huyiwen/qwen2.5_3b_mom_cpt/neb76hvj/file_stream\": EOF"} +{"time":"2026-10-04T00:17:21.940223945+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/huyiwen/qwen2.5_3b_mom_cpt/neb76hvj/file_stream\": EOF"} +{"time":"2026-10-04T00:19:48.767396206+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/huyiwen/qwen2.5_3b_mom_cpt/neb76hvj/file_stream\": EOF"} +{"time":"2026-10-04T00:20:02.042044451+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/huyiwen/qwen2.5_3b_mom_cpt/neb76hvj/file_stream\": EOF"} +{"time":"2026-10-04T00:20:17.40407131+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/huyiwen/qwen2.5_3b_mom_cpt/neb76hvj/file_stream\": EOF"} +{"time":"2026-10-04T00:20:29.883880501+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/huyiwen/qwen2.5_3b_mom_cpt/neb76hvj/file_stream\": EOF"} +{"time":"2026-10-04T00:20:30.485669138+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"} +{"time":"2026-10-04T00:20:44.994579313+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/huyiwen/qwen2.5_3b_mom_cpt/neb76hvj/file_stream\": EOF"} +{"time":"2026-10-04T00:21:02.900640914+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"} +{"time":"2026-10-04T00:21:37.575387697+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded (Client.Timeout exceeded while awaiting headers)"} +{"time":"2026-10-04T00:22:16.426157864+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"} +{"time":"2026-10-04T00:22:57.246240087+08:00","level":"INFO","msg":"flowcontrol: backed up, offloading to disk","recordNumber":6002} +{"time":"2026-10-04T00:23:04.708055607+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"} +{"time":"2026-10-04T00:23:40.822674886+08:00","level":"INFO","msg":"flowcontrol: unblocked","totalOffloaded":10} +{"time":"2026-10-04T00:51:37.946835389+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/huyiwen/qwen2.5_3b_mom_cpt/neb76hvj/file_stream\": EOF"} +{"time":"2026-10-04T00:53:50.608248186+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/huyiwen/qwen2.5_3b_mom_cpt/neb76hvj/file_stream\": dial tcp 34.8.250.101:443: connect: connection timed out"} +{"time":"2026-10-04T00:54:21.932925999+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/huyiwen/qwen2.5_3b_mom_cpt/neb76hvj/file_stream\": EOF"} +{"time":"2026-10-04T20:21:08.268580691+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/huyiwen/qwen2.5_3b_mom_cpt/neb76hvj/file_stream\": EOF"} +{"time":"2026-10-04T20:21:20.616873937+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/huyiwen/qwen2.5_3b_mom_cpt/neb76hvj/file_stream\": EOF"} +{"time":"2026-10-04T20:21:34.684804207+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": EOF"} +{"time":"2026-10-04T20:21:47.257725082+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": EOF"} +{"time":"2026-10-04T20:22:22.001280214+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded (Client.Timeout exceeded while awaiting headers)"} +{"time":"2026-10-04T20:22:40.639904893+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/huyiwen/qwen2.5_3b_mom_cpt/neb76hvj/file_stream\": EOF"} +{"time":"2026-10-04T20:23:00.036970039+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/huyiwen/qwen2.5_3b_mom_cpt/neb76hvj/file_stream\": EOF"} +{"time":"2026-10-04T20:23:29.015889448+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/huyiwen/qwen2.5_3b_mom_cpt/neb76hvj/file_stream\": EOF"} +{"time":"2026-10-04T20:24:12.722170873+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/huyiwen/qwen2.5_3b_mom_cpt/neb76hvj/file_stream\": EOF"} +{"time":"2026-10-04T20:25:23.128186804+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/huyiwen/qwen2.5_3b_mom_cpt/neb76hvj/file_stream\": EOF"} +{"time":"2026-10-04T20:26:35.250621719+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/huyiwen/qwen2.5_3b_mom_cpt/neb76hvj/file_stream\": EOF"} +{"time":"2026-10-04T20:26:47.734909018+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/huyiwen/qwen2.5_3b_mom_cpt/neb76hvj/file_stream\": EOF"} +{"time":"2026-10-04T20:27:04.853048393+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/huyiwen/qwen2.5_3b_mom_cpt/neb76hvj/file_stream\": EOF"} +{"time":"2026-10-04T20:27:17.427268502+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/huyiwen/qwen2.5_3b_mom_cpt/neb76hvj/file_stream\": EOF"} +{"time":"2026-10-04T20:28:38.822647668+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/huyiwen/qwen2.5_3b_mom_cpt/neb76hvj/file_stream\": EOF"} +{"time":"2026-10-04T20:28:53.402251459+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/huyiwen/qwen2.5_3b_mom_cpt/neb76hvj/file_stream\": EOF"} +{"time":"2026-10-04T20:29:06.242458788+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/huyiwen/qwen2.5_3b_mom_cpt/neb76hvj/file_stream\": EOF"} +{"time":"2026-10-05T09:35:42.961267638+08:00","level":"INFO","msg":"api: retrying HTTP error","status":500,"url":"https://api.wandb.ai/files/huyiwen/qwen2.5_3b_mom_cpt/neb76hvj/file_stream","body":"{\"error\":\"mysql query exceeded guard deadline: context deadline exceeded\"}"} +{"time":"2026-10-05T09:35:43.003831834+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"} +{"time":"2026-10-05T09:36:15.387744012+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"} +{"time":"2026-10-05T09:36:34.472084593+08:00","level":"INFO","msg":"api: retrying HTTP error","status":500,"url":"https://api.wandb.ai/graphql","body":"{\"error\":\"An internal error occurred. Please contact support.\"}"} +{"time":"2026-10-05T09:36:41.311002027+08:00","level":"INFO","msg":"api: retrying HTTP error","status":429,"url":"https://api.wandb.ai/files/huyiwen/qwen2.5_3b_mom_cpt/neb76hvj/file_stream","body":"{\"error\":\"rate limit exceeded: per_run limit on filestream requests\"}"} +{"time":"2026-10-05T14:23:22.622411251+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/huyiwen/qwen2.5_3b_mom_cpt/neb76hvj/file_stream\": EOF"} +{"time":"2026-10-05T14:23:35.332510685+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/huyiwen/qwen2.5_3b_mom_cpt/neb76hvj/file_stream\": EOF"} +{"time":"2026-10-05T14:23:50.703125116+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/huyiwen/qwen2.5_3b_mom_cpt/neb76hvj/file_stream\": EOF"} +{"time":"2026-10-05T14:24:22.554969727+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/huyiwen/qwen2.5_3b_mom_cpt/neb76hvj/file_stream\": EOF"} +{"time":"2026-10-05T14:24:37.512289017+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/huyiwen/qwen2.5_3b_mom_cpt/neb76hvj/file_stream\": EOF"} +{"time":"2026-10-05T14:24:50.156339635+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/huyiwen/qwen2.5_3b_mom_cpt/neb76hvj/file_stream\": EOF"} +{"time":"2026-10-05T14:25:05.235963369+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/huyiwen/qwen2.5_3b_mom_cpt/neb76hvj/file_stream\": EOF"} +{"time":"2026-10-05T14:25:24.075085054+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/huyiwen/qwen2.5_3b_mom_cpt/neb76hvj/file_stream\": EOF"} +{"time":"2026-10-05T14:27:07.531449591+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/huyiwen/qwen2.5_3b_mom_cpt/neb76hvj/file_stream\": EOF"} +{"time":"2026-10-05T14:27:22.510235052+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/huyiwen/qwen2.5_3b_mom_cpt/neb76hvj/file_stream\": EOF"} +{"time":"2026-10-05T14:27:34.977708119+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/huyiwen/qwen2.5_3b_mom_cpt/neb76hvj/file_stream\": EOF"} +{"time":"2026-10-05T14:27:49.525380822+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/huyiwen/qwen2.5_3b_mom_cpt/neb76hvj/file_stream\": EOF"} +{"time":"2026-10-05T14:27:55.277339507+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": EOF"} +{"time":"2026-10-05T14:28:08.330067714+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/huyiwen/qwen2.5_3b_mom_cpt/neb76hvj/file_stream\": EOF"} +{"time":"2026-10-05T14:30:36.176808349+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/huyiwen/qwen2.5_3b_mom_cpt/neb76hvj/file_stream\": dial tcp 34.8.250.101:443: connect: connection timed out"} +{"time":"2026-10-06T22:09:53.669948878+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded (Client.Timeout exceeded while awaiting headers)"} +{"time":"2026-10-06T22:10:20.275442496+08:00","level":"ERROR","msg":"request failed","error":"Put \"https://storage.googleapis.com/wandb-artifacts-prod/wandb_artifacts/1494869847/3827992217/f231f2107df69eab0a3862d50018a9b2?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer%40wandb-production.iam.gserviceaccount.com%2F20261006%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20261006T141019Z&X-Goog-Expires=86399&X-Goog-Signature=7d6629d602b0fcb4db2bc36e31fe1906e480dc486be14eae1b6514e00b2c934e1b88c80a7127704af63f2a3a549c813d2c64769e0515685c7fdbce6b005ed2eb4537bdf8ce3f07202d44d240e95df251274072e0103c4bfc9e70b14249d1ea36bfb3d7c64d42545990bd37c9a3139ce9926289be2dfde50f936a2030740769beb7054ec0291b9d5e900b36d785c2b71eb458fb728007628e9dce2a7b9a57216fa1506aced48542e228db7f6652a506ec275ea39f956dcfa4ad594e04601b1a87bbb442704eb767c3e484f7ec602a4b0990ad5fc6d47bf151ef35da4d8caabfc4f3b3503f81f2c172b86f28efcc5b1e2266a5a273d9f1c7bdaa4e6cc09cc21370&X-Goog-SignedHeaders=content-md5%3Bcontent-type%3Bhost&X-User=huyiwen\": read tcp 10.233.98.253:54508->142.250.73.155:443: read: connection reset by peer","method":"PUT","url":"https://storage.googleapis.com/wandb-artifacts-prod/wandb_artifacts/1494869847/3827992217/f231f2107df69eab0a3862d50018a9b2?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer%40wandb-production.iam.gserviceaccount.com%2F20261006%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20261006T141019Z&X-Goog-Expires=86399&X-Goog-Signature=7d6629d602b0fcb4db2bc36e31fe1906e480dc486be14eae1b6514e00b2c934e1b88c80a7127704af63f2a3a549c813d2c64769e0515685c7fdbce6b005ed2eb4537bdf8ce3f07202d44d240e95df251274072e0103c4bfc9e70b14249d1ea36bfb3d7c64d42545990bd37c9a3139ce9926289be2dfde50f936a2030740769beb7054ec0291b9d5e900b36d785c2b71eb458fb728007628e9dce2a7b9a57216fa1506aced48542e228db7f6652a506ec275ea39f956dcfa4ad594e04601b1a87bbb442704eb767c3e484f7ec602a4b0990ad5fc6d47bf151ef35da4d8caabfc4f3b3503f81f2c172b86f28efcc5b1e2266a5a273d9f1c7bdaa4e6cc09cc21370&X-Goog-SignedHeaders=content-md5%3Bcontent-type%3Bhost&X-User=huyiwen"} +{"time":"2026-10-10T02:17:49.028881163+08:00","level":"INFO","msg":"api: retrying HTTP error","status":502,"url":"https://api.wandb.ai/files/huyiwen/qwen2.5_3b_mom_cpt/neb76hvj/file_stream","body":"\n\n\n502 Server Error\n\n\n

Error: Server Error

\n

The server encountered a temporary error and could not complete your request.

Please try again in 30 seconds.

\n

\n\n"} +{"time":"2026-10-10T20:25:50.740816179+08:00","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"} +{"time":"2026-10-10T20:25:53.530647034+08:00","level":"INFO","msg":"handler: operation stats","stats":{}} +{"time":"2026-10-10T20:25:53.586303464+08:00","level":"INFO","msg":"stream: closing","id":"neb76hvj"} +{"time":"2026-10-10T20:25:53.586544397+08:00","level":"INFO","msg":"handler: closed","stream_id":"neb76hvj"} +{"time":"2026-10-10T20:25:53.587170191+08:00","level":"INFO","msg":"sender: closed","stream_id":"neb76hvj"} +{"time":"2026-10-10T20:25:53.587575033+08:00","level":"INFO","msg":"stream: closed","id":"neb76hvj"} diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/wandb/wandb/run-20261003_153655-neb76hvj/logs/debug.log b/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/wandb/wandb/run-20261003_153655-neb76hvj/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..71838880ec3cf6ecc627572581b08b5faf3b996b --- /dev/null +++ b/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/wandb/wandb/run-20261003_153655-neb76hvj/logs/debug.log @@ -0,0 +1,27 @@ +2026-10-03 15:36:55,089 INFO MainThread:730851 [wandb_setup.py:_flush():80] Current SDK version is 0.23.0 +2026-10-03 15:36:55,090 INFO MainThread:730851 [wandb_setup.py:_flush():80] Configure stats pid to 730851 +2026-10-03 15:36:55,090 INFO MainThread:730851 [wandb_setup.py:_flush():80] Loading settings from /root/.config/wandb/settings +2026-10-03 15:36:55,090 INFO MainThread:730851 [wandb_setup.py:_flush():80] Loading settings from /root/wandb/settings +2026-10-03 15:36:55,090 INFO MainThread:730851 [wandb_setup.py:_flush():80] Loading settings from environment variables +2026-10-03 15:36:55,090 INFO MainThread:730851 [wandb_init.py:setup_run_log_directory():713] Logging user logs to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/wandb/wandb/run-20261003_153655-neb76hvj/logs/debug.log +2026-10-03 15:36:55,090 INFO MainThread:730851 [wandb_init.py:setup_run_log_directory():714] Logging internal logs to /mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/wandb/wandb/run-20261003_153655-neb76hvj/logs/debug-internal.log +2026-10-03 15:36:55,090 INFO MainThread:730851 [wandb_init.py:init():840] calling init triggers +2026-10-03 15:36:55,091 INFO MainThread:730851 [wandb_init.py:init():845] wandb.init called with sweep_config: {} +config: {'num_layers': 56, 'encoder_num_layers': 56, 'decoder_num_layers': None, 'hidden_size': 2048, 'ffn_hidden_size': 4800, 'num_attention_heads': 32, 'attention_backend': , 'kv_channels': 64, 'group_query_attention': True, 'num_query_groups': 8, 'attention_output_gate': True, 'attention_output_gate_upcycling': False, 'gated_norm': True, 'gated_norm_upcycling': False, 'gated_norm_lora_rank': 16, 'softmax_type': 'vanilla', 'window_size': None, 'window_attn_skip_freq': None, 'max_position_embeddings': 131072, 'position_embedding_type': 'none', 'relative_attention_num_buckets': 32, 'relative_attention_max_distance': 128, 'use_rotary_position_embeddings': False, 'rotary_base': 10000, 'rotary_percent': 1.0, 'rotary_interleaved': False, 'rotary_seq_len_interpolation_factor': None, 'use_rope_scaling': False, 'rope_scaling_factor': 8.0, 'no_rope_freq': None, 'add_position_embedding': True, 'mrope_section': None, 'make_vocab_size_divisible_by': 128, 'normalization': 'RMSNorm', 'norm_epsilon': 1e-06, 'apply_layernorm_1p': False, 'freeze_layernorm_weight': False, 'scale_emb': 1.0, 'hidden_states_shrink': 1.0, 'lm_head_alpha': 1.0, 'mup_lr_mult': 1.0, 'apply_residual_connection_post_layernorm': False, 'openai_gelu': False, 'squared_relu': False, 'swiglu': True, 'quick_geglu': False, 'activation_func_clamp_value': 7.0, 'activation_func_clamp_mode': 'soft', 'glu_linear_offset': 0.0, 'sqreglu': False, 'geglu': False, 'onnx_safe': None, 'bert_binary_head': True, 'untie_embeddings_and_output_weights': True, 'multi_latent_attention': False, 'mtp_num_layers': 0, 'mtp_loss_scaling_factor': 0.1, 'mtp_linear_attention_type': 'gated_delta_net', 'ffn_token_shift': None, 'ffn_intermediate_token_shift': None, 'token_shift_conv_size': 4, 'token_shift_conv_init': 'default', 'attn_token_shift': None, 'attn_q_token_shift': None, 'attn_k_token_shift': None, 'attn_v_token_shift': None, 'attn_output_gate': None, 'emb_deviation_loss_coeff': 0, 'emb_deviation_type': None, 'reparam_keys': None, 'attention_dropout': 0.0, 'hidden_dropout': 0.0, 'word_embedding_dropout_prob': 0.0, 'weight_decay': 0.1, 'start_weight_decay': 0.1, 'end_weight_decay': 0.1, 'weight_decay_incr_style': 'constant', 'corrected_weight_decay': True, 'no_weight_decay_cond_type': None, 'clip_grad': 1.0, 'log_per_module_update_rms': False, 'log_spike_diagnostics': False, 'spike_diagnostics_start': 2, 'spike_diagnostics_end': 21, 'spike_diagnostics_dir': None, 'log_per_module_grad_rms': True, 'log_per_layer_grad_rms': True, 'adam_beta1': 0.9, 'adam_beta2': 0.95, 'adam_eps': 1e-15, 'adamw_lr_mup_scaler': False, 'sgd_momentum': 0.9, 'adamh_scope': 'lm_head', 'adamh_radius': None, 'adamh_eps': 1e-08, 'spectra_scope': 'matrices', 'spectra_hyperball': False, 'spectra_clip': 1.0, 'spectra_grad_clip': None, 'spectra_ns_steps': 10, 'muon_momentum': 0.9, 'muon_split_qkv': True, 'muon_qkv_split_mode': 'component', 'muon_split_fc1': True, 'muon_use_nesterov': False, 'muon_scale_mode': 'spectral_mup', 'muon_fp32_matmul_prec': 'medium', 'muon_num_ns_steps': 5, 'muon_tp_mode': 'blockwise', 'muon_extra_scale_factor': 1.0, 'muon_split_moe_experts': True, 'muon_vectorize': [], 'muon_vectorize_attn_dim': 'hidden_size', 'muon_scale_vectorized_mode': 'full', 'muon_ball_momentum': 0.9, 'muon_ball_use_nesterov': True, 'muon_ball_split_qkv': True, 'muon_ball_qkv_split_mode': 'component', 'muon_ball_split_fc1': True, 'muon_ball_split_moe_experts': True, 'muon_ball_msign_steps': 5, 'muon_ball_radius_mode': 'spectral_mup', 'muon_ball_power_iteration_steps': 10, 'muon_ball_scale_mode': 'spectral_mup', 'muon_ball_retract_mode': 'hard', 'muon_ball_retract_alpha': 0.05, 'spectral_ball_momentum': 0.9, 'spectral_ball_use_nesterov': True, 'spectral_ball_split_qkv': True, 'spectral_ball_qkv_split_mode': 'component', 'spectral_ball_split_fc1': True, 'spectral_ball_split_moe_experts': True, 'spectral_ball_msign_steps': 8, 'spectral_ball_solver': 'bisection', 'spectral_ball_solver_tolerance_f': 1e-08, 'spectral_ball_solver_max_iterations': 20, 'spectral_ball_radius_mode': 'spectral_mup', 'spectral_ball_power_iteration_steps': 20, 'spectral_ball_scale_mode': 'spectral_mup', 'spectral_ball_retract_mode': 'hard', 'spectral_ball_retract_alpha': 0.05, 'fast_thd_a2a': True, 'hybrid_adamh': True, 'reuse_training_communicators': True, 'runtime_audit_dir': '/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/runtime_audit', 'train_until_stopped': False, 'micro_batch_size': 1, 'global_batch_size': 128, 'rampup_batch_size': None, 'decrease_batch_size_if_needed': False, 'recompute_granularity': 'full', 'check_for_nan_in_loss_and_grad': True, 'check_for_spiky_loss': False, 'check_for_large_grads': False, 'distribute_saved_activations': False, 'recompute_method': 'block', 'recompute_num_layers': 56, 'pp_aware_finegrained_recompute': False, 'pp_aware_finegrained_recompute_pp_ranks': None, 'pp_aware_finegrained_recompute_global_layers': None, 'recompute_modules': None, 'clone_scatter_output_in_embedding': True, 'profile': False, 'profile_step_start': None, 'profile_step_end': None, 'iterations_to_skip': [], 'result_rejected_tracker_filename': None, 'enable_gloo_process_groups': True, 'use_pytorch_profiler': False, 'profile_ranks': [0], 'profile_microbatch_iteration': None, 'profile_microbatch_start': 0, 'profile_microbatch_count': 2, 'profile_microbatch_dir': None, 'record_memory_history': False, 'memory_snapshot_path': None, 'tp_comm_overlap': False, 'tp_comm_overlap_cfg': None, 'tp_comm_overlap_ag': True, 'tp_comm_overlap_rs': True, 'tp_comm_overlap_rs_dgrad': False, 'tp_comm_bulk_dgrad': True, 'tp_comm_bulk_wgrad': True, 'tp_comm_bootstrap_backend': 'nccl', 'use_cpu_initialization': None, 'empty_unused_memory_level': 0, 'deterministic_mode': False, 'check_weight_hash_across_dp_replicas_interval': None, 'calculate_per_token_loss': False, 'lm_loss_normalization': 'global-token', 'train_sync_interval': None, 'train_iters': None, 'train_samples': 877440, 'log_interval': 1, 'exit_interval': None, 'exit_duration_in_mins': None, 'exit_signal_handler': True, 'tensorboard_dir': '/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/tensorboard', 'masked_softmax_fusion': False, 'bias_gelu_fusion': False, 'bias_swiglu_fusion': True, 'use_fused_weighted_squared_relu': False, 'bias_dropout_fusion': True, 'apply_rope_fusion': False, 'rope_type': None, 'cross_entropy_loss_fusion': True, 'cross_entropy_fusion_impl': 'linear_triton', 'linear_cross_entropy_num_chunks': 8, 'final_logits_clamp_value': 30.0, 'linear_cross_entropy_accumulate_grad_in_fp32': True, 'use_flash_attn': True, 'add_bias_linear': False, 'add_qkv_bias': True, 'optimizer': 'adam', 'adam_bias_correction': False, 'optimizer_cpu_offload': False, 'optimizer_offload_fraction': 1.0, 'use_torch_optimizer_for_cpu_offload': False, 'overlap_cpu_optimizer_d2h_h2d': False, 'dump_param_to_param_group_map': None, 'pin_cpu_grads': True, 'pin_cpu_params': True, 'dataloader_type': 'cyclic', 'async_tensor_model_parallel_allreduce': True, 'no_persist_layer_norm': False, 'sequence_parallel': False, 'gradient_accumulation_fusion': True, 'deprecated_use_mcore_models': True, 'use_legacy_models': False, 'manual_gc': False, 'manual_gc_interval': 0, 'manual_gc_eval': True, 'tp_comm_split_ag': True, 'tp_comm_split_rs': True, 'pipeline_model_parallel_comm_backend': None, 'high_priority_stream_groups': [], 'use_te_activation_func': False, 'freeze_non_mamba': False, 'reset_iteration_one_to_zero': False, 'fine_grained_activation_offloading': False, 'offload_modules': [], 'min_offloaded_tensor_size': 1048576, 'perform_rl_step': False, 'rl_prompts_per_eval': 32, 'grpo_prompts_per_step': 32, 'grpo_group_size': 2, 'grpo_iterations': 2, 'grpo_clamp_eps_lower': 0.01, 'grpo_clamp_eps_upper': 0.01, 'grpo_kl_beta': 0.001, 'grpo_entropy_term_weight': 0.0, 'grpo_filter_groups_with_same_reward': False, 'grpo_default_temperature': 1.0, 'grpo_default_top_p': 0, 'langrl_inference_server_type': 'inplace_megatron', 'langrl_inference_server_conversation_template': None, 'langrl_external_server': False, 'langrl_env_config': None, 'rl_offload_optimizer_during_inference': False, 'rl_offload_kv_cache_during_training': False, 'rl_remove_kv_cache_during_training': False, 'rl_reset_cuda_graphs': False, 'rl_partial_rollouts': False, 'rl_inference_logprobs_is_correction': False, 'rl_importance_sampling_truncation_coef': None, 'rl_calculate_intra_group_similarity': False, 'rl_use_sequence_packing': False, 'rl_sequence_packing_bin_size': 8192, 'rl_sequence_packing_algo': 'fifo', 'seed': 1234, 'data_parallel_random_init': False, 'init_method_std': 0.02, 'split_qkv_init': True, 'split_qkv_init_mode': 'group', 'split_fc1_init': True, 'split_expert_init': True, 'embedding_init_method_std': None, 'spectral_mup_init': False, 'init_method_xavier_uniform': False, 'lr': 0.0005, 'lr_decay_style': 'constant', 'lr_wsd_decay_style': 'exponential', 'lr_decay_iters': None, 'lr_decay_samples': 877440, 'lr_wsd_decay_samples': None, 'lr_wsd_decay_iters': None, 'lr_warmup_fraction': None, 'lr_warmup_iters': 0, 'lr_warmup_samples': 0, 'lr_warmup_init': 0.0, 'min_lr': 0.0005, 'override_opt_param_scheduler': True, 'use_checkpoint_opt_param_scheduler': False, 'decoupled_lr': None, 'decoupled_min_lr': None, 'save': '/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/checkpoints', 'no_save_step_one': True, 'save_interval': 40, 'save_retain_interval': 1000000000, 'no_save_optim': None, 'no_save_rng': None, 'load': '/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage14_120B_8node_gdnlralpha_off_20260925a/checkpoints', 'no_load_optim': None, 'load_main_params_from_ckpt': None, 'no_load_rng': True, 'no_load_scheduler': None, 'save_after_load': False, 'strict_fsdp_dtensor_load': True, 'non_persistent_save_interval': None, 'non_persistent_keep_last': 1, 'non_persistent_ckpt_type': None, 'non_persistent_global_ckpt_dir': None, 'non_persistent_local_ckpt_dir': None, 'non_persistent_local_ckpt_algo': 'fully_parallel', 'finetune': False, 'continual_pretrain': True, 'continual_pretrain_preserve_opt_param_scheduler': False, 'pretrained_checkpoint': None, 'ckpt_step': None, 'perform_initialization': False, 'use_checkpoint_args': False, 'use_mp_args_from_checkpoint_args': False, 'use_tokenizer_model_from_checkpoint_args': True, 'exit_on_missing_checkpoint': True, 'use_dist_ckpt_deprecated': False, 'use_persistent_ckpt_worker': False, 'auto_detect_ckpt_format': True, 'dist_ckpt_format_deprecated': None, 'ckpt_format': 'torch_dist', 'ckpt_convert_format': None, 'ckpt_convert_save': None, 'ckpt_convert_update_legacy_dist_opt_format': False, 'ckpt_fully_parallel_save_deprecated': False, 'ckpt_fully_parallel_save': True, 'async_save': True, 'ckpt_fully_parallel_load': True, 'ckpt_assume_constant_structure': False, 'dist_ckpt_strictness': 'assume_ok_unexpected', 'dist_ckpt_save_pre_mcore_014': False, 'dist_ckpt_optim_fully_reshardable': False, 'distrib_optim_fully_reshardable_mem_efficient': False, 'fp16': False, 'bf16': True, 'grad_reduce_in_bf16': False, 'loss_scale': None, 'initial_loss_scale': 4294967296, 'min_loss_scale': 1.0, 'loss_scale_window': 1000, 'hysteresis': 2, 'fp32_residual_connection': False, 'apply_query_key_layer_scaling': False, 'attention_softmax_in_fp32': False, 'accumulate_allreduce_grads_in_fp32': True, 'fp16_lm_cross_entropy': False, 'disable_bf16_reduced_precision_matmul': False, 'reuse_grad_buf_for_mxfp8_param_ag': False, 'tensor_model_parallel_size': 1, 'pipeline_model_parallel_size': 1, 'decoder_first_pipeline_num_layers': None, 'decoder_last_pipeline_num_layers': None, 'pipeline_model_parallel_layout': None, 'num_layers_per_virtual_pipeline_stage': None, 'num_virtual_stages_per_pipeline_rank': None, 'microbatch_group_size_per_vp_stage': None, 'overlap_p2p_comm': False, 'overlap_p2p_comm_warmup_flush': False, 'distributed_backend': 'nccl', 'distributed_timeout_minutes': 120, 'distributed_timeout_seconds_after_init': None, 'overlap_grad_reduce': True, 'defer_grad_checks': False, 'defer_embedding_wgrad_compute': False, 'wgrad_deferral_limit': 0, 'align_grad_reduce': True, 'ddp_num_buckets': None, 'ddp_bucket_size': 40000000, 'ddp_pad_buckets_for_high_nccl_busbw': False, 'ddp_average_in_collective': False, 'overlap_param_gather': True, 'overlap_param_gather_with_optimizer_step': False, 'align_param_gather': False, 'scatter_gather_tensors_in_pipeline': True, 'use_ring_exchange_p2p': False, 'local_rank': 7, 'lazy_mpu_init': None, 'account_for_embedding_in_pipeline_split': False, 'account_for_loss_in_pipeline_split': False, 'use_distributed_optimizer': True, 'nccl_ub': False, 'disable_symmetric_registration': False, 'use_sharp': False, 'sharp_enabled_group': None, 'use_megatron_fsdp': False, 'init_model_with_meta_device': False, 'data_parallel_sharding_strategy': 'no_shard', 'gradient_reduce_div_fusion': True, 'fsdp_double_buffer': False, 'suggested_communication_unit_size': None, 'keep_fp8_transpose_cache': False, 'enable_full_sharding_in_hsdp': False, 'num_distributed_optimizer_instances': 1, 'use_torch_fsdp2': False, 'torch_fsdp2_reshard_after_forward': True, 'context_parallel_size': 8, 'cp_comm_type': ['a2a'], 'hierarchical_context_parallel_sizes': None, 'max_seqlen_per_cp_rank': None, 'hybrid_context_parallel': False, 'dynamic_context_parallel': False, 'dynamic_context_parallel_sizes': None, 'dynamic_cp_candidate_sizes': None, 'sequence_packing_scheduler': 'default_dynamic_cp', 'dynamic_cp_scheduler': 'default_dynamic_cp', 'dynamic_cp_moe_policy': None, 'dynamic_cp_gdn_policy': None, 'dynamic_cp_on_error': 'error', 'dynamic_cp_workspace_max_tokens': None, 'dynamic_cp_wave_coexecution': False, 'nccl_communicator_config_path': None, 'use_tp_pp_dp_mapping': False, 'replication': False, 'replication_jump': None, 'replication_factor': 2, 'full_validation': False, 'multiple_validation_sets': False, 'eval_iters': 10, 'eval_interval': 99999999, 'test_mode': False, 'skip_train': False, 'benchmark_eval': False, 'benchmark_tasks': None, 'benchmark_interval': None, 'benchmark_sequence_length': None, 'benchmark_micro_batch': None, 'benchmark_global_batch': None, 'data_path': ['/mnt/hdd/cpt/MoE_CPT_Data_0715-stage_15/stage_15_final_20261001_v2_link/worker_{0..78}_input_ids_tmp'], 'split': '999,1,0', 'train_data_path': None, 'valid_data_path': None, 'test_data_path': None, 'data_args_path': None, 'load_complemental_dataset': ['loss_mask'], 'per_split_data_args_path': None, 'dataloader_fast_cache_load': False, 'dataloader_defer_npy_index_mmap': False, 'data_cache_path': '/mnt/hdd/cpt/binidx_cache_stage15_final_20261001_v2', 'mmap_bin_files': True, 'mock_data': False, 'seq_length': 131072, 'encoder_seq_length': 131072, 'decoder_seq_length': None, 'retriever_seq_length': 256, 'sample_rate': 1.0, 'mask_prob': 0.15, 'short_seq_prob': 0.1, 'num_workers': 4, 'prefetch_factor': 8, 'reset_position_ids': False, 'reset_attention_mask': False, 'eod_mask_loss': False, 'create_attention_mask_in_dataloader': False, 'disable_explicit_attention_mask': False, 'num_dataset_builder_threads': 1, 'object_storage_cache_path': None, 'mid_level_dataset_surplus': 0.005, 'document_packing_algorithm': 'obfd', 'document_packing_dp_size': None, 'packing_tokens_per_shard': 100000000000, 'start_samples': 0, 'consumed_train_samples': 0, 'previous_stage_seq_length': None, 'allow_ambiguous_pad_tokens': False, 'auto_generate_cu_seqlens': True, 'vocab_size': None, 'padded_vocab_size': 99072, 'vocab_file': None, 'merge_file': None, 'vocab_extra_ids': 0, 'tokenizer_type': 'HuggingFaceTokenizer', 'tokenizer_model': '/mnt/ssd/yulan/pretrain-linear-moe/preprocess/modify_tokenizer/YuLan-Mini-Qwen-Template_new_sp', 'tokenizer_metadata': None, 'tiktoken_pattern': None, 'tiktoken_num_special_tokens': 1000, 'tiktoken_special_tokens': None, 'legacy_tokenizer': False, 'trust_remote_code': False, 'override_hf_eod_token_id': None, 'adlr_autoresume': False, 'adlr_autoresume_interval': 1000, 'ict_head_size': None, 'biencoder_projection_dim': 0, 'biencoder_shared_query_context_model': False, 'ict_load': None, 'bert_load': None, 'titles_data_path': None, 'query_in_block_prob': 0.1, 'use_one_sent_docs': False, 'evidence_data_path': None, 'retriever_report_topk_accuracies': [], 'retriever_score_scaling': False, 'block_data_path': None, 'embedding_path': None, 'indexer_batch_size': 128, 'indexer_log_interval': 1000, 'num_classes': 1000, 'img_h': 224, 'img_w': 224, 'num_channels': 3, 'patch_dim': 16, 'classes_fraction': 1.0, 'data_per_class_fraction': 1.0, 'data_sharding': True, 'head_lr_mult': 1.0, 'vision_pretraining': False, 'vision_pretraining_type': 'classify', 'vision_backbone_type': 'vit', 'swin_backbone_type': 'tiny', 'mask_type': 'random', 'mask_factor': 1.0, 'iter_per_epoch': 1250, 'dino_local_img_size': 96, 'dino_local_crops_number': 10, 'dino_head_hidden_size': 2048, 'dino_bottleneck_size': 256, 'dino_freeze_last_layer': 1, 'dino_norm_last_layer': False, 'dino_warmup_teacher_temp': 0.04, 'dino_teacher_temp': 0.07, 'dino_warmup_teacher_temp_epochs': 30, 'qk_layernorm': False, 'qk_l2_norm': False, 'expert_model_parallel_size': 8, 'expert_tensor_parallel_size': 1, 'num_experts': 192, 'moe_layer_freq': [0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], 'moe_ffn_hidden_size': 512, 'moe_shared_expert_intermediate_size': 512, 'moe_shared_expert_gate': True, 'moe_shared_expert_overlap': True, 'moe_triton_shared_expert_overlap': False, 'moe_grouped_gemm': True, 'moe_use_legacy_grouped_gemm': False, 'moe_layer_recompute': False, 'moe_extended_tp': False, 'moe_use_upcycling': False, 'moe_router_load_balancing_type': 'global_aux_loss', 'moe_router_dtype': 'fp32', 'moe_router_gemm_backend': 'te', 'moe_router_fusion': True, 'moe_router_aux_loss_fusion': 'auto', 'moe_gaux_overlap': False, 'moe_router_score_function': 'softmax', 'moe_router_sqrt_gate': True, 'moe_router_topk': 4, 'moe_router_pre_softmax': False, 'moe_router_num_groups': None, 'moe_router_group_topk': None, 'moe_router_topk_scaling_factor': None, 'moe_router_enable_expert_bias': False, 'moe_router_bias_update_rate': 0.001, 'moe_router_force_load_balancing': False, 'moe_router_padding_for_quantization': False, 'moe_router_bias_update_method': 'sign', 'moe_router_padding_for_fp8': False, 'moe_aux_loss_coeff': 0.05, 'moe_z_loss_coeff': None, 'moe_input_jitter_eps': None, 'moe_per_layer_logging': False, 'log_moe_activations': True, 'log_moe_params': False, 'log_moe_router_metrics': False, 'moe_monitor_layers': '', 'moe_monitor_log_interval': None, 'moe_token_dispatcher_type': 'alltoall', 'moe_triton_dispatcher_exclude_padding_tokens': False, 'moe_triton_workspace_expand': 4.0, 'moe_triton_mega_dispatch_sms': 48, 'moe_triton_mega_tail_sms': 16, 'moe_triton_mega_compute_sms': 24, 'moe_triton_mega_activation_sms': 4, 'moe_triton_mega_combine_sms': 32, 'moe_triton_mega_reduce_sms': 32, 'moe_triton_mega_num_warps': 16, 'moe_enable_deepep': False, 'moe_flex_dispatcher_backend': 'deepep', 'moe_deepep_num_sms': 20, 'moe_hybridep_num_sms': 16, 'moe_permute_fusion': True, 'moe_permute_backend': 'auto', 'moe_expert_capacity_factor': None, 'moe_pad_expert_input_to_capacity': False, 'moe_token_drop_policy': 'probs', 'moe_apply_probs_on_input': False, 'overlap_moe_expert_parallel_comm': False, 'delay_wgrad_compute': False, 'moe_upcycling_granularity': 1, 'moe_pad_experts_for_cuda_graph_inference': False, 'q_lora_rank': None, 'kv_lora_rank': 32, 'qk_head_dim': 128, 'qk_pos_emb_head_dim': 64, 'v_head_dim': 128, 'rotary_scaling_factor': 1.0, 'mscale': 1.0, 'mscale_all_dim': 0.0, 'cache_mla_latents': False, 'linear_attention_type': 'gated_delta_net', 'linear_attention_freq': [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 0, 0, 1, 1, 1, 1, 1, 1], 'linear_rope_mode': 'none', 'dc_shortconv': False, 'dc_shortconv_upcycling': False, 'dc_shortconv_optimizer_state': 'conservative', 'gdn_shortconv_separate_gate': False, 'gdn_shortconv_separate_gate_upcycling': False, 'linear_conv_kernel_dim': 4, 'linear_key_head_dim': 64, 'linear_value_head_dim': 64, 'linear_num_key_heads': 8, 'linear_num_value_heads': 32, 'gdn_pre_gdr_fusion': False, 'gdn_pre_gdr_fusion_high_precision_backward': False, 'gdn_pre_gdr_fusion_deterministic_weight_gradient': False, 'gdn_backend': 'fla', 'gdn_gated_norm_backend': 'linghe', 'gdn_split_in_proj': True, 'gdn_fused_split_forward': True, 'gdn_postinit_key_scale': 1.0, 'gdn_postinit_alpha_scale': 1.0, 'gdn_postinit_alog_shift': 0.0, 'gdn_alpha_weight_scale': 1.0, 'gdn_alpha_weight_scale_mode': 'matrix', 'gdn_lr_mult_q': 1.0, 'gdn_lr_mult_k': 1.0, 'gdn_lr_mult_v': 1.0, 'gdn_lr_mult_alpha': 0.5, 'gdn_lr_mult_beta': 1.0, 'heterogeneous_layers_config_path': None, 'heterogeneous_layers_config_encoded_json': None, 'log_params_norm': True, 'log_padding_stats': False, 'log_num_zeros_in_grad': False, 'log_throughput': True, 'log_progress': False, 'timing_log_level': 0, 'log_energy': False, 'increase_log_level_interval': 1000, 'increase_log_level_iters': 5, 'barrier_with_L1_time': False, 'timing_log_option': 'all', 'tensorboard_log_interval': 1, 'tensorboard_queue_size': 1000, 'log_timers_to_tensorboard': False, 'log_loss_scale_to_tensorboard': True, 'log_validation_ppl_to_tensorboard': False, 'log_memory_to_tensorboard': False, 'log_world_size_to_tensorboard': False, 'wandb_project': 'qwen2.5_3b_mom_cpt', 'wandb_entity': '', 'wandb_exp_name': 'stage14_120B_8node_gdnlralpha_off_20260925a', 'wandb_save_dir': '/mnt/ssd/cpt/moe-cpt-ebcloud/formal3/launcher_logs/f3-plain-shortconv-adamh-from32780-20260915a/stage15_115B_8node_gdnlralpha_off_20261003a/wandb', 'swanlab_project': '', 'swanlab_workspace': '', 'swanlab_exp_name': '', 'swanlab_save_dir': '', 'logging_level': None, 'enable_debug_logging': False, 'log_hidden_states': ['embeddings', 'residual_attn', 'input_layernorm', 'attention::q_layernorm', 'attention::k_layernorm', 'attention::core_attention', 'attention::o_proj', 'residual_mlp', 'pre_mlp_layernorm', 'mlp', 'moe::router', 'moe::shared_experts', 'moe::routed_experts', 'gated_delta::in_proj', 'gated_delta::conv1d', 'gated_delta::query', 'gated_delta::key', 'gated_delta::value', 'gated_delta::gate', 'gated_delta::beta', 'gated_delta::alpha', 'gated_delta::g', 'gated_delta::core_attention', 'gated_delta::out_norm', 'gated_delta::out_proj', 'final_layernorm'], 'log_params': ['attention::linear_qkv', 'attention::o_proj', 'mlp::linear_fc1', 'mlp::linear_fc2', 'input_layernorm', 'pre_mlp_layernorm', 'embedding', 'lm_head', 'gated_delta::in_proj', 'gated_delta::in_proj_bias', 'gated_delta::conv1d', 'gated_delta::conv1d_bias', 'gated_delta::dt_bias', 'gated_delta::A_log', 'gated_delta::out_norm', 'gated_delta::out_norm_bias', 'gated_delta::out_proj', 'gated_delta::out_proj_bias'], 'log_attn_logits': False, 'log_logits_z_loss': False, 'log_metrics_interval': 10, 'log_update_rms': True, 'output_logits_z_loss_coeff': None, 'log_straggler': False, 'disable_straggler_on_startup': False, 'straggler_ctrlr_port': 65535, 'straggler_minmax_count': 1, 'run_workload_inspector_server': False, 'inference_batch_times_seqlen_threshold': -1, 'max_tokens_to_oom': 12000, 'output_bert_embeddings': False, 'bert_embedder_type': 'megatron', 'flash_decode': False, 'enable_cuda_graph': False, 'cuda_graph_warmup_steps': 3, 'external_cuda_graph': False, 'cuda_graph_impl': 'none', 'cuda_graph_scope': [], 'inference_max_batch_size': 8, 'inference_max_seq_length': 2560, 'inference_dynamic_batching': False, 'inference_dynamic_batching_buffer_size_gb': 40.0, 'inference_dynamic_batching_block_size': 256, 'inference_dynamic_batching_buffer_guaranteed_fraction': 0.2, 'inference_dynamic_batching_buffer_overflow_factor': None, 'inference_dynamic_batching_max_requests_override': None, 'inference_dynamic_batching_max_tokens_override': None, 'inference_dynamic_batching_num_cuda_graphs': 16, 'inference_dynamic_batching_track_paused_request_events': False, 'decode_only_cuda_graphs': False, 'inference_dynamic_batching_unified_memory_level': 0, 'symmetric_ar_type': None, 'nccl_all_reduce_for_prefill': False, 'mlp_chunks_for_prefill': 1, 'disable_chunked_prefill': False, 'fp8': None, 'fp8_recipe': 'delayed', 'fp8_margin': 0, 'fp8_interval': 1, 'fp8_amax_history_len': 1, 'fp8_amax_compute_algo': 'most_recent', 'fp8_wgrad': True, 'transformer_impl': 'transformer_engine', 'fp8_param_gather': False, 'first_last_layers_bf16': False, 'num_layers_at_start_in_bf16': 1, 'num_layers_at_end_in_bf16': 1, 'fp4': None, 'fp4_recipe': 'nvfp4', 'fp4_param': False, 'te_rng_tracker': False, 'inference_rng_tracker': False, 'retro_project_dir': None, 'retro_add_retriever': False, 'retro_cyclic_train_iters': None, 'retro_encoder_layers': 2, 'retro_encoder_hidden_dropout': 0.1, 'retro_encoder_attention_dropout': 0.1, 'retro_num_neighbors': 2, 'retro_num_retrieved_chunks': 2, 'retro_attention_gate': 1, 'retro_verify_neighbor_count': True, 'enable_experimental': False, 'spec': None, 'hybrid_attention_ratio': 0.0625, 'hybrid_mlp_ratio': 0.5, 'hybrid_override_pattern': None, 'mamba_state_dim': 128, 'mamba_head_dim': 64, 'mamba_num_groups': 8, 'mamba_num_heads': None, 'mamba_expand': 2, 'is_hybrid_model': True, 'disable_mamba_mem_eff_path': False, 'mamba_disable_cp': False, 'yaml_cfg': None, 'use_precision_aware_optimizer': False, 'main_grads_dtype': torch.float32, 'main_params_dtype': torch.float32, 'exp_avg_dtype': torch.float32, 'exp_avg_sq_dtype': torch.float32, 'enable_one_logger': True, 'one_logger_project': 'megatron-lm', 'one_logger_run_name': None, 'one_logger_async': False, 'app_tag_run_name': None, 'app_tag_run_version': '0.0.0', 'inprocess_restart': False, 'inprocess_max_iterations': None, 'inprocess_monitor_thread_interval': 1.0, 'inprocess_monitor_process_interval': 1.0, 'inprocess_progress_watchdog_interval': 1.0, 'inprocess_heartbeat_interval': 30, 'inprocess_soft_timeout': 60, 'inprocess_hard_timeout': 90, 'inprocess_heartbeat_timeout': 60, 'inprocess_barrier_timeout': 120, 'inprocess_completion_timeout': 120, 'inprocess_last_call_wait': 1, 'inprocess_termination_grace_time': 1, 'inprocess_granularity': 'node', 'inprocess_active_world_size': 64, 'inprocess_empty_cuda_cache': False, 'enable_ft_package': False, 'calc_ft_timeouts': False, 'config_logger_dir': '', 'error_injection_rate': 0, 'error_injection_type': 'transient_error', 'rerun_mode': 'validate_results', 'enable_msc': True, 'kitchen_config_file': None, 'kitchen_recipe_number': None, 'sft': False, 'sft_tokenizer_prompt_format': 'nemotron-h-aligned', 'export_model_type': 'GPTModel', 'export_legacy_megatron': False, 'export_te_mcore_model': False, 'export_force_local_attention': False, 'export_kv_cache_quant': False, 'export_real_quant_cfg': 'None', 'export_quant_cfg': None, 'export_kd_cfg': None, 'teacher_model_config': None, 'export_kd_teacher_load': None, 'export_kd_teacher_ckpt_format': None, 'finetune_hf_dataset': None, 'finetune_data_split': 'train', 'export_qk_l2_norm': False, 'export_moe_apply_probs_on_input': False, 'export_offline_model': False, 'modelopt_enabled': False, 'rank': 63, 'world_size': 64, 'use_dist_ckpt': True, 'transformer_pipeline_model_parallel_size': 1, 'data_parallel_size': 8, 'virtual_pipeline_model_parallel_size': None, 'params_dtype': torch.bfloat16, 'skipped_train_samples': 0, 'consumed_valid_samples': 0, 'variable_seq_lengths': False, '_wandb': {}} +2026-10-03 15:36:55,091 INFO MainThread:730851 [wandb_init.py:init():888] starting backend +2026-10-03 15:36:55,382 INFO MainThread:730851 [wandb_init.py:init():891] sending inform_init request +2026-10-03 15:36:55,389 INFO MainThread:730851 [wandb_init.py:init():899] backend started and connected +2026-10-03 15:36:55,396 INFO MainThread:730851 [wandb_init.py:init():969] updated telemetry +2026-10-03 15:36:55,396 INFO MainThread:730851 [wandb_init.py:init():993] communicating run to backend with 90.0 second timeout +2026-10-03 15:36:57,237 INFO MainThread:730851 [wandb_init.py:init():1040] starting run threads in backend +2026-10-03 15:36:57,542 INFO MainThread:730851 [wandb_run.py:_console_start():2504] atexit reg +2026-10-03 15:36:57,542 INFO MainThread:730851 [wandb_run.py:_redirect():2352] redirect: wrap_raw +2026-10-03 15:36:57,542 INFO MainThread:730851 [wandb_run.py:_redirect():2421] Wrapping output streams. +2026-10-03 15:36:57,542 INFO MainThread:730851 [wandb_run.py:_redirect():2444] Redirects installed. +2026-10-03 15:36:57,547 INFO MainThread:730851 [wandb_init.py:init():1080] run started, returning control to user process +2026-10-03 15:47:07,644 INFO MainThread:730851 [wandb_run.py:_config_callback():1385] config_cb None None {'slurm_job_name': 'N/A'} +2026-10-10 20:25:46,273 INFO MainThread:730851 [wandb_run.py:_finish():2270] finishing run huyiwen/qwen2.5_3b_mom_cpt/neb76hvj +2026-10-10 20:25:46,275 INFO MainThread:730851 [wandb_run.py:_atexit_cleanup():2469] got exitcode: 0 +2026-10-10 20:25:46,275 INFO MainThread:730851 [wandb_run.py:_restore():2451] restore +2026-10-10 20:25:46,276 INFO MainThread:730851 [wandb_run.py:_restore():2457] restore done +2026-10-10 20:25:53,585 INFO MainThread:730851 [wandb_run.py:_footer_sync_info():3853] logging synced files diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints/iter_0002384/__18_0.distcp b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints/iter_0002384/__18_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..746163403e4847ee21c45761c01f61164822cba0 --- /dev/null +++ b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints/iter_0002384/__18_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:449d880a14c43ba7b4a7879dabf7504c9fccb1888ff98c46b9135193ac15151c +size 3725451014 diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints/iter_0002384/__21_0.distcp b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints/iter_0002384/__21_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..b15bdda72047fd98498329db70b3f5da174a5550 --- /dev/null +++ b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints/iter_0002384/__21_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9321c3c0c304b3d47b9834e9fd260c40a5b3b3efa796a760763529d841456a9e +size 3725472260 diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints/iter_0002384/__29_1.distcp b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints/iter_0002384/__29_1.distcp new file mode 100644 index 0000000000000000000000000000000000000000..c765128c476ffa20163392635f3eec768c98a4e1 --- /dev/null +++ b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints/iter_0002384/__29_1.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:adc9883d8aad9a4e90c3631842c433f02e0227a7e5a4360e1e442dfecdcb64a6 +size 3725405514 diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints/iter_0002384/__49_0.distcp b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints/iter_0002384/__49_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..566d603dd82039e5a7cf200dc693b53182728104 --- /dev/null +++ b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints/iter_0002384/__49_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7385d4c3dd6444331feb90763b617dd318b7135dd44b29429d4ed504d28e65fc +size 3725314012 diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints/iter_0002384/metadata.json b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints/iter_0002384/metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..c9dbde0bcf4d2d993122dcc7d6bcb1eef8b6fb77 --- /dev/null +++ b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints/iter_0002384/metadata.json @@ -0,0 +1 @@ +{"sharded_backend": "torch_dist", "sharded_backend_version": 1, "common_backend": "torch", "common_backend_version": 1} \ No newline at end of file diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints/iter_0002384/modelopt_run_config.yaml b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints/iter_0002384/modelopt_run_config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..45b440a96c3248c46df41f6efc245c668133ba26 --- /dev/null +++ b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints/iter_0002384/modelopt_run_config.yaml @@ -0,0 +1,833 @@ +activation_func: +activation_func_clamp_mode: soft +activation_func_clamp_value: '7.0' +add_bias_linear: false +add_qkv_bias: true +apply_query_key_layer_scaling: false +apply_residual_connection_post_layernorm: false +apply_rope_fusion: false +attention_backend: AttnBackend.auto +attention_dropout: '0.0' +attention_output_gate: true +attention_output_gate_upcycling: false +attention_softmax_in_fp32: false +attn_k_token_shift: None +attn_output_gate: None +attn_q_token_shift: None +attn_token_shift: None +attn_v_token_shift: None +autocast_dtype: torch.bfloat16 +barrier_with_L1_time: false +bf16: true +bias_activation_fusion: true +bias_dropout_fusion: true +calculate_per_token_loss: false +clone_scatter_output_in_embedding: true +config_logger_dir: '' +cross_entropy_fusion_impl: linear_triton +cross_entropy_loss_fusion: true +dc_shortconv: false +dc_shortconv_upcycling: false +defer_embedding_wgrad_compute: false +delay_wgrad_compute: false +deterministic_mode: false +disable_bf16_reduced_precision_matmul: false +disable_parameter_transpose_cache: false +distribute_saved_activations: false +dynamic_cp_candidate_sizes: None +dynamic_cp_gdn_policy: None +dynamic_cp_moe_policy: None +dynamic_cp_on_error: error +dynamic_cp_scheduler: default_dynamic_cp +dynamic_cp_wave_coexecution: false +dynamic_cp_workspace_max_tokens: None +emb_deviation_loss_coeff: 0 +emb_deviation_type: None +enable_autocast: false +fast_thd_a2a: true +ffn_hidden_size: 4800 +ffn_intermediate_token_shift: None +ffn_token_shift: None +final_logits_clamp_value: '30.0' +finalize_model_grads_func: +fine_grained_activation_offloading: false +first_last_layers_bf16: false +flash_decode: false +fp16: false +fp32_residual_connection: false +freeze_layernorm_weight: false +fused_single_qkv_rope: false +gated_linear_unit: true +gated_norm: true +gated_norm_lora_rank: 16 +gated_norm_upcycling: false +gdn_alpha_weight_scale: '1.0' +gdn_alpha_weight_scale_mode: matrix +gdn_backend: fla +gdn_fused_split_forward: true +gdn_gated_norm_backend: linghe +gdn_lr_mult_alpha: '1.0' +gdn_lr_mult_beta: '1.0' +gdn_lr_mult_k: '1.0' +gdn_lr_mult_q: '1.0' +gdn_lr_mult_v: '1.0' +gdn_pre_gdr_fusion: false +gdn_pre_gdr_fusion_deterministic_weight_gradient: false +gdn_pre_gdr_fusion_high_precision_backward: false +gdn_shortconv_separate_gate: false +gdn_shortconv_separate_gate_upcycling: false +gdn_split_in_proj: true +glu_linear_offset: '0.0' +grad_scale_func: > +grad_sync_func: "" +gradient_accumulation_fusion: true +hetereogenous_dist_checkpoint: false +heterogeneous_block_specs: false +hidden_dropout: '0.0' +hidden_size: 2048 +hidden_states_shrink: '1.0' +is_hybrid_model: true +kv_channels: 64 +layernorm_epsilon: 1e-06 +layernorm_zero_centered_gamma: false +linear_attention_freq: '[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, + 1, 0, 0, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 0, 1, 0, 0, 1, 1, 1, 1, 1, 1]' +linear_attention_type: gated_delta_net +linear_conv_kernel_dim: 4 +linear_cross_entropy_accumulate_grad_in_fp32: true +linear_cross_entropy_num_chunks: 32 +linear_key_head_dim: 64 +linear_num_key_heads: 8 +linear_num_value_heads: 32 +linear_rope_mode: none +linear_value_head_dim: 64 +lm_head_alpha: '1.0' +log_attn_logits: false +log_hidden_states: '[''embeddings'', ''residual_attn'', ''input_layernorm'', ''attention::q_layernorm'', + ''attention::k_layernorm'', ''attention::core_attention'', ''attention::o_proj'', + ''residual_mlp'', ''pre_mlp_layernorm'', ''mlp'', ''moe::router'', ''moe::shared_experts'', + ''moe::routed_experts'', ''gated_delta::in_proj'', ''gated_delta::conv1d'', ''gated_delta::query'', + ''gated_delta::key'', ''gated_delta::value'', ''gated_delta::gate'', ''gated_delta::beta'', + ''gated_delta::alpha'', ''gated_delta::g'', ''gated_delta::core_attention'', ''gated_delta::out_norm'', + ''gated_delta::out_proj'', ''final_layernorm'']' +log_logits_z_loss: false +log_metrics_interval: 10 +log_moe_activations: true +log_moe_params: false +log_moe_router_metrics: false +log_params: '[''attention::linear_qkv'', ''attention::o_proj'', ''mlp::linear_fc1'', + ''mlp::linear_fc2'', ''input_layernorm'', ''pre_mlp_layernorm'', ''embedding'', + ''lm_head'', ''gated_delta::in_proj'', ''gated_delta::in_proj_bias'', ''gated_delta::conv1d'', + ''gated_delta::conv1d_bias'', ''gated_delta::dt_bias'', ''gated_delta::A_log'', + ''gated_delta::out_norm'', ''gated_delta::out_norm_bias'', ''gated_delta::out_proj'', + ''gated_delta::out_proj_bias'']' +log_update_rms: true +mamba_disable_cp: false +mamba_expand: 2 +mamba_head_dim: 64 +mamba_num_groups: 8 +mamba_num_heads: None +mamba_state_dim: 128 +masked_softmax_fusion: false +max_seqlen_per_dp_cp_rank: None +memory_efficient_layer_norm: false +min_offloaded_tensor_size: 1048576 +mlp_chunks_for_prefill: 1 +moe_apply_probs_on_input: false +moe_aux_loss_coeff: '0.05' +moe_deepep_num_sms: 20 +moe_enable_deepep: false +moe_expert_capacity_factor: None +moe_extended_tp: false +moe_ffn_hidden_size: 512 +moe_flex_dispatcher_backend: deepep +moe_gaux_overlap: false +moe_grouped_gemm: true +moe_hybridep_num_sms: 16 +moe_input_jitter_eps: None +moe_layer_freq: '[0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1]' +moe_pad_expert_input_to_capacity: false +moe_per_layer_logging: false +moe_permute_backend: auto +moe_permute_fusion: true +moe_router_aux_loss_fusion: auto +moe_router_bias_update_method: sign +moe_router_bias_update_rate: '0.001' +moe_router_dtype: fp32 +moe_router_enable_expert_bias: false +moe_router_force_load_balancing: false +moe_router_fusion: true +moe_router_gemm_backend: te +moe_router_group_topk: None +moe_router_load_balancing_type: global_aux_loss +moe_router_num_groups: None +moe_router_padding_for_quantization: false +moe_router_pre_softmax: false +moe_router_score_function: softmax +moe_router_sqrt_gate: true +moe_router_topk: 4 +moe_router_topk_limited_devices: None +moe_router_topk_scaling_factor: None +moe_shared_expert_gate: true +moe_shared_expert_intermediate_size: 512 +moe_shared_expert_overlap: true +moe_token_dispatcher_type: alltoall +moe_token_drop_policy: probs +moe_token_dropping: false +moe_triton_dispatcher_exclude_padding_tokens: false +moe_triton_mega_activation_sms: 4 +moe_triton_mega_combine_sms: 32 +moe_triton_mega_compute_sms: 24 +moe_triton_mega_dispatch_sms: 48 +moe_triton_mega_num_warps: 16 +moe_triton_mega_reduce_sms: 32 +moe_triton_mega_tail_sms: 16 +moe_triton_shared_expert_overlap: false +moe_triton_workspace_expand: '4.0' +moe_triton_workspace_max_tokens: None +moe_use_legacy_grouped_gemm: false +moe_z_loss_coeff: None +mrope_section: None +multi_latent_attention: false +no_rope_freq: None +no_sync_func: "" +normalization: RMSNorm +num_attention_heads: 32 +num_layers: 56 +num_layers_at_end_in_bf16: 1 +num_layers_at_start_in_bf16: 1 +num_moe_experts: 192 +num_query_groups: 8 +nvidia_modelopt_version: 0.39.0 +offload_modules: '[]' +output_logits_z_loss_coeff: None +param_sync_func: None +params_dtype: torch.bfloat16 +perform_initialization: false +persist_layer_norm: true +qk_l2_norm: false +qk_layernorm: false +quant_recipe: None +reparam_keys: None +rotary_interleaved: false +scale_emb: '1.0' +sequence_packing_scheduler: default_dynamic_cp +softmax_scale: None +softmax_type: vanilla +spectral_mup_init: false +split_expert_init: true +split_fc1_init: true +split_qkv_init: true +symmetric_ar_type: None +test_mode: false +timers: +token_shift_conv_init: default +token_shift_conv_size: 4 +transformer_impl: transformer_engine +use_fused_weighted_squared_relu: false +use_kitchen: false +use_mamba_mem_eff_path: true +use_ring_exchange_p2p: false +use_te_activation_func: false +use_te_rng_tracker: false +variable_seq_lengths: false +wgrad_deferral_limit: 0 +window_attn_skip_freq: None +window_size: None +word_embedding_dropout_prob: '0.0' diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints/latest_checkpointed_iteration.txt b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints/latest_checkpointed_iteration.txt new file mode 100644 index 0000000000000000000000000000000000000000..614185b258cf7c7d61c81d887db0a382ea9520d1 --- /dev/null +++ b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints/latest_checkpointed_iteration.txt @@ -0,0 +1 @@ +2384 \ No newline at end of file diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints/latest_wandb_artifact_path.txt b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints/latest_wandb_artifact_path.txt new file mode 100644 index 0000000000000000000000000000000000000000..efa8eab56c990bdb03e16a2b711adba1588852a2 --- /dev/null +++ b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/checkpoints/latest_wandb_artifact_path.txt @@ -0,0 +1 @@ +huyiwen/qwen2.5_3b_mom_cpt \ No newline at end of file diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/communicators_rank0009.json b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/communicators_rank0009.json new file mode 100644 index 0000000000000000000000000000000000000000..6ec5a4d8c54aad2bc880b400272397137a7df221 --- /dev/null +++ b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/communicators_rank0009.json @@ -0,0 +1,84 @@ +{ + "rank": 9, + "world_diagnostics_gloo": true, + "aliases": [ + [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39, + 40, + 41, + 42, + 43, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 52, + 53, + 54, + 55, + 56, + 57, + 58, + 59, + 60, + 61, + 62, + 63 + ], + [ + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15 + ] + ], + "synchronous_singletons": true, + "async_work_handles_preserved": true +} diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/communicators_rank0014.json b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/communicators_rank0014.json new file mode 100644 index 0000000000000000000000000000000000000000..20978103690b74a1315b152288e229d43018615c --- /dev/null +++ b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/communicators_rank0014.json @@ -0,0 +1,84 @@ +{ + "rank": 14, + "world_diagnostics_gloo": true, + "aliases": [ + [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39, + 40, + 41, + 42, + 43, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 52, + 53, + 54, + 55, + 56, + 57, + 58, + 59, + 60, + 61, + 62, + 63 + ], + [ + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15 + ] + ], + "synchronous_singletons": true, + "async_work_handles_preserved": true +} diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/communicators_rank0038.json b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/communicators_rank0038.json new file mode 100644 index 0000000000000000000000000000000000000000..2269c4f1c449f875b1011582a7bfc0c3d9019077 --- /dev/null +++ b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/communicators_rank0038.json @@ -0,0 +1,84 @@ +{ + "rank": 38, + "world_diagnostics_gloo": true, + "aliases": [ + [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39, + 40, + 41, + 42, + 43, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 52, + 53, + 54, + 55, + 56, + 57, + 58, + 59, + 60, + 61, + 62, + 63 + ], + [ + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39 + ] + ], + "synchronous_singletons": true, + "async_work_handles_preserved": true +} diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/communicators_rank0043.json b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/communicators_rank0043.json new file mode 100644 index 0000000000000000000000000000000000000000..ea084f9c0e12cf4d72d3360aa5e4d206278a242a --- /dev/null +++ b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/communicators_rank0043.json @@ -0,0 +1,84 @@ +{ + "rank": 43, + "world_diagnostics_gloo": true, + "aliases": [ + [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39, + 40, + 41, + 42, + 43, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 52, + 53, + 54, + 55, + 56, + 57, + 58, + 59, + 60, + 61, + 62, + 63 + ], + [ + 40, + 41, + 42, + 43, + 44, + 45, + 46, + 47 + ] + ], + "synchronous_singletons": true, + "async_work_handles_preserved": true +} diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/communicators_rank0051.json b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/communicators_rank0051.json new file mode 100644 index 0000000000000000000000000000000000000000..92aac83c46d1955e90958eded3e77b17b815e104 --- /dev/null +++ b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/communicators_rank0051.json @@ -0,0 +1,84 @@ +{ + "rank": 51, + "world_diagnostics_gloo": true, + "aliases": [ + [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39, + 40, + 41, + 42, + 43, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 52, + 53, + 54, + 55, + 56, + 57, + 58, + 59, + 60, + 61, + 62, + 63 + ], + [ + 48, + 49, + 50, + 51, + 52, + 53, + 54, + 55 + ] + ], + "synchronous_singletons": true, + "async_work_handles_preserved": true +} diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/communicators_rank0055.json b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/communicators_rank0055.json new file mode 100644 index 0000000000000000000000000000000000000000..77fe6522c4fed52a7f8a9e7ecafd165e3f6555a3 --- /dev/null +++ b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/communicators_rank0055.json @@ -0,0 +1,84 @@ +{ + "rank": 55, + "world_diagnostics_gloo": true, + "aliases": [ + [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39, + 40, + 41, + 42, + 43, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 52, + 53, + 54, + 55, + 56, + 57, + 58, + 59, + 60, + 61, + 62, + 63 + ], + [ + 48, + 49, + 50, + 51, + 52, + 53, + 54, + 55 + ] + ], + "synchronous_singletons": true, + "async_work_handles_preserved": true +} diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/communicators_rank0061.json b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/communicators_rank0061.json new file mode 100644 index 0000000000000000000000000000000000000000..0887f775992029763c835c105e06bdd93acfe0d1 --- /dev/null +++ b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/communicators_rank0061.json @@ -0,0 +1,84 @@ +{ + "rank": 61, + "world_diagnostics_gloo": true, + "aliases": [ + [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39, + 40, + 41, + 42, + 43, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 52, + 53, + 54, + 55, + 56, + 57, + 58, + 59, + 60, + 61, + 62, + 63 + ], + [ + 56, + 57, + 58, + 59, + 60, + 61, + 62, + 63 + ] + ], + "synchronous_singletons": true, + "async_work_handles_preserved": true +} diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/communicators_rank0063.json b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/communicators_rank0063.json new file mode 100644 index 0000000000000000000000000000000000000000..1f66fc2e31068565d06f86ef97412623f76962ea --- /dev/null +++ b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/communicators_rank0063.json @@ -0,0 +1,84 @@ +{ + "rank": 63, + "world_diagnostics_gloo": true, + "aliases": [ + [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39, + 40, + 41, + 42, + 43, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 52, + 53, + 54, + 55, + 56, + 57, + 58, + 59, + 60, + 61, + 62, + 63 + ], + [ + 56, + 57, + 58, + 59, + 60, + 61, + 62, + 63 + ] + ], + "synchronous_singletons": true, + "async_work_handles_preserved": true +} diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/grouping_rank0000.json b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/grouping_rank0000.json new file mode 100644 index 0000000000000000000000000000000000000000..0a4f0e49c87158c79b0cba759af385f93ce49b9a --- /dev/null +++ b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/grouping_rank0000.json @@ -0,0 +1,13227 @@ +{ + "rank": 0, + "policy": "dc_adamh_named_matrices_v1", + "parameter_groups": [ + { + "name": "module.module.embedding.word_embeddings.weight", + "shape": [ + 99072, + 2048 + ], + "optimizer": "adamh", + "family": "embedding", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.mlp.linear_fc1.weight", + "shape": [ + 9600, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.mlp.linear_fc2.weight", + "shape": [ + 2048, + 4800 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.mlp.linear_fc1.weight", + "shape": [ + 9600, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.mlp.linear_fc2.weight", + "shape": [ + 2048, + 4800 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.2.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.2.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.3.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.3.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.4.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.4.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.5.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.5.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.6.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.6.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.7.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.7.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.8.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.8.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.9.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.9.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.10.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.10.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.11.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.11.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.12.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.12.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.12.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.12.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.12.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.12.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.13.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.13.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.14.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.14.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.15.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.15.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.16.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.16.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.17.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.17.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.18.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.18.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.19.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.19.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.20.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.20.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.20.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.20.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.20.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.20.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.21.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.21.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.21.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.21.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.21.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.21.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.22.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.22.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.23.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.23.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.23.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.23.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.23.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.23.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.24.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.24.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.25.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.25.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.26.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.26.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.27.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.27.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.28.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.28.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.29.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.29.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.30.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.30.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.31.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.31.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.32.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.32.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.33.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.33.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.34.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.34.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.35.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.35.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.36.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.36.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.37.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.37.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.38.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.38.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.39.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.39.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.40.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.40.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.41.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.41.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.42.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.42.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.43.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.43.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.44.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.44.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.45.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.45.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.46.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.46.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.46.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.46.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.46.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.46.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.47.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.47.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.48.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.48.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.48.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.48.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.48.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.48.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.49.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.49.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.49.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.49.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.49.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.49.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.50.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.50.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.51.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.51.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.52.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.52.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.53.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.53.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.54.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.54.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.55.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.55.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.final_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.final_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.final_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.final_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.output_layer.weight", + "shape": [ + 99072, + 2048 + ], + "optimizer": "adamh", + "family": "lm_head", + "expert": false + } + ], + "family_counts": { + "embedding": 1, + "in_proj": 294, + "out_proj": 49, + "linear_fc1": 56, + "linear_fc2": 56, + "linear_proj": 7, + "linear_qkv": 7, + "lm_head": 1 + }, + "moment_reset": false, + "loaded_step_values": { + "0": [ + 52788.0 + ], + "1": [] + }, + "native_optimizer_types": [ + "FusedAdam", + "FusedAdam" + ], + "full_matrix_norms_across_dp_shards": true, + "tp": 1, + "pp": 1 +} diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/grouping_rank0019.json b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/grouping_rank0019.json new file mode 100644 index 0000000000000000000000000000000000000000..9672da9653defa508791324b6fde8ef9effe7ea2 --- /dev/null +++ b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/grouping_rank0019.json @@ -0,0 +1,13227 @@ +{ + "rank": 19, + "policy": "dc_adamh_named_matrices_v1", + "parameter_groups": [ + { + "name": "module.module.embedding.word_embeddings.weight", + "shape": [ + 99072, + 2048 + ], + "optimizer": "adamh", + "family": "embedding", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.mlp.linear_fc1.weight", + "shape": [ + 9600, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.mlp.linear_fc2.weight", + "shape": [ + 2048, + 4800 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.mlp.linear_fc1.weight", + "shape": [ + 9600, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.mlp.linear_fc2.weight", + "shape": [ + 2048, + 4800 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.2.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.2.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.3.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.3.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.4.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.4.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.5.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.5.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.6.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.6.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.7.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.7.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.8.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.8.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.9.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.9.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.10.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.10.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.11.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.11.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.12.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.12.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.12.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.12.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.12.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.12.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.13.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.13.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.14.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.14.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.15.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.15.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.16.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.16.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.17.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.17.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.18.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.18.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.19.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.19.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.20.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.20.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.20.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.20.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.20.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.20.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.21.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.21.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.21.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.21.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.21.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.21.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.22.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.22.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.23.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.23.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.23.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.23.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.23.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.23.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.24.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.24.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.25.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.25.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.26.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.26.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.27.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.27.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.28.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.28.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.29.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.29.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.30.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.30.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.31.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.31.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.32.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.32.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.33.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.33.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.34.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.34.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.35.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.35.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.36.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.36.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.37.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.37.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.38.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.38.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.39.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.39.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.40.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.40.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.41.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.41.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.42.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.42.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.43.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.43.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.44.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.44.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.45.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.45.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.46.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.46.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.46.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.46.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.46.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.46.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.47.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.47.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.48.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.48.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.48.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.48.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.48.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.48.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.49.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.49.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.49.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.49.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.49.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.49.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.50.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.50.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.51.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.51.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.52.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.52.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.53.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.53.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.54.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.54.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.55.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.55.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.final_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.final_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.final_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.final_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.output_layer.weight", + "shape": [ + 99072, + 2048 + ], + "optimizer": "adamh", + "family": "lm_head", + "expert": false + } + ], + "family_counts": { + "embedding": 1, + "in_proj": 294, + "out_proj": 49, + "linear_fc1": 56, + "linear_fc2": 56, + "linear_proj": 7, + "linear_qkv": 7, + "lm_head": 1 + }, + "moment_reset": false, + "loaded_step_values": { + "0": [ + 52788.0 + ], + "1": [] + }, + "native_optimizer_types": [ + "FusedAdam", + "FusedAdam" + ], + "full_matrix_norms_across_dp_shards": true, + "tp": 1, + "pp": 1 +} diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/grouping_rank0033.json b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/grouping_rank0033.json new file mode 100644 index 0000000000000000000000000000000000000000..a535cf61f636e07330f7adae3eac8975e2e81d74 --- /dev/null +++ b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/grouping_rank0033.json @@ -0,0 +1,13227 @@ +{ + "rank": 33, + "policy": "dc_adamh_named_matrices_v1", + "parameter_groups": [ + { + "name": "module.module.embedding.word_embeddings.weight", + "shape": [ + 99072, + 2048 + ], + "optimizer": "adamh", + "family": "embedding", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.mlp.linear_fc1.weight", + "shape": [ + 9600, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.mlp.linear_fc2.weight", + "shape": [ + 2048, + 4800 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.mlp.linear_fc1.weight", + "shape": [ + 9600, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.mlp.linear_fc2.weight", + "shape": [ + 2048, + 4800 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.2.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.2.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.3.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.3.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.4.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.4.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.5.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.5.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.6.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.6.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.7.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.7.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.8.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.8.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.9.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.9.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.10.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.10.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.11.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.11.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.12.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.12.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.12.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.12.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.12.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.12.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.13.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.13.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.14.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.14.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.15.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.15.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.16.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.16.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.17.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.17.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.18.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.18.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.19.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.19.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.20.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.20.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.20.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.20.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.20.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.20.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.21.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.21.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.21.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.21.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.21.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.21.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.22.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.22.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.23.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.23.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.23.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.23.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.23.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.23.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.24.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.24.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.25.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.25.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.26.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.26.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.27.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.27.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.28.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.28.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.29.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.29.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.30.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.30.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.31.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.31.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.32.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.32.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.33.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.33.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.34.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.34.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.35.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.35.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.36.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.36.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.37.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.37.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.38.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.38.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.39.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.39.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.40.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.40.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.41.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.41.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.42.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.42.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.43.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.43.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.44.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.44.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.45.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.45.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.46.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.46.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.46.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.46.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.46.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.46.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.47.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.47.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.48.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.48.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.48.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.48.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.48.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.48.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.49.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.49.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.49.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.49.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.49.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.49.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.50.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.50.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.51.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.51.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.52.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.52.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.53.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.53.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.54.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.54.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.55.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.55.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.final_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.final_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.final_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.final_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.output_layer.weight", + "shape": [ + 99072, + 2048 + ], + "optimizer": "adamh", + "family": "lm_head", + "expert": false + } + ], + "family_counts": { + "embedding": 1, + "in_proj": 294, + "out_proj": 49, + "linear_fc1": 56, + "linear_fc2": 56, + "linear_proj": 7, + "linear_qkv": 7, + "lm_head": 1 + }, + "moment_reset": false, + "loaded_step_values": { + "0": [ + 52788.0 + ], + "1": [] + }, + "native_optimizer_types": [ + "FusedAdam", + "FusedAdam" + ], + "full_matrix_norms_across_dp_shards": true, + "tp": 1, + "pp": 1 +} diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/grouping_rank0046.json b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/grouping_rank0046.json new file mode 100644 index 0000000000000000000000000000000000000000..3006931298674fb0587dfcd4e07ec59427a419f8 --- /dev/null +++ b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/grouping_rank0046.json @@ -0,0 +1,13227 @@ +{ + "rank": 46, + "policy": "dc_adamh_named_matrices_v1", + "parameter_groups": [ + { + "name": "module.module.embedding.word_embeddings.weight", + "shape": [ + 99072, + 2048 + ], + "optimizer": "adamh", + "family": "embedding", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.mlp.linear_fc1.weight", + "shape": [ + 9600, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.mlp.linear_fc2.weight", + "shape": [ + 2048, + 4800 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.mlp.linear_fc1.weight", + "shape": [ + 9600, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.mlp.linear_fc2.weight", + "shape": [ + 2048, + 4800 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.2.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.2.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.3.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.3.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.4.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.4.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.5.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.5.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.6.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.6.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.7.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.7.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.8.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.8.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.9.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.9.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.10.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.10.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.11.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.11.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.12.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.12.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.12.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.12.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.12.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.12.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.13.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.13.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.14.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.14.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.15.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.15.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.16.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.16.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.17.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.17.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.18.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.18.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.19.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.19.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.20.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.20.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.20.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.20.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.20.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.20.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.21.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.21.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.21.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.21.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.21.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.21.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.22.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.22.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.23.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.23.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.23.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.23.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.23.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.23.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.24.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.24.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.25.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.25.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.26.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.26.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.27.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.27.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.28.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.28.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.29.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.29.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.30.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.30.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.31.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.31.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.32.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.32.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.33.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.33.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.34.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.34.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.35.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.35.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.36.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.36.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.37.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.37.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.38.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.38.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.39.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.39.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.40.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.40.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.41.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.41.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.42.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.42.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.43.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.43.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.44.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.44.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.45.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.45.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.46.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.46.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.46.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.46.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.46.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.46.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.47.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.47.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.48.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.48.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.48.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.48.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.48.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.48.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.49.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.49.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.49.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.49.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.49.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.49.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.50.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.50.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.51.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.51.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.52.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.52.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.53.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.53.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.54.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.54.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.55.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.55.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.final_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.final_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.final_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.final_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.output_layer.weight", + "shape": [ + 99072, + 2048 + ], + "optimizer": "adamh", + "family": "lm_head", + "expert": false + } + ], + "family_counts": { + "embedding": 1, + "in_proj": 294, + "out_proj": 49, + "linear_fc1": 56, + "linear_fc2": 56, + "linear_proj": 7, + "linear_qkv": 7, + "lm_head": 1 + }, + "moment_reset": false, + "loaded_step_values": { + "0": [ + 52788.0 + ], + "1": [] + }, + "native_optimizer_types": [ + "FusedAdam", + "FusedAdam" + ], + "full_matrix_norms_across_dp_shards": true, + "tp": 1, + "pp": 1 +} diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/grouping_rank0061.json b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/grouping_rank0061.json new file mode 100644 index 0000000000000000000000000000000000000000..9eba9facefaeba6ab2f85459a29377118ffe9d76 --- /dev/null +++ b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/grouping_rank0061.json @@ -0,0 +1,13227 @@ +{ + "rank": 61, + "policy": "dc_adamh_named_matrices_v1", + "parameter_groups": [ + { + "name": "module.module.embedding.word_embeddings.weight", + "shape": [ + 99072, + 2048 + ], + "optimizer": "adamh", + "family": "embedding", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.mlp.linear_fc1.weight", + "shape": [ + 9600, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.mlp.linear_fc2.weight", + "shape": [ + 2048, + 4800 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.mlp.linear_fc1.weight", + "shape": [ + 9600, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.mlp.linear_fc2.weight", + "shape": [ + 2048, + 4800 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.2.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.2.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.3.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.3.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.4.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.4.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.5.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.5.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.6.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.6.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.7.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.7.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.8.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.8.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.9.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.9.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.10.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.10.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.11.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.11.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.12.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.12.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.12.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.12.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.12.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.12.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.13.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.13.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.14.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.14.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.15.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.15.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.16.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.16.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.17.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.17.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.18.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.18.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.19.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.19.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.20.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.20.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.20.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.20.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.20.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.20.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.21.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.21.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.21.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.21.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.21.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.21.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.22.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.22.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.23.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.23.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.23.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.23.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.23.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.23.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.24.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.24.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.25.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.25.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.26.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.26.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.27.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.27.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.28.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.28.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.29.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.29.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.30.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.30.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.31.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.31.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.32.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.32.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.33.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.33.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.34.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.34.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.35.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.35.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.36.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.36.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.37.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.37.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.38.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.38.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.39.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.39.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.40.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.40.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.41.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.41.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.42.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.42.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.43.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.43.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.44.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.44.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.45.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.45.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.46.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.46.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.46.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.46.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.46.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.46.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.47.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.47.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.48.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.48.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.48.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.48.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.48.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.48.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.49.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.49.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.49.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.49.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.49.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.49.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.50.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.50.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.51.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.51.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.52.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.52.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.53.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.53.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.54.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.54.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.55.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.55.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.final_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.final_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.final_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.final_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.output_layer.weight", + "shape": [ + 99072, + 2048 + ], + "optimizer": "adamh", + "family": "lm_head", + "expert": false + } + ], + "family_counts": { + "embedding": 1, + "in_proj": 294, + "out_proj": 49, + "linear_fc1": 56, + "linear_fc2": 56, + "linear_proj": 7, + "linear_qkv": 7, + "lm_head": 1 + }, + "moment_reset": false, + "loaded_step_values": { + "0": [ + 52788.0 + ], + "1": [] + }, + "native_optimizer_types": [ + "FusedAdam", + "FusedAdam" + ], + "full_matrix_norms_across_dp_shards": true, + "tp": 1, + "pp": 1 +} diff --git a/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/grouping_rank0062.json b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/grouping_rank0062.json new file mode 100644 index 0000000000000000000000000000000000000000..df156ee17019de5f1a7cffebe3d393eda3ec1be3 --- /dev/null +++ b/f3-plain-shortconv-adamh-from32780-20260915a/stage16a_40B_8node_gdnlralpha_off_20261010a/runtime_audit/grouping_rank0062.json @@ -0,0 +1,13227 @@ +{ + "rank": 62, + "policy": "dc_adamh_named_matrices_v1", + "parameter_groups": [ + { + "name": "module.module.embedding.word_embeddings.weight", + "shape": [ + 99072, + 2048 + ], + "optimizer": "adamh", + "family": "embedding", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.0.mlp.linear_fc1.weight", + "shape": [ + 9600, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.0.mlp.linear_fc2.weight", + "shape": [ + 2048, + 4800 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.1.mlp.linear_fc1.weight", + "shape": [ + 9600, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.1.mlp.linear_fc2.weight", + "shape": [ + 2048, + 4800 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.2.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.2.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.2.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.2.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.3.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.3.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.3.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.3.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.4.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.4.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.4.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.4.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.5.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.5.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.5.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.5.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.6.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.6.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.6.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.6.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.7.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.7.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.7.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.7.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.8.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.8.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.8.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.8.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.9.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.9.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.9.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.9.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.10.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.10.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.10.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.10.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.11.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.11.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.11.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.11.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.12.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.12.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.12.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.12.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.12.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.12.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.12.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.13.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.13.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.13.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.13.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.14.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.14.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.14.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.14.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.15.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.15.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.15.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.15.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.16.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.16.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.16.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.16.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.17.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.17.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.17.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.17.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.18.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.18.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.18.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.18.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.19.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.19.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.19.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.19.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.20.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.20.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.20.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.20.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.20.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.20.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.20.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.21.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.21.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.21.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.21.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.21.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.21.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.21.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.22.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.22.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.22.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.22.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.23.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.23.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.23.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.23.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.23.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.23.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.23.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.24.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.24.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.24.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.24.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.25.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.25.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.25.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.25.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.26.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.26.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.26.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.26.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.27.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.27.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.27.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.27.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.28.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.28.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.28.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.28.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.29.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.29.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.29.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.29.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.30.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.30.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.30.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.30.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.31.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.31.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.31.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.31.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.32.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.32.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.32.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.32.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.33.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.33.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.33.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.33.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.34.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.34.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.34.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.34.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.35.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.35.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.35.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.35.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.36.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.36.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.36.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.36.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.37.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.37.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.37.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.37.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.38.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.38.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.38.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.38.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.39.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.39.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.39.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.39.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.40.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.40.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.40.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.40.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.41.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.41.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.41.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.41.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.42.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.42.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.42.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.42.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.43.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.43.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.43.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.43.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.44.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.44.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.44.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.44.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.45.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.45.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.45.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.45.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.46.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.46.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.46.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.46.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.46.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.46.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.46.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.47.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.47.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.47.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.47.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.48.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.48.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.48.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.48.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.48.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.48.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.48.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.49.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.self_attention.linear_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "linear_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.49.self_attention.linear_qkv.weight", + "shape": [ + 5120, + 2048 + ], + "optimizer": "adamh", + "family": "linear_qkv", + "expert": false + }, + { + "name": "module.module.decoder.layers.49.self_attention.linear_qkv.bias", + "shape": [ + 5120 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.49.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.49.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.49.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.49.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.50.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.50.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.50.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.50.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.51.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.51.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.51.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.51.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.52.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.52.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.52.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.52.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.53.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.53.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.53.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.53.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.54.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.54.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.54.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.54.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.input_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.input_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.input_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.input_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.dt_bias", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.A_log", + "shape": [ + 32 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.q_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.k_proj.weight", + "shape": [ + 512, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.v_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.z_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.b_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.a_proj.weight", + "shape": [ + 32, + 2048 + ], + "optimizer": "adamh", + "family": "in_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.conv1d.weight", + "shape": [ + 3072, + 1, + 4 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.out_norm.weight", + "shape": [ + 64 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.self_attention.out_proj.weight", + "shape": [ + 2048, + 2048 + ], + "optimizer": "adamh", + "family": "out_proj", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.pre_mlp_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.pre_mlp_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.pre_mlp_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.pre_mlp_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.mlp.router.weight", + "shape": [ + 192, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.mlp.experts.fc1_weight", + "shape": [ + 24, + 1024, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.55.mlp.experts.fc2_weight", + "shape": [ + 24, + 2048, + 512 + ], + "optimizer": "adamw", + "family": null, + "expert": true + }, + { + "name": "module.module.decoder.layers.55.mlp.shared_experts.gate_weight", + "shape": [ + 1, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.layers.55.mlp.shared_experts.linear_fc1.weight", + "shape": [ + 1024, + 2048 + ], + "optimizer": "adamh", + "family": "linear_fc1", + "expert": false + }, + { + "name": "module.module.decoder.layers.55.mlp.shared_experts.linear_fc2.weight", + "shape": [ + 2048, + 512 + ], + "optimizer": "adamh", + "family": "linear_fc2", + "expert": false + }, + { + "name": "module.module.decoder.final_layernorm.norm.weight", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.final_layernorm.gate_down.weight", + "shape": [ + 16, + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.final_layernorm.gate_up.weight", + "shape": [ + 2048, + 16 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.decoder.final_layernorm.gate_up.bias", + "shape": [ + 2048 + ], + "optimizer": "adamw", + "family": null, + "expert": false + }, + { + "name": "module.module.output_layer.weight", + "shape": [ + 99072, + 2048 + ], + "optimizer": "adamh", + "family": "lm_head", + "expert": false + } + ], + "family_counts": { + "embedding": 1, + "in_proj": 294, + "out_proj": 49, + "linear_fc1": 56, + "linear_fc2": 56, + "linear_proj": 7, + "linear_qkv": 7, + "lm_head": 1 + }, + "moment_reset": false, + "loaded_step_values": { + "0": [ + 52788.0 + ], + "1": [] + }, + "native_optimizer_types": [ + "FusedAdam", + "FusedAdam" + ], + "full_matrix_norms_across_dp_shards": true, + "tp": 1, + "pp": 1 +}